From 53f88894d52a845d1bd80a5917243261d3c2c462 Mon Sep 17 00:00:00 2001 From: Andri Yngvason Date: Thu, 14 Apr 2022 16:56:41 +0000 Subject: [PATCH] Add presentation timestamps --- include/encoder.h | 8 +++--- include/fb.h | 3 ++- include/h264-encoder.h | 17 ++++++++++++- include/neatvnc.h | 6 +++++ include/rfb-proto.h | 3 ++- src/encoder.c | 4 +-- src/fb.c | 18 +++++++++++++- src/h264-encoder.c | 19 ++++++++++++++- src/open-h264.c | 16 +++++++++--- src/raw-encoding.c | 5 ++-- src/server.c | 55 ++++++++++++++++++++++++++++++++++++------ src/tight.c | 9 +++++-- src/zrle.c | 5 ++-- 13 files changed, 140 insertions(+), 28 deletions(-) diff --git a/include/encoder.h b/include/encoder.h index 4d441f7..6aece2a 100644 --- a/include/encoder.h +++ b/include/encoder.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Andri Yngvason + * Copyright (c) 2021 - 2022 Andri Yngvason * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -52,7 +52,7 @@ struct encoder_impl { int (*push)(struct encoder*, struct nvnc_fb* fb, struct pixman_region16* damage); - struct rcbuf* (*pull)(struct encoder*); + struct rcbuf* (*pull)(struct encoder*, uint64_t* pts); void (*request_key_frame)(struct encoder*); }; @@ -65,7 +65,7 @@ struct encoder { int n_rects; - void (*on_done)(struct encoder*, struct rcbuf* result); + void (*on_done)(struct encoder*, struct rcbuf* result, uint64_t pts); void* userdata; }; @@ -87,6 +87,6 @@ int encoder_encode(struct encoder* self, struct nvnc_fb* fb, int encoder_push(struct encoder* self, struct nvnc_fb* fb, struct pixman_region16* damage); -struct rcbuf* encoder_pull(struct encoder* self); +struct rcbuf* encoder_pull(struct encoder* self, uint64_t* pts); void encoder_request_key_frame(struct encoder* self); diff --git a/include/fb.h b/include/fb.h index 1d19081..0bcc989 100644 --- a/include/fb.h +++ b/include/fb.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 - 2021 Andri Yngvason + * Copyright (c) 2019 - 2022 Andri Yngvason * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -38,6 +38,7 @@ struct nvnc_fb { uint16_t height; uint32_t fourcc_format; enum nvnc_transform transform; + uint64_t pts; // in micro seconds /* main memory buffer attributes */ void* addr; diff --git a/include/h264-encoder.h b/include/h264-encoder.h index 7465b10..1c33878 100644 --- a/include/h264-encoder.h +++ b/include/h264-encoder.h @@ -1,3 +1,18 @@ +/* + * Copyright (c) 2021 - 2022 Andri Yngvason + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, + * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM + * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE + * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THIS SOFTWARE. + */ #pragma once #include @@ -7,7 +22,7 @@ struct h264_encoder; struct nvnc_fb; typedef void (*h264_encoder_packet_handler_fn)(const void* payload, size_t size, - void* userdata); + uint64_t pts, void* userdata); struct h264_encoder* h264_encoder_create(uint32_t width, uint32_t height, uint32_t format); diff --git a/include/neatvnc.h b/include/neatvnc.h index bec11b1..6c88dfe 100644 --- a/include/neatvnc.h +++ b/include/neatvnc.h @@ -18,6 +18,9 @@ #include #include +#include + +#define NVNC_NO_PTS UINT64_MAX struct nvnc; struct nvnc_client; @@ -110,6 +113,8 @@ void nvnc_fb_set_release_fn(struct nvnc_fb* fb, nvnc_fb_release_fn fn, void* context); void nvnc_fb_set_transform(struct nvnc_fb* fb, enum nvnc_transform); +void nvnc_fb_set_pts(struct nvnc_fb* fb, uint64_t pts); + void* nvnc_fb_get_addr(const struct nvnc_fb* fb); uint16_t nvnc_fb_get_width(const struct nvnc_fb* fb); uint16_t nvnc_fb_get_height(const struct nvnc_fb* fb); @@ -119,6 +124,7 @@ int nvnc_fb_get_pixel_size(const struct nvnc_fb* fb); struct gbm_bo* nvnc_fb_get_gbm_bo(const struct nvnc_fb* fb); enum nvnc_transform nvnc_fb_get_transform(const struct nvnc_fb* fb); enum nvnc_fb_type nvnc_fb_get_type(const struct nvnc_fb* fb); +uint64_t nvnc_fb_get_pts(const struct nvnc_fb* fb); struct nvnc_fb_pool* nvnc_fb_pool_new(uint16_t width, uint16_t height, uint32_t fourcc_format, uint16_t stride); diff --git a/include/rfb-proto.h b/include/rfb-proto.h index 223e29f..fcec224 100644 --- a/include/rfb-proto.h +++ b/include/rfb-proto.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Andri Yngvason + * Copyright (c) 2019 - 2022 Andri Yngvason * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -66,6 +66,7 @@ enum rfb_encodings { RFB_ENCODING_JPEG_HIGHQ = -23, RFB_ENCODING_JPEG_LOWQ = -32, RFB_ENCODING_QEMU_EXT_KEY_EVENT = -258, + RFB_ENCODING_PTS = -1000, }; enum rfb_server_to_client_msg_type { diff --git a/src/encoder.c b/src/encoder.c index 7d93781..cbd2c1d 100644 --- a/src/encoder.c +++ b/src/encoder.c @@ -127,10 +127,10 @@ int encoder_push(struct encoder* self, struct nvnc_fb* fb, return -1; } -struct rcbuf* encoder_pull(struct encoder* self) +struct rcbuf* encoder_pull(struct encoder* self, uint64_t* pts) { if (self->impl->pull) - return self->impl->pull(self); + return self->impl->pull(self, pts); assert(self->impl->encode && !self->impl->push); return NULL; diff --git a/src/fb.c b/src/fb.c index 440e282..2f3db00 100644 --- a/src/fb.c +++ b/src/fb.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 - 2021 Andri Yngvason + * Copyright (c) 2019 - 2022 Andri Yngvason * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -48,6 +48,7 @@ struct nvnc_fb* nvnc_fb_new(uint16_t width, uint16_t height, fb->height = height; fb->fourcc_format = fourcc_format; fb->stride = stride; + fb->pts = NVNC_NO_PTS; size_t size = height * stride * 4; /* Assume 4 byte format for now */ size_t alignment = MAX(4, sizeof(void*)); @@ -78,6 +79,7 @@ struct nvnc_fb* nvnc_fb_from_buffer(void* buffer, uint16_t width, uint16_t heigh fb->height = height; fb->fourcc_format = fourcc_format; fb->stride = stride; + fb->pts = NVNC_NO_PTS; return fb; } @@ -97,6 +99,7 @@ struct nvnc_fb* nvnc_fb_from_gbm_bo(struct gbm_bo* bo) fb->height = gbm_bo_get_height(bo); fb->fourcc_format = gbm_bo_get_format(bo); fb->bo = bo; + fb->pts = NVNC_NO_PTS; return fb; #else @@ -159,6 +162,12 @@ enum nvnc_fb_type nvnc_fb_get_type(const struct nvnc_fb* fb) return fb->type; } +EXPORT +uint64_t nvnc_fb_get_pts(const struct nvnc_fb* fb) +{ + return fb->pts; +} + static void nvnc__fb_free(struct nvnc_fb* fb) { nvnc_cleanup_fn cleanup = fb->common.cleanup_fn; @@ -208,6 +217,12 @@ void nvnc_fb_set_transform(struct nvnc_fb* fb, enum nvnc_transform transform) fb->transform = transform; } +EXPORT +void nvnc_fb_set_pts(struct nvnc_fb* fb, uint64_t pts) +{ + fb->pts = pts; +} + void nvnc_fb_hold(struct nvnc_fb* fb) { fb->hold_count++; @@ -219,6 +234,7 @@ void nvnc_fb_release(struct nvnc_fb* fb) return; nvnc_fb_unmap(fb); + fb->pts = NVNC_NO_PTS; if (fb->on_release) fb->on_release(fb, fb->release_context); diff --git a/src/h264-encoder.c b/src/h264-encoder.c index fbbb780..a44c771 100644 --- a/src/h264-encoder.c +++ b/src/h264-encoder.c @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2021 - 2022 Andri Yngvason + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, + * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM + * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE + * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THIS SOFTWARE. + */ + #include "h264-encoder.h" #include "neatvnc.h" #include "fb.h" @@ -416,6 +432,7 @@ static void h264_encoder__on_work_done(void* handle) { struct h264_encoder* self = aml_get_userdata(handle); + uint64_t pts = nvnc_fb_get_pts(self->current_fb); nvnc_fb_release(self->current_fb); nvnc_fb_unref(self->current_fb); self->current_fb = NULL; @@ -430,7 +447,7 @@ static void h264_encoder__on_work_done(void* handle) return; self->on_packet_ready(self->current_packet.data, - self->current_packet.len, self->userdata); + self->current_packet.len, pts, self->userdata); vec_clear(&self->current_packet); h264_encoder__schedule_work(self); diff --git a/src/open-h264.c b/src/open-h264.c index 6929cce..a4f3518 100644 --- a/src/open-h264.c +++ b/src/open-h264.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Andri Yngvason + * Copyright (c) 2021 - 2022 Andri Yngvason * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -38,6 +38,7 @@ struct open_h264 { struct h264_encoder* encoder; struct vec pending; + uint64_t pts; uint32_t width; uint32_t height; @@ -60,7 +61,7 @@ static inline struct open_h264* open_h264(struct encoder* enc) return (struct open_h264*)enc; } -static void open_h264_handle_packet(const void* data, size_t size, +static void open_h264_handle_packet(const void* data, size_t size, uint64_t pts, void* userdata) { struct open_h264* self = userdata; @@ -73,9 +74,10 @@ static void open_h264_handle_packet(const void* data, size_t size, } vec_append(&self->pending, data, size); + self->pts = pts; if (self->parent.on_done) - self->parent.on_done(&self->parent, NULL); + self->parent.on_done(&self->parent, NULL, NVNC_NO_PTS); } static int open_h264_init_pending(struct open_h264* self) @@ -102,6 +104,8 @@ struct encoder* open_h264_new(void) return NULL; } + self->pts = NVNC_NO_PTS; + return (struct encoder*)self; } @@ -158,7 +162,7 @@ static int open_h264_push(struct encoder* enc, struct nvnc_fb* fb, return 0; } -static struct rcbuf* open_h264_pull(struct encoder* enc) +static struct rcbuf* open_h264_pull(struct encoder* enc, uint64_t* pts) { struct open_h264* self = open_h264(enc); @@ -168,6 +172,10 @@ static struct rcbuf* open_h264_pull(struct encoder* enc) if (payload_size == 0) return NULL; + if (pts) + *pts = self->pts; + self->pts = NVNC_NO_PTS; + uint32_t flags = self->needs_reset ? OPEN_H264_FLAG_RESET_CONTEXT : 0; self->needs_reset = false; diff --git a/src/raw-encoding.c b/src/raw-encoding.c index ba0482d..d29f8a0 100644 --- a/src/raw-encoding.c +++ b/src/raw-encoding.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 - 2021 Andri Yngvason + * Copyright (c) 2019 - 2022 Andri Yngvason * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -161,6 +161,7 @@ static void raw_encoder_on_done(void* obj) assert(self->current_result); + uint64_t pts = nvnc_fb_get_pts(self->current_fb); nvnc_fb_unref(self->current_fb); self->current_fb = NULL; @@ -173,7 +174,7 @@ static void raw_encoder_on_done(void* obj) self->work = NULL; if (self->encoder.on_done) - self->encoder.on_done(&self->encoder, result); + self->encoder.on_done(&self->encoder, result, pts); rcbuf_unref(result); } diff --git a/src/server.c b/src/server.c index bc2da9f..4a8d35a 100644 --- a/src/server.c +++ b/src/server.c @@ -45,6 +45,7 @@ #include #include #include +#include #ifdef ENABLE_TLS #include @@ -72,7 +73,7 @@ static int send_qemu_key_ext_frame(struct nvnc_client* client); static enum rfb_encodings choose_frame_encoding(struct nvnc_client* client, struct nvnc_fb*); static enum tight_quality client_get_tight_quality(struct nvnc_client* client); -static void on_encode_frame_done(struct encoder*, struct rcbuf*); +static void on_encode_frame_done(struct encoder*, struct rcbuf*, uint64_t pts); static bool client_has_encoding(const struct nvnc_client* client, enum rfb_encodings encoding); static void process_fb_update_requests(struct nvnc_client* client); @@ -87,6 +88,15 @@ EXPORT const char nvnc_version[] = "UNKNOWN"; extern const unsigned short code_map_qnum_to_linux[]; +static uint64_t nvnc__htonll(uint64_t x) +{ +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + return bswap_64(x); +#else + return x; +#endif +} + static void client_close(struct nvnc_client* client) { log_debug("client_close(%p): ref %d\n", client, client->ref); @@ -502,6 +512,7 @@ static int on_client_set_encodings(struct nvnc_client* client) case RFB_ENCODING_JPEG_HIGHQ: case RFB_ENCODING_JPEG_LOWQ: case RFB_ENCODING_QEMU_EXT_KEY_EVENT: + case RFB_ENCODING_PTS: client->encodings[n++] = encoding; } } @@ -511,9 +522,11 @@ static int on_client_set_encodings(struct nvnc_client* client) return sizeof(*msg) + 4 * n_encodings; } -static void on_encoder_push_done(struct encoder* encoder, struct rcbuf* payload) +static void on_encoder_push_done(struct encoder* encoder, struct rcbuf* payload, + uint64_t pts) { (void)payload; + (void)pts; struct nvnc_client* client = encoder->userdata; process_fb_update_requests(client); @@ -548,6 +561,25 @@ static void send_cursor_update(struct nvnc_client* client) NULL, NULL); } +static bool will_send_pts(const struct nvnc_client* client, uint64_t pts) +{ + return pts != NVNC_NO_PTS && client_has_encoding(client, RFB_ENCODING_PTS); +} + +static void send_pts_rect(struct nvnc_client* client, uint64_t pts) +{ + if (!will_send_pts(client, pts)) + return; + + uint8_t buf[sizeof(struct rfb_server_fb_rect) + 8] = { 0 }; + struct rfb_server_fb_rect* head = (struct rfb_server_fb_rect*)buf; + head->encoding = htonl(RFB_ENCODING_PTS); + uint64_t* msg_pts = (uint64_t*)&buf[sizeof(struct rfb_server_fb_rect)]; + *msg_pts = nvnc__htonll(pts); + + stream_write(client->net_stream, buf, sizeof(buf), NULL, NULL); +} + static void process_fb_update_requests(struct nvnc_client* client) { struct nvnc* server = client->server; @@ -628,17 +660,23 @@ static void process_fb_update_requests(struct nvnc_client* client) enum encoder_kind kind = encoder_get_kind(client->encoder); if (kind == ENCODER_KIND_PUSH_PULL) { - struct rcbuf* buf = encoder_pull(client->encoder); + uint64_t pts = NVNC_NO_PTS; + struct rcbuf* buf = encoder_pull(client->encoder, &pts); if (!buf) return; + int n_rects = client->encoder->n_rects; + n_rects += will_send_pts(client, pts) ? 1 : 0; + struct rfb_server_fb_update_msg update_msg = { .type = RFB_SERVER_TO_CLIENT_FRAMEBUFFER_UPDATE, - .n_rects = htons(client->encoder->n_rects), + .n_rects = htons(n_rects), }; stream_write(client->net_stream, &update_msg, sizeof(update_msg), NULL, NULL); + send_pts_rect(client, pts); + stream_send(client->net_stream, buf, NULL, NULL); pixman_region_clear(&client->damage); --client->n_pending_requests; @@ -1323,18 +1361,20 @@ static bool client_has_encoding(const struct nvnc_client* client, } static void finish_fb_update(struct nvnc_client* client, struct rcbuf* payload, - int n_rects) + int n_rects, uint64_t pts) { client_ref(client); if (client->net_stream->state != STREAM_STATE_CLOSED) { DTRACE_PROBE1(neatvnc, send_fb_start, client); + n_rects += will_send_pts(client, pts) ? 1 : 0; struct rfb_server_fb_update_msg update_msg = { .type = RFB_SERVER_TO_CLIENT_FRAMEBUFFER_UPDATE, .n_rects = htons(n_rects), }; stream_write(client->net_stream, &update_msg, sizeof(update_msg), NULL, NULL); + send_pts_rect(client, pts); rcbuf_ref(payload); stream_send(client->net_stream, payload, on_write_frame_done, client); @@ -1352,10 +1392,11 @@ static void finish_fb_update(struct nvnc_client* client, struct rcbuf* payload, DTRACE_PROBE1(neatvnc, update_fb_done, client); } -static void on_encode_frame_done(struct encoder* encoder, struct rcbuf* result) +static void on_encode_frame_done(struct encoder* encoder, struct rcbuf* result, + uint64_t pts) { struct nvnc_client* client = encoder->userdata; - finish_fb_update(client, result, encoder->n_rects); + finish_fb_update(client, result, encoder->n_rects, pts); client_unref(client); } diff --git a/src/tight.c b/src/tight.c index d10f52e..303da1c 100644 --- a/src/tight.c +++ b/src/tight.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 - 2021 Andri Yngvason + * Copyright (c) 2019 - 2022 Andri Yngvason * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -77,6 +77,7 @@ struct tight_encoder { struct rfb_pixel_format dfmt; struct rfb_pixel_format sfmt; struct nvnc_fb* fb; + uint64_t pts; uint32_t n_rects; uint32_t n_jobs; @@ -203,6 +204,8 @@ static int tight_encoder_init(struct tight_encoder* self, uint32_t width, aml_require_workers(aml_get_default(), 1); + self->pts = NVNC_NO_PTS; + return 0; } @@ -517,8 +520,9 @@ static void on_tight_finished(void* obj) self->encoder.n_rects = self->n_rects; if (self->encoder.on_done) - self->encoder.on_done(&self->encoder, result); + self->encoder.on_done(&self->encoder, result, self->pts); + self->pts = NVNC_NO_PTS; rcbuf_unref(result); } @@ -588,6 +592,7 @@ static int tight_encoder_encode(struct encoder* encoder, struct nvnc_fb* fb, assert(rc == 0); self->fb = fb; + self->pts = nvnc_fb_get_pts(fb); rc = nvnc_fb_map(self->fb); if (rc < 0) diff --git a/src/zrle.c b/src/zrle.c index 1642252..228d9f8 100644 --- a/src/zrle.c +++ b/src/zrle.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 - 2021 Andri Yngvason + * Copyright (c) 2019 - 2022 Andri Yngvason * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -363,6 +363,7 @@ static void zrle_encoder_on_done(void* obj) assert(self->current_result); + uint64_t pts = nvnc_fb_get_pts(self->current_fb); nvnc_fb_unref(self->current_fb); self->current_fb = NULL; @@ -375,7 +376,7 @@ static void zrle_encoder_on_done(void* obj) self->work = NULL; if (self->encoder.on_done) - self->encoder.on_done(&self->encoder, result); + self->encoder.on_done(&self->encoder, result, pts); rcbuf_unref(result); }