Add presentation timestamps

include/encoder.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Andri Yngvason
+ * Copyright (c) 2021 - 2022 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -52,7 +52,7 @@ struct encoder_impl {
 
 	int (*push)(struct encoder*, struct nvnc_fb* fb,
 			struct pixman_region16* damage);
-	struct rcbuf* (*pull)(struct encoder*);
+	struct rcbuf* (*pull)(struct encoder*, uint64_t* pts);
 
 	void (*request_key_frame)(struct encoder*);
 };
@@ -65,7 +65,7 @@ struct encoder {
 
 	int n_rects;
 
-	void (*on_done)(struct encoder*, struct rcbuf* result);
+	void (*on_done)(struct encoder*, struct rcbuf* result, uint64_t pts);
 	void* userdata;
 };
 
@@ -87,6 +87,6 @@ int encoder_encode(struct encoder* self, struct nvnc_fb* fb,
 
 int encoder_push(struct encoder* self, struct nvnc_fb* fb,
 		struct pixman_region16* damage);
-struct rcbuf* encoder_pull(struct encoder* self);
+struct rcbuf* encoder_pull(struct encoder* self, uint64_t* pts);
 
 void encoder_request_key_frame(struct encoder* self);

include/fb.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019 - 2021 Andri Yngvason
+ * Copyright (c) 2019 - 2022 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -38,6 +38,7 @@ struct nvnc_fb {
 	uint16_t height;
 	uint32_t fourcc_format;
 	enum nvnc_transform transform;
+	uint64_t pts; // in micro seconds
 
 	/* main memory buffer attributes */
 	void* addr;

include/h264-encoder.h

@@ -1,3 +1,18 @@
+/*
+ * Copyright (c) 2021 - 2022 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
 #pragma once
 
 #include <stdint.h>
@@ -7,7 +22,7 @@ struct h264_encoder;
 struct nvnc_fb;
 
 typedef void (*h264_encoder_packet_handler_fn)(const void* payload, size_t size,
-		void* userdata);
+		uint64_t pts, void* userdata);
 
 struct h264_encoder* h264_encoder_create(uint32_t width, uint32_t height,
 		uint32_t format);

include/neatvnc.h

@@ -18,6 +18,9 @@
 
 #include <stdint.h>
 #include <stdbool.h>
+#include <limits.h>
+
+#define NVNC_NO_PTS UINT64_MAX
 
 struct nvnc;
 struct nvnc_client;
@@ -110,6 +113,8 @@ void nvnc_fb_set_release_fn(struct nvnc_fb* fb, nvnc_fb_release_fn fn,
 			    void* context);
 void nvnc_fb_set_transform(struct nvnc_fb* fb, enum nvnc_transform);
 
+void nvnc_fb_set_pts(struct nvnc_fb* fb, uint64_t pts);
+
 void* nvnc_fb_get_addr(const struct nvnc_fb* fb);
 uint16_t nvnc_fb_get_width(const struct nvnc_fb* fb);
 uint16_t nvnc_fb_get_height(const struct nvnc_fb* fb);
@@ -119,6 +124,7 @@ int nvnc_fb_get_pixel_size(const struct nvnc_fb* fb);
 struct gbm_bo* nvnc_fb_get_gbm_bo(const struct nvnc_fb* fb);
 enum nvnc_transform nvnc_fb_get_transform(const struct nvnc_fb* fb);
 enum nvnc_fb_type nvnc_fb_get_type(const struct nvnc_fb* fb);
+uint64_t nvnc_fb_get_pts(const struct nvnc_fb* fb);
 
 struct nvnc_fb_pool* nvnc_fb_pool_new(uint16_t width, uint16_t height,
 				      uint32_t fourcc_format, uint16_t stride);

include/rfb-proto.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019 Andri Yngvason
+ * Copyright (c) 2019 - 2022 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -66,6 +66,7 @@ enum rfb_encodings {
 	RFB_ENCODING_JPEG_HIGHQ = -23,
 	RFB_ENCODING_JPEG_LOWQ = -32,
 	RFB_ENCODING_QEMU_EXT_KEY_EVENT = -258,
+	RFB_ENCODING_PTS = -1000,
 };
 
 enum rfb_server_to_client_msg_type {

src/encoder.c

@@ -127,10 +127,10 @@ int encoder_push(struct encoder* self, struct nvnc_fb* fb,
 	return -1;
 }
 
-struct rcbuf* encoder_pull(struct encoder* self)
+struct rcbuf* encoder_pull(struct encoder* self, uint64_t* pts)
 {
 	if (self->impl->pull)
-		return self->impl->pull(self);
+		return self->impl->pull(self, pts);
 
 	assert(self->impl->encode && !self->impl->push);
 	return NULL;

src/fb.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019 - 2021 Andri Yngvason
+ * Copyright (c) 2019 - 2022 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -48,6 +48,7 @@ struct nvnc_fb* nvnc_fb_new(uint16_t width, uint16_t height,
 	fb->height = height;
 	fb->fourcc_format = fourcc_format;
 	fb->stride = stride;
+	fb->pts = NVNC_NO_PTS;
 
 	size_t size = height * stride * 4; /* Assume 4 byte format for now */
 	size_t alignment = MAX(4, sizeof(void*));
@@ -78,6 +79,7 @@ struct nvnc_fb* nvnc_fb_from_buffer(void* buffer, uint16_t width, uint16_t heigh
 	fb->height = height;
 	fb->fourcc_format = fourcc_format;
 	fb->stride = stride;
+	fb->pts = NVNC_NO_PTS;
 
 	return fb;
 }
@@ -97,6 +99,7 @@ struct nvnc_fb* nvnc_fb_from_gbm_bo(struct gbm_bo* bo)
 	fb->height = gbm_bo_get_height(bo);
 	fb->fourcc_format = gbm_bo_get_format(bo);
 	fb->bo = bo;
+	fb->pts = NVNC_NO_PTS;
 
 	return fb;
 #else
@@ -159,6 +162,12 @@ enum nvnc_fb_type nvnc_fb_get_type(const struct nvnc_fb* fb)
 	return fb->type;
 }
 
+EXPORT
+uint64_t nvnc_fb_get_pts(const struct nvnc_fb* fb)
+{
+	return fb->pts;
+}
+
 static void nvnc__fb_free(struct nvnc_fb* fb)
 {
 	nvnc_cleanup_fn cleanup = fb->common.cleanup_fn;
@@ -208,6 +217,12 @@ void nvnc_fb_set_transform(struct nvnc_fb* fb, enum nvnc_transform transform)
 	fb->transform = transform;
 }
 
+EXPORT
+void nvnc_fb_set_pts(struct nvnc_fb* fb, uint64_t pts)
+{
+	fb->pts = pts;
+}
+
 void nvnc_fb_hold(struct nvnc_fb* fb)
 {
 	fb->hold_count++;
@@ -219,6 +234,7 @@ void nvnc_fb_release(struct nvnc_fb* fb)
 		return;
 
 	nvnc_fb_unmap(fb);
+	fb->pts = NVNC_NO_PTS;
 
 	if (fb->on_release)
 		fb->on_release(fb, fb->release_context);

src/h264-encoder.c

@@ -1,3 +1,19 @@
+/*
+ * Copyright (c) 2021 - 2022 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
 #include "h264-encoder.h"
 #include "neatvnc.h"
 #include "fb.h"
@@ -416,6 +432,7 @@ static void h264_encoder__on_work_done(void* handle)
 {
 	struct h264_encoder* self = aml_get_userdata(handle);
 
+	uint64_t pts = nvnc_fb_get_pts(self->current_fb);
 	nvnc_fb_release(self->current_fb);
 	nvnc_fb_unref(self->current_fb);
 	self->current_fb = NULL;
@@ -430,7 +447,7 @@ static void h264_encoder__on_work_done(void* handle)
 		return;
 
 	self->on_packet_ready(self->current_packet.data,
-			self->current_packet.len, self->userdata);
+			self->current_packet.len, pts, self->userdata);
 	vec_clear(&self->current_packet);
 
 	h264_encoder__schedule_work(self);

src/open-h264.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Andri Yngvason
+ * Copyright (c) 2021 - 2022 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -38,6 +38,7 @@ struct open_h264 {
 	struct h264_encoder* encoder;
 
 	struct vec pending;
+	uint64_t pts;
 
 	uint32_t width;
 	uint32_t height;
@@ -60,7 +61,7 @@ static inline struct open_h264* open_h264(struct encoder* enc)
 	return (struct open_h264*)enc;
 }
 
-static void open_h264_handle_packet(const void* data, size_t size,
+static void open_h264_handle_packet(const void* data, size_t size, uint64_t pts,
 		void* userdata)
 {
 	struct open_h264* self = userdata;
@@ -73,9 +74,10 @@ static void open_h264_handle_packet(const void* data, size_t size,
 	}
 
 	vec_append(&self->pending, data, size);
+	self->pts = pts;
 
 	if (self->parent.on_done)
-		self->parent.on_done(&self->parent, NULL);
+		self->parent.on_done(&self->parent, NULL, NVNC_NO_PTS);
 }
 
 static int open_h264_init_pending(struct open_h264* self)
@@ -102,6 +104,8 @@ struct encoder* open_h264_new(void)
 		return NULL;
 	}
 
+	self->pts = NVNC_NO_PTS;
+
 	return (struct encoder*)self;
 }
 
@@ -158,7 +162,7 @@ static int open_h264_push(struct encoder* enc, struct nvnc_fb* fb,
 	return 0;
 }
 
-static struct rcbuf* open_h264_pull(struct encoder* enc)
+static struct rcbuf* open_h264_pull(struct encoder* enc, uint64_t* pts)
 {
 	struct open_h264* self = open_h264(enc);
 
@@ -168,6 +172,10 @@ static struct rcbuf* open_h264_pull(struct encoder* enc)
 	if (payload_size == 0)
 		return NULL;
 
+	if (pts)
+		*pts = self->pts;
+	self->pts = NVNC_NO_PTS;
+
 	uint32_t flags = self->needs_reset ? OPEN_H264_FLAG_RESET_CONTEXT : 0;
 	self->needs_reset = false;
 

src/raw-encoding.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019 - 2021 Andri Yngvason
+ * Copyright (c) 2019 - 2022 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -161,6 +161,7 @@ static void raw_encoder_on_done(void* obj)
 
 	assert(self->current_result);
 
+	uint64_t pts = nvnc_fb_get_pts(self->current_fb);
 	nvnc_fb_unref(self->current_fb);
 	self->current_fb = NULL;
 
@@ -173,7 +174,7 @@ static void raw_encoder_on_done(void* obj)
 	self->work = NULL;
 
 	if (self->encoder.on_done)
-		self->encoder.on_done(&self->encoder, result);
+		self->encoder.on_done(&self->encoder, result, pts);
 
 	rcbuf_unref(result);
 }

src/server.c

@@ -45,6 +45,7 @@
 #include <sys/socket.h>
 #include <sys/un.h>
 #include <netdb.h>
+#include <byteswap.h>
 
 #ifdef ENABLE_TLS
 #include <gnutls/gnutls.h>
@@ -72,7 +73,7 @@ static int send_qemu_key_ext_frame(struct nvnc_client* client);
 static enum rfb_encodings choose_frame_encoding(struct nvnc_client* client,
 		struct nvnc_fb*);
 static enum tight_quality client_get_tight_quality(struct nvnc_client* client);
-static void on_encode_frame_done(struct encoder*, struct rcbuf*);
+static void on_encode_frame_done(struct encoder*, struct rcbuf*, uint64_t pts);
 static bool client_has_encoding(const struct nvnc_client* client,
 		enum rfb_encodings encoding);
 static void process_fb_update_requests(struct nvnc_client* client);
@@ -87,6 +88,15 @@ EXPORT const char nvnc_version[] = "UNKNOWN";
 
 extern const unsigned short code_map_qnum_to_linux[];
 
+static uint64_t nvnc__htonll(uint64_t x)
+{
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+	return bswap_64(x);
+#else
+	return x;
+#endif
+}
+
 static void client_close(struct nvnc_client* client)
 {
 	log_debug("client_close(%p): ref %d\n", client, client->ref);
@@ -502,6 +512,7 @@ static int on_client_set_encodings(struct nvnc_client* client)
 		case RFB_ENCODING_JPEG_HIGHQ:
 		case RFB_ENCODING_JPEG_LOWQ:
 		case RFB_ENCODING_QEMU_EXT_KEY_EVENT:
+		case RFB_ENCODING_PTS:
 			client->encodings[n++] = encoding;
 		}
 	}
@@ -511,9 +522,11 @@ static int on_client_set_encodings(struct nvnc_client* client)
 	return sizeof(*msg) + 4 * n_encodings;
 }
 
-static void on_encoder_push_done(struct encoder* encoder, struct rcbuf* payload)
+static void on_encoder_push_done(struct encoder* encoder, struct rcbuf* payload,
+		uint64_t pts)
 {
 	(void)payload;
+	(void)pts;
 
 	struct nvnc_client* client = encoder->userdata;
 	process_fb_update_requests(client);
@@ -548,6 +561,25 @@ static void send_cursor_update(struct nvnc_client* client)
 			NULL, NULL);
 }
 
+static bool will_send_pts(const struct nvnc_client* client, uint64_t pts)
+{
+	return pts != NVNC_NO_PTS && client_has_encoding(client, RFB_ENCODING_PTS);
+}
+
+static void send_pts_rect(struct nvnc_client* client, uint64_t pts)
+{
+	if (!will_send_pts(client, pts))
+		return;
+
+	uint8_t buf[sizeof(struct rfb_server_fb_rect) + 8] = { 0 };
+	struct rfb_server_fb_rect* head = (struct rfb_server_fb_rect*)buf;
+	head->encoding = htonl(RFB_ENCODING_PTS);
+	uint64_t* msg_pts = (uint64_t*)&buf[sizeof(struct rfb_server_fb_rect)];
+	*msg_pts = nvnc__htonll(pts);
+
+	stream_write(client->net_stream, buf, sizeof(buf), NULL, NULL);
+}
+
 static void process_fb_update_requests(struct nvnc_client* client)
 {
 	struct nvnc* server = client->server;
@@ -628,17 +660,23 @@ static void process_fb_update_requests(struct nvnc_client* client)
 
 	enum encoder_kind kind = encoder_get_kind(client->encoder);
 	if (kind == ENCODER_KIND_PUSH_PULL) {
-		struct rcbuf* buf = encoder_pull(client->encoder);
+		uint64_t pts = NVNC_NO_PTS;
+		struct rcbuf* buf = encoder_pull(client->encoder, &pts);
 		if (!buf)
 			return;
 
+		int n_rects = client->encoder->n_rects;
+		n_rects += will_send_pts(client, pts) ? 1 : 0;
+
 		struct rfb_server_fb_update_msg update_msg = {
 			.type = RFB_SERVER_TO_CLIENT_FRAMEBUFFER_UPDATE,
-			.n_rects = htons(client->encoder->n_rects),
+			.n_rects = htons(n_rects),
 		};
 		stream_write(client->net_stream, &update_msg,
 				sizeof(update_msg), NULL, NULL);
 
+		send_pts_rect(client, pts);
+
 		stream_send(client->net_stream, buf, NULL, NULL);
 		pixman_region_clear(&client->damage);
 		--client->n_pending_requests;
@@ -1323,18 +1361,20 @@ static bool client_has_encoding(const struct nvnc_client* client,
 }
 
 static void finish_fb_update(struct nvnc_client* client, struct rcbuf* payload,
-		int n_rects)
+		int n_rects, uint64_t pts)
 {
 	client_ref(client);
 
 	if (client->net_stream->state != STREAM_STATE_CLOSED) {
 		DTRACE_PROBE1(neatvnc, send_fb_start, client);
+		n_rects += will_send_pts(client, pts) ? 1 : 0;
 		struct rfb_server_fb_update_msg update_msg = {
 			.type = RFB_SERVER_TO_CLIENT_FRAMEBUFFER_UPDATE,
 			.n_rects = htons(n_rects),
 		};
 		stream_write(client->net_stream, &update_msg,
 				sizeof(update_msg), NULL, NULL);
+		send_pts_rect(client, pts);
 		rcbuf_ref(payload);
 		stream_send(client->net_stream, payload, on_write_frame_done,
 		            client);
@@ -1352,10 +1392,11 @@ static void finish_fb_update(struct nvnc_client* client, struct rcbuf* payload,
 	DTRACE_PROBE1(neatvnc, update_fb_done, client);
 }
 
-static void on_encode_frame_done(struct encoder* encoder, struct rcbuf* result)
+static void on_encode_frame_done(struct encoder* encoder, struct rcbuf* result,
+		uint64_t pts)
 {
 	struct nvnc_client* client = encoder->userdata;
-	finish_fb_update(client, result, encoder->n_rects);
+	finish_fb_update(client, result, encoder->n_rects, pts);
 	client_unref(client);
 }
 

src/tight.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019 - 2021 Andri Yngvason
+ * Copyright (c) 2019 - 2022 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -77,6 +77,7 @@ struct tight_encoder {
 	struct rfb_pixel_format dfmt;
 	struct rfb_pixel_format sfmt;
 	struct nvnc_fb* fb;
+	uint64_t pts;
 
 	uint32_t n_rects;
 	uint32_t n_jobs;
@@ -203,6 +204,8 @@ static int tight_encoder_init(struct tight_encoder* self, uint32_t width,
 
 	aml_require_workers(aml_get_default(), 1);
 
+	self->pts = NVNC_NO_PTS;
+
 	return 0;
 }
 
@@ -517,8 +520,9 @@ static void on_tight_finished(void* obj)
 	self->encoder.n_rects = self->n_rects;
 
 	if (self->encoder.on_done)
-		self->encoder.on_done(&self->encoder, result);
+		self->encoder.on_done(&self->encoder, result, self->pts);
 
+	self->pts = NVNC_NO_PTS;
 	rcbuf_unref(result);
 }
 
@@ -588,6 +592,7 @@ static int tight_encoder_encode(struct encoder* encoder, struct nvnc_fb* fb,
 	assert(rc == 0);
 
 	self->fb = fb;
+	self->pts = nvnc_fb_get_pts(fb);
 
 	rc = nvnc_fb_map(self->fb);
 	if (rc < 0)

src/zrle.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019 - 2021 Andri Yngvason
+ * Copyright (c) 2019 - 2022 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -363,6 +363,7 @@ static void zrle_encoder_on_done(void* obj)
 
 	assert(self->current_result);
 
+	uint64_t pts = nvnc_fb_get_pts(self->current_fb);
 	nvnc_fb_unref(self->current_fb);
 	self->current_fb = NULL;
 
@@ -375,7 +376,7 @@ static void zrle_encoder_on_done(void* obj)
 	self->work = NULL;
 
 	if (self->encoder.on_done)
-		self->encoder.on_done(&self->encoder, result);
+		self->encoder.on_done(&self->encoder, result, pts);
 
 	rcbuf_unref(result);
 }