main: WiP: Calculate delay based on rtt and jitter

.gitignore

@@ -1,3 +1,4 @@
 build*
 subprojects
 .clang_complete
+sandbox

include/buffer-pool.h

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2023 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#pragma once
+
+#include "buffer.h"
+
+#include <stdbool.h>
+#include <stdint.h>
+
+struct buffer_pool;
+struct buffer;
+
+struct buffer_pool* buffer_pool_create(enum buffer_type type, uint16_t width,
+		uint16_t height, uint32_t format, uint16_t stride, int scale);
+void buffer_pool_destroy(struct buffer_pool* self);
+
+bool buffer_pool_resize(struct buffer_pool* self, uint16_t width,
+		uint16_t height, uint32_t format, uint16_t stride, int scale);
+
+struct buffer* buffer_pool_acquire(struct buffer_pool* self);
+
+void buffer_pool_damage_all(struct buffer_pool* self,
+		struct pixman_region16* damage);

include/buffer.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Andri Yngvason
+ * Copyright (c) 2022 - 2023 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -16,6 +16,8 @@
 
 #pragma once
 
+#include "sys/queue.h"
+
 #include <stdbool.h>
 #include <unistd.h>
 #include <stdint.h>
@@ -31,6 +33,17 @@ enum buffer_type {
 };
 
 struct buffer {
+	int ref;
+	int hold;
+
+	TAILQ_ENTRY(buffer) queue_link;
+
+	LIST_ENTRY(buffer) registry_link;
+	TAILQ_ENTRY(buffer) pool_link;
+
+	void (*release_fn)(struct buffer*, void* ud);
+	void* release_ud;
+
 	enum buffer_type type;
 
 	int width, height;
@@ -38,10 +51,10 @@ struct buffer {
 	size_t size;
 	uint32_t format;
 	struct wl_buffer* wl_buffer;
-	bool is_attached;
-	bool please_clean_up;
 	struct pixman_region16 damage;
 
+	int32_t pts;
+
 	// wl_shm:
 	void* pixels;
 	int stride;
@@ -53,3 +66,12 @@ struct buffer {
 struct buffer* buffer_create_shm(int width, int height, int stride, uint32_t format);
 struct buffer* buffer_create_dmabuf(int width, int height, uint32_t format);
 void buffer_destroy(struct buffer* self);
+
+void buffer_ref(struct buffer*);
+void buffer_unref(struct buffer*);
+
+void buffer_set_release_fn(struct buffer*, void (*)(struct buffer*, void* ud),
+		void* userdata);
+
+void buffer_hold(struct buffer*);
+void buffer_release(struct buffer*);

include/ntp.h

@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2023 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#pragma once
+
+#include <stdint.h>
+#include <stdbool.h>
+
+#define NTP_SAMPLE_PERIOD 1000000 // µs
+#define NTP_MIN_SAMPLE_COUNT 3
+#define NTP_SAMPLE_SIZE 16
+
+struct aml_ticker;
+struct ntp_client;
+
+typedef void (*ntp_client_ping_fn)(struct ntp_client*, uint32_t t0, uint32_t t1,
+		uint32_t t2, uint32_t t3);
+
+struct ntp_sample {
+	int32_t theta;
+	uint32_t delta;
+};
+
+struct ntp_client {
+	struct ntp_sample samples[NTP_SAMPLE_SIZE];
+	int sample_index;
+	int sample_count;
+
+	struct aml_ticker* ping_ticker;
+
+	ntp_client_ping_fn send_ping;
+	void* userdata;
+};
+
+int ntp_client_init(struct ntp_client*, ntp_client_ping_fn send_ping,
+                void* userdata);
+void ntp_client_deinit(struct ntp_client*);
+
+void ntp_client_process_pong(struct ntp_client*, uint32_t t0, uint32_t t1,
+                uint32_t t2, uint32_t t3);
+
+bool ntp_client_get_best_sample(const struct ntp_client* self,
+		struct ntp_sample* sample);
+
+bool ntp_client_translate_server_time(const struct ntp_client* self,
+		uint32_t* dst, const uint32_t t);
+
+uint32_t ntp_client_get_jitter(const struct ntp_client* self);

include/performance.h

@@ -0,0 +1,30 @@
+#pragma once
+
+#define PERF_FRAME_LATENCY_SAMPLE_SIZE 60
+
+struct perf_sample_stats {
+	double min, max, average;
+};
+
+struct perf_sample_buffer {
+	int length;
+	int count;
+	int index;
+	double samples[];
+};
+
+struct perf {
+	struct perf_sample_buffer* frame_latency;
+};
+
+struct perf_sample_buffer* perf_sample_buffer_create(int length);
+
+void perf_sample_buffer_add(struct perf_sample_buffer* self, double sample);
+
+void perf_sample_buffer_get_stats(const struct perf_sample_buffer* self,
+		struct perf_sample_stats* stats);
+
+void perf_init(struct perf*);
+void perf_deinit(struct perf*);
+
+void perf_dump_latency_report(const struct perf*);

include/renderer-egl.h

@@ -11,4 +11,6 @@ void render_image_egl(struct buffer* dst, const struct image* src, double scale,
 		int pos_x, int pos_y);
 void render_av_frames_egl(struct buffer* dst, struct vnc_av_frame** src,
 		int n_av_frames, double scale, int x_pos, int y_pos);
+void render_buffer_egl(struct buffer* dst, const struct buffer* src,
+		double scale, int x_pos, int y_pos);
 

include/rfbclient.h

@@ -225,6 +225,9 @@ typedef char* (*GetUserProc)(struct _rfbClient* client);
 typedef char* (*GetSASLMechanismProc)(struct _rfbClient* client, char* mechlist);
 #endif /* LIBVNCSERVER_HAVE_SASL */
 
+typedef void (*NtpEventProc)(struct _rfbClient* client, uint32_t t0,
+                uint32_t t1, uint32_t t2, uint32_t t3);
+
 typedef struct _rfbClient {
 	uint8_t* frameBuffer;
 	int width, height;
@@ -467,6 +470,8 @@ typedef struct _rfbClient {
 
 	StartingFrameBufferUpdateProc StartingFrameBufferUpdate;
 	CancelledFrameBufferUpdateProc CancelledFrameBufferUpdate;
+
+	NtpEventProc NtpEvent;
 } rfbClient;
 
 /* cursor.c */
@@ -612,6 +617,9 @@ extern rfbBool HandleRFBServerMessage(rfbClient* client);
 
 extern rfbBool ReadToBuffer(rfbClient* client);
 
+extern rfbBool SendClientNtpEvent(rfbClient* client, uint32_t t0, uint32_t t1,
+		uint32_t t2, uint32_t t3);
+
 /**
  * Sends a text chat message to the server.
  * @param client The client through which to send the message

include/rfbproto.h

@@ -398,8 +398,7 @@ typedef struct {
 #define rfbSetDesktopSize 251
 #define rfbQemuEvent 255
 
-
-
+#define rfbNtpEvent 160
 
 /*****************************************************************************
  *
@@ -498,6 +497,8 @@ typedef struct {
 #define rfbEncodingSupportedEncodings 0xFFFE0002
 #define rfbEncodingServerIdentity     0xFFFE0003
 
+#define rfbEncodingPts -1000
+#define rfbEncodingNtp -1001
 
 /*****************************************************************************
  *
@@ -1138,6 +1139,12 @@ typedef struct rfbExtDesktopScreen {
     uint32_t flags;
 } rfbExtDesktopScreen;
 
+struct rfbNtpMsg {
+	uint8_t type;
+	uint8_t padding[3];
+	uint32_t t0, t1, t2, t3;
+};
+
 #define sz_rfbExtDesktopSizeMsg (4)
 #define sz_rfbExtDesktopScreen (16)
 
@@ -1228,6 +1235,7 @@ typedef union {
 	rfbTextChatMsg tc;
 	rfbXvpMsg xvp;
 	rfbExtDesktopSizeMsg eds;
+	struct rfbNtpMsg ntp;
 } rfbServerToClientMsg;
 
 

include/vnc.h

@@ -45,6 +45,8 @@ struct vnc_client {
 	int (*alloc_fb)(struct vnc_client*);
 	void (*update_fb)(struct vnc_client*);
 	void (*cut_text)(struct vnc_client*, const char*, size_t);
+        void (*ntp_event)(struct vnc_client*, uint32_t t0, uint32_t t1,
+                        uint32_t t2, uint32_t t3);
 
 	void* userdata;
 	struct pixman_region16 damage;
@@ -79,3 +81,5 @@ void vnc_client_set_compression_level(struct vnc_client* self, int value);
 void vnc_client_send_cut_text(struct vnc_client* self, const char* text,
 		size_t len);
 void vnc_client_clear_av_frames(struct vnc_client* self);
+void vnc_client_send_ntp_event(struct vnc_client* self, uint32_t t0,
+                uint32_t t1, uint32_t t2, uint32_t t3);

meson.build

@@ -77,11 +77,14 @@ sources = [
 	'src/renderer.c',
 	'src/renderer-egl.c',
 	'src/buffer.c',
+	'src/buffer-pool.c',
 	'src/open-h264.c',
 	'src/cursor.c',
 	'src/rfbproto.c',
 	'src/sockets.c',
 	'src/vncviewer.c',
+	'src/ntp.c',
+	'src/performance.c',
 ]
 
 dependencies = [

src/buffer-pool.c

@@ -0,0 +1,163 @@
+/*
+ * Copyright (c) 2023 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "buffer-pool.h"
+#include "buffer.h"
+
+#include "sys/queue.h"
+
+#include <stdlib.h>
+#include <assert.h>
+
+LIST_HEAD(buffer_list, buffer);
+TAILQ_HEAD(bufferq, buffer);
+
+struct buffer_pool {
+	int ref;
+
+	struct buffer_list registry;
+	struct bufferq buffers;
+
+	enum buffer_type type;
+	uint16_t width;
+	uint16_t height;
+	int32_t stride;
+	uint32_t format;
+	int scale;
+};
+
+struct buffer_pool* buffer_pool_create(enum buffer_type type, uint16_t width,
+		uint16_t height, uint32_t format, uint16_t stride, int scale)
+{
+	struct buffer_pool* self = calloc(1, sizeof(*self));
+	if (!self)
+		return NULL;
+
+	self->ref = 1;
+
+	LIST_INIT(&self->registry);
+	TAILQ_INIT(&self->buffers);
+	self->type = type;
+	self->width = width;
+	self->height = height;
+	self->stride = stride;
+	self->format = format;
+	self->scale = scale;
+
+	return self;
+}
+
+static void buffer_pool__unref_buffers(struct buffer_pool* self)
+{
+	while (!LIST_EMPTY(&self->registry)) {
+		struct buffer* buffer = LIST_FIRST(&self->registry);
+		LIST_REMOVE(buffer, registry_link);
+		buffer_set_release_fn(buffer, NULL, NULL);
+		buffer_unref(buffer);
+	}
+	TAILQ_INIT(&self->buffers);
+}
+
+void buffer_pool_destroy(struct buffer_pool* self)
+{
+	buffer_pool__unref_buffers(self);
+	free(self);
+}
+
+bool buffer_pool_resize(struct buffer_pool* self, uint16_t width,
+		uint16_t height, uint32_t format, uint16_t stride, int scale)
+{
+	if (width == self->width && height == self->height &&
+			format == self->format && stride == self->stride &&
+			scale == self->scale)
+		return false;
+
+	buffer_pool__unref_buffers(self);
+
+	self->width = width;
+	self->height = height;
+	self->stride = stride;
+	self->format = format;
+	self->scale = scale;
+
+	return true;
+}
+
+static void buffer_pool__on_buffer_release(struct buffer* buffer, void* userdata)
+{
+	struct buffer_pool* self = userdata;
+
+	if (buffer->width != self->width || buffer->height != self->height ||
+			buffer->format != self->format ||
+			(buffer->type == BUFFER_WL_SHM && buffer->stride != self->stride) ||
+			buffer->scale != self->scale) {
+		buffer_unref(buffer);
+	} else {
+		TAILQ_INSERT_TAIL(&self->buffers, buffer, pool_link);
+	}
+}
+
+static struct buffer* buffer_pool__acquire_new(struct buffer_pool* self)
+{
+	struct buffer* buffer;
+	switch (self->type) {
+	case BUFFER_WL_SHM:
+		buffer = buffer_create_shm(self->width, self->height,
+				self->stride, self->format);
+		break;
+	case BUFFER_DMABUF:
+		buffer = buffer_create_dmabuf(self->width, self->height,
+				self->format);
+		break;
+	case BUFFER_UNSPEC:
+		abort();
+	}
+	if (!buffer)
+		return NULL;
+
+	buffer->scale = self->scale;
+
+	LIST_INSERT_HEAD(&self->registry, buffer, registry_link);
+	buffer_set_release_fn(buffer, buffer_pool__on_buffer_release, self);
+
+	return buffer;
+}
+
+static struct buffer* buffer_pool__acquire_from_list(struct buffer_pool* self)
+{
+	struct buffer* buffer = TAILQ_FIRST(&self->buffers);
+	assert(buffer);
+
+	TAILQ_REMOVE(&self->buffers, buffer, pool_link);
+
+	return buffer;
+}
+
+struct buffer* buffer_pool_acquire(struct buffer_pool* self)
+{
+	return TAILQ_EMPTY(&self->buffers) ?
+		buffer_pool__acquire_new(self) :
+		buffer_pool__acquire_from_list(self);
+}
+
+void buffer_pool_damage_all(struct buffer_pool* self,
+		struct pixman_region16* damage)
+{
+	struct buffer* buffer;
+	LIST_FOREACH(buffer, &self->registry, registry_link) {
+		pixman_region_union(&buffer->damage, damage, damage);
+	}
+}

src/buffer.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Andri Yngvason
+ * Copyright (c) 2022 - 2023 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -25,24 +25,22 @@
 #include <gbm.h>
 #include <assert.h>
 
+// TODO: Some buffers needn't be wayland buffers.
+
 /* Origin: main.c */
 extern struct wl_shm* wl_shm;
 extern struct gbm_device* gbm_device;
 extern struct zwp_linux_dmabuf_v1* zwp_linux_dmabuf_v1;
 
-static void buffer_release(void* data, struct wl_buffer* wl_buffer)
+static void buffer_wl_release(void* data, struct wl_buffer* wl_buffer)
 {
 	(void)wl_buffer;
 	struct buffer* self = data;
-	self->is_attached = false;
-
-	if (self->please_clean_up) {
-		buffer_destroy(self);
-	}
+	buffer_release(self);
 }
 
 static const struct wl_buffer_listener buffer_listener = {
-	.release = buffer_release,
+	.release = buffer_wl_release,
 };
 
 struct buffer* buffer_create_shm(int width, int height, int stride,
@@ -54,6 +52,7 @@ struct buffer* buffer_create_shm(int width, int height, int stride,
 	if (!self)
 		return NULL;
 
+	self->ref = 1;
 	self->type = BUFFER_WL_SHM;
 	self->width = width;
 	self->height = height;
@@ -106,6 +105,7 @@ struct buffer* buffer_create_dmabuf(int width, int height, uint32_t format)
 	if (!self)
 		return NULL;
 
+	self->ref = 1;
 	self->type = BUFFER_DMABUF;
 	self->width = width;
 	self->height = height;
@@ -159,9 +159,6 @@ void buffer_destroy(struct buffer* self)
 	if (!self)
 		return;
 
-	if (self->is_attached)
-		self->please_clean_up = true;
-
 	pixman_region_fini(&self->damage);
 	wl_buffer_destroy(self->wl_buffer);
 
@@ -179,3 +176,41 @@ void buffer_destroy(struct buffer* self)
 
 	free(self);
 }
+
+void buffer_ref(struct buffer* self)
+{
+	++self->ref;
+}
+
+void buffer_unref(struct buffer* self)
+{
+	if (self && --self->ref == 0)
+		buffer_destroy(self);
+}
+
+void buffer_set_release_fn(struct buffer* self,
+		void (*fn)(struct buffer*, void* ud), void* userdata)
+{
+	self->release_fn = fn;
+	self->release_ud = userdata;
+}
+
+void buffer_hold(struct buffer* self)
+{
+	assert(self);
+	++self->hold;
+}
+
+void buffer_release(struct buffer* self)
+{
+	assert(self);
+
+	int hold = --self->hold;
+	if (hold != 0)
+	       return;
+
+	if (self->release_fn)
+		self->release_fn(self, self->release_ud);
+	else
+		buffer_unref(self);
+}

src/main.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020 - 2022 Andri Yngvason
+ * Copyright (c) 2020 - 2023 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -42,34 +42,44 @@
 #include "pixels.h"
 #include "region.h"
 #include "buffer.h"
+#include "buffer-pool.h"
 #include "renderer.h"
 #include "renderer-egl.h"
 #include "linux-dmabuf-unstable-v1.h"
 #include "time-util.h"
 #include "output.h"
+#include "ntp.h"
+#include "performance.h"
 
 #define CANARY_TICK_PERIOD INT64_C(100000) // us
 #define CANARY_LETHALITY_LEVEL INT64_C(8000) // us
+#define LATENCY_REPORT_PERIOD INT64_C(250000) // us
 
 struct point {
 	double x, y;
 };
 
+TAILQ_HEAD(frame_queue, buffer);
+
 struct window {
 	struct wl_surface* wl_surface;
 	struct xdg_surface* xdg_surface;
 	struct xdg_toplevel* xdg_toplevel;
 
-	struct buffer* buffers[3];
+	struct buffer_pool* buffers;
 	struct buffer* back_buffer;
-	int buffer_index;
 
 	struct pixman_region16 current_damage;
 
+	struct buffer_pool* frame_pool;
+	struct frame_queue frame_queue;
+
 	struct vnc_client* vnc;
 	void* vnc_fb;
 
 	bool is_frame_committed;
+	bool is_frame_callback_registered;
+	bool is_not_first_frame;
 };
 
 static void register_frame_callback(void);
@@ -87,6 +97,8 @@ struct pointer_collection* pointers;
 struct keyboard_collection* keyboards;
 static int drm_fd = -1;
 static uint64_t last_canary_tick;
+static struct ntp_client ntp;
+static struct perf perf;
 
 static bool have_egl = false;
 
@@ -238,8 +250,10 @@ void on_wayland_event(void* obj)
 		}
 	}
 
-	if (wl_display_dispatch_pending(wl_display) < 0)
+	if (wl_display_dispatch_pending(wl_display) < 0) {
 		fprintf(stderr, "Failed to dispatch pending\n");
+		abort();
+	}
 }
 
 static int init_wayland_event_handler(void)
@@ -274,7 +288,7 @@ static int init_signal_handler(void)
 
 static void window_attach(struct window* w, int x, int y)
 {
-	w->back_buffer->is_attached = true;
+	buffer_hold(w->back_buffer);
 	wl_surface_attach(w->wl_surface, w->back_buffer->wl_buffer, x, y);
 	wl_surface_set_buffer_scale(window->wl_surface,
 			window->back_buffer->scale);
@@ -365,8 +379,9 @@ static void window_commit(struct window* w)
 
 static void window_swap(struct window* w)
 {
-	w->buffer_index = (w->buffer_index + 1) % 3;
-	w->back_buffer = w->buffers[w->buffer_index];
+	buffer_unref(w->back_buffer);
+	w->back_buffer = buffer_pool_acquire(w->buffers);
+	buffer_ref(w->back_buffer);
 }
 
 static void window_damage(struct window* w, int x, int y, int width, int height)
@@ -396,24 +411,19 @@ static void window_resize(struct window* w, int width, int height, int scale)
 	if (width == 0 || height == 0 || scale == 0)
 		return;
 
-	if (w->back_buffer && w->back_buffer->width == width &&
-			w->back_buffer->height == height &&
-			w->back_buffer->scale == scale)
-		return;
-
-	for (int i = 0; i < 3; ++i)
-		buffer_destroy(w->buffers[i]);
-
-	for (int i = 0; i < 3; ++i) {
-		w->buffers[i] = have_egl
-			? buffer_create_dmabuf(scale * width, scale * height,
-					dmabuf_format)
-			: buffer_create_shm(scale * width, scale * height,
-					scale * 4 * width, shm_format);
-		w->buffers[i]->scale = scale;
+	uint32_t format = have_egl ? dmabuf_format : shm_format;
+	if (!w->buffers) {
+		enum buffer_type type = have_egl ? BUFFER_DMABUF : BUFFER_WL_SHM;
+		w->buffers = buffer_pool_create(type, scale * width,
+				scale * height, format, scale * 4 * width,
+				scale);
+	} else {
+		buffer_pool_resize(w->buffers, scale * width, scale * height,
+				format, scale * 4 * width, scale);
 	}
 
-	w->back_buffer = w->buffers[0];
+	w->back_buffer = buffer_pool_acquire(w->buffers);
+	buffer_ref(w->back_buffer);
 }
 
 static void xdg_toplevel_configure(void* data, struct xdg_toplevel* toplevel,
@@ -474,9 +484,8 @@ wl_surface_failure:
 
 static void window_destroy(struct window* w)
 {
-	for (int i = 0; i < 3; ++i)
-		buffer_destroy(w->buffers[i]);
-
+	buffer_unref(w->back_buffer);
+	buffer_pool_destroy(w->buffers);
 	free(w->vnc_fb);
 	xdg_toplevel_destroy(w->xdg_toplevel);
 	xdg_surface_destroy(w->xdg_surface);
@@ -560,6 +569,8 @@ int on_vnc_client_alloc_fb(struct vnc_client* client)
 		window = window_create(app_id, vnc_client_get_desktop_name(client));
 		window->vnc = client;
 
+		TAILQ_INIT(&window->frame_queue);
+
 		int32_t scale = output_list_get_max_scale(&outputs);
 		window_resize(window, width, height, scale);
 	}
@@ -587,9 +598,7 @@ static void get_frame_damage(struct vnc_client* client,
 
 static void apply_buffer_damage(struct pixman_region16* damage)
 {
-	for (int i = 0; i < 3; ++i)
-		pixman_region_union(&window->buffers[i]->damage,
-				&window->buffers[i]->damage, damage);
+	buffer_pool_damage_all(window->buffers, damage);
 }
 
 static void window_damage_region(struct window* w,
@@ -608,6 +617,20 @@ static void window_damage_region(struct window* w,
 	}
 }
 
+static void update_frame_latency_stats(void)
+{
+	uint32_t server_pts = window->vnc->pts;
+
+	uint32_t client_pts = 0;
+	if (!ntp_client_translate_server_time(&ntp, &client_pts, server_pts))
+		return;
+
+	uint32_t now = gettime_us();
+	int32_t latency = (int32_t)(now - client_pts);
+
+	perf_sample_buffer_add(perf.frame_latency, latency);
+}
+
 static void render_from_vnc(void)
 {
 	if (!pixman_region_not_empty(&window->current_damage) &&
@@ -617,9 +640,6 @@ static void render_from_vnc(void)
 	if (window->is_frame_committed)
 		return;
 
-	if (window->back_buffer->is_attached)
-		fprintf(stderr, "Oops, back-buffer is still attached.\n");
-
 	window_attach(window, 0, 0);
 
 	double scale;
@@ -653,18 +673,98 @@ static void render_from_vnc(void)
 	window_commit(window);
 	window_swap(window);
 
+	update_frame_latency_stats();
+
 	pixman_region_clear(&window->current_damage);
 	vnc_client_clear_av_frames(window->vnc);
 }
 
-void on_vnc_client_update_fb(struct vnc_client* client)
+void on_vnc_client_update_fb_immediate(struct vnc_client* client)
 {
 	get_frame_damage(window->vnc, &window->current_damage);
 	render_from_vnc();
 }
 
-static void handle_frame_callback(void* data, struct wl_callback* callback,
-		uint32_t time)
+void on_vnc_client_update_fb_queued(struct vnc_client* client)
+{
+	struct window* w = window;
+	struct pixman_region16 damage;
+	pixman_region_init(&damage);
+
+	get_frame_damage(w->vnc, &damage);
+	if (!pixman_region_not_empty(&damage)) {
+		return;
+	}
+
+	int width = vnc_client_get_width(client);
+	int height = vnc_client_get_height(client);
+	int stride = vnc_client_get_stride(client);
+
+	if (!w->frame_pool) {
+		// TODO: Delete this pool somewhere
+		enum buffer_type type = have_egl ? BUFFER_DMABUF : BUFFER_WL_SHM;
+		uint32_t format = have_egl ? dmabuf_format : shm_format;
+		int scale = 1;
+		w->frame_pool = buffer_pool_create(type, width, height, format,
+				stride, scale);
+	}
+
+	struct buffer* frame = buffer_pool_acquire(w->frame_pool);
+	assert(frame);
+
+	// TODO: Implement damage tracking
+	pixman_region_union_rect(&damage, &damage, 0, 0, frame->width, frame->height);
+
+	buffer_pool_damage_all(w->frame_pool, &damage);
+
+	if (!w->is_not_first_frame) {
+		w->is_not_first_frame = true;
+		pixman_region_union(&w->current_damage, &w->current_damage,
+				&damage);
+		render_from_vnc();
+	}
+
+	if (w->vnc->n_av_frames != 0) {
+		assert(have_egl);
+
+		render_av_frames_egl(frame, w->vnc->av_frames,
+				w->vnc->n_av_frames, 1.0, 0, 0);
+	} else {
+		struct image image = {
+			.pixels = w->vnc_fb,
+			.width = width,
+			.height = height,
+			.stride = stride,
+			// TODO: Get the format from the vnc module
+			.format = frame->format,
+			.damage = &frame->damage,
+		};
+
+		if (have_egl)
+			render_image_egl(frame, &image, 1.0, 0, 0);
+		else
+			render_image(frame, &image, 1.0, 0, 0);
+	}
+
+	uint32_t pts;
+	if (ntp_client_translate_server_time(&ntp, &pts, client->pts)) {
+		frame->pts = pts;
+	} else {
+		frame->pts = gettime_us();
+	}
+
+	buffer_ref(frame);
+	buffer_hold(frame);
+	TAILQ_INSERT_TAIL(&window->frame_queue, frame, queue_link);
+
+	register_frame_callback();
+
+	pixman_region_fini(&damage);
+	vnc_client_clear_av_frames(window->vnc);
+}
+
+static void handle_frame_callback_immediate(void* data,
+		struct wl_callback* callback, uint32_t time)
 {
 	wl_callback_destroy(callback);
 	window->is_frame_committed = false;
@@ -673,14 +773,129 @@ static void handle_frame_callback(void* data, struct wl_callback* callback,
 		render_from_vnc();
 }
 
-static const struct wl_callback_listener frame_listener = {
-	.done = handle_frame_callback
+static const struct wl_callback_listener frame_listener_immediate = {
+	.done = handle_frame_callback_immediate
 };
 
+static int32_t abs32(int32_t v)
+{
+	return v >= 0 ? v : -v;
+}
+
+static struct buffer* choose_frame(struct window* w)
+{
+	int32_t delay = 0;
+
+	struct ntp_sample best_sample;
+	if (ntp_client_get_best_sample(&ntp, &best_sample)) {
+		// Half rtt + jitter
+		delay = ntp_client_get_jitter(&ntp) + best_sample.theta / 2;
+	}
+
+	int32_t now_us = gettime_us();
+
+	int32_t smallest_diff = INT32_MAX;
+	struct buffer* chosen_frame = NULL;
+	struct buffer* frame;
+	TAILQ_FOREACH(frame, &w->frame_queue, queue_link) {
+		int32_t diff = abs32(now_us - (frame->pts + delay));
+		if (diff >= smallest_diff) {
+			continue;
+		}
+
+		smallest_diff = diff;
+		chosen_frame = frame;
+	}
+
+	return chosen_frame;
+}
+
+// TODO: Discard all queued frames on exit
+static void discard_older_frames(struct window* w, struct buffer* chosen)
+{
+	while (!TAILQ_EMPTY(&w->frame_queue)) {
+		struct buffer* frame = TAILQ_FIRST(&w->frame_queue);
+		int32_t diff = frame->pts - chosen->pts;
+		if (diff >= 0) {
+			return;
+		}
+
+		pixman_region_union(&chosen->damage, &chosen->damage,
+				&frame->damage);
+
+		TAILQ_REMOVE(&w->frame_queue, frame, queue_link);
+		buffer_release(frame);
+		buffer_unref(frame);
+	}
+}
+
+static void handle_frame_callback_queued(void* data,
+		struct wl_callback* callback, uint32_t time)
+{
+	wl_callback_destroy(callback);
+	window->is_frame_callback_registered = false;
+
+	struct buffer* frame = choose_frame(window);
+	if (!frame)
+		return;
+
+	discard_older_frames(window, frame);
+	pixman_region_union(&window->current_damage, &window->current_damage,
+			&frame->damage);
+
+	// TODO: Try not to render frames that are already rendered.
+
+	window_attach(window, 0, 0);
+
+	register_frame_callback();
+
+	// TODO: Consolidate all this scaling and translating
+	double scale;
+	int x_pos, y_pos;
+	window_calculate_transform(window, &scale, &x_pos, &y_pos);
+
+	struct pixman_region16 damage_scaled = { 0 }, buffer_damage = { 0 },
+			       surface_damage = { 0 };
+	region_scale(&damage_scaled, &window->current_damage, scale);
+	region_translate(&buffer_damage, &damage_scaled, x_pos, y_pos);
+	pixman_region_clear(&damage_scaled);
+
+	double output_scale = output_list_get_max_scale(&outputs);
+	struct point scoord = buffer_coord_to_surface_coord(x_pos, y_pos);
+	region_scale(&damage_scaled, &window->current_damage,
+			scale / output_scale);
+	region_translate(&surface_damage, &damage_scaled, scoord.x, scoord.y);
+	pixman_region_fini(&damage_scaled);
+
+	apply_buffer_damage(&buffer_damage);
+	window_damage_region(window, &surface_damage);
+
+	pixman_region_fini(&surface_damage);
+	pixman_region_fini(&buffer_damage);
+
+	// TODO: Implement software rendering
+	render_buffer_egl(window->back_buffer, frame, scale, x_pos, y_pos);
+
+	window_commit(window);
+	window_swap(window);
+
+	pixman_region_clear(&window->current_damage);
+}
+
+static const struct wl_callback_listener frame_listener_queued = {
+	.done = handle_frame_callback_queued
+};
+
+// TODO: Maybe try avoid leaking the callback on exit
 static void register_frame_callback(void)
 {
+	if (window->is_frame_callback_registered)
+		return;
+	window->is_frame_callback_registered = true;
+
 	struct wl_callback* callback = wl_surface_frame(window->wl_surface);
-	wl_callback_add_listener(callback, &frame_listener, NULL);
+	//wl_callback_add_listener(callback, &frame_listener_immediate, NULL);
+	wl_callback_add_listener(callback, &frame_listener_queued, NULL);
 }
 
 void on_vnc_client_event(void* obj)
@@ -804,6 +1019,18 @@ static void on_canary_tick(void* obj)
 				delay);
 }
 
+static void send_ntp_ping(struct ntp_client* ntp_client, uint32_t t0,
+		uint32_t t1, uint32_t t2, uint32_t t3)
+{
+	vnc_client_send_ntp_event(window->vnc, t0, t1, t2, t3);
+}
+
+static void on_ntp_event(struct vnc_client* vnc, uint32_t t0, uint32_t t1,
+		uint32_t t2, uint32_t t3)
+{
+	ntp_client_process_pong(&ntp, t0, t1, t2, t3);
+}
+
 static void create_canary_ticker(void)
 {
 	last_canary_tick = gettime_us();
@@ -815,6 +1042,22 @@ static void create_canary_ticker(void)
 	aml_unref(ticker);
 }
 
+static void on_latency_report_tick(void* handler)
+{
+	(void)handler;
+
+	perf_dump_latency_report(&perf);
+}
+
+static void create_latency_report_ticker(void)
+{
+	struct aml* aml = aml_get_default();
+	struct aml_ticker* ticker = aml_ticker_new(LATENCY_REPORT_PERIOD,
+			on_latency_report_tick, NULL, NULL);
+	aml_start(aml, ticker);
+	aml_unref(ticker);
+}
+
 void run_main_loop_once(void)
 {
 	struct aml* aml = aml_get_default();
@@ -968,8 +1211,12 @@ int main(int argc, char* argv[])
 	if (!vnc)
 		goto vnc_failure;
 
+	vnc->userdata = window;
+
 	vnc->alloc_fb = on_vnc_client_alloc_fb;
-	vnc->update_fb = on_vnc_client_update_fb;
+	//vnc->update_fb = on_vnc_client_update_fb_immediate;
+	vnc->update_fb = on_vnc_client_update_fb_queued;
+	vnc->ntp_event = on_ntp_event;
 
 	if (vnc_client_set_pixel_format(vnc, shm_format) < 0) {
 		fprintf(stderr, "Unsupported pixel format\n");
@@ -1008,12 +1255,16 @@ int main(int argc, char* argv[])
 		goto vnc_setup_failure;
 	}
 
+	perf_init(&perf);
+	ntp_client_init(&ntp, send_ntp_ping, window);
+
 	pointers->userdata = vnc;
 	keyboards->userdata = vnc;
 
 	wl_display_dispatch(wl_display);
 
 	create_canary_ticker();
+	//create_latency_report_ticker();
 
 	while (do_run)
 		run_main_loop_once();
@@ -1021,6 +1272,10 @@ int main(int argc, char* argv[])
 	rc = 0;
 	if (window)
 		window_destroy(window);
+
+	ntp_client_deinit(&ntp);
+	perf_deinit(&perf);
+
 vnc_setup_failure:
 	vnc_client_destroy(vnc);
 vnc_failure:

src/ntp.c

@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2023 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "ntp.h"
+#include "time-util.h"
+
+#include <assert.h>
+#include <aml.h>
+#include <stdio.h>
+
+static inline int32_t clamp_to_zero(int32_t v)
+{
+	return v >= 0 ? v : 0;
+}
+
+void ntp_client__tick(void* handler)
+{
+	struct ntp_client* self = aml_get_userdata(handler);
+	assert(self);
+	assert(self->send_ping);
+
+	uint32_t t0 = gettime_us();
+	self->send_ping(self, t0, 0, 0, 0);
+}
+
+int ntp_client_init(struct ntp_client* self, ntp_client_ping_fn send_ping,
+		void* userdata)
+{
+	self->send_ping = send_ping;
+	self->userdata = userdata;
+
+	struct aml_ticker* ticker = aml_ticker_new(NTP_SAMPLE_PERIOD,
+			ntp_client__tick, self, NULL);
+	if (!ticker)
+		return -1;
+
+	int rc = aml_start(aml_get_default(), ticker);
+	if (rc >= 0)
+		self->ping_ticker = ticker;
+	else
+		aml_unref(ticker);
+
+	return rc;
+}
+
+void ntp_client_deinit(struct ntp_client* self)
+{
+	if (self->ping_ticker) {
+		aml_stop(aml_get_default(), self->ping_ticker);
+		aml_unref(self->ping_ticker);
+	}
+}
+
+void ntp_client_process_pong(struct ntp_client* self, uint32_t t0, uint32_t t1,
+                uint32_t t2, uint32_t t3)
+{
+	t3 = gettime_us();
+
+	int32_t theta = ((int32_t)(t1 - t0) + (int32_t)(t2 - t3)) / 2;
+	uint32_t delta = clamp_to_zero((int32_t)(t3 - t0) - (int32_t)(t2 - t1));
+
+	struct ntp_sample sample = {
+		.theta = theta,
+		.delta = delta,
+	};
+
+	self->samples[self->sample_index] = sample;
+	self->sample_index = (self->sample_index + 1) % NTP_SAMPLE_SIZE;
+
+	if (self->sample_count < NTP_SAMPLE_SIZE)
+		self->sample_count++;
+
+//	printf("%.3f %.3f\n", delta / 1e3, theta / 1e3);
+}
+
+bool ntp_client_get_best_sample(const struct ntp_client* self,
+		struct ntp_sample* out)
+{
+	if (self->sample_count < NTP_MIN_SAMPLE_COUNT)
+		return false;
+
+	struct ntp_sample result = {
+		.theta = 0,
+		.delta = UINT32_MAX,
+	};
+
+	for (int i = 0; i < self->sample_count; ++i) {
+		const struct ntp_sample *sample = &self->samples[i];
+		if (sample->delta < result.delta) {
+			result = *sample;
+		}
+	}
+
+	*out = result;
+	return true;
+}
+
+bool ntp_client_translate_server_time(const struct ntp_client* self,
+		uint32_t* dst, const uint32_t t)
+{
+	struct ntp_sample sample;
+	if (!ntp_client_get_best_sample(self, &sample))
+		return false;
+
+	*dst = (int32_t)t - sample.theta;
+	return true;
+}
+
+uint32_t ntp_client_get_jitter(const struct ntp_client* self)
+{
+	uint32_t max_delta = 0;
+	for (int i = 0; i < self->sample_count; ++i) {
+		const struct ntp_sample *sample = &self->samples[i];
+		if (sample->delta > max_delta) {
+			max_delta = sample->delta;
+		}
+	}
+	return max_delta;
+}

src/performance.c

@@ -0,0 +1,72 @@
+#include "performance.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+
+struct perf_sample_buffer* perf_sample_buffer_create(int length)
+{
+	struct perf_sample_buffer* self = malloc(sizeof(*self)
+			+ length * sizeof(self->samples[0]));
+	self->count = 0;
+	self->index = 0;
+	self->length = length;
+	return self;
+}
+
+void perf_sample_buffer_add(struct perf_sample_buffer* self, double sample)
+{
+	self->samples[self->index] = sample;
+	self->index = (self->index + 1) % self->length;
+	if (self->count < self->length)
+		self->count += 1;
+}
+
+void perf_sample_buffer_get_stats(const struct perf_sample_buffer* self,
+		struct perf_sample_stats* stats)
+{
+	memset(stats, 0, sizeof(*stats));
+
+	double sum = 0;
+	double minimum = INFINITY;
+	double maximum = 0;
+
+	for (int i = 0; i < self->count; ++i) {
+		double sample = self->samples[i];
+
+		sum += sample;
+
+		if (sample < minimum)
+			minimum = sample;
+
+		if (sample > maximum)
+			maximum = sample;
+	}
+
+	stats->min = minimum;
+	stats->max = maximum;
+	stats->average = sum / (double)self->count;
+}
+
+void perf_init(struct perf* self)
+{
+	memset(self, 0, sizeof(*self));
+
+	self->frame_latency =
+		perf_sample_buffer_create(PERF_FRAME_LATENCY_SAMPLE_SIZE);
+}
+
+void perf_deinit(struct perf* self)
+{
+	free(self->frame_latency);
+}
+
+void perf_dump_latency_report(const struct perf* self)
+{
+	struct perf_sample_stats stats;
+	perf_sample_buffer_get_stats(self->frame_latency, &stats);
+
+	printf("Latency report: frame-latency (min, avg, max): %.1f, %.1f, %.1f\n",
+			stats.min / 1e3, stats.average / 1e3, stats.max / 1e3);
+}

src/renderer-egl.c

@@ -374,6 +374,47 @@ static GLuint texture_from_av_frame(const struct AVFrame* frame)
 	return tex;
 }
 
+static GLuint texture_from_buffer(const struct buffer* buffer)
+{
+	int index = 0;
+	EGLint attr[128];
+
+	append_attr(attr, &index, EGL_WIDTH, buffer->width);
+	append_attr(attr, &index, EGL_HEIGHT, buffer->height);
+	append_attr(attr, &index, EGL_LINUX_DRM_FOURCC_EXT, buffer->format);
+	append_attr(attr, &index, EGL_IMAGE_PRESERVED_KHR, EGL_TRUE);
+
+	struct gbm_bo* bo = buffer->bo;
+	int fd = gbm_bo_get_fd(bo);
+
+	append_attr(attr, &index, plane_fd_key(0), fd);
+	append_attr(attr, &index, plane_offset_key(0), gbm_bo_get_offset(bo, 0));
+	append_attr(attr, &index, plane_pitch_key(0), gbm_bo_get_stride(bo));
+
+	uint64_t mod = gbm_bo_get_modifier(buffer->bo);
+	append_attr(attr, &index, plane_modifier_lo_key(0), mod & UINT32_MAX);
+	append_attr(attr, &index, plane_modifier_hi_key(0), mod >> 32);
+
+	attr[index++] = EGL_NONE;
+
+	EGLImageKHR image = eglCreateImageKHR(egl_display, EGL_NO_CONTEXT,
+				EGL_LINUX_DMA_BUF_EXT, NULL, attr);
+	assert(image != EGL_NO_IMAGE_KHR);
+
+	GLuint tex = 0;
+	glGenTextures(1, &tex);
+
+	glBindTexture(GL_TEXTURE_EXTERNAL_OES, tex);
+
+	glEGLImageTargetTexture2DOES(GL_TEXTURE_EXTERNAL_OES, image);
+
+	eglDestroyImageKHR(egl_display, image);
+
+	glBindTexture(GL_TEXTURE_EXTERNAL_OES, 0);
+	close(fd);
+	return tex;
+}
+
 void gl_draw(void)
 {
 	static const GLfloat s_vertices[4][2] = {
@@ -542,3 +583,42 @@ void render_av_frames_egl(struct buffer* dst, struct vnc_av_frame** src,
 
 	pixman_region_clear(&dst->damage);
 }
+
+void render_buffer_egl(struct buffer* dst, const struct buffer* src,
+		double scale, int x_pos, int y_pos)
+{
+	struct fbo_info fbo;
+	fbo_from_gbm_bo(&fbo, dst->bo);
+
+	// Make sure we've completed drawing into the source buffer:
+	glFinish();
+
+	glBindFramebuffer(GL_FRAMEBUFFER, fbo.fbo);
+
+	struct pixman_box16* ext = pixman_region_extents(&dst->damage);
+	glScissor(ext->x1, ext->y1, ext->x2 - ext->x1, ext->y2 - ext->y1);
+	glEnable(GL_SCISSOR_TEST);
+
+	glUseProgram(shader_program_ext);
+
+	int width = round((double)src->width * scale);
+	int height = round((double)src->height * scale);
+	glViewport(x_pos, y_pos, width, height);
+
+	GLuint tex = texture_from_buffer(src);
+	glBindTexture(GL_TEXTURE_EXTERNAL_OES, tex);
+
+	gl_draw();
+
+	glBindTexture(GL_TEXTURE_EXTERNAL_OES, 0);
+
+	glDisable(GL_SCISSOR_TEST);
+
+	glFlush();
+
+	glBindFramebuffer(GL_FRAMEBUFFER, 0);
+	glDeleteFramebuffers(1, &fbo.fbo);
+	glDeleteRenderbuffers(1, &fbo.rbo);
+
+	pixman_region_clear(&dst->damage);
+}

src/rfbproto.c

@@ -1396,7 +1396,11 @@ rfbBool SetFormatAndEncodings(rfbClient* client)
 
 	/* pts */
 	if (se->nEncodings < MAX_ENCODINGS)
-		encs[se->nEncodings++] = rfbClientSwap32IfLE(-1000);
+		encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingPts);
+
+	/* ntp */
+	if (se->nEncodings < MAX_ENCODINGS)
+		encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingNtp);
 
 	len = sz_rfbSetEncodingsMsg + se->nEncodings * 4;
 
@@ -1742,6 +1746,22 @@ rfbBool SendClientCutText(rfbClient* client, char* str, int len)
 	        WriteToRFBServer(client, str, len));
 }
 
+rfbBool SendClientNtpEvent(rfbClient* client, uint32_t t0, uint32_t t1,
+		uint32_t t2, uint32_t t3)
+{
+	if (!SupportsClient2Server(client, rfbNtpEvent))
+		return FALSE;
+
+	struct rfbNtpMsg msg = {
+		.type = rfbNtpEvent,
+		.t0 = rfbClientSwap32IfLE(t0),
+		.t1 = rfbClientSwap32IfLE(t1),
+		.t2 = rfbClientSwap32IfLE(t2),
+		.t3 = rfbClientSwap32IfLE(t3),
+	};
+	return WriteToRFBServer(client, (char*)&msg, sizeof(msg));
+}
+
 static rfbBool HandleFramebufferUpdate(rfbClient* client,
                                        rfbServerToClientMsg* msg)
 {
@@ -2300,6 +2320,10 @@ static rfbBool HandleFramebufferUpdate(rfbClient* client,
 			SetClient2Server(client, rfbQemuEvent);
 			break;
 
+		case rfbEncodingNtp:
+			SetClient2Server(client, rfbNtpEvent);
+			break;
+
 		default: {
 			rfbBool handled = FALSE;
 			rfbClientProtocolExtension* e;
@@ -2339,6 +2363,22 @@ failure:
 	return FALSE;
 }
 
+rfbBool handleNtpEvent(rfbClient* client, struct rfbNtpMsg* msg)
+{
+	if (!ReadFromRFBServer(client, (char*)msg + 1, sizeof(*msg) - 1))
+		return FALSE;
+
+	uint32_t t0 = rfbClientSwap32IfLE(msg->t0);
+	uint32_t t1 = rfbClientSwap32IfLE(msg->t1);
+	uint32_t t2 = rfbClientSwap32IfLE(msg->t2);
+	uint32_t t3 = rfbClientSwap32IfLE(msg->t3);
+
+	if (client->NtpEvent)
+		client->NtpEvent(client, t0, t1, t2, t3);
+
+	return TRUE;
+}
+
 /*
  * HandleRFBServerMessage.
  */
@@ -2527,6 +2567,9 @@ rfbBool HandleRFBServerMessage(rfbClient* client)
 		break;
 	}
 
+	case rfbNtpEvent:
+		return handleNtpEvent(client, &msg.ntp);
+
 	default: {
 		rfbBool handled = FALSE;
 		rfbClientProtocolExtension* e;

src/vnc.c

@@ -136,6 +136,16 @@ static void vnc_client_got_cut_text(rfbClient* client, const char* text,
 		self->cut_text(self, text, len);
 }
 
+static void vnc_client_ntp_event(rfbClient* client, uint32_t t0, uint32_t t1,
+		uint32_t t2, uint32_t t3)
+{
+	struct vnc_client* self = rfbClientGetClientData(client, NULL);
+	assert(self);
+
+	if (self->ntp_event)
+		self->ntp_event(self, t0, t1, t2, t3);
+}
+
 static rfbBool vnc_client_handle_open_h264_rect(rfbClient* client,
 		rfbFramebufferUpdateRectHeader* rect_header)
 {
@@ -246,6 +256,7 @@ struct vnc_client* vnc_client_create(void)
 	client->StartingFrameBufferUpdate = vnc_client_start_update;
 	client->CancelledFrameBufferUpdate = vnc_client_cancel_update;
 	client->GotXCutText = vnc_client_got_cut_text;
+	client->NtpEvent = vnc_client_ntp_event;
 
 	self->pts = NO_PTS;
 
@@ -443,3 +454,9 @@ void vnc_client_send_cut_text(struct vnc_client* self, const char* text,
 	// libvncclient doesn't modify text, so typecast is OK.
 	SendClientCutText(self->client, (char*)text, len);
 }
+
+void vnc_client_send_ntp_event(struct vnc_client* self, uint32_t t0,
+                uint32_t t1, uint32_t t2, uint32_t t3)
+{
+	SendClientNtpEvent(self->client, t0, t1, t2, t3);
+}