HACK

.gitignore

@@ -1,3 +1,4 @@
 build*
 subprojects
 .clang_complete
+sandbox

include/ntp.h

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2023 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#pragma once
+
+#include <stdint.h>
+#include <stdbool.h>
+
+#define NTP_SAMPLE_PERIOD 1000000 // µs
+#define NTP_MIN_SAMPLE_COUNT 3
+#define NTP_SAMPLE_SIZE 16
+
+struct aml_ticker;
+struct ntp_client;
+
+typedef void (*ntp_client_ping_fn)(struct ntp_client*, uint32_t t0, uint32_t t1,
+		uint32_t t2, uint32_t t3);
+
+struct ntp_sample {
+	int32_t theta;
+	uint32_t delta;
+};
+
+struct ntp_client {
+	struct ntp_sample samples[NTP_SAMPLE_SIZE];
+	int sample_index;
+	int sample_count;
+
+	struct aml_ticker* ping_ticker;
+
+	ntp_client_ping_fn send_ping;
+	void* userdata;
+};
+
+int ntp_client_init(struct ntp_client*, ntp_client_ping_fn send_ping,
+                void* userdata);
+void ntp_client_deinit(struct ntp_client*);
+
+void ntp_client_process_pong(struct ntp_client*, uint32_t t0, uint32_t t1,
+                uint32_t t2, uint32_t t3);
+
+bool ntp_client_get_best_sample(const struct ntp_client* self,
+		struct ntp_sample* sample);
+
+bool ntp_client_translate_server_time(const struct ntp_client* self,
+		uint32_t* dst, const uint32_t t);

include/performance.h

@@ -0,0 +1,30 @@
+#pragma once
+
+#define PERF_FRAME_LATENCY_SAMPLE_SIZE 60
+
+struct perf_sample_stats {
+	double min, max, average;
+};
+
+struct perf_sample_buffer {
+	int length;
+	int count;
+	int index;
+	double samples[];
+};
+
+struct perf {
+	struct perf_sample_buffer* frame_latency;
+};
+
+struct perf_sample_buffer* perf_sample_buffer_create(int length);
+
+void perf_sample_buffer_add(struct perf_sample_buffer* self, double sample);
+
+void perf_sample_buffer_get_stats(const struct perf_sample_buffer* self,
+		struct perf_sample_stats* stats);
+
+void perf_init(struct perf*);
+void perf_deinit(struct perf*);
+
+void perf_dump_latency_report(const struct perf*);

include/rfbclient.h

@@ -225,6 +225,9 @@ typedef char* (*GetUserProc)(struct _rfbClient* client);
 typedef char* (*GetSASLMechanismProc)(struct _rfbClient* client, char* mechlist);
 #endif /* LIBVNCSERVER_HAVE_SASL */
 
+typedef void (*NtpEventProc)(struct _rfbClient* client, uint32_t t0,
+                uint32_t t1, uint32_t t2, uint32_t t3);
+
 typedef struct _rfbClient {
 	uint8_t* frameBuffer;
 	int width, height;
@@ -467,6 +470,8 @@ typedef struct _rfbClient {
 
 	StartingFrameBufferUpdateProc StartingFrameBufferUpdate;
 	CancelledFrameBufferUpdateProc CancelledFrameBufferUpdate;
+
+	NtpEventProc NtpEvent;
 } rfbClient;
 
 /* cursor.c */
@@ -612,6 +617,9 @@ extern rfbBool HandleRFBServerMessage(rfbClient* client);
 
 extern rfbBool ReadToBuffer(rfbClient* client);
 
+extern rfbBool SendClientNtpEvent(rfbClient* client, uint32_t t0, uint32_t t1,
+		uint32_t t2, uint32_t t3);
+
 /**
  * Sends a text chat message to the server.
  * @param client The client through which to send the message

include/rfbproto.h

@@ -398,8 +398,7 @@ typedef struct {
 #define rfbSetDesktopSize 251
 #define rfbQemuEvent 255
 
-
+#define rfbNtpEvent 160
-
 
 /*****************************************************************************
  *
@@ -498,6 +497,8 @@ typedef struct {
 #define rfbEncodingSupportedEncodings 0xFFFE0002
 #define rfbEncodingServerIdentity     0xFFFE0003
 
+#define rfbEncodingPts -1000
+#define rfbEncodingNtp -1001
 
 /*****************************************************************************
  *
@@ -1138,6 +1139,12 @@ typedef struct rfbExtDesktopScreen {
     uint32_t flags;
 } rfbExtDesktopScreen;
 
+struct rfbNtpMsg {
+	uint8_t type;
+	uint8_t padding[3];
+	uint32_t t0, t1, t2, t3;
+};
+
 #define sz_rfbExtDesktopSizeMsg (4)
 #define sz_rfbExtDesktopScreen (16)
 
@@ -1228,6 +1235,7 @@ typedef union {
 	rfbTextChatMsg tc;
 	rfbXvpMsg xvp;
 	rfbExtDesktopSizeMsg eds;
+	struct rfbNtpMsg ntp;
 } rfbServerToClientMsg;
 
 

include/vnc.h

@@ -45,6 +45,8 @@ struct vnc_client {
 	int (*alloc_fb)(struct vnc_client*);
 	void (*update_fb)(struct vnc_client*);
 	void (*cut_text)(struct vnc_client*, const char*, size_t);
+        void (*ntp_event)(struct vnc_client*, uint32_t t0, uint32_t t1,
+                        uint32_t t2, uint32_t t3);
 
 	void* userdata;
 	struct pixman_region16 damage;
@@ -79,3 +81,5 @@ void vnc_client_set_compression_level(struct vnc_client* self, int value);
 void vnc_client_send_cut_text(struct vnc_client* self, const char* text,
 		size_t len);
 void vnc_client_clear_av_frames(struct vnc_client* self);
+void vnc_client_send_ntp_event(struct vnc_client* self, uint32_t t0,
+                uint32_t t1, uint32_t t2, uint32_t t3);

meson.build

@@ -82,6 +82,8 @@ sources = [
 	'src/rfbproto.c',
 	'src/sockets.c',
 	'src/vncviewer.c',
+	'src/ntp.c',
+	'src/performance.c',
 ]
 
 dependencies = [

src/main.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020 - 2022 Andri Yngvason
+ * Copyright (c) 2020 - 2023 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -47,9 +47,12 @@
 #include "linux-dmabuf-unstable-v1.h"
 #include "time-util.h"
 #include "output.h"
+#include "ntp.h"
+#include "performance.h"
 
 #define CANARY_TICK_PERIOD INT64_C(100000) // us
 #define CANARY_LETHALITY_LEVEL INT64_C(8000) // us
+#define LATENCY_REPORT_PERIOD INT64_C(250000) // us
 
 struct point {
 	double x, y;
@@ -87,6 +90,8 @@ struct pointer_collection* pointers;
 struct keyboard_collection* keyboards;
 static int drm_fd = -1;
 static uint64_t last_canary_tick;
+static struct ntp_client ntp;
+static struct perf perf;
 
 static bool have_egl = false;
 
@@ -608,6 +613,20 @@ static void window_damage_region(struct window* w,
 	}
 }
 
+static void update_frame_latency_stats(void)
+{
+	uint32_t server_pts = window->vnc->pts;
+
+	uint32_t client_pts = 0;
+	if (!ntp_client_translate_server_time(&ntp, &client_pts, server_pts))
+		return;
+
+	uint32_t now = gettime_us();
+	int32_t latency = (int32_t)(now - client_pts);
+
+	perf_sample_buffer_add(perf.frame_latency, latency);
+}
+
 static void render_from_vnc(void)
 {
 	if (!pixman_region_not_empty(&window->current_damage) &&
@@ -653,6 +672,8 @@ static void render_from_vnc(void)
 	window_commit(window);
 	window_swap(window);
 
+	update_frame_latency_stats();
+
 	pixman_region_clear(&window->current_damage);
 	vnc_client_clear_av_frames(window->vnc);
 }
@@ -804,6 +825,18 @@ static void on_canary_tick(void* obj)
 				delay);
 }
 
+static void send_ntp_ping(struct ntp_client* ntp_client, uint32_t t0,
+		uint32_t t1, uint32_t t2, uint32_t t3)
+{
+	vnc_client_send_ntp_event(window->vnc, t0, t1, t2, t3);
+}
+
+static void on_ntp_event(struct vnc_client* vnc, uint32_t t0, uint32_t t1,
+		uint32_t t2, uint32_t t3)
+{
+	ntp_client_process_pong(&ntp, t0, t1, t2, t3);
+}
+
 static void create_canary_ticker(void)
 {
 	last_canary_tick = gettime_us();
@@ -815,6 +848,22 @@ static void create_canary_ticker(void)
 	aml_unref(ticker);
 }
 
+static void on_latency_report_tick(void* handler)
+{
+	(void)handler;
+
+	perf_dump_latency_report(&perf);
+}
+
+static void create_latency_report_ticker(void)
+{
+	struct aml* aml = aml_get_default();
+	struct aml_ticker* ticker = aml_ticker_new(LATENCY_REPORT_PERIOD,
+			on_latency_report_tick, NULL, NULL);
+	aml_start(aml, ticker);
+	aml_unref(ticker);
+}
+
 void run_main_loop_once(void)
 {
 	struct aml* aml = aml_get_default();
@@ -968,8 +1017,11 @@ int main(int argc, char* argv[])
 	if (!vnc)
 		goto vnc_failure;
 
+	vnc->userdata = window;
+
 	vnc->alloc_fb = on_vnc_client_alloc_fb;
 	vnc->update_fb = on_vnc_client_update_fb;
+	vnc->ntp_event = on_ntp_event;
 
 	if (vnc_client_set_pixel_format(vnc, shm_format) < 0) {
 		fprintf(stderr, "Unsupported pixel format\n");
@@ -1008,12 +1060,16 @@ int main(int argc, char* argv[])
 		goto vnc_setup_failure;
 	}
 
+	perf_init(&perf);
+	ntp_client_init(&ntp, send_ntp_ping, window);
+
 	pointers->userdata = vnc;
 	keyboards->userdata = vnc;
 
 	wl_display_dispatch(wl_display);
 
 	create_canary_ticker();
+	create_latency_report_ticker();
 
 	while (do_run)
 		run_main_loop_once();
@@ -1021,6 +1077,10 @@ int main(int argc, char* argv[])
 	rc = 0;
 	if (window)
 		window_destroy(window);
+
+	ntp_client_deinit(&ntp);
+	perf_deinit(&perf);
+
 vnc_setup_failure:
 	vnc_client_destroy(vnc);
 vnc_failure:

src/ntp.c

@@ -0,0 +1,120 @@
+/*
+ * Copyright (c) 2023 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "ntp.h"
+#include "time-util.h"
+
+#include <assert.h>
+#include <aml.h>
+#include <stdio.h>
+
+static inline int32_t clamp_to_zero(int32_t v)
+{
+	return v >= 0 ? v : 0;
+}
+
+void ntp_client__tick(void* handler)
+{
+	struct ntp_client* self = aml_get_userdata(handler);
+	assert(self);
+	assert(self->send_ping);
+
+	uint32_t t0 = gettime_us();
+	self->send_ping(self, t0, 0, 0, 0);
+}
+
+int ntp_client_init(struct ntp_client* self, ntp_client_ping_fn send_ping,
+		void* userdata)
+{
+	self->send_ping = send_ping;
+	self->userdata = userdata;
+
+	struct aml_ticker* ticker = aml_ticker_new(NTP_SAMPLE_PERIOD,
+			ntp_client__tick, self, NULL);
+	if (!ticker)
+		return -1;
+
+	int rc = aml_start(aml_get_default(), ticker);
+	if (rc >= 0)
+		self->ping_ticker = ticker;
+	else
+		aml_unref(ticker);
+
+	return rc;
+}
+
+void ntp_client_deinit(struct ntp_client* self)
+{
+	if (self->ping_ticker) {
+		aml_stop(aml_get_default(), self->ping_ticker);
+		aml_unref(self->ping_ticker);
+	}
+}
+
+void ntp_client_process_pong(struct ntp_client* self, uint32_t t0, uint32_t t1,
+                uint32_t t2, uint32_t t3)
+{
+	t3 = gettime_us();
+
+	int32_t theta = ((int32_t)(t1 - t0) + (int32_t)(t2 - t3)) / 2;
+	uint32_t delta = clamp_to_zero((int32_t)(t3 - t0) - (int32_t)(t2 - t1));
+
+	struct ntp_sample sample = {
+		.theta = theta,
+		.delta = delta,
+	};
+
+	self->samples[self->sample_index] = sample;
+	self->sample_index = (self->sample_index + 1) % NTP_SAMPLE_SIZE;
+
+	if (self->sample_count < NTP_SAMPLE_SIZE)
+		self->sample_count++;
+
+//	printf("%.3f %.3f\n", delta / 1e3, theta / 1e3);
+}
+
+bool ntp_client_get_best_sample(const struct ntp_client* self,
+		struct ntp_sample* out)
+{
+	if (self->sample_count < NTP_MIN_SAMPLE_COUNT)
+		return false;
+
+	struct ntp_sample result = {
+		.theta = 0,
+		.delta = UINT32_MAX,
+	};
+
+	for (int i = 0; i < self->sample_count; ++i) {
+		const struct ntp_sample *sample = &self->samples[i];
+		if (sample->delta < result.delta) {
+			result = *sample;
+		}
+	}
+
+	*out = result;
+	return true;
+}
+
+bool ntp_client_translate_server_time(const struct ntp_client* self,
+		uint32_t* dst, const uint32_t t)
+{
+	struct ntp_sample sample;
+	if (!ntp_client_get_best_sample(self, &sample))
+		return false;
+
+	*dst = (int32_t)t - sample.theta;
+	return true;
+}

src/performance.c

@@ -0,0 +1,72 @@
+#include "performance.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+
+struct perf_sample_buffer* perf_sample_buffer_create(int length)
+{
+	struct perf_sample_buffer* self = malloc(sizeof(*self)
+			+ length * sizeof(self->samples[0]));
+	self->count = 0;
+	self->index = 0;
+	self->length = length;
+	return self;
+}
+
+void perf_sample_buffer_add(struct perf_sample_buffer* self, double sample)
+{
+	self->samples[self->index] = sample;
+	self->index = (self->index + 1) % self->length;
+	if (self->count < self->length)
+		self->count += 1;
+}
+
+void perf_sample_buffer_get_stats(const struct perf_sample_buffer* self,
+		struct perf_sample_stats* stats)
+{
+	memset(stats, 0, sizeof(*stats));
+
+	double sum = 0;
+	double minimum = INFINITY;
+	double maximum = 0;
+
+	for (int i = 0; i < self->count; ++i) {
+		double sample = self->samples[i];
+
+		sum += sample;
+
+		if (sample < minimum)
+			minimum = sample;
+
+		if (sample > maximum)
+			maximum = sample;
+	}
+
+	stats->min = minimum;
+	stats->max = maximum;
+	stats->average = sum / (double)self->count;
+}
+
+void perf_init(struct perf* self)
+{
+	memset(self, 0, sizeof(*self));
+
+	self->frame_latency =
+		perf_sample_buffer_create(PERF_FRAME_LATENCY_SAMPLE_SIZE);
+}
+
+void perf_deinit(struct perf* self)
+{
+	free(self->frame_latency);
+}
+
+void perf_dump_latency_report(const struct perf* self)
+{
+	struct perf_sample_stats stats;
+	perf_sample_buffer_get_stats(self->frame_latency, &stats);
+
+	printf("Latency report: frame-latency (min, avg, max): %.1f, %.1f, %.1f\n",
+			stats.min / 1e3, stats.average / 1e3, stats.max / 1e3);
+}

src/rfbproto.c

@@ -1396,7 +1396,11 @@ rfbBool SetFormatAndEncodings(rfbClient* client)
 
 	/* pts */
 	if (se->nEncodings < MAX_ENCODINGS)
-		encs[se->nEncodings++] = rfbClientSwap32IfLE(-1000);
+		encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingPts);
+
+	/* ntp */
+	if (se->nEncodings < MAX_ENCODINGS)
+		encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingNtp);
 
 	len = sz_rfbSetEncodingsMsg + se->nEncodings * 4;
 
@@ -1742,6 +1746,22 @@ rfbBool SendClientCutText(rfbClient* client, char* str, int len)
 	        WriteToRFBServer(client, str, len));
 }
 
+rfbBool SendClientNtpEvent(rfbClient* client, uint32_t t0, uint32_t t1,
+		uint32_t t2, uint32_t t3)
+{
+//	if (!SupportsClient2Server(client, rfbNtpEvent))
+//		return FALSE;
+
+	struct rfbNtpMsg msg = {
+		.type = rfbNtpEvent,
+		.t0 = rfbClientSwap32IfLE(t0),
+		.t1 = rfbClientSwap32IfLE(t1),
+		.t2 = rfbClientSwap32IfLE(t2),
+		.t3 = rfbClientSwap32IfLE(t3),
+	};
+	return WriteToRFBServer(client, (char*)&msg, sizeof(msg));
+}
+
 static rfbBool HandleFramebufferUpdate(rfbClient* client,
                                        rfbServerToClientMsg* msg)
 {
@@ -2300,6 +2320,10 @@ static rfbBool HandleFramebufferUpdate(rfbClient* client,
 			SetClient2Server(client, rfbQemuEvent);
 			break;
 
+		case rfbEncodingNtp:
+			SetClient2Server(client, rfbNtpEvent);
+			break;
+
 		default: {
 			rfbBool handled = FALSE;
 			rfbClientProtocolExtension* e;
@@ -2339,6 +2363,22 @@ failure:
 	return FALSE;
 }
 
+rfbBool handleNtpEvent(rfbClient* client, struct rfbNtpMsg* msg)
+{
+	if (!ReadFromRFBServer(client, (char*)msg + 1, sizeof(*msg) - 1))
+		return FALSE;
+
+	uint32_t t0 = rfbClientSwap32IfLE(msg->t0);
+	uint32_t t1 = rfbClientSwap32IfLE(msg->t1);
+	uint32_t t2 = rfbClientSwap32IfLE(msg->t2);
+	uint32_t t3 = rfbClientSwap32IfLE(msg->t3);
+
+	if (client->NtpEvent)
+		client->NtpEvent(client, t0, t1, t2, t3);
+
+	return TRUE;
+}
+
 /*
  * HandleRFBServerMessage.
  */
@@ -2527,6 +2567,9 @@ rfbBool HandleRFBServerMessage(rfbClient* client)
 		break;
 	}
 
+	case rfbNtpEvent:
+		return handleNtpEvent(client, &msg.ntp);
+
 	default: {
 		rfbBool handled = FALSE;
 		rfbClientProtocolExtension* e;

src/vnc.c

@@ -136,6 +136,16 @@ static void vnc_client_got_cut_text(rfbClient* client, const char* text,
 		self->cut_text(self, text, len);
 }
 
+static void vnc_client_ntp_event(rfbClient* client, uint32_t t0, uint32_t t1,
+		uint32_t t2, uint32_t t3)
+{
+	struct vnc_client* self = rfbClientGetClientData(client, NULL);
+	assert(self);
+
+	if (self->ntp_event)
+		self->ntp_event(self, t0, t1, t2, t3);
+}
+
 static rfbBool vnc_client_handle_open_h264_rect(rfbClient* client,
 		rfbFramebufferUpdateRectHeader* rect_header)
 {
@@ -246,6 +256,7 @@ struct vnc_client* vnc_client_create(void)
 	client->StartingFrameBufferUpdate = vnc_client_start_update;
 	client->CancelledFrameBufferUpdate = vnc_client_cancel_update;
 	client->GotXCutText = vnc_client_got_cut_text;
+	client->NtpEvent = vnc_client_ntp_event;
 
 	self->pts = NO_PTS;
 
@@ -443,3 +454,9 @@ void vnc_client_send_cut_text(struct vnc_client* self, const char* text,
 	// libvncclient doesn't modify text, so typecast is OK.
 	SendClientCutText(self->client, (char*)text, len);
 }
+
+void vnc_client_send_ntp_event(struct vnc_client* self, uint32_t t0,
+                uint32_t t1, uint32_t t2, uint32_t t3)
+{
+	SendClientNtpEvent(self->client, t0, t1, t2, t3);
+}