From 9fa1027353cb5c66439d4a8ebacf4bab3069ae48 Mon Sep 17 00:00:00 2001
From: Andri Yngvason <andri@yngvason.is>
Date: Fri, 2 Feb 2024 22:24:01 +0000
Subject: [PATCH] server: Drop current frame if formats change

If the currently in-flight frame was dispatched before a format change,
it might be the wrong format for the client, so it's better to drop it.
---
 include/common.h |  1 +
 src/server.c     | 15 +++++++++++++++
 2 files changed, 16 insertions(+)

diff --git a/include/common.h b/include/common.h
index b178de3..50ed1b0 100644
--- a/include/common.h
+++ b/include/common.h
@@ -108,6 +108,7 @@ struct nvnc_client {
 	struct encoder* tight_encoder;
 	uint32_t cursor_seq;
 	int quality;
+	bool formats_changed;
 
 #ifdef HAVE_CRYPTO
 	struct crypto_key* apple_dh_secret;
diff --git a/src/server.c b/src/server.c
index 89268a4..204e976 100644
--- a/src/server.c
+++ b/src/server.c
@@ -938,6 +938,7 @@ static int on_client_set_pixel_format(struct nvnc_client* client)
 	memcpy(&client->pixfmt, fmt, sizeof(client->pixfmt));
 
 	client->has_pixfmt = true;
+	client->formats_changed = true;
 
 	nvnc_log(NVNC_LOG_DEBUG, "Client %p chose pixel format: %s", client,
 			rfb_pixfmt_to_string(fmt));
@@ -1007,6 +1008,7 @@ static int on_client_set_encodings(struct nvnc_client* client)
 			encoding_list);
 
 	client->n_encodings = n;
+	client->formats_changed = true;
 
 	return sizeof(*msg) + 4 * n_encodings;
 }
@@ -1186,6 +1188,7 @@ static void process_fb_update_requests(struct nvnc_client* client)
 	pixman_region_init(&client->damage);
 
 	client->is_updating = true;
+	client->formats_changed = false;
 	client->current_fb = fb;
 	nvnc_fb_hold(fb);
 	nvnc_fb_ref(fb);
@@ -1207,6 +1210,7 @@ static void process_fb_update_requests(struct nvnc_client* client)
 		nvnc_log(NVNC_LOG_ERROR, "Failed to encode current frame");
 		client_unref(client);
 		client->is_updating = false;
+		client->formats_changed = false;
 		assert(client->current_fb);
 		nvnc_fb_release(client->current_fb);
 		nvnc_fb_unref(client->current_fb);
@@ -2105,6 +2109,17 @@ static void finish_fb_update(struct nvnc_client* client, struct rcbuf* payload,
 	if (client->net_stream->state == STREAM_STATE_CLOSED)
 		goto complete;
 
+	if (client->formats_changed) {
+		/* Client has requested new pixel format or encoding in the
+		 * meantime, so it probably won't know what to do with this
+		 * frame. Pending requests get incremented because this one is
+		 * dropped.
+		 */
+		nvnc_log(NVNC_LOG_DEBUG, "Client changed pixel format or encoding with in-flight buffer");
+		client->n_pending_requests++;
+		goto complete;
+	}
+
 	DTRACE_PROBE2(neatvnc, send_fb_start, client, pts);
 	n_rects += will_send_pts(client, pts) ? 1 : 0;
 	struct rfb_server_fb_update_msg update_msg = {