Implement desktop resizing

Implement minimal support for ExtendedDesktopSize pseudo-encoding
and SetDesktopSize client message.

The opaque nvnc_desktop_layout structure contains all information
from the SetDesktopSize client message.

include/common.h

@@ -110,6 +110,7 @@ struct nvnc {
 	nvnc_fb_req_fn fb_req_fn;
 	nvnc_client_fn new_client_fn;
 	nvnc_cut_text_fn cut_text_fn;
+	nvnc_desktop_layout_fn desktop_layout_fn;
 	struct nvnc_display* display;
 	struct {
 		struct nvnc_fb* buffer;

include/desktop-layout.h

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2023 Philipp Zabel
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#pragma once
+
+#include <stdint.h>
+
+struct nvnc_display;
+struct rfb_screen;
+
+struct nvnc_display_layout {
+	struct nvnc_display* display;
+	uint32_t id;
+	uint16_t x_pos, y_pos;
+	uint16_t width, height;
+};
+
+struct nvnc_desktop_layout {
+	uint16_t width, height;
+	uint8_t n_display_layouts;
+	struct nvnc_display_layout display_layouts[0];
+};
+
+void nvnc_display_layout_init(
+		struct nvnc_display_layout* display, struct rfb_screen* screen);

include/neatvnc.h

@@ -50,6 +50,7 @@
 
 struct nvnc;
 struct nvnc_client;
+struct nvnc_desktop_layout;
 struct nvnc_display;
 struct nvnc_fb;
 struct nvnc_fb_pool;
@@ -117,6 +118,8 @@ typedef struct nvnc_fb* (*nvnc_fb_alloc_fn)(uint16_t width, uint16_t height,
 		uint32_t format, uint16_t stride);
 typedef void (*nvnc_cleanup_fn)(void* userdata);
 typedef void (*nvnc_log_fn)(const struct nvnc_log_data*, const char* message);
+typedef bool (*nvnc_desktop_layout_fn)(
+		struct nvnc_client*, const struct nvnc_desktop_layout*);
 
 extern const char nvnc_version[];
 
@@ -148,6 +151,7 @@ void nvnc_set_fb_req_fn(struct nvnc* self, nvnc_fb_req_fn);
 void nvnc_set_new_client_fn(struct nvnc* self, nvnc_client_fn);
 void nvnc_set_client_cleanup_fn(struct nvnc_client* self, nvnc_client_fn fn);
 void nvnc_set_cut_text_fn(struct nvnc*, nvnc_cut_text_fn fn);
+void nvnc_set_desktop_layout_fn(struct nvnc* self, nvnc_desktop_layout_fn);
 
 bool nvnc_has_auth(void);
 int nvnc_enable_auth(struct nvnc* self, const char* privkey_path,
@@ -202,6 +206,20 @@ struct nvnc* nvnc_display_get_server(const struct nvnc_display*);
 void nvnc_display_feed_buffer(struct nvnc_display*, struct nvnc_fb*,
 			      struct pixman_region16* damage);
 
+uint16_t nvnc_desktop_layout_get_width(const struct nvnc_desktop_layout*);
+uint16_t nvnc_desktop_layout_get_height(const struct nvnc_desktop_layout*);
+uint8_t nvnc_desktop_layout_get_display_count(const struct nvnc_desktop_layout*);
+uint16_t nvnc_desktop_layout_get_display_x_pos(
+		const struct nvnc_desktop_layout*, uint8_t display_index);
+uint16_t nvnc_desktop_layout_get_display_y_pos(
+		const struct nvnc_desktop_layout*, uint8_t display_index);
+uint16_t nvnc_desktop_layout_get_display_width(
+		const struct nvnc_desktop_layout*, uint8_t display_index);
+uint16_t nvnc_desktop_layout_get_display_height(
+		const struct nvnc_desktop_layout*, uint8_t display_index);
+struct nvnc_display* nvnc_desktop_layout_get_display(
+		const struct nvnc_desktop_layout*, uint8_t display_index);
+
 void nvnc_send_cut_text(struct nvnc*, const char* text, uint32_t len);
 
 void nvnc_set_cursor(struct nvnc*, struct nvnc_fb*, uint16_t width,

include/rfb-proto.h

@@ -45,6 +45,7 @@ enum rfb_client_to_server_msg_type {
 	RFB_CLIENT_TO_SERVER_KEY_EVENT = 4,
 	RFB_CLIENT_TO_SERVER_POINTER_EVENT = 5,
 	RFB_CLIENT_TO_SERVER_CLIENT_CUT_TEXT = 6,
+	RFB_CLIENT_TO_SERVER_SET_DESKTOP_SIZE = 251,
 	RFB_CLIENT_TO_SERVER_QEMU = 255,
 };
 
@@ -64,6 +65,7 @@ enum rfb_encodings {
 	RFB_ENCODING_CURSOR = -239,
 	RFB_ENCODING_DESKTOPSIZE = -223,
 	RFB_ENCODING_QEMU_EXT_KEY_EVENT = -258,
+	RFB_ENCODING_EXTENDEDDESKTOPSIZE = -308,
 	RFB_ENCODING_PTS = -1000,
 };
 
@@ -87,6 +89,20 @@ enum rfb_vencrypt_subtype {
 	RFB_VENCRYPT_X509_PLAIN,
 };
 
+enum rfb_resize_initiator {
+	RFB_RESIZE_INITIATOR_SERVER = 0,
+	RFB_RESIZE_INITIATOR_THIS_CLIENT = 1,
+	RFB_RESIZE_INITIATOR_OTHER_CLIENT = 2,
+};
+
+enum rfb_resize_status {
+	RFB_RESIZE_STATUS_SUCCESS = 0,
+	RFB_RESIZE_STATUS_PROHIBITED = 1,
+	RFB_RESIZE_STATUS_OUT_OF_RESOURCES = 2,
+	RFB_RESIZE_STATUS_INVALID_LAYOUT = 3,
+	RFB_RESIZE_STATUS_REQUEST_FORWARDED = 4,
+};
+
 struct rfb_security_types_msg {
 	uint8_t n;
 	uint8_t types[1];
@@ -172,6 +188,25 @@ struct rfb_server_fb_rect {
 	int32_t encoding;
 } RFB_PACKED;
 
+struct rfb_screen {
+	uint32_t id;
+	uint16_t x;
+	uint16_t y;
+	uint16_t width;
+	uint16_t height;
+	uint32_t flags;
+} RFB_PACKED;
+
+struct rfb_client_set_desktop_size_event_msg {
+	uint8_t type;
+	uint8_t padding;
+	uint16_t width;
+	uint16_t height;
+	uint8_t number_of_screens;
+	uint8_t padding2;
+	struct rfb_screen screens[0];
+} RFB_PACKED;
+
 struct rfb_server_fb_update_msg {
 	uint8_t type;
 	uint8_t padding;

meson.build

@@ -77,6 +77,7 @@ sources = [
 	'src/fb_pool.c',
 	'src/rcbuf.c',
 	'src/stream.c',
+	'src/desktop-layout.c',
 	'src/display.c',
 	'src/tight.c',
 	'src/enc-util.c',

src/desktop-layout.c

@@ -0,0 +1,96 @@
+/*
+ * Copyright (c) 2023 Philipp Zabel
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "desktop-layout.h"
+#include "neatvnc.h"
+#include "rfb-proto.h"
+
+#define EXPORT __attribute__((visibility("default")))
+
+void nvnc_display_layout_init(
+		struct nvnc_display_layout* display, struct rfb_screen* screen)
+{
+	display->display = NULL;
+	display->id = ntohl(screen->id);
+	display->x_pos = ntohs(screen->x);
+	display->y_pos = ntohs(screen->y);
+	display->width = ntohs(screen->width);
+	display->height = ntohs(screen->height);
+}
+
+EXPORT
+uint16_t nvnc_desktop_layout_get_width(const struct nvnc_desktop_layout* layout)
+{
+	return layout->width;
+}
+
+EXPORT
+uint16_t nvnc_desktop_layout_get_height(const struct nvnc_desktop_layout* layout)
+{
+	return layout->height;
+}
+
+EXPORT
+uint8_t nvnc_desktop_layout_get_display_count(
+		const struct nvnc_desktop_layout* layout)
+{
+	return layout->n_display_layouts;
+}
+
+EXPORT
+uint16_t nvnc_desktop_layout_get_display_x_pos(
+		const struct nvnc_desktop_layout* layout, uint8_t display_index)
+{
+	if (display_index >= layout->n_display_layouts)
+		return 0;
+	return layout->display_layouts[display_index].x_pos;
+}
+
+EXPORT
+uint16_t nvnc_desktop_layout_get_display_y_pos(
+		const struct nvnc_desktop_layout* layout, uint8_t display_index)
+{
+	if (display_index >= layout->n_display_layouts)
+		return 0;
+	return layout->display_layouts[display_index].y_pos;
+}
+
+EXPORT
+uint16_t nvnc_desktop_layout_get_display_width(
+		const struct nvnc_desktop_layout* layout, uint8_t display_index)
+{
+	if (display_index >= layout->n_display_layouts)
+		return 0;
+	return layout->display_layouts[display_index].width;
+}
+
+EXPORT
+uint16_t nvnc_desktop_layout_get_display_height(
+		const struct nvnc_desktop_layout* layout, uint8_t display_index)
+{
+	if (display_index >= layout->n_display_layouts)
+		return 0;
+	return layout->display_layouts[display_index].height;
+}
+
+EXPORT
+struct nvnc_display* nvnc_desktop_layout_get_display(
+		const struct nvnc_desktop_layout* layout, uint8_t display_index)
+{
+	if (display_index >= layout->n_display_layouts)
+		return NULL;
+	return layout->display_layouts[display_index].display;
+}

src/server.c

@@ -18,6 +18,7 @@
 #include "vec.h"
 #include "type-macros.h"
 #include "fb.h"
+#include "desktop-layout.h"
 #include "display.h"
 #include "neatvnc.h"
 #include "common.h"
@@ -513,6 +514,7 @@ static int on_client_set_encodings(struct nvnc_client* client)
 		case RFB_ENCODING_OPEN_H264:
 		case RFB_ENCODING_CURSOR:
 		case RFB_ENCODING_DESKTOPSIZE:
+		case RFB_ENCODING_EXTENDEDDESKTOPSIZE:
 		case RFB_ENCODING_QEMU_EXT_KEY_EVENT:
 		case RFB_ENCODING_PTS:
 			client->encodings[n++] = encoding;
@@ -737,6 +739,12 @@ static int on_client_fb_update_request(struct nvnc_client* client)
 	if (fn)
 		fn(client, incremental, x, y, width, height);
 
+	if (!incremental &&
+	    client_has_encoding(client, RFB_ENCODING_EXTENDEDDESKTOPSIZE)) {
+		client->known_width = 0;
+		client->known_height = 0;
+	}
+
 	process_fb_update_requests(client);
 
 	return sizeof(*msg);
@@ -932,6 +940,105 @@ static void process_big_cut_text(struct nvnc_client* client)
 	client->cut_text.buffer = NULL;
 }
 
+static enum rfb_resize_status check_desktop_layout(struct nvnc_client* client,
+		uint16_t width, uint16_t height, uint8_t n_screens,
+		struct rfb_screen* screens)
+{
+	struct nvnc* server = client->server;
+	struct nvnc_desktop_layout* layout;
+	enum rfb_resize_status status = RFB_RESIZE_STATUS_SUCCESS;
+
+	layout = malloc(sizeof(*layout) +
+			n_screens * sizeof(*layout->display_layouts));
+	if (!layout)
+		return RFB_RESIZE_STATUS_OUT_OF_RESOURCES;
+
+	layout->width = width;
+	layout->height = height;
+	layout->n_display_layouts = n_screens;
+
+	for (size_t i = 0; i < n_screens; ++i) {
+		struct nvnc_display_layout* display;
+		struct rfb_screen* screen;
+
+		display = &layout->display_layouts[i];
+		screen = &screens[i];
+
+		nvnc_display_layout_init(display, screen);
+
+		if (screen->id == 0)
+			display->display = server->display;
+
+		if (display->x_pos + display->width > width ||
+		    display->y_pos + display->height > height) {
+			status = RFB_RESIZE_STATUS_INVALID_LAYOUT;
+			goto out;
+		}
+	}
+
+	if (!server->desktop_layout_fn ||
+	    !server->desktop_layout_fn(client, layout))
+		status = RFB_RESIZE_STATUS_PROHIBITED;
+out:
+	free(layout);
+	return status;
+}
+
+static void send_extended_desktop_size(struct nvnc_client* client,
+		enum rfb_resize_initiator initiator,
+		enum rfb_resize_status status)
+{
+	struct rfb_server_fb_update_msg head = {
+		.type = RFB_SERVER_TO_CLIENT_FRAMEBUFFER_UPDATE,
+		.n_rects = htons(1),
+	};
+
+	struct rfb_server_fb_rect rect = {
+		.encoding = htonl(RFB_ENCODING_EXTENDEDDESKTOPSIZE),
+		.x = htons(initiator),
+		.y = htons(status),
+		.width = htons(client->known_width),
+		.height = htons(client->known_height),
+	};
+
+	uint8_t number_of_screens = 1;
+	uint8_t buf[4] = { number_of_screens };
+
+	struct rfb_screen screen = {
+		.width = htons(client->known_width),
+		.height = htons(client->known_height),
+	};
+
+	stream_write(client->net_stream, &head, sizeof(head), NULL, NULL);
+	stream_write(client->net_stream, &rect, sizeof(rect), NULL, NULL);
+	stream_write(client->net_stream, &buf, sizeof(buf), NULL, NULL);
+	stream_write(client->net_stream, &screen, sizeof(screen), NULL, NULL);
+}
+
+static int on_client_set_desktop_size_event(struct nvnc_client* client)
+{
+	struct rfb_client_set_desktop_size_event_msg* msg;
+	enum rfb_resize_status status;
+	uint16_t width, height;
+
+	if (client->buffer_len - client->buffer_index < sizeof(*msg))
+		return 0;
+
+	msg = (struct rfb_client_set_desktop_size_event_msg*)
+	      (client->msg_buffer + client->buffer_index);
+
+	width = ntohs(msg->width);
+	height = ntohs(msg->height);
+
+	status = check_desktop_layout(client, width, height,
+			msg->number_of_screens, msg->screens);
+
+	send_extended_desktop_size(client, RFB_RESIZE_INITIATOR_THIS_CLIENT,
+				   status);
+
+	return sizeof(*msg) + msg->number_of_screens * sizeof(struct rfb_screen);
+}
+
 static int on_client_message(struct nvnc_client* client)
 {
 	if (client->buffer_len - client->buffer_index < 1)
@@ -955,6 +1062,8 @@ static int on_client_message(struct nvnc_client* client)
 		return on_client_cut_text(client);
 	case RFB_CLIENT_TO_SERVER_QEMU:
 		return on_client_qemu_event(client);
+	case RFB_CLIENT_TO_SERVER_SET_DESKTOP_SIZE:
+		return on_client_set_desktop_size_event(client);
 	}
 
 	nvnc_log(NVNC_LOG_WARNING, "Got uninterpretable message from client: %p (ref %d)",
@@ -1428,7 +1537,8 @@ static void on_encode_frame_done(struct encoder* encoder, struct rcbuf* result,
 
 static int send_desktop_resize(struct nvnc_client* client, struct nvnc_fb* fb)
 {
-	if (!client_has_encoding(client, RFB_ENCODING_DESKTOPSIZE)) {
+	if (!client_has_encoding(client, RFB_ENCODING_DESKTOPSIZE) &&
+	    !client_has_encoding(client, RFB_ENCODING_EXTENDEDDESKTOPSIZE)) {
 		nvnc_log(NVNC_LOG_ERROR, "Client does not support desktop resizing. Closing connection...");
 		nvnc_client_close(client);
 		return -1;
@@ -1443,6 +1553,13 @@ static int send_desktop_resize(struct nvnc_client* client, struct nvnc_fb* fb)
 	pixman_region_union_rect(&client->damage, &client->damage, 0, 0,
 			fb->width, fb->height);
 
+	if (client_has_encoding(client, RFB_ENCODING_EXTENDEDDESKTOPSIZE)) {
+		send_extended_desktop_size(client,
+				RFB_RESIZE_INITIATOR_SERVER,
+				RFB_RESIZE_STATUS_SUCCESS);
+		return 0;
+	}
+
 	struct rfb_server_fb_update_msg head = {
 		.type = RFB_SERVER_TO_CLIENT_FRAMEBUFFER_UPDATE,
 		.n_rects = htons(1),
@@ -1545,6 +1662,12 @@ void nvnc_set_cut_text_fn(struct nvnc* self, nvnc_cut_text_fn fn)
 	self->cut_text_fn = fn;
 }
 
+EXPORT
+void nvnc_set_desktop_layout_fn(struct nvnc* self, nvnc_desktop_layout_fn fn)
+{
+	self->desktop_layout_fn = fn;
+}
+
 EXPORT
 void nvnc_add_display(struct nvnc* self, struct nvnc_display* display)
 {