Change H264 encoder from baseline to main

Add a `struct nvnc* nvnc_open_from_fd(int fd)` function which takes an
existing connection-based socket file descriptor bound by the library
user or a parent process and just calls listen() on it, as an
alternative to letting neatvnc handle socket configuration.

.gitignore

@@ -8,3 +8,4 @@ build
 experiments
 subprojects
 sandbox
+.vscode

include/common.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019 - 2020 Andri Yngvason
+ * Copyright (c) 2019 - 2024 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -109,6 +109,8 @@ struct nvnc_client {
 	uint32_t cursor_seq;
 	int quality;
 	bool formats_changed;
+	enum nvnc_keyboard_led_state led_state;
+	enum nvnc_keyboard_led_state pending_led_state;
 
 #ifdef HAVE_CRYPTO
 	struct crypto_key* apple_dh_secret;
@@ -129,6 +131,7 @@ enum nvnc__socket_type {
 	NVNC__SOCKET_TCP,
 	NVNC__SOCKET_UNIX,
 	NVNC__SOCKET_WEBSOCKET,
+	NVNC__SOCKET_FROM_FD,
 };
 
 struct nvnc {

include/neatvnc.h

@@ -86,6 +86,12 @@ enum nvnc_transform {
 	NVNC_TRANSFORM_FLIPPED_270 = 7,
 };
 
+enum nvnc_keyboard_led_state {
+	NVNC_KEYBOARD_LED_SCROLL_LOCK = 1 << 0,
+	NVNC_KEYBOARD_LED_NUM_LOCK = 1 << 1,
+	NVNC_KEYBOARD_LED_CAPS_LOCK = 1 << 2,
+};
+
 enum nvnc_log_level {
 	NVNC_LOG_PANIC = 0,
 	NVNC_LOG_ERROR = 1,
@@ -132,6 +138,7 @@ extern const char nvnc_version[];
 struct nvnc* nvnc_open(const char* addr, uint16_t port);
 struct nvnc* nvnc_open_unix(const char *addr);
 struct nvnc* nvnc_open_websocket(const char* addr, uint16_t port);
+struct nvnc* nvnc_open_from_fd(int fd);
 void nvnc_close(struct nvnc* self);
 
 void nvnc_add_display(struct nvnc*, struct nvnc_display*);
@@ -150,6 +157,9 @@ struct nvnc_client* nvnc_client_first(struct nvnc* self);
 struct nvnc_client* nvnc_client_next(struct nvnc_client* client);
 void nvnc_client_close(struct nvnc_client* client);
 
+void nvnc_client_set_led_state(struct nvnc_client*,
+		enum nvnc_keyboard_led_state);
+
 void nvnc_set_name(struct nvnc* self, const char* name);
 
 void nvnc_set_key_fn(struct nvnc* self, nvnc_key_fn);

include/rfb-proto.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019 - 2022 Andri Yngvason
+ * Copyright (c) 2019 - 2024 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -69,9 +69,11 @@ enum rfb_encodings {
 	RFB_ENCODING_CURSOR = -239,
 	RFB_ENCODING_DESKTOPSIZE = -223,
 	RFB_ENCODING_QEMU_EXT_KEY_EVENT = -258,
+	RFB_ENCODING_QEMU_LED_STATE = -261,
 	RFB_ENCODING_EXTENDEDDESKTOPSIZE = -308,
 	RFB_ENCODING_PTS = -1000,
 	RFB_ENCODING_NTP = -1001,
+	RFB_ENCODING_VMWARE_LED_STATE = 0x574d5668,
 };
 
 #define RFB_ENCODING_JPEG_HIGHQ -23
@@ -114,6 +116,13 @@ enum rfb_rsa_aes_cred_subtype {
 	RFB_RSA_AES_CRED_SUBTYPE_ONLY_PASS = 2,
 };
 
+// This is the same for both qemu and vmware extensions
+enum rfb_led_state {
+	RFB_LED_STATE_SCROLL_LOCK = 1 << 0,
+	RFB_LED_STATE_NUM_LOCK = 1 << 1,
+	RFB_LED_STATE_CAPS_LOCK = 1 << 2,
+};
+
 struct rfb_security_types_msg {
 	uint8_t n;
 	uint8_t types[0];

src/h264-encoder-ffmpeg-impl.c

@@ -310,9 +310,11 @@ static int h264_encoder__init_codec_context(struct h264_encoder_ffmpeg* self,
 	c->global_quality = quality;
 
 	/* open-h264 requires baseline profile, so we use constrained
-	 * baseline.
+	 * baseline: AV_PROFILE_H264_BASELINE.
+	 * But that is not supported by many clients. So we use a "DEFAULT" profile.
+	 * 
 	 */
-	c->profile = 578;
+	c->profile = AV_PROFILE_H264_MAIN;
 
 	return 0;
 }

src/server.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019 - 2022 Andri Yngvason
+ * Copyright (c) 2019 - 2024 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -85,6 +85,7 @@ static void process_fb_update_requests(struct nvnc_client* client);
 static void sockaddr_to_string(char* dst, size_t sz,
 		const struct sockaddr* addr);
 static const char* encoding_to_string(enum rfb_encodings encoding);
+static bool client_send_led_state(struct nvnc_client* client);
 
 #if defined(PROJECT_VERSION)
 EXPORT const char nvnc_version[] = PROJECT_VERSION;
@@ -1025,6 +1026,8 @@ static int on_client_set_encodings(struct nvnc_client* client)
 		case RFB_ENCODING_DESKTOPSIZE:
 		case RFB_ENCODING_EXTENDEDDESKTOPSIZE:
 		case RFB_ENCODING_QEMU_EXT_KEY_EVENT:
+		case RFB_ENCODING_QEMU_LED_STATE:
+		case RFB_ENCODING_VMWARE_LED_STATE:
 #ifdef ENABLE_EXPERIMENTAL
 		case RFB_ENCODING_PTS:
 		case RFB_ENCODING_NTP:
@@ -1117,6 +1120,8 @@ static const char* encoding_to_string(enum rfb_encodings encoding)
 	case RFB_ENCODING_DESKTOPSIZE: return "desktop-size";
 	case RFB_ENCODING_EXTENDEDDESKTOPSIZE: return "extended-desktop-size";
 	case RFB_ENCODING_QEMU_EXT_KEY_EVENT: return "qemu-extended-key-event";
+	case RFB_ENCODING_QEMU_LED_STATE: return "qemu-led-state";
+	case RFB_ENCODING_VMWARE_LED_STATE: return "vmware-led-state";
 	case RFB_ENCODING_PTS: return "pts";
 	case RFB_ENCODING_NTP: return "ntp";
 	}
@@ -1225,6 +1230,11 @@ static void process_fb_update_requests(struct nvnc_client* client)
 			return;
 	}
 
+	if (client_send_led_state(client)) {
+		if (--client->n_pending_requests <= 0)
+			return;
+	}
+
 	if (!pixman_region_not_empty(&client->damage))
 		return;
 
@@ -1796,6 +1806,7 @@ static void on_connection(void* obj)
 	client->ref = 1;
 	client->server = server;
 	client->quality = 10; /* default to lossless */
+	client->led_state = -1; /* trigger sending of initial state */
 
 	int fd = accept(server->fd, NULL, 0);
 	if (fd < 0) {
@@ -1964,7 +1975,7 @@ static int bind_address_unix(const char* name)
 }
 
 static int bind_address(const char* name, uint16_t port,
-		enum nvnc__socket_type type)
+		int fd, enum nvnc__socket_type type)
 {
 	switch (type) {
 	case NVNC__SOCKET_TCP:
@@ -1972,6 +1983,9 @@ static int bind_address(const char* name, uint16_t port,
 		return bind_address_tcp(name, port);
 	case NVNC__SOCKET_UNIX:
 		return bind_address_unix(name);
+	case NVNC__SOCKET_FROM_FD:
+		// nothing to bind
+		return fd;
 	}
 
 	nvnc_log(NVNC_LOG_PANIC, "Unknown socket address type");
@@ -1979,7 +1993,7 @@ static int bind_address(const char* name, uint16_t port,
 }
 
 static struct nvnc* open_common(const char* address, uint16_t port,
-		enum nvnc__socket_type type)
+		int fd, enum nvnc__socket_type type)
 {
 	nvnc__log_init();
 
@@ -1995,7 +2009,7 @@ static struct nvnc* open_common(const char* address, uint16_t port,
 
 	LIST_INIT(&self->clients);
 
-	self->fd = bind_address(address, port, type);
+	self->fd = bind_address(address, port, fd, type);
 	if (self->fd < 0)
 		goto bind_failure;
 
@@ -2028,14 +2042,14 @@ bind_failure:
 EXPORT
 struct nvnc* nvnc_open(const char* address, uint16_t port)
 {
-	return open_common(address, port, NVNC__SOCKET_TCP);
+	return open_common(address, port, -1, NVNC__SOCKET_TCP);
 }
 
 EXPORT
 struct nvnc* nvnc_open_websocket(const char *address, uint16_t port)
 {
 #ifdef ENABLE_WEBSOCKET
-	return open_common(address, port, NVNC__SOCKET_WEBSOCKET);
+	return open_common(address, port, -1, NVNC__SOCKET_WEBSOCKET);
 #else
 	return NULL;
 #endif
@@ -2044,7 +2058,13 @@ struct nvnc* nvnc_open_websocket(const char *address, uint16_t port)
 EXPORT
 struct nvnc* nvnc_open_unix(const char* address)
 {
-	return open_common(address, 0, NVNC__SOCKET_UNIX);
+	return open_common(address, 0, -1, NVNC__SOCKET_UNIX);
+}
+
+EXPORT
+struct nvnc* nvnc_open_from_fd(int fd)
+{
+	return open_common(NULL, 0, fd, NVNC__SOCKET_FROM_FD);
 }
 
 static void unlink_fd_path(int fd)
@@ -2425,6 +2445,60 @@ bool nvnc_client_supports_cursor(const struct nvnc_client* client)
 	return false;
 }
 
+static bool client_send_led_state(struct nvnc_client* client)
+{
+	if (client->pending_led_state == client->led_state)
+		return false;
+
+	bool have_qemu_led_state =
+		client_has_encoding(client, RFB_ENCODING_QEMU_LED_STATE);
+	bool have_vmware_led_state =
+		client_has_encoding(client, RFB_ENCODING_VMWARE_LED_STATE);
+
+	if (!have_qemu_led_state && !have_vmware_led_state)
+		return false;
+
+	nvnc_log(NVNC_LOG_DEBUG, "Keyboard LED state changed: %x -> %x",
+			client->led_state, client->pending_led_state);
+
+	struct vec payload;
+	vec_init(&payload, 4096);
+
+	struct rfb_server_fb_update_msg head = {
+		.type = RFB_SERVER_TO_CLIENT_FRAMEBUFFER_UPDATE,
+		.n_rects = htons(1),
+	};
+
+	struct rfb_server_fb_rect rect = {
+		.encoding = htonl(RFB_ENCODING_QEMU_LED_STATE),
+	};
+
+	vec_append(&payload, &head, sizeof(head));
+	vec_append(&payload, &rect, sizeof(rect));
+
+	if (have_qemu_led_state) {
+		uint8_t data = client->pending_led_state;
+		vec_append(&payload, &data, sizeof(data));
+	} else if (have_vmware_led_state) {
+		uint32_t data = htonl(client->pending_led_state);
+		vec_append(&payload, &data, sizeof(data));
+	}
+
+	stream_send(client->net_stream, rcbuf_new(payload.data, payload.len),
+			NULL, NULL);
+	client->led_state = client->pending_led_state;
+
+	return true;
+}
+
+EXPORT
+void nvnc_client_set_led_state(struct nvnc_client* client,
+		enum nvnc_keyboard_led_state state)
+{
+	client->pending_led_state = state;
+	process_fb_update_requests(client);
+}
+
 EXPORT
 void nvnc_set_name(struct nvnc* self, const char* name)
 {