Change H264 encoder from baseline to main

Add a `struct nvnc* nvnc_open_from_fd(int fd)` function which takes an
existing connection-based socket file descriptor bound by the library
user or a parent process and just calls listen() on it, as an
alternative to letting neatvnc handle socket configuration.

.github/pull_request_template.md

@@ -0,0 +1 @@
+Please read CONTRIBUTING.md before making a pull request.

.gitignore

@@ -1,8 +1,11 @@
 .clang_complete
 .ycm_extra_conf.py
+.vimrc
 vgcore.*
 perf.data
 perf.data.old
 build
 experiments
 subprojects
+sandbox
+.vscode

CONTRIBUTING.md

@@ -0,0 +1 @@
+See wayvnc's [CONTRIBUTING.md](https://github.com/any1/wayvnc/blob/master/CONTRIBUTING.md).

FUNDING.yml

@@ -1 +1,2 @@
+github: any1
 patreon: andriyngvason

README.md

@@ -12,29 +12,25 @@ neat.
 ## Building
 
 ### Runtime Dependencies
- * pixman
  * aml - https://github.com/any1/aml/
- * zlib
+ * ffmpeg (optional)
+ * gbm (optional)
  * gnutls (optional)
+ * libdrm (optional)
  * libturbojpeg (optional)
+ * nettle (optional)
+ * hogweed (optional)
+ * gmp (optional)
+ * pixman
+ * zlib
 
 ### Build Dependencies
+ * libdrm
  * meson
  * pkg-config
- * libdrm
 
 To build just run:
 ```
 meson build
 ninja -C build
 ```
-
-## Client Compatibility
- Name    | ZRLE Encoding | Tight Encoding | Crypto & Auth | SSH Tunneling
----------|---------------|----------------|---------------|--------------
-bVNC     |           Yes |              ? |           Yes |          Yes
-RealVNC  |           Yes |              ? |             ? |            ?
-Remmina  |           Yes |            Yes |             ? |          Yes
-TigerVNC |           Yes |            Yes |           Yes |            ?
-TightVNC |            No |            Yes |             ? |            ?
-UltraVNC |             ? |              ? |             ? |            ?

_clang-format

@@ -1,24 +0,0 @@
----
-BasedOnStyle: LLVM
-SortIncludes: false
-
-IndentWidth: 8
-ContinuationIndentWidth: 8
-
-UseTab: ForIndentation
-BreakBeforeBraces: Linux
-
-AllowShortIfStatementsOnASingleLine: false
-AllowShortFunctionsOnASingleLine: false
-
-IndentCaseLabels: false
-#IndentGotoLabels: false
-
-PointerAlignment: Left
-
-ForEachMacros:
-  - LIST_FOREACH
-
-PenaltyBreakAssignment: 2
-PenaltyExcessCharacter: 10
-...

bench/zrle-bench.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019 Andri Yngvason
+ * Copyright (c) 2019 - 2021 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -55,6 +55,7 @@ static int run_benchmark(const char *image)
 	void *addr = nvnc_fb_get_addr(fb);
 	int width = nvnc_fb_get_width(fb);
 	int height = nvnc_fb_get_height(fb);
+	int stride = nvnc_fb_get_stride(fb);
 
 	struct rfb_pixel_format pixfmt;
 	rfb_pixfmt_from_fourcc(&pixfmt, DRM_FORMAT_ARGB8888);
@@ -65,27 +66,27 @@ static int run_benchmark(const char *image)
 	pixman_region_union_rect(&region, &region, 0, 0, width, height);
 
 	struct vec frame;
-	vec_init(&frame, width * height * 3 / 2);
+	vec_init(&frame, stride * height * 3 / 2);
 
 	z_stream zs = { 0 };
 
 	deflateInit2(&zs, /* compression level: */ 1,
-			  /*            method: */ Z_DEFLATED,
-			  /*       window bits: */ 15,
-			  /*         mem level: */ 9,
-			  /*          strategy: */ Z_DEFAULT_STRATEGY);
+			/*            method: */ Z_DEFLATED,
+			/*       window bits: */ 15,
+			/*         mem level: */ 9,
+			/*          strategy: */ Z_DEFAULT_STRATEGY);
 
-	void *dummy = malloc(width * height * 4);
+	void *dummy = malloc(stride * height * 4);
 	if (!dummy)
 		goto failure;
 
 	uint64_t start_time = gettime_us(CLOCK_PROCESS_CPUTIME_ID);
 
-	memcpy_unoptimized(dummy, addr, width * height * 4);
+	memcpy_unoptimized(dummy, addr, stride * height * 4);
 
 	uint64_t end_time = gettime_us(CLOCK_PROCESS_CPUTIME_ID);
 	printf("memcpy baseline for %s took %"PRIu64" micro seconds\n", image,
-	       end_time - start_time);
+			end_time - start_time);
 
 	free(dummy);
 
@@ -94,9 +95,9 @@ static int run_benchmark(const char *image)
 
 	end_time = gettime_us(CLOCK_PROCESS_CPUTIME_ID);
 	printf("Encoding %s took %"PRIu64" micro seconds\n", image,
-	       end_time - start_time);
+			end_time - start_time);
 
-	double orig_size = width * height * 4;
+	double orig_size = stride * height * 4;
 	double compressed_size = frame.len;
 
 	double reduction = (orig_size - compressed_size) / orig_size;

examples/draw.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019 - 2020 Andri Yngvason
+ * Copyright (c) 2019 - 2022 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -28,19 +28,97 @@
 #include <sys/param.h>
 #include <libdrm/drm_fourcc.h>
 
-#define MAX_COORD 128
+#include "sys/queue.h"
+
+// TODO: Align pixel formats
 
 struct coord {
 	int x, y;
 };
 
-struct draw {
-	struct nvnc_display* display;
-	struct nvnc_fb* fb;
-	struct coord coord[MAX_COORD];
-	int index;
+struct fb_side_data {
+	struct pixman_region16 damage;
+	LIST_ENTRY(fb_side_data) link;
 };
 
+LIST_HEAD(fb_side_data_list, fb_side_data);
+
+struct draw {
+	int width;
+	int height;
+	uint32_t format;
+
+	pixman_image_t* whiteboard;
+	uint32_t* whiteboard_buffer;
+
+	struct nvnc_display* display;
+	struct nvnc_fb_pool* fb_pool;
+
+	struct fb_side_data_list fb_side_data_list;
+};
+
+static struct nvnc_fb* create_cursor()
+{
+	static char ascii_art[] =
+		"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+		"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX "
+		"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXX  "
+		"XXXXXXXXXXXXXXXXXXXXXXXXXXXXX   "
+		"XXXXXXXXXXXXXXXXXXXXXXXXXXXX    "
+		"XXXXXXXXXXXXXXXXXXXXXXXXXXX     "
+		"XXXXXXXXXXXXXXXXXXXXXXXXXX      "
+		"XXXXXXXXXXXXXXXXXXXXXXXXX       "
+		"XXXXXXXX                        "
+		"XXXXXXXX                        "
+		"XXXXXXXX                        "
+		"XXXXXXXX                        "
+		"XXXXXXXX                        "
+		"XXXXXXXX                        "
+		"XXXXXXXX                        "
+		"XXXXXXXX                        "
+		"XXXXXXXX                        "
+		"XXXXXXXX                        "
+		"XXXXXXXX                        "
+		"XXXXXXXX                        "
+		"XXXXXXXX                        "
+		"XXXXXXXX                        "
+		"XXXXXXXX                        "
+		"XXXXXXXX                        "
+		"XXXXXXXX                        "
+		"XXXXXXX                         "
+		"XXXXXX                          "
+		"XXXXX                           "
+		"XXXX                            "
+		"XXX                             "
+		"XX                              "
+		"X                               ";
+
+	struct nvnc_fb* fb = nvnc_fb_new(32, 32, DRM_FORMAT_RGBA8888, 32);
+	assert(fb);
+
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+	uint32_t colour = 0x00ff00ffULL;
+#else
+	uint32_t colour = 0xff00ff00ULL;
+#endif
+
+	uint32_t* pixels = nvnc_fb_get_addr(fb);
+
+	for (int i = 0; i < 32 * 32; ++i) {
+		pixels[i] = ascii_art[i] != ' ' ? colour : 0;
+	}
+
+	return fb;
+}
+
+static void fb_side_data_destroy(void* userdata)
+{
+	struct fb_side_data* fb_side_data = userdata;
+	LIST_REMOVE(fb_side_data, link);
+	pixman_region_fini(&fb_side_data->damage);
+	free(fb_side_data);
+}
+
 static int coord_distance_between(struct coord a, struct coord b)
 {
 	float x = abs(a.x - b.x);
@@ -48,12 +126,61 @@ static int coord_distance_between(struct coord a, struct coord b)
 	return round(sqrt(x * x + y * y));
 }
 
-static void draw_dot(struct nvnc_display* display, struct nvnc_fb* fb,
-                     struct coord coord, int radius, uint32_t colour)
+static void damage_all_buffers(struct draw* draw,
+		struct pixman_region16* region)
 {
-	uint32_t* image = nvnc_fb_get_addr(fb);
-	int width = nvnc_fb_get_width(fb);
-	int height = nvnc_fb_get_height(fb);
+	struct fb_side_data *item;
+	LIST_FOREACH(item, &draw->fb_side_data_list, link)
+		pixman_region_union(&item->damage, &item->damage, region);
+}
+
+static void update_vnc_buffer(struct draw* draw,
+		struct pixman_region16* frame_damage)
+{
+	struct nvnc_fb *fb = nvnc_fb_pool_acquire(draw->fb_pool);
+	assert(fb);
+
+	struct fb_side_data* fb_side_data = nvnc_get_userdata(fb);
+	if (!fb_side_data) {
+		fb_side_data = calloc(1, sizeof(*fb_side_data));
+		assert(fb_side_data);
+
+		/* This is a new buffer, so the whole surface is damaged. */
+		pixman_region_init_rect(&fb_side_data->damage, 0, 0,
+				draw->width, draw->height);
+
+		nvnc_set_userdata(fb, fb_side_data, fb_side_data_destroy);
+		LIST_INSERT_HEAD(&draw->fb_side_data_list, fb_side_data, link);
+	}
+
+	pixman_image_t* dstimg = pixman_image_create_bits_no_clear(
+			PIXMAN_r8g8b8x8, draw->width, draw->height,
+			nvnc_fb_get_addr(fb), 4 * draw->width);
+
+	/* Clip region is set to limit copying to only the damaged region. */
+	pixman_image_set_clip_region(dstimg, &fb_side_data->damage);
+
+	pixman_image_composite(PIXMAN_OP_OVER, draw->whiteboard, NULL, dstimg,
+			0, 0,
+			0, 0,
+			0, 0,
+			draw->width, draw->height);
+
+	pixman_image_unref(dstimg);
+
+	/* The buffer is now up to date, so the damage region can be cleared. */
+	pixman_region_clear(&fb_side_data->damage);
+
+	nvnc_display_feed_buffer(draw->display, fb, frame_damage);
+	nvnc_fb_unref(fb);
+}
+
+static void composite_dot(struct draw *draw, uint32_t* image,
+		struct coord coord, int radius, uint32_t colour,
+		struct pixman_region16* damage)
+{
+	int width = draw->width;
+	int height = draw->height;
 
 	struct coord start, stop;
 
@@ -62,12 +189,6 @@ static void draw_dot(struct nvnc_display* display, struct nvnc_fb* fb,
 	stop.x = MIN(width, coord.x + radius);
 	stop.y = MIN(height, coord.y + radius);
 
-	struct pixman_region16 region;
-	pixman_region_init_rect(&region, start.x, start.y,
-	                        stop.x - start.x, stop.y - start.y);
-	nvnc_display_damage_region(display, &region);
-	pixman_region_fini(&region);
-
 	/* The brute force method. ;) */
 	for (int y = start.y; y < stop.y; ++y)
 		for (int x = start.x; x < stop.x; ++x) {
@@ -75,10 +196,30 @@ static void draw_dot(struct nvnc_display* display, struct nvnc_fb* fb,
 			if (coord_distance_between(point, coord) <= radius)
 				image[x + y * width] = colour;
 		}
+
+	pixman_region_init_rect(damage, start.x, start.y,
+			stop.x - start.x, stop.y - start.y);
+}
+
+static void draw_dot(struct draw *draw, struct coord coord, int radius,
+		uint32_t colour)
+{
+	struct pixman_region16 region;
+	composite_dot(draw, draw->whiteboard_buffer, coord, radius, colour,
+			&region);
+
+	/* All the buffers that are currently in the pool will need to be
+	 * upgraded in this region before being sent to nvnc.
+	 */
+	damage_all_buffers(draw, &region);
+
+	update_vnc_buffer(draw, &region);
+
+	pixman_region_fini(&region);
 }
 
 static void on_pointer_event(struct nvnc_client* client, uint16_t x, uint16_t y,
-                             enum nvnc_button_mask buttons)
+		enum nvnc_button_mask buttons)
 {
 	if (!(buttons & NVNC_BUTTON_LEFT))
 		return;
@@ -89,35 +230,52 @@ static void on_pointer_event(struct nvnc_client* client, uint16_t x, uint16_t y,
 	struct draw* draw = nvnc_get_userdata(server);
 	assert(draw);
 
-	int width = nvnc_fb_get_width(draw->fb);
-	int height = nvnc_fb_get_height(draw->fb);
-
-	if (draw->index >= MAX_COORD)
-		return;
-
-	draw->coord[draw->index].x = x;
-	draw->coord[draw->index].y = y;
-	draw->index++;
-
-	struct pixman_region16 region;
-	pixman_region_init_rect(&region, 0, 0, width, height);
-	pixman_region_intersect_rect(&region, &region, x, y, 1, 1);
-	nvnc_display_damage_region(draw->display, &region);
-	pixman_region_fini(&region);
+	struct coord coord = { x, y };
+	draw_dot(draw, coord, 16, 0);
 }
 
-static void on_render(struct nvnc_display* display, struct nvnc_fb* fb)
+static bool on_desktop_layout_event(struct nvnc_client* client,
+		const struct nvnc_desktop_layout* layout)
 {
-	struct nvnc* server = nvnc_display_get_server(display);
+	uint16_t width = nvnc_desktop_layout_get_width(layout);
+	uint16_t height = nvnc_desktop_layout_get_height(layout);
+	struct nvnc* server = nvnc_client_get_server(client);
 	assert(server);
 
 	struct draw* draw = nvnc_get_userdata(server);
 	assert(draw);
 
-	for (int i = 0; i < draw->index; ++i)
-		draw_dot(draw->display, fb, draw->coord[i], 16, 0);
+	nvnc_fb_pool_resize(draw->fb_pool, width, height, draw->format, width);
 
-	draw->index = 0;
+	uint32_t* buffer = malloc(width * height * 4);
+	assert(buffer);
+
+	memset(buffer, 0xff, width * height * 4);
+
+	pixman_image_t* image = pixman_image_create_bits_no_clear(
+			PIXMAN_r8g8b8x8, width, height, buffer, width * 4);
+	assert(image);
+
+	pixman_image_composite(PIXMAN_OP_OVER, draw->whiteboard, NULL, image, 0,
+			0, 0, 0,
+			width > draw->width ? (width - draw->width) / 2 : 0,
+			height > draw->height ? (height - draw->height) / 2 : 0,
+			draw->width, draw->height);
+
+	pixman_image_unref(draw->whiteboard);
+	free(draw->whiteboard_buffer);
+
+	draw->whiteboard_buffer = buffer;
+	draw->whiteboard = image;
+	draw->width = width;
+	draw->height = height;
+
+	struct pixman_region16 damage;
+	pixman_region_init_rect(&damage, 0, 0, width, height);
+	update_vnc_buffer(draw, &damage);
+	pixman_region_fini(&damage);
+
+	return true;
 }
 
 static void on_sigint()
@@ -129,40 +287,63 @@ int main(int argc, char* argv[])
 {
 	struct draw draw = { 0 };
 
-	int width = 500, height = 500;
-	uint32_t format = DRM_FORMAT_RGBX8888;
-	draw.fb = nvnc_fb_new(width, height, format);
-	assert(draw.fb);
-
-	void* addr = nvnc_fb_get_addr(draw.fb);
-
-	memset(addr, 0xff, width * height * 4);
+	LIST_INIT(&draw.fb_side_data_list);
 
 	struct aml* aml = aml_new();
 	aml_set_default(aml);
 
+	draw.width = 500;
+	draw.height = 500;
+	draw.format = DRM_FORMAT_RGBX8888;
+
+	draw.whiteboard_buffer = malloc(draw.width * draw.height * 4);
+	assert(draw.whiteboard_buffer);
+
+	memset(draw.whiteboard_buffer, 0xff, draw.width * draw.height * 4);
+
+	draw.whiteboard = pixman_image_create_bits_no_clear(PIXMAN_r8g8b8x8,
+			draw.width, draw.height, draw.whiteboard_buffer,
+			draw.width * 4);
+	assert(draw.whiteboard);
+
+	draw.fb_pool = nvnc_fb_pool_new(draw.width, draw.height, draw.format,
+			draw.width);
+	assert(draw.fb_pool);
+
 	struct nvnc* server = nvnc_open("127.0.0.1", 5900);
 	assert(server);
 
 	draw.display = nvnc_display_new(0, 0);
 	assert(draw.display);
-	nvnc_display_set_buffer(draw.display, draw.fb);
-	nvnc_display_set_render_fn(draw.display, on_render);
 
 	nvnc_add_display(server, draw.display);
 
 	nvnc_set_name(server, "Draw");
 	nvnc_set_pointer_fn(server, on_pointer_event);
-	nvnc_set_userdata(server, &draw);
+	nvnc_set_desktop_layout_fn(server, on_desktop_layout_event);
+	nvnc_set_userdata(server, &draw, NULL);
+
+	struct nvnc_fb* cursor = create_cursor();
+	assert(cursor);
+
+	nvnc_set_cursor(server, cursor, 32, 32, 0, 0, true);
+	nvnc_fb_unref(cursor);
 
 	struct aml_signal* sig = aml_signal_new(SIGINT, on_sigint, NULL, NULL);
 	aml_start(aml_get_default(), sig);
 	aml_unref(sig);
 
+	struct pixman_region16 damage;
+	pixman_region_init_rect(&damage, 0, 0, draw.width, draw.height);
+	update_vnc_buffer(&draw, &damage);
+	pixman_region_fini(&damage);
+
 	aml_run(aml);
 
 	nvnc_close(server);
 	nvnc_display_unref(draw.display);
-	nvnc_fb_unref(draw.fb);
+	nvnc_fb_pool_unref(draw.fb_pool);
+	pixman_image_unref(draw.whiteboard);
+	free(draw.whiteboard_buffer);
 	aml_unref(aml);
 }

examples/png-server.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019 Andri Yngvason
+ * Copyright (c) 2019 - 2021 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -53,12 +53,14 @@ int main(int argc, char* argv[])
 	struct nvnc_display* display = nvnc_display_new(0, 0);
 	assert(display);
 
-	nvnc_display_set_buffer(display, fb);
-
 	nvnc_add_display(server, display);
 	nvnc_set_name(server, file);
 
-	nvnc_display_damage_whole(display);
+	struct pixman_region16 damage;
+	pixman_region_init_rect(&damage, 0, 0, nvnc_fb_get_width(fb),
+			nvnc_fb_get_height(fb));
+	nvnc_display_feed_buffer(display, fb, &damage);
+	pixman_region_fini(&damage);
 
 	struct aml_signal* sig = aml_signal_new(SIGINT, on_sigint, NULL, NULL);
 	aml_start(aml_get_default(), sig);

include/base64.h

@@ -1,5 +1,4 @@
-/*
- * Copyright (c) 2019 Andri Yngvason
+/* Copyright (c) 2023 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -18,15 +17,9 @@
 
 #include <stdint.h>
 #include <unistd.h>
-#include <zlib.h>
 
-struct nvnc_fb;
-struct rfb_pixel_format;
-struct pixman_region16;
-struct vec;
+#define BASE64_ENCODED_SIZE(x) ((((x) + 2) / 3) * 4 + 1)
+#define BASE64_DECODED_MAX_SIZE(x) ((((x) + 3) / 4) * 3)
 
-int zrle_encode_frame(z_stream* zs, struct vec* dst,
-                      const struct rfb_pixel_format* dst_fmt,
-                      const struct nvnc_fb* src,
-                      const struct rfb_pixel_format* src_fmt,
-                      struct pixman_region16* region);
+void base64_encode(char* dst, const uint8_t* src, size_t src_len);
+ssize_t base64_decode(uint8_t* dst, const char* src);

include/common.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019 - 2020 Andri Yngvason
+ * Copyright (c) 2019 - 2024 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -24,9 +24,12 @@
 #include "sys/queue.h"
 
 #include "neatvnc.h"
-#include "tight.h"
 #include "config.h"
 
+#ifdef HAVE_CRYPTO
+#include "crypto.h"
+#endif
+
 #ifdef ENABLE_TLS
 #include <gnutls/gnutls.h>
 #endif
@@ -44,6 +47,13 @@ enum nvnc_client_state {
 	VNC_CLIENT_STATE_WAITING_FOR_VENCRYPT_VERSION,
 	VNC_CLIENT_STATE_WAITING_FOR_VENCRYPT_SUBTYPE,
 	VNC_CLIENT_STATE_WAITING_FOR_VENCRYPT_PLAIN_AUTH,
+#endif
+#ifdef HAVE_CRYPTO
+	VNC_CLIENT_STATE_WAITING_FOR_APPLE_DH_RESPONSE,
+	VNC_CLIENT_STATE_WAITING_FOR_RSA_AES_PUBLIC_KEY,
+	VNC_CLIENT_STATE_WAITING_FOR_RSA_AES_CHALLENGE,
+	VNC_CLIENT_STATE_WAITING_FOR_RSA_AES_CLIENT_HASH,
+	VNC_CLIENT_STATE_WAITING_FOR_RSA_AES_CREDENTIALS,
 #endif
 	VNC_CLIENT_STATE_WAITING_FOR_INIT,
 	VNC_CLIENT_STATE_READY,
@@ -54,9 +64,13 @@ struct stream;
 struct aml_handler;
 struct aml_idle;
 struct nvnc_display;
+struct crypto_key;
+struct crypto_rsa_pub_key;
+struct crypto_rsa_priv_key;
 
 struct nvnc_common {
 	void* userdata;
+	nvnc_cleanup_fn cleanup_fn;
 };
 
 struct cut_text {
@@ -69,6 +83,7 @@ struct nvnc_client {
 	struct nvnc_common common;
 	int ref;
 	struct stream* net_stream;
+	char username[256];
 	struct nvnc* server;
 	enum nvnc_client_state state;
 	bool has_pixfmt;
@@ -79,25 +94,51 @@ struct nvnc_client {
 	struct pixman_region16 damage;
 	int n_pending_requests;
 	bool is_updating;
+	struct nvnc_fb* current_fb;
 	nvnc_client_fn cleanup_fn;
-	z_stream z_stream;
-	struct tight_encoder tight_encoder;
 	size_t buffer_index;
 	size_t buffer_len;
 	uint8_t msg_buffer[MSG_BUFFER_SIZE];
 	uint32_t known_width;
 	uint32_t known_height;
 	struct cut_text cut_text;
-	bool is_qemu_key_ext_notified;
+	bool is_ext_notified;
+	struct encoder* encoder;
+	struct encoder* zrle_encoder;
+	struct encoder* tight_encoder;
+	uint32_t cursor_seq;
+	int quality;
+	bool formats_changed;
+	enum nvnc_keyboard_led_state led_state;
+	enum nvnc_keyboard_led_state pending_led_state;
+
+#ifdef HAVE_CRYPTO
+	struct crypto_key* apple_dh_secret;
+
+	struct {
+		enum crypto_hash_type hash_type;
+		enum crypto_cipher_type cipher_type;
+		size_t challenge_len;
+		uint8_t challenge[32];
+		struct crypto_rsa_pub_key* pub;
+	} rsa;
+#endif
 };
 
 LIST_HEAD(nvnc_client_list, nvnc_client);
 
+enum nvnc__socket_type {
+	NVNC__SOCKET_TCP,
+	NVNC__SOCKET_UNIX,
+	NVNC__SOCKET_WEBSOCKET,
+	NVNC__SOCKET_FROM_FD,
+};
+
 struct nvnc {
 	struct nvnc_common common;
 	int fd;
+	enum nvnc__socket_type socket_type;
 	struct aml_handler* poll_handle;
-	struct aml_idle* dispatch_handler;
 	struct nvnc_client_list clients;
 	char name[256];
 	void* userdata;
@@ -107,13 +148,29 @@ struct nvnc {
 	nvnc_fb_req_fn fb_req_fn;
 	nvnc_client_fn new_client_fn;
 	nvnc_cut_text_fn cut_text_fn;
+	nvnc_desktop_layout_fn desktop_layout_fn;
 	struct nvnc_display* display;
+	struct {
+		struct nvnc_fb* buffer;
+		uint32_t width, height;
+		uint32_t hotspot_x, hotspot_y;
+	} cursor;
+	uint32_t cursor_seq;
+
+	enum nvnc_auth_flags auth_flags;
+	nvnc_auth_fn auth_fn;
+	void* auth_ud;
 
 #ifdef ENABLE_TLS
 	gnutls_certificate_credentials_t tls_creds;
-	nvnc_auth_fn auth_fn;
-	void* auth_ud;
 #endif
+
+#ifdef HAVE_CRYPTO
+	struct crypto_rsa_pub_key* rsa_pub;
+	struct crypto_rsa_priv_key* rsa_priv;
+#endif
+
+	uint32_t n_damage_clients;
 };
 
 void nvnc__damage_region(struct nvnc* self,

include/crypto.h

@@ -0,0 +1,113 @@
+#pragma once
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <unistd.h>
+
+struct crypto_key;
+struct crypto_cipher;
+struct crypto_hash;
+struct crypto_rsa_pub_key;
+struct crypto_rsa_priv_key;
+struct vec;
+
+enum crypto_cipher_type {
+	CRYPTO_CIPHER_INVALID = 0,
+	CRYPTO_CIPHER_AES128_ECB,
+	CRYPTO_CIPHER_AES_EAX,
+	CRYPTO_CIPHER_AES256_EAX,
+};
+
+enum crypto_hash_type {
+	CRYPTO_HASH_INVALID = 0,
+	CRYPTO_HASH_MD5,
+	CRYPTO_HASH_SHA1,
+	CRYPTO_HASH_SHA256,
+};
+
+struct crypto_data_entry {
+	uint8_t* data;
+	size_t len;
+};
+
+void crypto_dump_base16(const char* msg, const uint8_t* bytes, size_t len);
+void crypto_dump_base64(const char* msg, const uint8_t* bytes, size_t len);
+
+void crypto_random(uint8_t* dst, size_t len);
+
+// Key generation
+struct crypto_key* crypto_key_new(int g, const uint8_t *p, uint32_t p_len,
+		const uint8_t* q, uint32_t q_len);
+void crypto_key_del(struct crypto_key* key);
+
+int crypto_key_g(const struct crypto_key* key);
+uint32_t crypto_key_p(const struct crypto_key* key, uint8_t* dst,
+		uint32_t dst_size);
+uint32_t crypto_key_q(const struct crypto_key* key, uint8_t* dst,
+		uint32_t dst_size);
+
+struct crypto_key* crypto_keygen(void);
+
+// Diffie-Hellman
+struct crypto_key* crypto_derive_public_key(const struct crypto_key* priv);
+struct crypto_key* crypto_derive_shared_secret(
+		const struct crypto_key* own_secret,
+		const struct crypto_key* remote_public_key);
+
+// Ciphers
+struct crypto_cipher* crypto_cipher_new(const uint8_t* enc_key,
+		const uint8_t* dec_key, enum crypto_cipher_type type);
+void crypto_cipher_del(struct crypto_cipher* self);
+
+bool crypto_cipher_encrypt(struct crypto_cipher* self, struct vec* dst,
+		uint8_t* mac, const uint8_t* src, size_t len,
+		const uint8_t* ad, size_t ad_len);
+ssize_t crypto_cipher_decrypt(struct crypto_cipher* self, uint8_t* dst,
+		uint8_t* mac, const uint8_t* src, size_t len,
+		const uint8_t* ad, size_t ad_len);
+
+// Hashing
+struct crypto_hash* crypto_hash_new(enum crypto_hash_type type);
+void crypto_hash_del(struct crypto_hash* self);
+
+void crypto_hash_append(struct crypto_hash* self, const uint8_t* src,
+		size_t len);
+void crypto_hash_digest(struct crypto_hash* self, uint8_t* dst,
+		size_t len);
+
+void crypto_hash_one(uint8_t* dst, size_t dst_len, enum crypto_hash_type type,
+		const uint8_t* src, size_t src_len);
+void crypto_hash_many(uint8_t* dst, size_t dst_len, enum crypto_hash_type type,
+		const struct crypto_data_entry *src);
+
+// RSA
+struct crypto_rsa_pub_key* crypto_rsa_pub_key_new(void);
+void crypto_rsa_pub_key_del(struct crypto_rsa_pub_key*);
+
+// Returns length in bytes
+size_t crypto_rsa_pub_key_length(const struct crypto_rsa_pub_key* key);
+
+struct crypto_rsa_pub_key* crypto_rsa_pub_key_import(const uint8_t* modulus,
+		const uint8_t* exponent, size_t size);
+
+void crypto_rsa_pub_key_modulus(const struct crypto_rsa_pub_key* key,
+		uint8_t* dst, size_t dst_size);
+void crypto_rsa_pub_key_exponent(const struct crypto_rsa_pub_key* key,
+		uint8_t* dst, size_t dst_size);
+
+bool crypto_rsa_priv_key_import_pkcs1_der(struct crypto_rsa_priv_key* priv,
+		struct crypto_rsa_pub_key* pub, const uint8_t* key,
+		size_t size);
+
+bool crypto_rsa_priv_key_load(struct crypto_rsa_priv_key* priv,
+		struct crypto_rsa_pub_key* pub, const char* path);
+
+struct crypto_rsa_priv_key *crypto_rsa_priv_key_new(void);
+void crypto_rsa_priv_key_del(struct crypto_rsa_priv_key*);
+
+bool crypto_rsa_keygen(struct crypto_rsa_pub_key*, struct crypto_rsa_priv_key*);
+
+ssize_t crypto_rsa_encrypt(struct crypto_rsa_pub_key* pub, uint8_t* dst,
+		size_t dst_size, const uint8_t* src, size_t src_size);
+ssize_t crypto_rsa_decrypt(struct crypto_rsa_priv_key* priv, uint8_t* dst,
+		size_t dst_size, const uint8_t* src, size_t src_size);

include/cursor.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019 - 2020 Andri Yngvason
+ * Copyright (c) 2022 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -16,12 +16,12 @@
 
 #pragma once
 
+#include <stdint.h>
+
+struct vec;
 struct nvnc_fb;
 struct rfb_pixel_format;
-struct pixman_region16;
-struct vec;
 
-int raw_encode_frame(struct vec* dst, const struct rfb_pixel_format* dst_fmt,
-                     const struct nvnc_fb* src,
-                     const struct rfb_pixel_format* src_fmt,
-                     struct pixman_region16* region);
+int cursor_encode(struct vec* dst, struct rfb_pixel_format* pixfmt,
+		struct nvnc_fb* image, uint32_t width, uint32_t height,
+		uint32_t x_hotspot, uint32_t y_hotspot);

include/damage-refinery.h

@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2020 - 2021 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#pragma once
+
+#include <stdint.h>
+
+struct pixman_region16;
+struct nvnc_fb;
+struct XXH3_state_s;
+
+struct damage_refinery {
+	struct XXH3_state_s* state;
+	uint32_t* hashes;
+	uint32_t width;
+	uint32_t height;
+};
+
+int damage_refinery_init(struct damage_refinery* self, uint32_t width,
+		uint32_t height);
+int damage_refinery_resize(struct damage_refinery* self, uint32_t width,
+		uint32_t height);
+void damage_refinery_destroy(struct damage_refinery* self);
+
+void damage_refine(struct damage_refinery* self,
+		struct pixman_region16* refined, 
+		struct pixman_region16* hint,
+		struct nvnc_fb* buffer);

include/desktop-layout.h

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2023 Philipp Zabel
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#pragma once
+
+#include <stdint.h>
+
+struct nvnc_display;
+struct rfb_screen;
+
+struct nvnc_display_layout {
+	struct nvnc_display* display;
+	uint32_t id;
+	uint16_t x_pos, y_pos;
+	uint16_t width, height;
+};
+
+struct nvnc_desktop_layout {
+	uint16_t width, height;
+	uint8_t n_display_layouts;
+	struct nvnc_display_layout display_layouts[0];
+};
+
+void nvnc_display_layout_init(
+		struct nvnc_display_layout* display, struct rfb_screen* screen);

include/display.h

@@ -17,17 +17,20 @@
 #pragma once
 
 #include "neatvnc.h"
+#include "damage-refinery.h"
 
 #include <stdint.h>
 #include <pixels.h>
 
 struct nvnc;
 struct nvnc_fb;
+struct resampler;
 
 struct nvnc_display {
 	int ref;
 	struct nvnc* server;
 	uint16_t x_pos, y_pos;
 	struct nvnc_fb* buffer;
-	nvnc_render_fn render_fn;
+	struct resampler* resampler;
+	struct damage_refinery damage_refinery;
 };

include/enc-util.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019 - 2020 Andri Yngvason
+ * Copyright (c) 2019 - 2022 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -21,8 +21,9 @@
 #include <stdint.h>
 
 struct vec;
+struct pixman_region16;
 
-int encode_rect_count(struct vec* dst, uint32_t count);
 int encode_rect_head(struct vec* dst, enum rfb_encodings encoding,
 		uint32_t x, uint32_t y, uint32_t width, uint32_t height);
 uint32_t calc_bytes_per_cpixel(const struct rfb_pixel_format* fmt);
+uint32_t calculate_region_area(struct pixman_region16* region);

include/encoder.h

@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2021 - 2022 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+#pragma once
+
+#include "rfb-proto.h"
+
+#include <stdint.h>
+#include <unistd.h>
+
+struct encoder;
+struct nvnc_fb;
+struct pixman_region16;
+struct rcbuf;
+
+enum encoder_impl_flags {
+	ENCODER_IMPL_FLAG_NONE = 0,
+	ENCODER_IMPL_FLAG_IGNORES_DAMAGE = 1 << 0,
+};
+
+struct encoder_impl {
+	enum encoder_impl_flags flags;
+
+	void (*destroy)(struct encoder*);
+
+	void (*set_output_format)(struct encoder*,
+			const struct rfb_pixel_format*);
+	void (*set_quality)(struct encoder*, int quality);
+
+	int (*resize)(struct encoder*, uint16_t width, uint16_t height);
+
+	int (*encode)(struct encoder*, struct nvnc_fb* fb,
+			struct pixman_region16* damage);
+
+	void (*request_key_frame)(struct encoder*);
+};
+
+struct encoder {
+	struct encoder_impl* impl;
+
+	int ref;
+
+	uint16_t x_pos;
+	uint16_t y_pos;
+
+	int n_rects;
+
+	void (*on_done)(struct encoder*, struct rcbuf* result, uint64_t pts);
+	void* userdata;
+};
+
+struct encoder* encoder_new(enum rfb_encodings type, uint16_t width,
+		uint16_t height);
+void encoder_ref(struct encoder* self);
+void encoder_unref(struct encoder* self);
+
+void encoder_init(struct encoder* self, struct encoder_impl*);
+
+enum rfb_encodings encoder_get_type(const struct encoder* self);
+enum encoder_kind encoder_get_kind(const struct encoder* self);
+
+void encoder_set_output_format(struct encoder* self,
+		const struct rfb_pixel_format*);
+void encoder_set_quality(struct encoder* self, int value);
+
+int encoder_resize(struct encoder* self, uint16_t width, uint16_t height);
+
+int encoder_encode(struct encoder* self, struct nvnc_fb* fb,
+		struct pixman_region16* damage);
+
+void encoder_request_key_frame(struct encoder* self);
+
+void encoder_finish_frame(struct encoder* self, struct rcbuf* result,
+		uint64_t pts);

include/fb.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019 - 2020 Andri Yngvason
+ * Copyright (c) 2019 - 2022 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -19,16 +19,37 @@
 #include <unistd.h>
 #include <stdint.h>
 #include <stdatomic.h>
+#include <stdbool.h>
 
 #include "neatvnc.h"
+#include "common.h"
+
+struct gbm_bo;
 
 struct nvnc_fb {
+	struct nvnc_common common;
+	enum nvnc_fb_type type;
 	int ref;
-	void* addr;
-	enum nvnc_fb_flags flags;
-	size_t size;
+	int hold_count;
+	nvnc_fb_release_fn on_release;
+	void* release_context;
+	bool is_external;
 	uint16_t width;
 	uint16_t height;
 	uint32_t fourcc_format;
-	uint64_t fourcc_modifier;
+	enum nvnc_transform transform;
+	uint64_t pts; // in micro seconds
+
+	/* main memory buffer attributes */
+	void* addr;
+	int32_t stride;
+
+	/* dmabuf attributes */
+	struct gbm_bo* bo;
+	void* bo_map_handle;
 };
+
+void nvnc_fb_hold(struct nvnc_fb* fb);
+void nvnc_fb_release(struct nvnc_fb* fb);
+int nvnc_fb_map(struct nvnc_fb* fb);
+void nvnc_fb_unmap(struct nvnc_fb* fb);

include/h264-encoder.h

@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2021 - 2024 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+#pragma once
+
+#include <stdint.h>
+#include <unistd.h>
+#include <stdbool.h>
+
+struct nvnc_fb;
+struct h264_encoder;
+
+typedef void (*h264_encoder_packet_handler_fn)(const void* payload, size_t size,
+		uint64_t pts, void* userdata);
+
+struct h264_encoder_impl {
+	struct h264_encoder* (*create)(uint32_t width, uint32_t height,
+			uint32_t format, int quality);
+	void (*destroy)(struct h264_encoder*);
+	void (*feed)(struct h264_encoder*, struct nvnc_fb*);
+};
+
+struct h264_encoder {
+	struct h264_encoder_impl *impl;
+	h264_encoder_packet_handler_fn on_packet_ready;
+	void* userdata;
+	bool next_frame_should_be_keyframe;
+};
+
+struct h264_encoder* h264_encoder_create(uint32_t width, uint32_t height,
+		uint32_t format, int quality);
+
+void h264_encoder_destroy(struct h264_encoder*);
+
+void h264_encoder_set_packet_handler_fn(struct h264_encoder*,
+		h264_encoder_packet_handler_fn);
+void h264_encoder_set_userdata(struct h264_encoder*, void* userdata);
+
+void h264_encoder_feed(struct h264_encoder*, struct nvnc_fb*);
+
+void h264_encoder_request_keyframe(struct h264_encoder*);

include/http.h

@@ -0,0 +1,37 @@
+/* Copyright (c) 2014-2016, Marel
+ * Copyright (c) 2023, Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#pragma once
+
+#define HTTP_FIELD_INDEX_MAX 32
+
+#include <stddef.h>
+
+struct http_kv {
+	char* key;
+	char* value;
+};
+
+struct http_req {
+	size_t header_length;
+	size_t content_length;
+	char* content_type;
+	size_t field_index;
+	struct http_kv field[HTTP_FIELD_INDEX_MAX];
+};
+
+int http_req_parse(struct http_req* req, const char* head);
+void http_req_free(struct http_req* req);

include/logging.h

@@ -1,5 +1,7 @@
+#pragma once
+
 /*
- * Copyright (c) 2019 - 2020 Andri Yngvason
+ * Copyright (c) 2022 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -14,16 +16,4 @@
  * PERFORMANCE OF THIS SOFTWARE.
  */
 
-#pragma once
-
-#include <stdio.h>
-
-#ifdef NDEBUG
-#define log_debug(...)
-#else
-#define log_debug(fmt, ...) \
-	fprintf(stderr, "%s:%d: " fmt, __FILE__, __LINE__, ## __VA_ARGS__)
-#endif
-
-#define log_error(...) \
-	fprintf(stderr, "ERROR: " __VA_ARGS__)
+void nvnc__log_init(void);

include/neatvnc.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019 - 2020 Andri Yngvason
+ * Copyright (c) 2019 - 2022 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -18,12 +18,45 @@
 
 #include <stdint.h>
 #include <stdbool.h>
+#include <limits.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <sys/socket.h>
+
+#define NVNC_NO_PTS UINT64_MAX
+
+#define nvnc_log(lvl, fmt, ...) do { \
+	assert(lvl != NVNC_LOG_TRACE); \
+	struct nvnc_log_data ld = { \
+		.level = lvl, \
+		.file = __FILE__, \
+		.line = __LINE__, \
+	}; \
+	nvnc__log(&ld, fmt, ## __VA_ARGS__); \
+} while(0)
+
+#ifndef NDEBUG
+#define nvnc_trace(fmt, ...) do { \
+	struct nvnc_log_data ld = { \
+		.level = NVNC_LOG_TRACE, \
+		.file = __FILE__, \
+		.line = __LINE__, \
+	}; \
+	nvnc__log(&ld, fmt, ## __VA_ARGS__); \
+} while(0)
+#else
+#define nvnc_trace(...)
+#endif
 
 struct nvnc;
 struct nvnc_client;
+struct nvnc_desktop_layout;
 struct nvnc_display;
 struct nvnc_fb;
+struct nvnc_fb_pool;
 struct pixman_region16;
+struct gbm_bo;
 
 enum nvnc_button_mask {
 	NVNC_BUTTON_LEFT = 1 << 0,
@@ -31,10 +64,52 @@ enum nvnc_button_mask {
 	NVNC_BUTTON_RIGHT = 1 << 2,
 	NVNC_SCROLL_UP = 1 << 3,
 	NVNC_SCROLL_DOWN = 1 << 4,
+	NVNC_SCROLL_LEFT = 1 << 5,
+	NVNC_SCROLL_RIGHT = 1 << 6,
 };
 
-enum nvnc_fb_flags {
-	NVNC_FB_PARTIAL = 1 << 0, // The buffer contains only the damaged region
+enum nvnc_fb_type {
+	NVNC_FB_UNSPEC = 0,
+	NVNC_FB_SIMPLE,
+	NVNC_FB_GBM_BO,
+};
+
+/* This is the same as wl_output_transform */
+enum nvnc_transform {
+	NVNC_TRANSFORM_NORMAL = 0,
+	NVNC_TRANSFORM_90 = 1,
+	NVNC_TRANSFORM_180 = 2,
+	NVNC_TRANSFORM_270 = 3,
+	NVNC_TRANSFORM_FLIPPED = 4,
+	NVNC_TRANSFORM_FLIPPED_90 = 5,
+	NVNC_TRANSFORM_FLIPPED_180 = 6,
+	NVNC_TRANSFORM_FLIPPED_270 = 7,
+};
+
+enum nvnc_keyboard_led_state {
+	NVNC_KEYBOARD_LED_SCROLL_LOCK = 1 << 0,
+	NVNC_KEYBOARD_LED_NUM_LOCK = 1 << 1,
+	NVNC_KEYBOARD_LED_CAPS_LOCK = 1 << 2,
+};
+
+enum nvnc_log_level {
+	NVNC_LOG_PANIC = 0,
+	NVNC_LOG_ERROR = 1,
+	NVNC_LOG_WARNING = 2,
+	NVNC_LOG_INFO = 3,
+	NVNC_LOG_DEBUG = 4,
+	NVNC_LOG_TRACE = 5,
+};
+
+enum nvnc_auth_flags {
+	NVNC_AUTH_REQUIRE_AUTH = 1 << 0,
+	NVNC_AUTH_REQUIRE_ENCRYPTION = 1 << 1,
+};
+
+struct nvnc_log_data {
+	enum nvnc_log_level level;
+	const char* file;
+	int line;
 };
 
 typedef void (*nvnc_key_fn)(struct nvnc_client*, uint32_t key,
@@ -48,21 +123,42 @@ typedef void (*nvnc_client_fn)(struct nvnc_client*);
 typedef void (*nvnc_damage_fn)(struct pixman_region16* damage, void* userdata);
 typedef bool (*nvnc_auth_fn)(const char* username, const char* password,
                              void* userdata);
-typedef void (*nvnc_render_fn)(struct nvnc_display*, struct nvnc_fb*);
-typedef void (*nvnc_cut_text_fn)(struct nvnc*, const char* text, uint32_t len);
+typedef void (*nvnc_cut_text_fn)(struct nvnc_client*, const char* text,
+		uint32_t len);
+typedef void (*nvnc_fb_release_fn)(struct nvnc_fb*, void* context);
+typedef struct nvnc_fb* (*nvnc_fb_alloc_fn)(uint16_t width, uint16_t height,
+		uint32_t format, uint16_t stride);
+typedef void (*nvnc_cleanup_fn)(void* userdata);
+typedef void (*nvnc_log_fn)(const struct nvnc_log_data*, const char* message);
+typedef bool (*nvnc_desktop_layout_fn)(
+		struct nvnc_client*, const struct nvnc_desktop_layout*);
 
 extern const char nvnc_version[];
 
 struct nvnc* nvnc_open(const char* addr, uint16_t port);
+struct nvnc* nvnc_open_unix(const char *addr);
+struct nvnc* nvnc_open_websocket(const char* addr, uint16_t port);
+struct nvnc* nvnc_open_from_fd(int fd);
 void nvnc_close(struct nvnc* self);
 
 void nvnc_add_display(struct nvnc*, struct nvnc_display*);
 void nvnc_remove_display(struct nvnc*, struct nvnc_display*);
 
-void nvnc_set_userdata(void* self, void* userdata);
+void nvnc_set_userdata(void* self, void* userdata, nvnc_cleanup_fn);
 void* nvnc_get_userdata(const void* self);
 
 struct nvnc* nvnc_client_get_server(const struct nvnc_client* client);
+bool nvnc_client_supports_cursor(const struct nvnc_client* client);
+int nvnc_client_get_address(const struct nvnc_client* client,
+		struct sockaddr* restrict addr, socklen_t* restrict addrlen);
+const char* nvnc_client_get_auth_username(const struct nvnc_client* client);
+
+struct nvnc_client* nvnc_client_first(struct nvnc* self);
+struct nvnc_client* nvnc_client_next(struct nvnc_client* client);
+void nvnc_client_close(struct nvnc_client* client);
+
+void nvnc_client_set_led_state(struct nvnc_client*,
+		enum nvnc_keyboard_led_state);
 
 void nvnc_set_name(struct nvnc* self, const char* name);
 
@@ -72,25 +168,55 @@ void nvnc_set_pointer_fn(struct nvnc* self, nvnc_pointer_fn);
 void nvnc_set_fb_req_fn(struct nvnc* self, nvnc_fb_req_fn);
 void nvnc_set_new_client_fn(struct nvnc* self, nvnc_client_fn);
 void nvnc_set_client_cleanup_fn(struct nvnc_client* self, nvnc_client_fn fn);
-void nvnc_set_cut_text_receive_fn(struct nvnc* self, nvnc_cut_text_fn fn);
+void nvnc_set_cut_text_fn(struct nvnc*, nvnc_cut_text_fn fn);
+void nvnc_set_desktop_layout_fn(struct nvnc* self, nvnc_desktop_layout_fn);
 
 bool nvnc_has_auth(void);
-int nvnc_enable_auth(struct nvnc* self, const char* privkey_path,
-                     const char* cert_path, nvnc_auth_fn, void* userdata);
+int nvnc_enable_auth(struct nvnc* self, enum nvnc_auth_flags flags,
+		nvnc_auth_fn, void* userdata);
+int nvnc_set_tls_creds(struct nvnc* self, const char* privkey_path,
+                     const char* cert_path);
+int nvnc_set_rsa_creds(struct nvnc* self, const char* private_key_path);
 
 struct nvnc_fb* nvnc_fb_new(uint16_t width, uint16_t height,
-                            uint32_t fourcc_format);
+                            uint32_t fourcc_format, uint16_t stride);
+struct nvnc_fb* nvnc_fb_from_buffer(void* buffer, uint16_t width,
+				    uint16_t height, uint32_t fourcc_format,
+				    int32_t stride);
+struct nvnc_fb* nvnc_fb_from_gbm_bo(struct gbm_bo* bo);
 
 void nvnc_fb_ref(struct nvnc_fb* fb);
 void nvnc_fb_unref(struct nvnc_fb* fb);
 
-enum nvnc_fb_flags nvnc_fb_get_flags(const struct nvnc_fb*);
-void nvnc_fb_set_flags(struct nvnc_fb*, enum nvnc_fb_flags);
+void nvnc_fb_set_release_fn(struct nvnc_fb* fb, nvnc_fb_release_fn fn,
+			    void* context);
+void nvnc_fb_set_transform(struct nvnc_fb* fb, enum nvnc_transform);
+
+void nvnc_fb_set_pts(struct nvnc_fb* fb, uint64_t pts);
 
 void* nvnc_fb_get_addr(const struct nvnc_fb* fb);
 uint16_t nvnc_fb_get_width(const struct nvnc_fb* fb);
 uint16_t nvnc_fb_get_height(const struct nvnc_fb* fb);
 uint32_t nvnc_fb_get_fourcc_format(const struct nvnc_fb* fb);
+int32_t nvnc_fb_get_stride(const struct nvnc_fb* fb);
+int nvnc_fb_get_pixel_size(const struct nvnc_fb* fb);
+struct gbm_bo* nvnc_fb_get_gbm_bo(const struct nvnc_fb* fb);
+enum nvnc_transform nvnc_fb_get_transform(const struct nvnc_fb* fb);
+enum nvnc_fb_type nvnc_fb_get_type(const struct nvnc_fb* fb);
+uint64_t nvnc_fb_get_pts(const struct nvnc_fb* fb);
+
+struct nvnc_fb_pool* nvnc_fb_pool_new(uint16_t width, uint16_t height,
+				      uint32_t fourcc_format, uint16_t stride);
+bool nvnc_fb_pool_resize(struct nvnc_fb_pool*, uint16_t width, uint16_t height,
+			 uint32_t fourcc_format, uint16_t stride);
+
+void nvnc_fb_pool_set_alloc_fn(struct nvnc_fb_pool*, nvnc_fb_alloc_fn);
+
+void nvnc_fb_pool_ref(struct nvnc_fb_pool*);
+void nvnc_fb_pool_unref(struct nvnc_fb_pool*);
+
+struct nvnc_fb* nvnc_fb_pool_acquire(struct nvnc_fb_pool*);
+void nvnc_fb_pool_release(struct nvnc_fb_pool*, struct nvnc_fb*);
 
 struct nvnc_display* nvnc_display_new(uint16_t x_pos, uint16_t y_pos);
 void nvnc_display_ref(struct nvnc_display*);
@@ -98,21 +224,32 @@ void nvnc_display_unref(struct nvnc_display*);
 
 struct nvnc* nvnc_display_get_server(const struct nvnc_display*);
 
-void nvnc_display_set_render_fn(struct nvnc_display* self, nvnc_render_fn fn);
-void nvnc_display_set_buffer(struct nvnc_display*, struct nvnc_fb*);
+void nvnc_display_feed_buffer(struct nvnc_display*, struct nvnc_fb*,
+			      struct pixman_region16* damage);
 
-void nvnc_display_damage_region(struct nvnc_display*,
-                                const struct pixman_region16*);
-void nvnc_display_damage_whole(struct nvnc_display*);
-
-/*
- * Find the regions that differ between fb0 and fb1. Regions outside the hinted
- * rectangle region are not guaranteed to be checked.
- *
- * This is a utility function that may be used to reduce network traffic.
- */
-int nvnc_check_damage(struct nvnc_fb* fb0, struct nvnc_fb* fb1,
-                      int x_hint, int y_hint, int width_hint, int height_hint,
-                      nvnc_damage_fn on_check_done, void* userdata);
+uint16_t nvnc_desktop_layout_get_width(const struct nvnc_desktop_layout*);
+uint16_t nvnc_desktop_layout_get_height(const struct nvnc_desktop_layout*);
+uint8_t nvnc_desktop_layout_get_display_count(const struct nvnc_desktop_layout*);
+uint16_t nvnc_desktop_layout_get_display_x_pos(
+		const struct nvnc_desktop_layout*, uint8_t display_index);
+uint16_t nvnc_desktop_layout_get_display_y_pos(
+		const struct nvnc_desktop_layout*, uint8_t display_index);
+uint16_t nvnc_desktop_layout_get_display_width(
+		const struct nvnc_desktop_layout*, uint8_t display_index);
+uint16_t nvnc_desktop_layout_get_display_height(
+		const struct nvnc_desktop_layout*, uint8_t display_index);
+struct nvnc_display* nvnc_desktop_layout_get_display(
+		const struct nvnc_desktop_layout*, uint8_t display_index);
 
 void nvnc_send_cut_text(struct nvnc*, const char* text, uint32_t len);
+
+void nvnc_set_cursor(struct nvnc*, struct nvnc_fb*, uint16_t width,
+		     uint16_t height, uint16_t hotspot_x, uint16_t hotspot_y,
+		     bool is_damaged);
+
+void nvnc_default_logger(const struct nvnc_log_data* meta, const char* message);
+
+void nvnc_set_log_fn(nvnc_log_fn);
+void nvnc_set_log_fn_thread_local(nvnc_log_fn fn);
+void nvnc_set_log_level(enum nvnc_log_level);
+void nvnc__log(const struct nvnc_log_data*, const char* fmt, ...);

include/pixels.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019 Andri Yngvason
+ * Copyright (c) 2019 - 2021 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -18,14 +18,28 @@
 
 #include <stdint.h>
 #include <unistd.h>
+#include <pixman.h>
+#include <stdbool.h>
 
 struct rfb_pixel_format;
+struct rfb_set_colour_map_entries_msg;
 
-void pixel32_to_cpixel(uint8_t* restrict dst,
+void pixel_to_cpixel(uint8_t* restrict dst,
                        const struct rfb_pixel_format* dst_fmt,
-                       const uint32_t* restrict src,
+                       const uint8_t* restrict src,
                        const struct rfb_pixel_format* src_fmt,
                        size_t bytes_per_cpixel, size_t len);
 
 int rfb_pixfmt_from_fourcc(struct rfb_pixel_format *dst, uint32_t src);
 uint32_t rfb_pixfmt_to_fourcc(const struct rfb_pixel_format* fmt);
+
+int pixel_size_from_fourcc(uint32_t fourcc);
+
+bool fourcc_to_pixman_fmt(pixman_format_code_t* dst, uint32_t src);
+
+bool extract_alpha_mask(uint8_t* dst, const void* src, uint32_t format,
+		size_t len);
+
+const char* drm_format_to_string(uint32_t fmt);
+const char* rfb_pixfmt_to_string(const struct rfb_pixel_format* fmt);
+void make_rgb332_pal8_map(struct rfb_set_colour_map_entries_msg* msg);

include/resampler.h

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2021 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#pragma once
+
+#include <stdint.h>
+
+struct nvnc_fb;
+struct pixman_region16;
+
+struct resampler;
+
+typedef void (*resampler_fn)(struct nvnc_fb*, struct pixman_region16* damage,
+			     void* userdata);
+
+struct resampler* resampler_create(void);
+void resampler_destroy(struct resampler*);
+
+int resampler_feed(struct resampler*, struct nvnc_fb* fb,
+		struct pixman_region16* damage, resampler_fn on_done,
+		void* userdata);
+
+void resample_now(struct nvnc_fb* dst, struct nvnc_fb* src,
+		struct pixman_region16* damage);

include/rfb-proto.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019 Andri Yngvason
+ * Copyright (c) 2019 - 2024 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -29,8 +29,11 @@ enum rfb_security_type {
 	RFB_SECURITY_TYPE_INVALID = 0,
 	RFB_SECURITY_TYPE_NONE = 1,
 	RFB_SECURITY_TYPE_VNC_AUTH = 2,
+	RFB_SECURITY_TYPE_RSA_AES = 5,
 	RFB_SECURITY_TYPE_TIGHT = 16,
 	RFB_SECURITY_TYPE_VENCRYPT = 19,
+	RFB_SECURITY_TYPE_APPLE_DH = 30,
+	RFB_SECURITY_TYPE_RSA_AES256 = 129,
 };
 
 enum rfb_security_handshake_result {
@@ -45,6 +48,8 @@ enum rfb_client_to_server_msg_type {
 	RFB_CLIENT_TO_SERVER_KEY_EVENT = 4,
 	RFB_CLIENT_TO_SERVER_POINTER_EVENT = 5,
 	RFB_CLIENT_TO_SERVER_CLIENT_CUT_TEXT = 6,
+	RFB_CLIENT_TO_SERVER_NTP = 160,
+	RFB_CLIENT_TO_SERVER_SET_DESKTOP_SIZE = 251,
 	RFB_CLIENT_TO_SERVER_QEMU = 255,
 };
 
@@ -60,18 +65,26 @@ enum rfb_encodings {
 	RFB_ENCODING_TIGHT = 7,
 	RFB_ENCODING_TRLE = 15,
 	RFB_ENCODING_ZRLE = 16,
+	RFB_ENCODING_OPEN_H264 = 50,
 	RFB_ENCODING_CURSOR = -239,
 	RFB_ENCODING_DESKTOPSIZE = -223,
-	RFB_ENCODING_JPEG_HIGHQ = -23,
-	RFB_ENCODING_JPEG_LOWQ = -32,
 	RFB_ENCODING_QEMU_EXT_KEY_EVENT = -258,
+	RFB_ENCODING_QEMU_LED_STATE = -261,
+	RFB_ENCODING_EXTENDEDDESKTOPSIZE = -308,
+	RFB_ENCODING_PTS = -1000,
+	RFB_ENCODING_NTP = -1001,
+	RFB_ENCODING_VMWARE_LED_STATE = 0x574d5668,
 };
 
+#define RFB_ENCODING_JPEG_HIGHQ -23
+#define RFB_ENCODING_JPEG_LOWQ -32
+
 enum rfb_server_to_client_msg_type {
 	RFB_SERVER_TO_CLIENT_FRAMEBUFFER_UPDATE = 0,
 	RFB_SERVER_TO_CLIENT_SET_COLOUR_MAP_ENTRIES = 1,
 	RFB_SERVER_TO_CLIENT_BELL = 2,
 	RFB_SERVER_TO_CLIENT_SERVER_CUT_TEXT = 3,
+	RFB_SERVER_TO_CLIENT_NTP = 160,
 };
 
 enum rfb_vencrypt_subtype {
@@ -84,9 +97,35 @@ enum rfb_vencrypt_subtype {
 	RFB_VENCRYPT_X509_PLAIN,
 };
 
+enum rfb_resize_initiator {
+	RFB_RESIZE_INITIATOR_SERVER = 0,
+	RFB_RESIZE_INITIATOR_THIS_CLIENT = 1,
+	RFB_RESIZE_INITIATOR_OTHER_CLIENT = 2,
+};
+
+enum rfb_resize_status {
+	RFB_RESIZE_STATUS_SUCCESS = 0,
+	RFB_RESIZE_STATUS_PROHIBITED = 1,
+	RFB_RESIZE_STATUS_OUT_OF_RESOURCES = 2,
+	RFB_RESIZE_STATUS_INVALID_LAYOUT = 3,
+	RFB_RESIZE_STATUS_REQUEST_FORWARDED = 4,
+};
+
+enum rfb_rsa_aes_cred_subtype {
+	RFB_RSA_AES_CRED_SUBTYPE_USER_AND_PASS = 1,
+	RFB_RSA_AES_CRED_SUBTYPE_ONLY_PASS = 2,
+};
+
+// This is the same for both qemu and vmware extensions
+enum rfb_led_state {
+	RFB_LED_STATE_SCROLL_LOCK = 1 << 0,
+	RFB_LED_STATE_NUM_LOCK = 1 << 1,
+	RFB_LED_STATE_CAPS_LOCK = 1 << 2,
+};
+
 struct rfb_security_types_msg {
 	uint8_t n;
-	uint8_t types[1];
+	uint8_t types[0];
 } RFB_PACKED;
 
 struct rfb_error_reason {
@@ -169,6 +208,25 @@ struct rfb_server_fb_rect {
 	int32_t encoding;
 } RFB_PACKED;
 
+struct rfb_screen {
+	uint32_t id;
+	uint16_t x;
+	uint16_t y;
+	uint16_t width;
+	uint16_t height;
+	uint32_t flags;
+} RFB_PACKED;
+
+struct rfb_client_set_desktop_size_event_msg {
+	uint8_t type;
+	uint8_t padding;
+	uint16_t width;
+	uint16_t height;
+	uint8_t number_of_screens;
+	uint8_t padding2;
+	struct rfb_screen screens[0];
+} RFB_PACKED;
+
 struct rfb_server_fb_update_msg {
 	uint8_t type;
 	uint8_t padding;
@@ -190,3 +248,42 @@ struct rfb_vencrypt_plain_auth_msg {
 	uint32_t password_len;
 	char text[0];
 } RFB_PACKED;
+
+struct rfb_ntp_msg {
+	uint8_t type;
+	uint8_t padding[3];
+	uint32_t t0, t1, t2, t3;
+} RFB_PACKED;
+
+struct rfb_apple_dh_server_msg {
+	uint16_t generator;
+	uint16_t key_size;
+	uint8_t modulus_and_key[0];
+} RFB_PACKED;
+
+struct rfb_apple_dh_client_msg {
+	uint8_t encrypted_credentials[128];
+	uint8_t public_key[0];
+} RFB_PACKED;
+
+struct rfb_rsa_aes_pub_key_msg {
+	uint32_t length;
+	uint8_t modulus_and_exponent[0];
+} RFB_PACKED;
+
+struct rfb_rsa_aes_challenge_msg {
+	uint16_t length;
+	uint8_t challenge[0];
+} RFB_PACKED;
+
+struct rfb_colour_map_entry {
+	uint16_t r, g, b;
+} RFB_PACKED;
+
+struct rfb_set_colour_map_entries_msg {
+	uint8_t type;
+	uint8_t padding;
+	uint16_t first_colour;
+	uint16_t n_colours;
+	struct rfb_colour_map_entry colours[0];
+} RFB_PACKED;

include/stream-common.h

@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2020 - 2023 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#pragma once
+
+#include "stream.h"
+
+#include <aml.h>
+
+static inline void stream__poll_r(struct stream* self)
+{
+	aml_set_event_mask(self->handler, AML_EVENT_READ);
+}
+
+static inline void stream__poll_w(struct stream* self)
+{
+	aml_set_event_mask(self->handler, AML_EVENT_WRITE);
+}
+
+static inline void stream__poll_rw(struct stream* self)
+{
+	aml_set_event_mask(self->handler, AML_EVENT_READ | AML_EVENT_WRITE);
+}
+
+void stream_req__finish(struct stream_req* req, enum stream_req_status status);
+void stream__remote_closed(struct stream* self);

include/stream-tcp.h

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2023 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#pragma once
+
+#include "stream.h"
+
+#include <unistd.h>
+
+struct stream;
+
+int stream_tcp_init(struct stream* self, int fd, stream_event_fn on_event,
+		void* userdata);
+int stream_tcp_close(struct stream* self);
+void stream_tcp_destroy(struct stream* self);
+ssize_t stream_tcp_read(struct stream* self, void* dst, size_t size);
+int stream_tcp_send(struct stream* self, struct rcbuf* payload,
+                stream_req_fn on_done, void* userdata);
+int stream_tcp_send_first(struct stream* self, struct rcbuf* payload);
+void stream_tcp_exec_and_send(struct stream* self,
+		stream_exec_fn exec_fn, void* userdata);
+int stream_tcp_install_cipher(struct stream* self,
+		struct crypto_cipher* cipher);

include/stream.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020 Andri Yngvason
+ * Copyright (c) 2020 - 2023 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -14,15 +14,21 @@
  * PERFORMANCE OF THIS SOFTWARE.
  */
 
+#pragma once
+
 #include "config.h"
 #include "sys/queue.h"
 #include "rcbuf.h"
+#include "vec.h"
+
+#ifdef HAVE_CRYPTO
+#include "crypto.h"
+#endif
 
 #include <stdint.h>
+#include <stdbool.h>
 
-#ifdef ENABLE_TLS
-#include <gnutls/gnutls.h>
-#endif
+#define STREAM_ALLOC_SIZE 4096
 
 enum stream_state {
 	STREAM_STATE_NORMAL = 0,
@@ -33,11 +39,6 @@ enum stream_state {
 #endif
 };
 
-enum stream_status {
-	STREAM_READY = 0,
-	STREAM_CLOSED,
-};
-
 enum stream_req_status {
 	STREAM_REQ_DONE = 0,
 	STREAM_REQ_FAILED,
@@ -49,20 +50,35 @@ enum stream_event {
 };
 
 struct stream;
+struct crypto_cipher;
 
 typedef void (*stream_event_fn)(struct stream*, enum stream_event);
 typedef void (*stream_req_fn)(void*, enum stream_req_status);
+typedef struct rcbuf* (*stream_exec_fn)(struct stream*, void* userdata);
 
 struct stream_req {
 	struct rcbuf* payload;
 	stream_req_fn on_done;
+	stream_exec_fn exec;
 	void* userdata;
 	TAILQ_ENTRY(stream_req) link;
 };
 
 TAILQ_HEAD(stream_send_queue, stream_req);
 
+struct stream_impl {
+	int (*close)(struct stream*);
+	void (*destroy)(struct stream*);
+	ssize_t (*read)(struct stream*, void* dst, size_t size);
+	int (*send)(struct stream*, struct rcbuf* payload,
+			stream_req_fn on_done, void* userdata);
+	int (*send_first)(struct stream*, struct rcbuf* payload);
+	void (*exec_and_send)(struct stream*, stream_exec_fn, void* userdata);
+};
+
 struct stream {
+	struct stream_impl *impl;
+
 	enum stream_state state;
 
 	int fd;
@@ -72,14 +88,19 @@ struct stream {
 
 	struct stream_send_queue send_queue;
 
-#ifdef ENABLE_TLS
-	gnutls_session_t tls_session;
-#endif
-
 	uint32_t bytes_sent;
 	uint32_t bytes_received;
+
+	bool cork;
+
+	struct crypto_cipher* cipher;
+	struct vec tmp_buf;
 };
 
+#ifdef ENABLE_WEBSOCKET
+struct stream* stream_ws_new(int fd, stream_event_fn on_event, void* userdata);
+#endif
+
 struct stream* stream_new(int fd, stream_event_fn on_event, void* userdata);
 int stream_close(struct stream* self);
 void stream_destroy(struct stream* self);
@@ -88,7 +109,17 @@ int stream_write(struct stream* self, const void* payload, size_t len,
                  stream_req_fn on_done, void* userdata);
 int stream_send(struct stream* self, struct rcbuf* payload,
                 stream_req_fn on_done, void* userdata);
+int stream_send_first(struct stream* self, struct rcbuf* payload);
+
+// Queue a pure function to be executed when time comes to send it.
+void stream_exec_and_send(struct stream* self, stream_exec_fn, void* userdata);
 
 #ifdef ENABLE_TLS
 int stream_upgrade_to_tls(struct stream* self, void* context);
 #endif
+
+#ifdef HAVE_CRYPTO
+int stream_upgrade_to_rsa_eas(struct stream* base,
+		enum crypto_cipher_type cipher_type,
+		const uint8_t* enc_key, const uint8_t* dec_key);
+#endif

include/tight.h

@@ -1,78 +0,0 @@
-/*
- * Copyright (c) 2019 - 2020 Andri Yngvason
- *
- * Permission to use, copy, modify, and/or distribute this software for any
- * purpose with or without fee is hereby granted, provided that the above
- * copyright notice and this permission notice appear in all copies.
- *
- * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
- * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
- * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
- * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
- * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
- * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
- * PERFORMANCE OF THIS SOFTWARE.
- */
-
-#pragma once
-
-#include "rfb-proto.h"
-#include "vec.h"
-
-#include <unistd.h>
-#include <stdint.h>
-#include <zlib.h>
-#include <pthread.h>
-
-struct tight_tile;
-struct pixman_region16;
-struct aml_work;
-
-typedef void (*tight_done_fn)(struct vec* frame, void*);
-
-enum tight_quality {
-	TIGHT_QUALITY_UNSPEC = 0,
-	TIGHT_QUALITY_LOSSLESS,
-	TIGHT_QUALITY_LOW,
-	TIGHT_QUALITY_HIGH,
-};
-
-struct tight_encoder {
-	uint32_t width;
-	uint32_t height;
-	uint32_t grid_width;
-	uint32_t grid_height;
-	enum tight_quality quality;
-
-	struct tight_tile* grid;
-
-	z_stream zs[4];
-	struct aml_work* zs_worker[4];
-
-	struct rfb_pixel_format dfmt;
-	struct rfb_pixel_format sfmt;
-	struct nvnc_fb* fb;
-
-	uint32_t n_rects;
-	uint32_t n_jobs;
-
-	struct vec dst;
-
-	tight_done_fn on_frame_done;
-	void* userdata;
-};
-
-int tight_encoder_init(struct tight_encoder* self, uint32_t width,
-		uint32_t height);
-void tight_encoder_destroy(struct tight_encoder* self);
-
-int tight_encoder_resize(struct tight_encoder* self, uint32_t width,
-		uint32_t height);
-
-int tight_encode_frame(struct tight_encoder* self,
-		const struct rfb_pixel_format* dfmt,
-		struct nvnc_fb* src,
-		const struct rfb_pixel_format* sfmt,
-		struct pixman_region16* damage,
-		enum tight_quality quality,
-		tight_done_fn on_done, void* userdata);

include/transform-util.h

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2020 - 2021 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#pragma once
+
+#include "neatvnc.h"
+
+#include <pixman.h>
+
+void nvnc_transform_to_pixman_transform(pixman_transform_t* dst,
+		enum nvnc_transform src, int width, int height);
+
+void nvnc_transform_dimensions(enum nvnc_transform transform, uint32_t* width,
+		uint32_t* height);
+
+void nvnc_transform_region(struct pixman_region16* dst,
+		struct pixman_region16* src, enum nvnc_transform transform,
+		int width, int height);

include/websocket.h

@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2023 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#pragma once
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <unistd.h>
+
+#define WS_HEADER_MIN_SIZE 14
+
+enum ws_opcode {
+	WS_OPCODE_CONT = 0,
+	WS_OPCODE_TEXT,
+	WS_OPCODE_BIN,
+	WS_OPCODE_CLOSE = 8,
+	WS_OPCODE_PING,
+	WS_OPCODE_PONG,
+};
+
+struct ws_frame_header {
+	bool fin;
+	enum ws_opcode opcode;
+	bool mask;
+	uint64_t payload_length;
+	uint8_t masking_key[4];
+	size_t header_length;
+};
+
+ssize_t ws_handshake(char* output, size_t output_maxlen, const char* input);
+
+const char *ws_opcode_name(enum ws_opcode op);
+
+bool ws_parse_frame_header(struct ws_frame_header* header,
+		const uint8_t* payload, size_t length);
+void ws_apply_mask(const struct ws_frame_header* header,
+		uint8_t* restrict payload);
+void ws_copy_payload(const struct ws_frame_header* header,
+		uint8_t* restrict dst, const uint8_t* restrict src, size_t len);
+int ws_write_frame_header(uint8_t* dst, const struct ws_frame_header* header);

meson.build

@@ -1,10 +1,11 @@
 project(
 	'neatvnc',
 	'c',
-	version: '0.4.0',
+	version: '0.9-dev',
 	license: 'ISC',
 	default_options: [
 		'c_std=gnu11',
+		'warning_level=2',
 	],
 )
 
@@ -12,31 +13,35 @@ buildtype = get_option('buildtype')
 host_system = host_machine.system()
 
 c_args = [
-	'-DPROJECT_VERSION="@0@"'.format(meson.project_version()),
 	'-D_GNU_SOURCE',
 	'-fvisibility=hidden',
+	'-DAML_UNSTABLE_API=1',
+
 	'-Wmissing-prototypes',
+	'-Wno-unused-parameter',
+	'-Wno-format-truncation',
 ]
 
 if buildtype != 'debug' and buildtype != 'debugoptimized'
 	c_args += '-DNDEBUG'
 endif
 
+version = '"@0@"'.format(meson.project_version())
 git = find_program('git', native: true, required: false)
 if git.found()
-	git_describe = run_command([git, 'describe', '--tags', '--long'])
+	git_commit = run_command([git, 'rev-parse', '--short', 'HEAD'])
 	git_branch = run_command([git, 'rev-parse', '--abbrev-ref', 'HEAD'])
-	if git_describe.returncode() == 0 and git_branch.returncode() == 0
-		c_args += '-DGIT_VERSION="@0@ (@1@)"'.format(
-			git_describe.stdout().strip(),
+	if git_commit.returncode() == 0 and git_branch.returncode() == 0
+		version = '"v@0@-@1@ (@2@)"'.format(
+			meson.project_version(),
+			git_commit.stdout().strip(),
 			git_branch.stdout().strip(),
 		)
 	endif
 endif
+add_project_arguments('-DPROJECT_VERSION=@0@'.format(version), language: 'c')
 
-libdrm_include_dir = dependency('libdrm').get_variable(pkgconfig: 'includedir')
-
-c_args += '-I' + libdrm_include_dir
+libdrm_inc = dependency('libdrm').partial_dependency(compile_args: true)
 
 add_project_arguments(c_args, language: 'c')
 
@@ -47,13 +52,23 @@ libm = cc.find_library('m', required: false)
 pixman = dependency('pixman-1')
 libturbojpeg = dependency('libturbojpeg', required: get_option('jpeg'))
 gnutls = dependency('gnutls', required: get_option('tls'))
+nettle = dependency('nettle', required: get_option('nettle'))
+hogweed = dependency('hogweed', required: get_option('nettle'))
+gmp = dependency('gmp', required: get_option('nettle'))
 zlib = dependency('zlib')
+gbm = dependency('gbm', required: get_option('gbm'))
+libdrm = dependency('libdrm', required: get_option('h264'))
 
-aml_project = subproject('aml', required: false)
+libavcodec = dependency('libavcodec', required: get_option('h264'))
+libavfilter = dependency('libavfilter', required: get_option('h264'))
+libavutil = dependency('libavutil', required: get_option('h264'))
+
+aml_version = ['>=0.3.0', '<0.4.0']
+aml_project = subproject('aml', required: false, version: aml_version)
 if aml_project.found()
 	aml = aml_project.get_variable('aml_dep')
 else
-	aml = dependency('aml')
+	aml = dependency('aml', version: aml_version)
 endif
 
 inc = include_directories('include')
@@ -64,14 +79,24 @@ sources = [
 	'src/zrle.c',
 	'src/raw-encoding.c',
 	'src/pixels.c',
-	'src/damage.c',
 	'src/fb.c',
+	'src/fb_pool.c',
 	'src/rcbuf.c',
 	'src/stream.c',
+	'src/stream-common.c',
+	'src/stream-tcp.c',
+	'src/desktop-layout.c',
 	'src/display.c',
 	'src/tight.c',
 	'src/enc-util.c',
 	'src/qnum-to-evdev.c',
+	'src/resampler.c',
+	'src/transform-util.c',
+	'src/damage-refinery.c',
+	'src/encoder.c',
+	'src/cursor.c',
+	'src/logging.c',
+	'src/base64.c',
 ]
 
 dependencies = [
@@ -79,8 +104,11 @@ dependencies = [
 	pixman,
 	aml,
 	zlib,
+	libdrm_inc,
 ]
 
+enable_websocket = false
+
 config = configuration_data()
 
 if libturbojpeg.found()
@@ -89,14 +117,64 @@ if libturbojpeg.found()
 endif
 
 if gnutls.found()
+	sources += 'src/stream-gnutls.c'
 	dependencies += gnutls
 	config.set('ENABLE_TLS', true)
 endif
 
+if nettle.found() and hogweed.found() and gmp.found()
+	dependencies += [ nettle, hogweed, gmp ]
+	enable_websocket = true
+	config.set('HAVE_CRYPTO', true)
+	sources += ['src/crypto-nettle.c', 'src/stream-rsa-aes.c']
+endif
+
 if host_system == 'linux' and get_option('systemtap') and cc.has_header('sys/sdt.h')
 	config.set('HAVE_USDT', true)
 endif
 
+if gbm.found()
+	dependencies += gbm
+	config.set('HAVE_GBM', true)
+endif
+
+have_ffmpeg = gbm.found() and libdrm.found() and libavcodec.found() and libavfilter.found() and libavutil.found()
+have_v4l2 = gbm.found() and libdrm.found() and cc.check_header('linux/videodev2.h')
+
+if have_ffmpeg
+	sources += [ 'src/h264-encoder-ffmpeg-impl.c' ]
+	dependencies += [libdrm, libavcodec, libavfilter, libavutil]
+	config.set('HAVE_FFMPEG', true)
+	config.set('HAVE_LIBAVUTIL', true)
+endif
+
+if have_v4l2
+	sources += [ 'src/h264-encoder-v4l2m2m-impl.c' ]
+	config.set('HAVE_V4L2', true)
+endif
+
+if have_ffmpeg or have_v4l2
+	sources += [ 'src/h264-encoder.c', 'src/open-h264.c' ]
+	config.set('ENABLE_OPEN_H264', true)
+endif
+
+if enable_websocket
+	sources += [
+		'src/ws-handshake.c',
+		'src/ws-framing.c',
+		'src/http.c',
+		'src/stream-ws.c',
+	]
+	config.set('ENABLE_WEBSOCKET', true)
+endif
+
+if get_option('experimental')
+	if buildtype == 'release'
+		warning('Experimental features enabled in release build')
+	endif
+	config.set('ENABLE_EXPERIMENTAL', true)
+endif
+
 configure_file(
 	output: 'config.h',
 	configuration: config,
@@ -124,6 +202,10 @@ if get_option('benchmarks')
 	subdir('bench')
 endif
 
+if get_option('tests')
+	subdir('test')
+endif
+
 install_headers('include/neatvnc.h')
 
 pkgconfig = import('pkgconfig')

meson_options.txt

@@ -1,5 +1,10 @@
 option('benchmarks', type: 'boolean', value: false, description: 'Build benchmarks')
 option('examples', type: 'boolean', value: false, description: 'Build examples')
+option('tests', type: 'boolean', value: false, description: 'Build unit tests')
 option('jpeg', type: 'feature', value: 'auto', description: 'Enable JPEG compression')
 option('tls', type: 'feature', value: 'auto', description: 'Enable encryption & authentication')
+option('nettle', type: 'feature', value: 'auto', description: 'Enable nettle low level encryption library')
 option('systemtap', type: 'boolean', value: false, description: 'Enable tracing using sdt')
+option('gbm', type: 'feature', value: 'auto', description: 'Enable GBM integration')
+option('h264', type: 'feature', value: 'auto', description: 'Enable open h264 encoding')
+option('experimental', type: 'boolean', value: false, description: 'Enable experimental features')

src/base64.c

@@ -0,0 +1,155 @@
+/* Copyright (c) 2023 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "base64.h"
+
+#include <unistd.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <string.h>
+
+static const char base64_enc_lut[] =
+	"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+
+static const uint8_t base64_validation_lut[256] = {
+	['A'] = 1, ['B'] = 1, ['C'] = 1, ['D'] = 1,
+	['E'] = 1, ['F'] = 1, ['G'] = 1, ['H'] = 1,
+	['I'] = 1, ['J'] = 1, ['K'] = 1, ['L'] = 1,
+	['M'] = 1, ['N'] = 1, ['O'] = 1, ['P'] = 1,
+	['Q'] = 1, ['R'] = 1, ['S'] = 1, ['T'] = 1,
+	['U'] = 1, ['V'] = 1, ['W'] = 1, ['X'] = 1,
+	['Y'] = 1, ['Z'] = 1, ['a'] = 1, ['b'] = 1,
+	['c'] = 1, ['d'] = 1, ['e'] = 1, ['f'] = 1,
+	['g'] = 1, ['h'] = 1, ['i'] = 1, ['j'] = 1,
+	['k'] = 1, ['l'] = 1, ['m'] = 1, ['n'] = 1,
+	['o'] = 1, ['p'] = 1, ['q'] = 1, ['r'] = 1,
+	['s'] = 1, ['t'] = 1, ['u'] = 1, ['v'] = 1,
+	['w'] = 1, ['x'] = 1, ['y'] = 1, ['z'] = 1,
+	['0'] = 1, ['1'] = 1, ['2'] = 1, ['3'] = 1,
+	['4'] = 1, ['5'] = 1, ['6'] = 1, ['7'] = 1,
+	['8'] = 1, ['9'] = 1, ['+'] = 1, ['/'] = 1,
+	['-'] = 1, ['_'] = 1, ['='] = 1,
+};
+
+static const uint8_t base64_dec_lut[256] = {
+	['A'] = 0x00, ['B'] = 0x01, ['C'] = 0x02, ['D'] = 0x03,
+	['E'] = 0x04, ['F'] = 0x05, ['G'] = 0x06, ['H'] = 0x07,
+	['I'] = 0x08, ['J'] = 0x09, ['K'] = 0x0a, ['L'] = 0x0b,
+	['M'] = 0x0c, ['N'] = 0x0d, ['O'] = 0x0e, ['P'] = 0x0f,
+	['Q'] = 0x10, ['R'] = 0x11, ['S'] = 0x12, ['T'] = 0x13,
+	['U'] = 0x14, ['V'] = 0x15, ['W'] = 0x16, ['X'] = 0x17,
+	['Y'] = 0x18, ['Z'] = 0x19, ['a'] = 0x1a, ['b'] = 0x1b,
+	['c'] = 0x1c, ['d'] = 0x1d, ['e'] = 0x1e, ['f'] = 0x1f,
+	['g'] = 0x20, ['h'] = 0x21, ['i'] = 0x22, ['j'] = 0x23,
+	['k'] = 0x24, ['l'] = 0x25, ['m'] = 0x26, ['n'] = 0x27,
+	['o'] = 0x28, ['p'] = 0x29, ['q'] = 0x2a, ['r'] = 0x2b,
+	['s'] = 0x2c, ['t'] = 0x2d, ['u'] = 0x2e, ['v'] = 0x2f,
+	['w'] = 0x30, ['x'] = 0x31, ['y'] = 0x32, ['z'] = 0x33,
+	['0'] = 0x34, ['1'] = 0x35, ['2'] = 0x36, ['3'] = 0x37,
+	['4'] = 0x38, ['5'] = 0x39, ['6'] = 0x3a, ['7'] = 0x3b,
+	['8'] = 0x3c, ['9'] = 0x3d, ['+'] = 0x3e, ['/'] = 0x3f,
+	['-'] = 0x3e, ['_'] = 0x3f,
+};
+
+void base64_encode(char* dst, const uint8_t* src, size_t src_len)
+{
+	size_t i = 0;
+
+	for (; i < src_len / 3; ++i) {
+		uint32_t tmp = 0;
+		tmp |= (uint32_t)src[i * 3 + 0] << 16;
+		tmp |= (uint32_t)src[i * 3 + 1] << 8;
+		tmp |= (uint32_t)src[i * 3 + 2];
+
+		dst[i * 4 + 0] = base64_enc_lut[tmp >> 18];
+		dst[i * 4 + 1] = base64_enc_lut[(tmp >> 12) & 0x3f];
+		dst[i * 4 + 2] = base64_enc_lut[(tmp >> 6) & 0x3f];
+		dst[i * 4 + 3] = base64_enc_lut[tmp & 0x3f];
+	}
+
+	size_t rem = src_len % 3;
+	if (rem == 0) {
+		dst[i * 4] = '\0';
+		return;
+	}
+
+	uint32_t tmp = 0;
+	for (size_t r = 0; r < rem; ++r) {
+		size_t s = (2 - r) * 8;
+		tmp |= (uint32_t)src[i * 3 + r] << s;
+	}
+
+	size_t di = 0;
+	for (; di < rem + 1; ++di) {
+		size_t s = (3 - di) * 6;
+		dst[i * 4 + di] = base64_enc_lut[(tmp >> s) & 0x3f];
+	}
+
+	for (; di < 4; ++di) {
+		dst[i * 4 + di] = '=';
+	}
+
+	dst[i * 4 + di] = '\0';
+
+}
+
+static bool base64_is_valid(const char* src)
+{
+	for (int i = 0; src[i]; i++)
+		if (!base64_validation_lut[(uint8_t)src[i]])
+			return false;
+	return true;
+}
+
+ssize_t base64_decode(uint8_t* dst, const char* src)
+{
+	if (!base64_is_valid(src))
+		return -1;
+
+	size_t src_len = strcspn(src, "=");
+	size_t i = 0;
+
+	for (; i < src_len / 4; ++i) {
+		uint32_t tmp = 0;
+		tmp |= (uint32_t)base64_dec_lut[(uint8_t)src[i * 4 + 0]] << 18;
+		tmp |= (uint32_t)base64_dec_lut[(uint8_t)src[i * 4 + 1]] << 12;
+		tmp |= (uint32_t)base64_dec_lut[(uint8_t)src[i * 4 + 2]] << 6;
+		tmp |= (uint32_t)base64_dec_lut[(uint8_t)src[i * 4 + 3]];
+
+		dst[i * 3 + 0] = tmp >> 16;
+		dst[i * 3 + 1] = (tmp >> 8) & 0xff;
+		dst[i * 3 + 2] = tmp & 0xff;
+	}
+
+	size_t rem = src_len % 4;
+	if (rem == 0)
+		return i * 3;
+
+	size_t di = 0;
+
+	uint32_t tmp = 0;
+	for (size_t r = 0; r < rem; ++r) {
+		size_t s = (3 - r) * 6;
+		tmp |= (uint32_t)base64_dec_lut[(uint8_t)src[i * 4 + r]] << s;
+	}
+
+	for (; di < (rem * 3) / 4; ++di) {
+		size_t s = (2 - di) * 8;
+		dst[i * 3 + di] = (tmp >> s) & 0xff;
+	}
+
+	return i * 3 + di;
+}

src/crypto-nettle.c

@@ -0,0 +1,739 @@
+#include "crypto.h"
+#include "neatvnc.h"
+#include "vec.h"
+#include "base64.h"
+
+#include <gmp.h>
+#include <nettle/base64.h>
+#include <nettle/base16.h>
+#include <nettle/aes.h>
+#include <nettle/eax.h>
+#include <nettle/md5.h>
+#include <nettle/sha1.h>
+#include <nettle/sha.h>
+#include <nettle/rsa.h>
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdbool.h>
+#include <string.h>
+#include <sys/param.h>
+#include <arpa/inet.h>
+
+// TODO: This is linux specific
+#include <sys/random.h>
+
+#define UDIV_UP(a, b) (((a) + (b) - 1) / (b))
+
+struct vec;
+
+struct crypto_key {
+	int g;
+	mpz_t p;
+	mpz_t q;
+};
+
+struct crypto_aes_eax {
+	struct eax_aes128_ctx ctx;
+	uint64_t count[2];
+};
+
+struct crypto_aes256_eax {
+	struct EAX_CTX(struct aes256_ctx) ctx;
+	uint64_t count[2];
+};
+
+struct crypto_cipher {
+	union {
+		struct aes128_ctx aes128_ecb;
+		struct crypto_aes_eax aes_eax;
+		struct crypto_aes256_eax aes256_eax;
+	} enc_ctx;
+
+	union {
+		struct aes128_ctx aes128_ecb;
+		struct crypto_aes_eax aes_eax;
+		struct crypto_aes256_eax aes256_eax;
+	} dec_ctx;
+
+	bool (*encrypt)(struct crypto_cipher*, struct vec* dst, uint8_t* mac,
+			const uint8_t* src, size_t src_len, const uint8_t* ad,
+			size_t ad_len);
+	ssize_t (*decrypt)(struct crypto_cipher*, uint8_t* dst, uint8_t* mac,
+			const uint8_t* src, size_t src_len, const uint8_t* ad,
+			size_t ad_len);
+};
+
+struct crypto_hash {
+	union {
+		struct md5_ctx md5;
+		struct sha1_ctx sha1;
+		struct sha256_ctx sha256;
+	} ctx;
+
+	void (*update)(void* ctx, size_t len, const uint8_t* src);
+	void (*digest)(void* ctx, size_t len, uint8_t* dst);
+};
+
+struct crypto_rsa_pub_key {
+	struct rsa_public_key key;
+};
+
+struct crypto_rsa_priv_key {
+	struct rsa_private_key key;
+};
+
+void crypto_dump_base64(const char* msg, const uint8_t* bytes, size_t len)
+{
+	struct base64_encode_ctx ctx = {};
+	size_t buflen = BASE64_ENCODE_LENGTH(len);
+	char* buffer = malloc(buflen + BASE64_ENCODE_FINAL_LENGTH + 1);
+	assert(buffer);
+	nettle_base64_encode_init(&ctx);
+	size_t count = nettle_base64_encode_update(&ctx, buffer, len, bytes);
+	count += nettle_base64_encode_final(&ctx, buffer + count);
+	buffer[count] = '\0';
+
+	nvnc_log(NVNC_LOG_DEBUG, "%s: %s", msg, buffer);
+	free(buffer);
+}
+
+void crypto_dump_base16(const char* msg, const uint8_t* bytes, size_t len)
+{
+	size_t buflen = BASE16_ENCODE_LENGTH(len);
+	char* buffer = calloc(1, buflen + 1);
+	assert(buffer);
+	nettle_base16_encode_update(buffer, len, bytes);
+
+	nvnc_log(NVNC_LOG_DEBUG, "%s: %s", msg, buffer);
+	free(buffer);
+}
+
+void crypto_random(uint8_t* dst, size_t len)
+{
+	getrandom(dst, len, 0);
+}
+
+static void crypto_import(mpz_t n, const uint8_t* src, size_t len)
+{
+	int order = 1;
+	int unit_size = 1;
+	int endian = 1;
+	int skip_bits = 0;
+
+	mpz_import(n, len, order, unit_size, endian, skip_bits, src);
+}
+
+struct crypto_key *crypto_key_new(int g, const uint8_t* p, uint32_t p_len,
+		const uint8_t* q, uint32_t q_len)
+{
+	struct crypto_key* self = calloc(1, sizeof(*self));
+	if (!self)
+		return NULL;
+
+	self->g = g;
+
+	mpz_init(self->p);
+	crypto_import(self->p, p, p_len);
+
+	mpz_init(self->q);
+	crypto_import(self->q, q, q_len);
+
+	return self;
+}
+
+void crypto_key_del(struct crypto_key* key)
+{
+	if (!key)
+		return;
+	mpz_clear(key->q);
+	mpz_clear(key->p);
+	free(key);
+}
+
+int crypto_key_g(const struct crypto_key* key)
+{
+	return key->g;
+}
+
+static size_t crypto_export(uint8_t* dst, size_t dst_size, const mpz_t n)
+{
+	int order = 1; // msb first
+	int unit_size = 1; // byte
+	int endian = 1; // msb first
+	int skip_bits = 0;
+
+	size_t bitsize = mpz_sizeinbase(n, 2);
+	size_t bytesize = (bitsize + 7) / 8;
+
+	assert(bytesize <= dst_size);
+
+	memset(dst, 0, dst_size);
+	mpz_export(dst + dst_size - bytesize, &bytesize, order, unit_size,
+			endian, skip_bits, n);
+
+	return bytesize;
+}
+
+uint32_t crypto_key_p(const struct crypto_key* key, uint8_t* dst,
+		uint32_t dst_size)
+{
+	return crypto_export(dst, dst_size, key->p);
+}
+
+uint32_t crypto_key_q(const struct crypto_key* key, uint8_t* dst,
+		uint32_t dst_size)
+{
+	return crypto_export(dst, dst_size, key->q);
+}
+
+static void initialise_p(mpz_t p)
+{
+	// RFC 3526, section 3
+	static const char s[] =
+		"FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD1"
+		"29024E088A67CC74020BBEA63B139B22514A08798E3404DD"
+		"EF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245"
+		"E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7ED"
+		"EE386BFB5A899FA5AE9F24117C4B1FE649286651ECE45B3D"
+		"C2007CB8A163BF0598DA48361C55D39A69163FA8FD24CF5F"
+		"83655D23DCA3AD961C62F356208552BB9ED529077096966D"
+		"670C354E4ABC9804F1746C08CA18217C32905E462E36CE3B"
+		"E39E772C180E86039B2783A2EC07A28FB5C55DF06F4C52C9"
+		"DE2BCBF6955817183995497CEA956AE515D2261898FA0510"
+		"15728E5A8AACAA68FFFFFFFFFFFFFFFF";
+
+	char buf[256];
+	size_t len = 0;
+	struct base16_decode_ctx ctx;
+	nettle_base16_decode_init(&ctx);
+	nettle_base16_decode_update(&ctx, &len, (uint8_t*)buf, sizeof(s) - 1, s);
+	nettle_base16_decode_final(&ctx);
+	assert(len == sizeof(buf));
+
+	crypto_import(p, (const uint8_t*)buf, sizeof(buf));
+}
+
+static void generate_random(mpz_t n)
+{
+	uint8_t buf[256];
+	getrandom(buf, sizeof(buf), 0);
+	crypto_import(n, buf, sizeof(buf));
+}
+
+struct crypto_key* crypto_keygen(void)
+{
+	struct crypto_key* self = calloc(1, sizeof(*self));
+	if (!self)
+		return NULL;
+
+	self->g = 2;
+
+	mpz_init(self->p);
+	initialise_p(self->p);
+
+	mpz_init(self->q);
+	generate_random(self->q);
+
+	return self;
+}
+
+struct crypto_key* crypto_derive_public_key(const struct crypto_key* priv)
+{
+	struct crypto_key* pub = calloc(1, sizeof(*pub));
+	if (!pub)
+		return NULL;
+
+	pub->g = priv->g;
+	mpz_set(pub->p, priv->p);
+	mpz_init(pub->q);
+
+	mpz_t g;
+	mpz_init(g);
+	mpz_set_ui(g, priv->g);
+
+	mpz_powm_sec(pub->q, g, priv->q, priv->p);
+	mpz_clear(g);
+
+	return pub;
+}
+
+struct crypto_key* crypto_derive_shared_secret(
+		const struct crypto_key* own_secret,
+		const struct crypto_key* remote_public_key)
+{
+	if (own_secret->g != remote_public_key->g) {
+		return NULL;
+	}
+
+	if (mpz_cmp(own_secret->p, remote_public_key->p) != 0) {
+		return NULL;
+	}
+
+	struct crypto_key* shared = calloc(1, sizeof(*shared));
+	if (!shared)
+		return NULL;
+
+	shared->g = own_secret->g;
+	mpz_set(shared->p, own_secret->p);
+
+	mpz_t g;
+	mpz_init(g);
+	mpz_set_ui(g, own_secret->g);
+
+	mpz_powm_sec(shared->q, remote_public_key->q, own_secret->q,
+			own_secret->p);
+	mpz_clear(g);
+
+	return shared;
+}
+
+static bool crypto_cipher_aes128_ecb_encrypt(struct crypto_cipher* self,
+		struct vec* dst, uint8_t* mac, const uint8_t* src,
+		size_t len, const uint8_t* ad, size_t ad_len)
+{
+	vec_reserve(dst, dst->len + len);
+	aes128_encrypt(&self->enc_ctx.aes128_ecb, len, dst->data, src);
+	dst->len = len;
+	return true;
+}
+
+static ssize_t crypto_cipher_aes128_ecb_decrypt(struct crypto_cipher* self,
+		uint8_t* dst, uint8_t* mac, const uint8_t* src, size_t len,
+		const uint8_t* ad, size_t ad_len)
+{
+	aes128_decrypt(&self->dec_ctx.aes128_ecb, len, dst, src);
+	return len;
+}
+
+static struct crypto_cipher* crypto_cipher_new_aes128_ecb(
+		const uint8_t* enc_key, const uint8_t* dec_key)
+{
+	struct crypto_cipher* self = calloc(1, sizeof(*self));
+	if (!self)
+		return NULL;
+
+	if (enc_key)
+		aes128_set_encrypt_key(&self->enc_ctx.aes128_ecb, enc_key);
+
+	if (dec_key)
+		aes128_set_decrypt_key(&self->dec_ctx.aes128_ecb, dec_key);
+
+	self->encrypt = crypto_cipher_aes128_ecb_encrypt;
+	self->decrypt = crypto_cipher_aes128_ecb_decrypt;
+
+	return self;
+}
+
+static void crypto_aes_eax_update_nonce(struct crypto_aes_eax* self)
+{
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+	nettle_eax_aes128_set_nonce(&self->ctx, 16, (const uint8_t*)self->count);
+#else
+	uint64_t c[2];
+	c[0] = __builtin_bswap64(self->count[0]);
+	c[1] = __builtin_bswap64(self->count[1]);
+	nettle_eax_aes128_set_nonce(&self->ctx, 16, (const uint8_t*)c);
+#endif
+
+	if (++self->count[0] == 0)
+		++self->count[1];
+}
+
+static bool crypto_cipher_aes_eax_encrypt(struct crypto_cipher* self,
+		struct vec* dst, uint8_t* mac, const uint8_t* src,
+		size_t src_len, const uint8_t* ad, size_t ad_len)
+{
+	vec_reserve(dst, dst->len + src_len);
+
+	crypto_aes_eax_update_nonce(&self->enc_ctx.aes_eax);
+	nettle_eax_aes128_update(&self->enc_ctx.aes_eax.ctx, ad_len,
+			(uint8_t*)ad);
+	nettle_eax_aes128_encrypt(&self->enc_ctx.aes_eax.ctx, src_len,
+			(uint8_t*)dst->data + dst->len, src);
+	dst->len += src_len;
+
+	nettle_eax_aes128_digest(&self->enc_ctx.aes_eax.ctx, 16, mac);
+
+	return true;
+}
+
+static ssize_t crypto_cipher_aes_eax_decrypt(struct crypto_cipher* self,
+		uint8_t* dst, uint8_t* mac, const uint8_t* src, size_t len,
+		const uint8_t* ad, size_t ad_len)
+{
+	crypto_aes_eax_update_nonce(&self->dec_ctx.aes_eax);
+	nettle_eax_aes128_update(&self->dec_ctx.aes_eax.ctx, ad_len, ad);
+	nettle_eax_aes128_decrypt(&self->dec_ctx.aes_eax.ctx, len, dst, src);
+	nettle_eax_aes128_digest(&self->dec_ctx.aes_eax.ctx, 16, mac);
+	return len;
+}
+
+static struct crypto_cipher* crypto_cipher_new_aes_eax(const uint8_t* enc_key,
+		const uint8_t* dec_key)
+{
+	struct crypto_cipher* self = calloc(1, sizeof(*self));
+	if (!self)
+		return NULL;
+
+	eax_aes128_set_key(&self->enc_ctx.aes_eax.ctx, enc_key);
+	eax_aes128_set_key(&self->dec_ctx.aes_eax.ctx, dec_key);
+
+	self->encrypt = crypto_cipher_aes_eax_encrypt;
+	self->decrypt = crypto_cipher_aes_eax_decrypt;
+
+	return self;
+}
+
+static void crypto_aes256_eax_update_nonce(struct crypto_aes256_eax* self)
+{
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+	EAX_SET_NONCE(&self->ctx, aes256_encrypt, 16, (const uint8_t*)self->count);
+#else
+	uint64_t c[2];
+	c[0] = __builtin_bswap64(self->count[0]);
+	c[1] = __builtin_bswap64(self->count[1]);
+	EAX_SET_NONCE(&self->ctx, aes256_encrypt, 16, (const uint8_t*)c);
+#endif
+
+	if (++self->count[0] == 0)
+		++self->count[1];
+}
+
+static bool crypto_cipher_aes256_eax_encrypt(struct crypto_cipher* self,
+		struct vec* dst, uint8_t* mac, const uint8_t* src,
+		size_t src_len, const uint8_t* ad, size_t ad_len)
+{
+	vec_reserve(dst, dst->len + src_len);
+
+	crypto_aes256_eax_update_nonce(&self->enc_ctx.aes256_eax);
+
+	EAX_UPDATE(&self->enc_ctx.aes256_eax.ctx, aes256_encrypt, ad_len, ad);
+	EAX_ENCRYPT(&self->enc_ctx.aes256_eax.ctx, aes256_encrypt, src_len,
+			(uint8_t*)dst->data + dst->len, src);
+	dst->len += src_len;
+
+	EAX_DIGEST(&self->enc_ctx.aes256_eax.ctx, aes256_encrypt, 16, mac);
+
+	return true;
+}
+
+static ssize_t crypto_cipher_aes256_eax_decrypt(struct crypto_cipher* self,
+		uint8_t* dst, uint8_t* mac, const uint8_t* src, size_t len,
+		const uint8_t* ad, size_t ad_len)
+{
+	crypto_aes256_eax_update_nonce(&self->dec_ctx.aes256_eax);
+	EAX_UPDATE(&self->dec_ctx.aes256_eax.ctx, aes256_encrypt, ad_len, ad);
+	EAX_DECRYPT(&self->dec_ctx.aes256_eax.ctx, aes256_encrypt, len, dst, src);
+	EAX_DIGEST(&self->dec_ctx.aes256_eax.ctx, aes256_encrypt, 16, mac);
+	return len;
+}
+
+static struct crypto_cipher* crypto_cipher_new_aes256_eax(const uint8_t* enc_key,
+		const uint8_t* dec_key)
+{
+	struct crypto_cipher* self = calloc(1, sizeof(*self));
+	if (!self)
+		return NULL;
+
+	EAX_SET_KEY(&self->enc_ctx.aes256_eax.ctx, aes256_set_encrypt_key,
+			aes256_encrypt, enc_key);
+	EAX_SET_KEY(&self->dec_ctx.aes256_eax.ctx, aes256_set_encrypt_key,
+			aes256_encrypt, dec_key);
+
+	self->encrypt = crypto_cipher_aes256_eax_encrypt;
+	self->decrypt = crypto_cipher_aes256_eax_decrypt;
+
+	return self;
+}
+
+struct crypto_cipher* crypto_cipher_new(const uint8_t* enc_key,
+		const uint8_t* dec_key, enum crypto_cipher_type type)
+{
+	switch (type) {
+	case CRYPTO_CIPHER_AES128_ECB:
+		return crypto_cipher_new_aes128_ecb(enc_key, dec_key);
+	case CRYPTO_CIPHER_AES_EAX:
+		return crypto_cipher_new_aes_eax(enc_key, dec_key);
+	case CRYPTO_CIPHER_AES256_EAX:
+		return crypto_cipher_new_aes256_eax(enc_key, dec_key);
+	case CRYPTO_CIPHER_INVALID:
+		break;
+	}
+
+	nvnc_log(NVNC_LOG_PANIC, "Invalid type: %d", type);
+	return NULL;
+}
+
+void crypto_cipher_del(struct crypto_cipher* self)
+{
+	free(self);
+}
+
+bool crypto_cipher_encrypt(struct crypto_cipher* self, struct vec* dst,
+		uint8_t* mac, const uint8_t* src, size_t src_len,
+		const uint8_t* ad, size_t ad_len)
+{
+	return self->encrypt(self, dst, mac, src, src_len, ad, ad_len);
+}
+
+ssize_t crypto_cipher_decrypt(struct crypto_cipher* self, uint8_t* dst,
+		uint8_t* mac, const uint8_t* src, size_t src_len,
+		const uint8_t* ad, size_t ad_len)
+{
+	return self->decrypt(self, dst, mac, src, src_len, ad, ad_len);
+}
+
+struct crypto_hash* crypto_hash_new(enum crypto_hash_type type)
+{
+	struct crypto_hash* self = calloc(1, sizeof(*self));
+	if (!self)
+		return NULL;
+
+	switch (type) {
+	case CRYPTO_HASH_INVALID:
+		nvnc_log(NVNC_LOG_PANIC, "Invalid hash type");
+		break;
+	case CRYPTO_HASH_MD5:
+		md5_init(&self->ctx.md5);
+		self->update = (void*)nettle_md5_update;
+		self->digest = (void*)nettle_md5_digest;
+		break;
+	case CRYPTO_HASH_SHA1:
+		sha1_init(&self->ctx.sha1);
+		self->update = (void*)nettle_sha1_update;
+		self->digest = (void*)nettle_sha1_digest;
+		break;
+	case CRYPTO_HASH_SHA256:
+		sha256_init(&self->ctx.sha256);
+		self->update = (void*)nettle_sha256_update;
+		self->digest = (void*)nettle_sha256_digest;
+		break;
+	}
+
+	return self;
+}
+
+void crypto_hash_del(struct crypto_hash* self)
+{
+	free(self);
+}
+
+void crypto_hash_append(struct crypto_hash* self, const uint8_t* src,
+		size_t len)
+{
+	self->update(&self->ctx, len, src);
+}
+
+void crypto_hash_digest(struct crypto_hash* self, uint8_t* dst, size_t len)
+{
+	self->digest(&self->ctx, len, dst);
+}
+
+void crypto_hash_one(uint8_t* dst, size_t dst_len, enum crypto_hash_type type,
+		const uint8_t* src, size_t src_len)
+{
+	struct crypto_hash *hash = crypto_hash_new(type);
+	crypto_hash_append(hash, src, src_len);
+	crypto_hash_digest(hash, dst, dst_len);
+	crypto_hash_del(hash);
+}
+
+void crypto_hash_many(uint8_t* dst, size_t dst_len, enum crypto_hash_type type,
+		const struct crypto_data_entry *src)
+{
+	struct crypto_hash *hash = crypto_hash_new(type);
+	for (int i = 0; src[i].data && src[i].len; ++i)
+		crypto_hash_append(hash, src[i].data, src[i].len);
+	crypto_hash_digest(hash, dst, dst_len);
+	crypto_hash_del(hash);
+}
+
+struct crypto_rsa_pub_key *crypto_rsa_pub_key_new(void)
+{
+	struct crypto_rsa_pub_key* self = calloc(1, sizeof(*self));
+	if (!self)
+		return NULL;
+
+	rsa_public_key_init(&self->key);
+	return self;
+}
+
+void crypto_rsa_pub_key_del(struct crypto_rsa_pub_key* self)
+{
+	if (!self)
+		return;
+	rsa_public_key_clear(&self->key);
+	free(self);
+}
+
+struct crypto_rsa_pub_key* crypto_rsa_pub_key_import(const uint8_t* modulus,
+		const uint8_t* exponent, size_t size)
+{
+	struct crypto_rsa_pub_key* self = crypto_rsa_pub_key_new();
+	if (!self)
+		return NULL;
+
+	rsa_public_key_init(&self->key);
+	mpz_init(self->key.n);
+	crypto_import(self->key.n, modulus, size);
+	mpz_init(self->key.e);
+	crypto_import(self->key.e, exponent, size);
+	rsa_public_key_prepare(&self->key);
+
+	return self;
+}
+
+bool crypto_rsa_priv_key_import_pkcs1_der(struct crypto_rsa_priv_key* priv,
+		struct crypto_rsa_pub_key* pub, const uint8_t* key,
+		size_t size)
+{
+	return rsa_keypair_from_der(&pub->key, &priv->key, 0, size, key);
+}
+
+bool crypto_rsa_priv_key_load(struct crypto_rsa_priv_key* priv,
+		struct crypto_rsa_pub_key* pub, const char* path)
+{
+	FILE* stream = fopen(path, "r");
+	if (!stream) {
+		nvnc_log(NVNC_LOG_ERROR, "Could not open file: %m");
+		return false;
+	}
+
+	char* line = NULL;
+	size_t n = 0;
+	if (getline(&line, &n, stream) < 0) {
+		nvnc_log(NVNC_LOG_ERROR, "RSA private key file is not PEM");
+		return false;
+	}
+
+	char head[128];
+	strncpy(head, line, sizeof(head));
+	head[sizeof(head) - 1] = '\0';
+	char* end = strchr(head, '\n');
+	if (end)
+		*end = '\0';
+
+	nvnc_trace("Read PEM head: \"%s\"\n", head);
+
+	struct vec base64_der;
+	vec_init(&base64_der, 4096);
+
+	while (getline(&line, &n, stream) >= 0) {
+		if (strncmp(line, "-----END", 8) == 0)
+			break;
+
+
+		vec_append(&base64_der, line, strcspn(line, "\n"));
+	}
+	free(line);
+	fclose(stream);
+
+	vec_append_zero(&base64_der, 1);
+
+	uint8_t* der = malloc(BASE64_DECODED_MAX_SIZE(base64_der.len));
+	assert(der);
+
+	ssize_t der_len = base64_decode(der, base64_der.data);
+	vec_destroy(&base64_der);
+	if (der_len < 0) {
+		free(der);
+		return false;
+	}
+
+	bool ok = false;
+	if (strcmp(head, "-----BEGIN RSA PRIVATE KEY-----") == 0) {
+		ok = crypto_rsa_priv_key_import_pkcs1_der(priv, pub, der, der_len);
+	} else {
+		nvnc_log(NVNC_LOG_ERROR, "Unsupported RSA private key format");
+	}
+
+	nvnc_trace("Private key is %d bits long", priv->key.size * 8);
+
+	free(der);
+	return ok;
+}
+
+void crypto_rsa_pub_key_modulus(const struct crypto_rsa_pub_key* key,
+		uint8_t* dst, size_t dst_size)
+{
+	crypto_export(dst, dst_size, key->key.n);
+}
+
+void crypto_rsa_pub_key_exponent(const struct crypto_rsa_pub_key* key,
+		uint8_t* dst, size_t dst_size)
+{
+	char* str = mpz_get_str(NULL, 16, key->key.e);
+	free(str);
+
+	crypto_export(dst, dst_size, key->key.e);
+}
+
+struct crypto_rsa_priv_key *crypto_rsa_priv_key_new(void)
+{
+	struct crypto_rsa_priv_key* self = calloc(1, sizeof(*self));
+	if (!self)
+		return NULL;
+
+	rsa_private_key_init(&self->key);
+	return self;
+}
+
+void crypto_rsa_priv_key_del(struct crypto_rsa_priv_key* self)
+{
+	if (!self)
+		return;
+	rsa_private_key_clear(&self->key);
+	free(self);
+}
+
+size_t crypto_rsa_pub_key_length(const struct crypto_rsa_pub_key* key)
+{
+	return key->key.size;
+}
+
+static void generate_random_for_rsa(void* random_ctx, size_t len, uint8_t* dst)
+{
+	getrandom(dst, len, 0);
+}
+
+bool crypto_rsa_keygen(struct crypto_rsa_pub_key* pub,
+		struct crypto_rsa_priv_key* priv)
+{
+	void* random_ctx = NULL;
+	nettle_random_func* random_func = generate_random_for_rsa;
+	void* progress_ctx = NULL;
+	nettle_progress_func* progress = NULL;
+
+	int rc = rsa_generate_keypair(&pub->key, &priv->key, random_ctx,
+			random_func, progress_ctx, progress, 2048, 30);
+	return rc != 0;
+}
+
+ssize_t crypto_rsa_encrypt(struct crypto_rsa_pub_key* pub, uint8_t* dst,
+		size_t dst_size, const uint8_t* src, size_t src_size)
+{
+	mpz_t ciphertext;
+	mpz_init(ciphertext);
+	int r = rsa_encrypt(&pub->key, NULL, generate_random_for_rsa,
+			src_size, src, ciphertext);
+	if (r == 0) {
+		mpz_clear(ciphertext);
+		return -1;
+	}
+	size_t len = crypto_export(dst, dst_size, ciphertext);
+	mpz_clear(ciphertext);
+	return len;
+}
+
+ssize_t crypto_rsa_decrypt(struct crypto_rsa_priv_key* priv, uint8_t* dst,
+		size_t dst_size, const uint8_t* src, size_t src_size)
+{
+	mpz_t ciphertext;
+	mpz_init(ciphertext);
+	crypto_import(ciphertext, src, src_size);
+	int r = rsa_decrypt(&priv->key, &dst_size, dst, ciphertext);
+	mpz_clear(ciphertext);
+	return r != 0 ? (ssize_t)dst_size : -1;
+}

src/cursor.c

@@ -0,0 +1,126 @@
+/*
+ * Copyright (c) 2022 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "cursor.h"
+
+#include "fb.h"
+#include "pixels.h"
+#include "rfb-proto.h"
+#include "vec.h"
+#include "enc-util.h"
+#include "resampler.h"
+#include "transform-util.h"
+
+#include <stdlib.h>
+#include <unistd.h>
+#include <assert.h>
+
+#define UDIV_UP(a, b) (((a) + (b) - 1) / (b))
+
+static struct nvnc_fb* apply_transform(struct nvnc_fb* fb)
+{
+	if (fb->transform == NVNC_TRANSFORM_NORMAL) {
+		nvnc_fb_ref(fb);
+		return fb;
+	}
+
+	uint32_t width = fb->width;
+	uint32_t height = fb->height;
+
+	nvnc_transform_dimensions(fb->transform, &width, &height);
+	struct nvnc_fb* dst = nvnc_fb_new(width, height, fb->fourcc_format,
+			width);
+	assert(dst);
+
+	// TODO: Don't assume bpp
+	memset(dst->addr, 0, width * height * 4);
+
+	resample_now(dst, fb, NULL);
+
+	return dst;
+}
+
+int cursor_encode(struct vec* dst, struct rfb_pixel_format* pixfmt,
+		struct nvnc_fb* image, uint32_t width, uint32_t height,
+		uint32_t hotspot_x, uint32_t hotspot_y)
+{
+	int rc = -1;
+
+	// Empty cursor
+	if (!image)
+		return encode_rect_head(dst, RFB_ENCODING_CURSOR, 0, 0, 0, 0);
+
+	nvnc_transform_dimensions(image->transform, &width, &height);
+	nvnc_transform_dimensions(image->transform, &hotspot_x, &hotspot_y);
+
+	if (nvnc_fb_map(image) < 0)
+		goto failure;
+
+	image = apply_transform(image);
+
+	assert(width <= image->width);
+	assert(height <= image->height);
+
+	struct rfb_pixel_format srcfmt = { 0 };
+	rc = rfb_pixfmt_from_fourcc(&srcfmt, image->fourcc_format);
+	if (rc < 0)
+		goto failure;
+
+	rc = encode_rect_head(dst, RFB_ENCODING_CURSOR, hotspot_x, hotspot_y,
+			width, height);
+	if (rc < 0)
+		goto failure;
+
+	int bpp = pixfmt->bits_per_pixel / 8;
+	size_t size = width * height;
+
+	rc = vec_reserve(dst, dst->len + size * bpp + UDIV_UP(size, 8));
+	if (rc < 0)
+		goto failure;
+
+	uint8_t* dstdata = dst->data;
+	dstdata += dst->len;
+
+	int32_t src_byte_stride = image->stride * (srcfmt.bits_per_pixel / 8);
+
+	if((int32_t)width == image->stride) {
+		pixel_to_cpixel(dstdata, pixfmt, image->addr, &srcfmt, bpp, size);
+	} else {
+		for (uint32_t y = 0; y < height; ++y) {
+			pixel_to_cpixel(dstdata + y * bpp * width, pixfmt,
+					(uint8_t*)image->addr + y * src_byte_stride,
+					&srcfmt, bpp, width);
+		}
+	}
+
+	dst->len += size * bpp;
+	dstdata = dst->data;
+	dstdata += dst->len;
+
+	for (uint32_t y = 0; y < height; ++y) {
+		if (!extract_alpha_mask(dstdata + y * UDIV_UP(width, 8),
+				(uint32_t*)image->addr + y * image->stride,
+				image->fourcc_format, width))
+			goto failure;
+
+		dst->len += UDIV_UP(width, 8);
+	}
+
+	rc = 0;
+failure:
+	nvnc_fb_unref(image);
+	return rc;
+}

src/damage-refinery.c

@@ -0,0 +1,161 @@
+/*
+ * Copyright (c) 2020 - 2021 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <stdlib.h>
+#include <unistd.h>
+#include <stdint.h>
+#include <assert.h>
+#include <pixman.h>
+#include <sys/param.h>
+
+#include "fb.h"
+#include "pixels.h"
+#include "damage-refinery.h"
+
+#define XXH_STATIC_LINKING_ONLY
+#define XXH_IMPLEMENTATION
+#define XXH_VECTOR XXH_SCALAR
+#include "xxhash.h"
+
+#define UDIV_UP(a, b) (((a) + (b) - 1) / (b))
+
+#define HASH_SEED 0
+
+int damage_refinery_init(struct damage_refinery* self, uint32_t width,
+		uint32_t height)
+{
+	self->width = width;
+	self->height = height;
+
+	uint32_t twidth = UDIV_UP(width, 32);
+	uint32_t theight = UDIV_UP(height, 32);
+
+	self->state = XXH3_createState();
+	if (!self->state)
+		return -1;
+
+	self->hashes = calloc(twidth * theight, sizeof(*self->hashes));
+	if (!self->hashes) {
+		XXH3_freeState(self->state);
+		return -1;
+	}
+
+	return 0;
+}
+
+int damage_refinery_resize(struct damage_refinery* self, uint32_t width,
+		uint32_t height)
+{
+	if (width == self->width && height == self->height)
+		return 0;
+
+	damage_refinery_destroy(self);
+	return damage_refinery_init(self, width, height);
+}
+
+void damage_refinery_destroy(struct damage_refinery* self)
+{
+	XXH3_freeState(self->state);
+	free(self->hashes);
+}
+
+static uint32_t damage_hash_tile(struct damage_refinery* self, uint32_t tx,
+		uint32_t ty, const struct nvnc_fb* buffer)
+{
+	uint8_t* pixels = buffer->addr;
+	int bpp = pixel_size_from_fourcc(buffer->fourcc_format);
+	int byte_stride = buffer->stride * bpp;
+
+	int x_start = tx * 32;
+	int x_stop = MIN((tx + 1) * 32, self->width);
+	int y_start = ty * 32;
+	int y_stop = MIN((ty + 1) * 32, self->height);
+
+	int32_t xoff = x_start * bpp;
+
+	XXH3_64bits_reset(self->state);
+	for (int y = y_start; y < y_stop; ++y) {
+		XXH3_64bits_update(self->state, pixels + xoff + y * byte_stride,
+				bpp * (x_stop - x_start));
+	}
+
+	return XXH3_64bits_digest(self->state);
+}
+
+static uint32_t* damage_tile_hash_ptr(struct damage_refinery* self,
+		uint32_t tx, uint32_t ty)
+{
+	uint32_t twidth = UDIV_UP(self->width, 32);
+	return &self->hashes[tx + ty * twidth];
+}
+
+static void damage_refine_tile(struct damage_refinery* self,
+		struct pixman_region16* refined, uint32_t tx, uint32_t ty,
+		const struct nvnc_fb* buffer)
+{
+	uint32_t hash = damage_hash_tile(self, tx, ty, buffer);
+	uint32_t* old_hash_ptr = damage_tile_hash_ptr(self, tx, ty);
+	int is_damaged = hash != *old_hash_ptr;
+	*old_hash_ptr = hash;
+
+	if (is_damaged)
+		pixman_region_union_rect(refined, refined, tx * 32, ty * 32, 32,
+				32);
+}
+
+static void tile_region_from_region(struct pixman_region16* dst,
+		struct pixman_region16* src)
+{
+	int n_rects = 0;
+	struct pixman_box16* rects = pixman_region_rectangles(src, &n_rects);
+
+	for (int i = 0; i < n_rects; ++i) {
+		int x1 = rects[i].x1 / 32;
+		int y1 = rects[i].y1 / 32;
+		int x2 = UDIV_UP(rects[i].x2, 32);
+		int y2 = UDIV_UP(rects[i].y2, 32);
+
+		pixman_region_union_rect(dst, dst, x1, y1, x2 - x1, y2 - y1);
+	}
+}
+
+void damage_refine(struct damage_refinery* self,
+		struct pixman_region16* refined,
+		struct pixman_region16* hint,
+		struct nvnc_fb* buffer)
+{
+	assert(self->width == (uint32_t)buffer->width &&
+			self->height == (uint32_t)buffer->height);
+
+	nvnc_fb_map(buffer);
+
+	struct pixman_region16 tile_region;
+	pixman_region_init(&tile_region);
+	tile_region_from_region(&tile_region, hint);
+
+	int n_rects = 0;
+	struct pixman_box16* rects = pixman_region_rectangles(&tile_region,
+			&n_rects);
+
+	for (int i = 0; i < n_rects; ++i)
+		for (int ty = rects[i].y1; ty < rects[i].y2; ++ty)
+			for (int tx = rects[i].x1; tx < rects[i].x2; ++tx)
+				damage_refine_tile(self, refined, tx, ty, buffer);
+
+	pixman_region_fini(&tile_region);
+	pixman_region_intersect_rect(refined, refined, 0, 0, self->width,
+			self->height);
+}

src/damage.c

@@ -1,187 +0,0 @@
-/*
- * Copyright (c) 2019 - 2020 Andri Yngvason
- *
- * Permission to use, copy, modify, and/or distribute this software for any
- * purpose with or without fee is hereby granted, provided that the above
- * copyright notice and this permission notice appear in all copies.
- *
- * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
- * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
- * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
- * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
- * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
- * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
- * PERFORMANCE OF THIS SOFTWARE.
- */
-
-#include "neatvnc.h"
-#include "fb.h"
-
-#include <stdlib.h>
-#include <stdbool.h>
-#include <unistd.h>
-#include <pixman.h>
-#include <string.h>
-#include <sys/param.h>
-#include <libdrm/drm_fourcc.h>
-#include <aml.h>
-#include <assert.h>
-
-#define EXPORT __attribute__((visibility("default")))
-#define ALIGN_DOWN(a, b) (((a) / (b)) * (b))
-
-struct damage_check {
-	struct aml_work* work;
-	struct nvnc_fb* fb0;
-	struct nvnc_fb* fb1;
-	int x_hint;
-	int y_hint;
-	int width_hint;
-	int height_hint;
-	nvnc_damage_fn on_done;
-	struct pixman_region16 damage;
-	void* userdata;
-};
-
-static bool fbs_are_compatible(const struct nvnc_fb* fb0,
-                               const struct nvnc_fb* fb1)
-{
-	return fb0->fourcc_format == fb1->fourcc_format &&
-	       fb0->width == fb1->width && fb0->height == fb1->height;
-}
-
-static inline bool are_tiles_equal(const uint32_t* a, const uint32_t* b,
-                                   int stride, int width, int height)
-{
-	for (int y = 0; y < height; ++y)
-		if (memcmp(a + y * stride, b + y * stride, width * 4) != 0)
-			return false;
-
-	return true;
-}
-
-#define TILE_SIDE_LENGTH 32
-static int check_damage_linear(struct pixman_region16* damage,
-                               const struct nvnc_fb* fb0,
-                               const struct nvnc_fb* fb1,
-                               int x_hint, int y_hint,
-                               int width_hint, int height_hint)
-{
-	uint32_t* b0 = fb0->addr;
-	uint32_t* b1 = fb1->addr;
-
-	int width = fb0->width;
-	int height = fb0->height;
-
-	assert(x_hint + width_hint <= width);
-	assert(y_hint + height_hint <= height);
-
-	int x_start = ALIGN_DOWN(x_hint, TILE_SIDE_LENGTH);
-	int y_start = ALIGN_DOWN(y_hint, TILE_SIDE_LENGTH);
-
-	width_hint += x_hint - x_start;
-	height_hint += y_hint - y_start;
-
-	for (int y = y_start; y < y_start + height_hint; y += TILE_SIDE_LENGTH) {
-		int tile_height = MIN(TILE_SIDE_LENGTH, height - y);
-
-		for (int x = x_start; x < x_start + width_hint;
-		     x += TILE_SIDE_LENGTH) {
-			int tile_width = MIN(TILE_SIDE_LENGTH, width - x);
-
-			int offset = x + y * width;
-
-			if (are_tiles_equal(b0 + offset, b1 + offset, width,
-			                    tile_width, tile_height))
-				continue;
-
-			pixman_region_union_rect(damage, damage, x, y,
-			                         tile_width, tile_height);
-		}
-	}
-
-	return 0;
-}
-#undef TILE_SIDE_LENGTH
-
-static void do_damage_check_linear(void* work)
-{
-	struct damage_check* check = aml_get_userdata(work);
-
-	check_damage_linear(&check->damage, check->fb0, check->fb1,
-	                    check->x_hint, check->y_hint, check->width_hint,
-	                    check->height_hint);
-}
-
-static void on_damage_check_done_linear(void* work)
-{
-	struct damage_check* check = aml_get_userdata(work);
-
-	check->on_done(&check->damage, check->userdata);
-
-	nvnc_fb_unref(check->fb1);
-	nvnc_fb_unref(check->fb0);
-
-	pixman_region_fini(&check->damage);
-}
-
-static int check_damage_linear_threaded(struct nvnc_fb* fb0,
-                                        struct nvnc_fb* fb1,
-                                        int x_hint, int y_hint,
-                                        int width_hint, int height_hint,
-                                        nvnc_damage_fn on_check_done,
-                                        void* userdata)
-{
-	struct damage_check* work = calloc(1, sizeof(*work));
-	if (!work)
-		return -1;
-
-	work->on_done = on_check_done;
-	work->userdata = userdata;
-	work->fb0 = fb0;
-	work->fb1 = fb1;
-	work->x_hint = x_hint;
-	work->y_hint = y_hint;
-	work->width_hint = width_hint;
-	work->height_hint = height_hint;
-	pixman_region_init(&work->damage);
-
-	/* TODO: Spread the work into more tasks */
-	struct aml_work* obj =
-		aml_work_new(do_damage_check_linear, on_damage_check_done_linear,
-		             work, free);
-	if (!obj) {
-		free(work);
-		return -1;
-	}
-
-	int rc = aml_start(aml_get_default(), obj);
-	aml_unref(obj);
-	if (rc < 0)
-		return -1;
-
-	work->work = obj;
-
-	nvnc_fb_ref(fb0);
-	nvnc_fb_ref(fb1);
-
-	return 0;
-}
-
-EXPORT
-int nvnc_check_damage(struct nvnc_fb* fb0, struct nvnc_fb* fb1,
-                      int x_hint, int y_hint, int width_hint, int height_hint,
-                      nvnc_damage_fn on_check_done, void* userdata)
-{
-	if (!fbs_are_compatible(fb0, fb1))
-		return -1;
-
-	switch (fb0->fourcc_modifier) {
-	case DRM_FORMAT_MOD_LINEAR:
-		return check_damage_linear_threaded(fb0, fb1, x_hint, y_hint,
-		                                    width_hint, height_hint,
-		                                    on_check_done, userdata);
-	}
-
-	return -1;
-}

src/desktop-layout.c

@@ -0,0 +1,96 @@
+/*
+ * Copyright (c) 2023 Philipp Zabel
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "desktop-layout.h"
+#include "neatvnc.h"
+#include "rfb-proto.h"
+
+#define EXPORT __attribute__((visibility("default")))
+
+void nvnc_display_layout_init(
+		struct nvnc_display_layout* display, struct rfb_screen* screen)
+{
+	display->display = NULL;
+	display->id = ntohl(screen->id);
+	display->x_pos = ntohs(screen->x);
+	display->y_pos = ntohs(screen->y);
+	display->width = ntohs(screen->width);
+	display->height = ntohs(screen->height);
+}
+
+EXPORT
+uint16_t nvnc_desktop_layout_get_width(const struct nvnc_desktop_layout* layout)
+{
+	return layout->width;
+}
+
+EXPORT
+uint16_t nvnc_desktop_layout_get_height(const struct nvnc_desktop_layout* layout)
+{
+	return layout->height;
+}
+
+EXPORT
+uint8_t nvnc_desktop_layout_get_display_count(
+		const struct nvnc_desktop_layout* layout)
+{
+	return layout->n_display_layouts;
+}
+
+EXPORT
+uint16_t nvnc_desktop_layout_get_display_x_pos(
+		const struct nvnc_desktop_layout* layout, uint8_t display_index)
+{
+	if (display_index >= layout->n_display_layouts)
+		return 0;
+	return layout->display_layouts[display_index].x_pos;
+}
+
+EXPORT
+uint16_t nvnc_desktop_layout_get_display_y_pos(
+		const struct nvnc_desktop_layout* layout, uint8_t display_index)
+{
+	if (display_index >= layout->n_display_layouts)
+		return 0;
+	return layout->display_layouts[display_index].y_pos;
+}
+
+EXPORT
+uint16_t nvnc_desktop_layout_get_display_width(
+		const struct nvnc_desktop_layout* layout, uint8_t display_index)
+{
+	if (display_index >= layout->n_display_layouts)
+		return 0;
+	return layout->display_layouts[display_index].width;
+}
+
+EXPORT
+uint16_t nvnc_desktop_layout_get_display_height(
+		const struct nvnc_desktop_layout* layout, uint8_t display_index)
+{
+	if (display_index >= layout->n_display_layouts)
+		return 0;
+	return layout->display_layouts[display_index].height;
+}
+
+EXPORT
+struct nvnc_display* nvnc_desktop_layout_get_display(
+		const struct nvnc_desktop_layout* layout, uint8_t display_index)
+{
+	if (display_index >= layout->n_display_layouts)
+		return NULL;
+	return layout->display_layouts[display_index].display;
+}

src/display.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020 Andri Yngvason
+ * Copyright (c) 2020 - 2021 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -17,12 +17,39 @@
 #include "display.h"
 #include "neatvnc.h"
 #include "common.h"
+#include "fb.h"
+#include "resampler.h"
+#include "transform-util.h"
+#include "encoder.h"
+#include "usdt.h"
 
 #include <assert.h>
 #include <stdlib.h>
 
 #define EXPORT __attribute__((visibility("default")))
 
+static void nvnc_display__on_resampler_done(struct nvnc_fb* fb,
+		struct pixman_region16* damage, void* userdata)
+{
+	struct nvnc_display* self = userdata;
+
+	DTRACE_PROBE2(neatvnc, nvnc_display__on_resampler_done, self, fb->pts);
+
+	if (self->buffer) {
+		nvnc_fb_release(self->buffer);
+		nvnc_fb_unref(self->buffer);
+	}
+
+	self->buffer = fb;
+	nvnc_fb_ref(fb);
+	nvnc_fb_hold(fb);
+
+	assert(self->server);
+
+	// TODO: Shift according to display position
+	nvnc__damage_region(self->server, damage);
+}
+
 EXPORT
 struct nvnc_display* nvnc_display_new(uint16_t x_pos, uint16_t y_pos)
 {
@@ -30,17 +57,35 @@ struct nvnc_display* nvnc_display_new(uint16_t x_pos, uint16_t y_pos)
 	if (!self)
 		return NULL;
 
+	self->resampler = resampler_create();
+	if (!self->resampler)
+		goto resampler_failure;
+
+	if (damage_refinery_init(&self->damage_refinery, 0, 0) < 0)
+		goto refinery_failure;
+
 	self->ref = 1;
 	self->x_pos = x_pos;
 	self->y_pos = y_pos;
 
 	return self;
+
+refinery_failure:
+	resampler_destroy(self->resampler);
+resampler_failure:
+	free(self);
+
+	return NULL;
 }
 
 static void nvnc__display_free(struct nvnc_display* self)
 {
-	if (self->buffer)
+	if (self->buffer) {
+		nvnc_fb_release(self->buffer);
 		nvnc_fb_unref(self->buffer);
+	}
+	damage_refinery_destroy(&self->damage_refinery);
+	resampler_destroy(self->resampler);
 	free(self);
 }
 
@@ -64,40 +109,38 @@ struct nvnc* nvnc_display_get_server(const struct nvnc_display* self)
 }
 
 EXPORT
-void nvnc_display_set_buffer(struct nvnc_display* self, struct nvnc_fb* fb)
+void nvnc_display_feed_buffer(struct nvnc_display* self, struct nvnc_fb* fb,
+		struct pixman_region16* damage)
 {
-	if (self->buffer)
-		nvnc_fb_unref(self->buffer);
+	DTRACE_PROBE2(neatvnc, nvnc_display_feed_buffer, self, fb->pts);
 
-	self->buffer = fb;
-	nvnc_fb_ref(fb);
-}
-
-EXPORT
-void nvnc_display_set_render_fn(struct nvnc_display* self, nvnc_render_fn fn)
-{
-	self->render_fn = fn;
-}
-
-EXPORT
-void nvnc_display_damage_region(struct nvnc_display* self,
-                                const struct pixman_region16* region)
-{
-	// TODO: Shift according to display position
-	assert(self->server);
-	nvnc__damage_region(self->server, region);
-}
-
-EXPORT
-void nvnc_display_damage_whole(struct nvnc_display* self)
-{
-	assert(self->server);
-
-	uint16_t width = nvnc_fb_get_width(self->buffer);
-	uint16_t height = nvnc_fb_get_height(self->buffer);
-
-	struct pixman_region16 damage;
-	pixman_region_init_rect(&damage, 0, 0, width, height);
-	nvnc_display_damage_region(self, &damage);
-	pixman_region_fini(&damage);
+	struct nvnc* server = self->server;
+	assert(server);
+
+	struct pixman_region16 refined_damage;
+	pixman_region_init(&refined_damage);
+
+	if (server->n_damage_clients != 0) {
+		damage_refinery_resize(&self->damage_refinery, fb->width,
+				fb->height);
+
+		// TODO: Run the refinery in a worker thread?
+		damage_refine(&self->damage_refinery, &refined_damage, damage, fb);
+		damage = &refined_damage;
+	} else {
+		// Resizing to zero causes the damage refinery to be reset when
+		// it's needed.
+		damage_refinery_resize(&self->damage_refinery, 0, 0);
+	}
+
+	struct pixman_region16 transformed_damage;
+	pixman_region_init(&transformed_damage);
+	nvnc_transform_region(&transformed_damage, damage, fb->transform,
+			fb->width, fb->height);
+
+	resampler_feed(self->resampler, fb, &transformed_damage,
+			nvnc_display__on_resampler_done, self);
+
+	pixman_region_fini(&transformed_damage);
+	pixman_region_fini(&refined_damage);
 }

src/enc-util.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019 - 2020 Andri Yngvason
+ * Copyright (c) 2019 - 2022 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -19,16 +19,10 @@
 #include "vec.h"
 
 #include <arpa/inet.h>
+#include <stdint.h>
+#include <pixman.h>
 
-int encode_rect_count(struct vec* dst, uint32_t count)
-{
-	struct rfb_server_fb_update_msg msg = {
-		.type = RFB_SERVER_TO_CLIENT_FRAMEBUFFER_UPDATE,
-		.n_rects = htons(count),
-	};
-
-	return vec_append(dst, &msg, sizeof(msg));
-}
+#define UDIV_UP(a, b) (((a) + (b) - 1) / (b))
 
 int encode_rect_head(struct vec* dst, enum rfb_encodings encoding,
 		uint32_t x, uint32_t y, uint32_t width, uint32_t height)
@@ -46,6 +40,23 @@ int encode_rect_head(struct vec* dst, enum rfb_encodings encoding,
 
 uint32_t calc_bytes_per_cpixel(const struct rfb_pixel_format* fmt)
 {
-	return fmt->bits_per_pixel == 32 ? fmt->depth / 8
-	                                 : fmt->bits_per_pixel / 8;
+	return fmt->bits_per_pixel == 32 ? UDIV_UP(fmt->depth, 8)
+	                                 : UDIV_UP(fmt->bits_per_pixel, 8);
+}
+
+uint32_t calculate_region_area(struct pixman_region16* region)
+{
+	uint32_t area = 0;
+
+	int n_rects = 0;
+	struct pixman_box16* rects = pixman_region_rectangles(region,
+			&n_rects);
+
+	for (int i = 0; i < n_rects; ++i) {
+		int width = rects[i].x2 - rects[i].x1;
+		int height = rects[i].y2 - rects[i].y1;
+		area += width * height;
+	}
+
+	return area;
 }

src/encoder.c

@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2021 - 2022 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+#include "encoder.h"
+#include "config.h"
+
+#include <stdlib.h>
+#include <assert.h>
+
+struct encoder* raw_encoder_new(void);
+struct encoder* zrle_encoder_new(void);
+struct encoder* tight_encoder_new(uint16_t width, uint16_t height);
+#ifdef ENABLE_OPEN_H264
+struct encoder* open_h264_new(void);
+#endif
+
+extern struct encoder_impl encoder_impl_raw;
+extern struct encoder_impl encoder_impl_zrle;
+extern struct encoder_impl encoder_impl_tight;
+#ifdef ENABLE_OPEN_H264
+extern struct encoder_impl encoder_impl_open_h264;
+#endif
+
+struct encoder* encoder_new(enum rfb_encodings type, uint16_t width,
+		uint16_t height)
+{
+	switch (type) {
+	case RFB_ENCODING_RAW: return raw_encoder_new();
+	case RFB_ENCODING_ZRLE: return zrle_encoder_new();
+	case RFB_ENCODING_TIGHT: return tight_encoder_new(width, height);
+#ifdef ENABLE_OPEN_H264
+	case RFB_ENCODING_OPEN_H264: return open_h264_new();
+#endif
+	default: break;
+	}
+
+	return NULL;
+}
+
+void encoder_init(struct encoder* self, struct encoder_impl* impl)
+{
+	self->ref = 1;
+	self->impl = impl;
+}
+
+enum rfb_encodings encoder_get_type(const struct encoder* self)
+{
+	if (self->impl == &encoder_impl_raw)
+		return RFB_ENCODING_RAW;
+	if (self->impl == &encoder_impl_zrle)
+		return RFB_ENCODING_ZRLE;
+	if (self->impl == &encoder_impl_tight)
+		return RFB_ENCODING_TIGHT;
+#ifdef ENABLE_OPEN_H264
+	if (self->impl == &encoder_impl_open_h264)
+		return RFB_ENCODING_OPEN_H264;
+#endif
+
+	abort();
+	return 0;
+}
+
+void encoder_ref(struct encoder* self)
+{
+	assert(self->ref > 0);
+	self->ref++;
+}
+
+void encoder_unref(struct encoder* self)
+{
+	if (!self)
+		return;
+
+	if (--self->ref != 0)
+		return;
+
+	if (self->impl->destroy)
+		self->impl->destroy(self);
+}
+
+void encoder_set_output_format(struct encoder* self,
+		const struct rfb_pixel_format* pixfmt)
+{
+	if (self->impl->set_output_format)
+		self->impl->set_output_format(self, pixfmt);
+}
+
+void encoder_set_quality(struct encoder* self, int value)
+{
+	if (self->impl->set_quality)
+		self->impl->set_quality(self, value);
+}
+
+int encoder_resize(struct encoder* self, uint16_t width, uint16_t height)
+{
+	if (self->impl->resize)
+		return self->impl->resize(self, width, height);
+
+	return 0;
+}
+
+int encoder_encode(struct encoder* self, struct nvnc_fb* fb,
+		struct pixman_region16* damage)
+{
+	assert(self->impl->encode);
+	return self->impl->encode(self, fb, damage);
+}
+
+void encoder_request_key_frame(struct encoder* self)
+{
+	if (self->impl->request_key_frame)
+		return self->impl->request_key_frame(self);
+}
+
+void encoder_finish_frame(struct encoder* self, struct rcbuf* result,
+		uint64_t pts)
+{
+	if (self->on_done)
+		self->on_done(self, result, pts);
+}

src/fb.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019 - 2020 Andri Yngvason
+ * Copyright (c) 2019 - 2022 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -15,6 +15,7 @@
  */
 
 #include "fb.h"
+#include "pixels.h"
 #include "neatvnc.h"
 
 #include <stdlib.h>
@@ -22,31 +23,38 @@
 #include <sys/param.h>
 #include <stdatomic.h>
 
+#include "config.h"
+
+#ifdef HAVE_GBM
+#include <gbm.h>
+#endif
+
 #define UDIV_UP(a, b) (((a) + (b) - 1) / (b))
 #define ALIGN_UP(n, a) (UDIV_UP(n, a) * a)
 #define EXPORT __attribute__((visibility("default")))
 
 EXPORT
 struct nvnc_fb* nvnc_fb_new(uint16_t width, uint16_t height,
-                            uint32_t fourcc_format)
+		uint32_t fourcc_format, uint16_t stride)
 {
 	struct nvnc_fb* fb = calloc(1, sizeof(*fb));
 	if (!fb)
 		return NULL;
 
+	uint32_t bpp = pixel_size_from_fourcc(fourcc_format);
+
+	fb->type = NVNC_FB_SIMPLE;
 	fb->ref = 1;
 	fb->width = width;
 	fb->height = height;
 	fb->fourcc_format = fourcc_format;
-	fb->size = width * height * 4; /* Assume 4 byte format for now */
+	fb->stride = stride;
+	fb->pts = NVNC_NO_PTS;
 
+	size_t size = height * stride * bpp;
 	size_t alignment = MAX(4, sizeof(void*));
-	size_t aligned_size = ALIGN_UP(fb->size, alignment);
+	size_t aligned_size = ALIGN_UP(size, alignment);
 
-	/* fb could be allocated in single allocation, but I want to reserve
-	 * the possiblity to create an fb with a pixel buffer passed from the
-	 * user.
-	 */
 	fb->addr = aligned_alloc(alignment, aligned_size);
 	if (!fb->addr) {
 		free(fb);
@@ -57,15 +65,48 @@ struct nvnc_fb* nvnc_fb_new(uint16_t width, uint16_t height,
 }
 
 EXPORT
-enum nvnc_fb_flags nvnc_fb_get_flags(const struct nvnc_fb* fb)
+struct nvnc_fb* nvnc_fb_from_buffer(void* buffer, uint16_t width, uint16_t height,
+		uint32_t fourcc_format, int32_t stride)
 {
-	return fb->flags;
+	struct nvnc_fb* fb = calloc(1, sizeof(*fb));
+	if (!fb)
+		return NULL;
+
+	fb->type = NVNC_FB_SIMPLE;
+	fb->ref = 1;
+	fb->addr = buffer;
+	fb->is_external = true;
+	fb->width = width;
+	fb->height = height;
+	fb->fourcc_format = fourcc_format;
+	fb->stride = stride;
+	fb->pts = NVNC_NO_PTS;
+
+	return fb;
 }
 
 EXPORT
-void nvnc_fb_set_flags(struct nvnc_fb* fb, enum nvnc_fb_flags flags)
+struct nvnc_fb* nvnc_fb_from_gbm_bo(struct gbm_bo* bo)
 {
-	fb->flags = flags;
+#ifdef HAVE_GBM
+	struct nvnc_fb* fb = calloc(1, sizeof(*fb));
+	if (!fb)
+		return NULL;
+
+	fb->type = NVNC_FB_GBM_BO;
+	fb->ref = 1;
+	fb->is_external = true;
+	fb->width = gbm_bo_get_width(bo);
+	fb->height = gbm_bo_get_height(bo);
+	fb->fourcc_format = gbm_bo_get_format(bo);
+	fb->bo = bo;
+	fb->pts = NVNC_NO_PTS;
+
+	return fb;
+#else
+	nvnc_log(NVNC_LOG_ERROR, "nvnc_fb_from_gbm_bo was not enabled during build time");
+	return NULL;
+#endif
 }
 
 EXPORT
@@ -92,9 +133,66 @@ uint32_t nvnc_fb_get_fourcc_format(const struct nvnc_fb* fb)
 	return fb->fourcc_format;
 }
 
+EXPORT
+int32_t nvnc_fb_get_stride(const struct nvnc_fb* fb)
+{
+	return fb->stride;
+}
+
+EXPORT
+int nvnc_fb_get_pixel_size(const struct nvnc_fb* fb)
+{
+	return pixel_size_from_fourcc(fb->fourcc_format);
+}
+
+EXPORT
+struct gbm_bo* nvnc_fb_get_gbm_bo(const struct nvnc_fb* fb)
+{
+	return fb->bo;
+}
+
+EXPORT
+enum nvnc_transform nvnc_fb_get_transform(const struct nvnc_fb* fb)
+{
+	return fb->transform;
+}
+
+EXPORT
+enum nvnc_fb_type nvnc_fb_get_type(const struct nvnc_fb* fb)
+{
+	return fb->type;
+}
+
+EXPORT
+uint64_t nvnc_fb_get_pts(const struct nvnc_fb* fb)
+{
+	return fb->pts;
+}
+
 static void nvnc__fb_free(struct nvnc_fb* fb)
 {
-	free(fb->addr);
+	nvnc_cleanup_fn cleanup = fb->common.cleanup_fn;
+	if (cleanup)
+		cleanup(fb->common.userdata);
+
+	nvnc_fb_unmap(fb);
+
+	if (!fb->is_external)
+		switch (fb->type) {
+		case NVNC_FB_UNSPEC:
+			abort();
+		case NVNC_FB_SIMPLE:
+			free(fb->addr);
+			break;
+		case NVNC_FB_GBM_BO:
+#ifdef HAVE_GBM
+			gbm_bo_destroy(fb->bo);
+#else
+			abort();
+#endif
+			break;
+		}
+
 	free(fb);
 }
 
@@ -110,3 +208,74 @@ void nvnc_fb_unref(struct nvnc_fb* fb)
 	if (--fb->ref == 0)
 		nvnc__fb_free(fb);
 }
+
+EXPORT
+void nvnc_fb_set_release_fn(struct nvnc_fb* fb, nvnc_fb_release_fn fn, void* context)
+{
+	fb->on_release = fn;
+	fb->release_context = context;
+}
+
+EXPORT
+void nvnc_fb_set_transform(struct nvnc_fb* fb, enum nvnc_transform transform)
+{
+	fb->transform = transform;
+}
+
+EXPORT
+void nvnc_fb_set_pts(struct nvnc_fb* fb, uint64_t pts)
+{
+	fb->pts = pts;
+}
+
+void nvnc_fb_hold(struct nvnc_fb* fb)
+{
+	fb->hold_count++;
+}
+
+void nvnc_fb_release(struct nvnc_fb* fb)
+{
+	if (--fb->hold_count != 0)
+		return;
+
+	nvnc_fb_unmap(fb);
+	fb->pts = NVNC_NO_PTS;
+
+	if (fb->on_release)
+		fb->on_release(fb, fb->release_context);
+}
+
+int nvnc_fb_map(struct nvnc_fb* fb)
+{
+#ifdef HAVE_GBM
+	if (fb->type != NVNC_FB_GBM_BO || fb->bo_map_handle)
+		return 0;
+
+	uint32_t stride = 0;
+	fb->addr = gbm_bo_map(fb->bo, 0, 0, fb->width, fb->height,
+			GBM_BO_TRANSFER_READ, &stride, &fb->bo_map_handle);
+	fb->stride = stride / nvnc_fb_get_pixel_size(fb);
+	if (fb->addr)
+		return 0;
+
+	fb->bo_map_handle = NULL;
+	return -1;
+#else
+	return 0;
+#endif
+}
+
+void nvnc_fb_unmap(struct nvnc_fb* fb)
+{
+#ifdef HAVE_GBM
+	if (fb->type != NVNC_FB_GBM_BO)
+		return;
+
+	if (fb->bo_map_handle)
+		gbm_bo_unmap(fb->bo, fb->bo_map_handle);
+
+	fb->bo_map_handle = NULL;
+	fb->addr = NULL;
+	fb->stride = 0;
+#endif
+}

src/fb_pool.c

@@ -0,0 +1,179 @@
+/*
+ * Copyright (c) 2021 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "fb.h"
+#include "neatvnc.h"
+
+#include "sys/queue.h"
+
+#include <stdlib.h>
+#include <assert.h>
+
+#define EXPORT __attribute__((visibility("default")))
+
+struct fbq_item {
+	struct nvnc_fb* fb;
+	TAILQ_ENTRY(fbq_item) link;
+};
+
+TAILQ_HEAD(fbq, fbq_item);
+
+struct nvnc_fb_pool {
+	int ref;
+
+	struct fbq fbs;
+
+	uint16_t width;
+	uint16_t height;
+	int32_t stride;
+	uint32_t fourcc_format;
+
+	nvnc_fb_alloc_fn alloc_fn;
+};
+
+EXPORT
+struct nvnc_fb_pool* nvnc_fb_pool_new(uint16_t width, uint16_t height,
+		uint32_t fourcc_format, uint16_t stride)
+{
+	struct nvnc_fb_pool* self = calloc(1, sizeof(*self));
+	if (!self)
+		return NULL;
+
+	self->ref = 1;
+
+	TAILQ_INIT(&self->fbs);
+	self->width = width;
+	self->height = height;
+	self->stride = stride;
+	self->fourcc_format = fourcc_format;
+	self->alloc_fn = nvnc_fb_new;
+
+	return self;
+}
+
+static void nvnc_fb_pool__destroy_fbs(struct nvnc_fb_pool* self)
+{
+	while (!TAILQ_EMPTY(&self->fbs)) {
+		struct fbq_item* item = TAILQ_FIRST(&self->fbs);
+		TAILQ_REMOVE(&self->fbs, item, link);
+		nvnc_fb_unref(item->fb);
+		free(item);
+	}
+}
+
+static void nvnc_fb_pool__destroy(struct nvnc_fb_pool* self)
+{
+	nvnc_fb_pool__destroy_fbs(self);
+	free(self);
+}
+
+EXPORT
+bool nvnc_fb_pool_resize(struct nvnc_fb_pool* self, uint16_t width,
+		uint16_t height, uint32_t fourcc_format, uint16_t stride)
+{
+	if (width == self->width && height == self->height &&
+			fourcc_format == self->fourcc_format &&
+			stride == self->stride)
+		return false;
+
+	nvnc_fb_pool__destroy_fbs(self);
+
+	self->width = width;
+	self->height = height;
+	self->stride = stride;
+	self->fourcc_format = fourcc_format;
+
+	return true;
+}
+
+EXPORT
+void nvnc_fb_pool_ref(struct nvnc_fb_pool* self)
+{
+	self->ref++;
+}
+
+EXPORT
+void nvnc_fb_pool_unref(struct nvnc_fb_pool* self)
+{
+	if (--self->ref == 0)
+		nvnc_fb_pool__destroy(self);
+}
+
+static void nvnc_fb_pool__on_fb_release(struct nvnc_fb* fb, void* userdata)
+{
+	struct nvnc_fb_pool* pool = userdata;
+
+	nvnc_fb_pool_release(pool, fb);
+	nvnc_fb_pool_unref(pool);
+}
+
+static struct nvnc_fb* nvnc_fb_pool__acquire_new(struct nvnc_fb_pool* self)
+{
+	struct nvnc_fb* fb = self->alloc_fn(self->width, self->height,
+			self->fourcc_format, self->stride);
+	if (!fb)
+		return NULL;
+
+	nvnc_fb_set_release_fn(fb, nvnc_fb_pool__on_fb_release, self);
+	nvnc_fb_pool_ref(self);
+
+	return fb;
+}
+
+static struct nvnc_fb* nvnc_fb_pool__acquire_from_list(struct nvnc_fb_pool* self)
+{
+	struct fbq_item* item = TAILQ_FIRST(&self->fbs);
+	struct nvnc_fb* fb = item->fb;
+	assert(item && fb);
+
+	TAILQ_REMOVE(&self->fbs, item, link);
+	free(item);
+
+	nvnc_fb_pool_ref(self);
+
+	return fb;
+}
+
+EXPORT
+struct nvnc_fb* nvnc_fb_pool_acquire(struct nvnc_fb_pool* self)
+{
+	return TAILQ_EMPTY(&self->fbs) ?
+		nvnc_fb_pool__acquire_new(self) :
+		nvnc_fb_pool__acquire_from_list(self);
+}
+
+EXPORT
+void nvnc_fb_pool_release(struct nvnc_fb_pool* self, struct nvnc_fb* fb)
+{
+	if (fb->width != self->width || fb->height != self->height ||
+			fb->fourcc_format != self->fourcc_format ||
+			fb->stride != self->stride) {
+		return;
+	}
+
+	nvnc_fb_ref(fb);
+
+	struct fbq_item* item = calloc(1, sizeof(*item));
+	assert(item);
+	item->fb = fb;
+	TAILQ_INSERT_TAIL(&self->fbs, item, link);
+}
+
+EXPORT
+void nvnc_fb_pool_set_alloc_fn(struct nvnc_fb_pool* self, nvnc_fb_alloc_fn fn)
+{
+	self->alloc_fn = fn;
+}

src/h264-encoder-ffmpeg-impl.c

@@ -0,0 +1,627 @@
+/*
+ * Copyright (c) 2021 - 2024 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "h264-encoder.h"
+#include "neatvnc.h"
+#include "fb.h"
+#include "sys/queue.h"
+#include "vec.h"
+#include "usdt.h"
+
+#include <stdlib.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <unistd.h>
+#include <assert.h>
+#include <gbm.h>
+#include <xf86drm.h>
+#include <aml.h>
+
+#include <libavcodec/avcodec.h>
+#include <libavutil/hwcontext.h>
+#include <libavutil/hwcontext_drm.h>
+#include <libavutil/pixdesc.h>
+#include <libavutil/dict.h>
+#include <libavfilter/avfilter.h>
+#include <libavfilter/buffersink.h>
+#include <libavfilter/buffersrc.h>
+
+#include <libdrm/drm_fourcc.h>
+
+struct h264_encoder;
+
+struct fb_queue_entry {
+	struct nvnc_fb* fb;
+	TAILQ_ENTRY(fb_queue_entry) link;
+};
+
+TAILQ_HEAD(fb_queue, fb_queue_entry);
+
+struct h264_encoder_ffmpeg {
+	struct h264_encoder base;
+
+	uint32_t width;
+	uint32_t height;
+	uint32_t format;
+
+	AVRational timebase;
+	AVRational sample_aspect_ratio;
+	enum AVPixelFormat av_pixel_format;
+
+	/* type: AVHWDeviceContext */
+	AVBufferRef* hw_device_ctx;
+
+	/* type: AVHWFramesContext */
+	AVBufferRef* hw_frames_ctx;
+
+	AVCodecContext* codec_ctx;
+
+	AVFilterGraph* filter_graph;
+	AVFilterContext* filter_in;
+	AVFilterContext* filter_out;
+
+	struct fb_queue fb_queue;
+
+	struct aml_work* work;
+	struct nvnc_fb* current_fb;
+	struct vec current_packet;
+	bool current_frame_is_keyframe;
+
+	bool please_destroy;
+};
+
+struct h264_encoder_impl h264_encoder_ffmpeg_impl;
+
+static enum AVPixelFormat drm_to_av_pixel_format(uint32_t format)
+{
+	switch (format) {
+	case DRM_FORMAT_XRGB8888:
+	case DRM_FORMAT_ARGB8888:
+		return AV_PIX_FMT_BGR0;
+	case DRM_FORMAT_XBGR8888:
+	case DRM_FORMAT_ABGR8888:
+		return AV_PIX_FMT_RGB0;
+	case DRM_FORMAT_RGBX8888:
+	case DRM_FORMAT_RGBA8888:
+		return AV_PIX_FMT_0BGR;
+	case DRM_FORMAT_BGRX8888:
+	case DRM_FORMAT_BGRA8888:
+		return AV_PIX_FMT_0RGB;
+	}
+
+	return AV_PIX_FMT_NONE;
+}
+
+static void hw_frame_desc_free(void* opaque, uint8_t* data)
+{
+	struct AVDRMFrameDescriptor* desc = (void*)data;
+	assert(desc);
+
+	for (int i = 0; i < desc->nb_objects; ++i)
+		close(desc->objects[i].fd);
+
+	free(desc);
+}
+
+// TODO: Maybe do this once per frame inside nvnc_fb?
+static AVFrame* fb_to_avframe(struct nvnc_fb* fb)
+{
+	struct gbm_bo* bo = fb->bo;
+
+	int n_planes = gbm_bo_get_plane_count(bo);
+
+	AVDRMFrameDescriptor* desc = calloc(1, sizeof(*desc));
+	desc->nb_objects = n_planes;
+
+	desc->nb_layers = 1;
+	desc->layers[0].format = gbm_bo_get_format(bo);
+	desc->layers[0].nb_planes = n_planes;
+
+	for (int i = 0; i < n_planes; ++i) {
+		uint32_t stride = gbm_bo_get_stride_for_plane(bo, i);
+
+		desc->objects[i].fd = gbm_bo_get_fd_for_plane(bo, i);
+		desc->objects[i].size = stride * fb->height;
+		desc->objects[i].format_modifier = gbm_bo_get_modifier(bo);
+
+		desc->layers[0].format = gbm_bo_get_format(bo);
+		desc->layers[0].planes[i].object_index = i;
+		desc->layers[0].planes[i].offset = gbm_bo_get_offset(bo, i);
+		desc->layers[0].planes[i].pitch = stride;
+	}
+
+	AVFrame* frame = av_frame_alloc();
+	if (!frame) {
+		hw_frame_desc_free(NULL, (void*)desc);
+		return NULL;
+	}
+
+	frame->opaque = fb;
+	frame->width = fb->width;
+	frame->height = fb->height;
+	frame->format = AV_PIX_FMT_DRM_PRIME;
+	frame->sample_aspect_ratio = (AVRational){1, 1};
+
+	AVBufferRef* desc_ref = av_buffer_create((void*)desc, sizeof(*desc),
+			hw_frame_desc_free, NULL, 0);
+	if (!desc_ref) {
+		hw_frame_desc_free(NULL, (void*)desc);
+		av_frame_free(&frame);
+		return NULL;
+	}
+
+	frame->buf[0] = desc_ref;
+	frame->data[0] = (void*)desc_ref->data;
+
+	// TODO: Set colorspace?
+
+	return frame;
+}
+
+static struct nvnc_fb* fb_queue_dequeue(struct fb_queue* queue)
+{
+	if (TAILQ_EMPTY(queue))
+		return NULL;
+
+	struct fb_queue_entry* entry = TAILQ_FIRST(queue);
+	TAILQ_REMOVE(queue, entry, link);
+	struct nvnc_fb* fb = entry->fb;
+	free(entry);
+
+	return fb;
+}
+
+static int fb_queue_enqueue(struct fb_queue* queue, struct nvnc_fb* fb)
+{
+	struct fb_queue_entry* entry = calloc(1, sizeof(*entry));
+	if (!entry)
+		return -1;
+
+	entry->fb = fb;
+	nvnc_fb_ref(fb);
+	TAILQ_INSERT_TAIL(queue, entry, link);
+
+	return 0;
+}
+
+static int h264_encoder__init_buffersrc(struct h264_encoder_ffmpeg* self)
+{
+	int rc;
+
+	/* Placeholder values are used to pacify input checking and the real
+	 * values are set below.
+	 */
+	rc = avfilter_graph_create_filter(&self->filter_in,
+			avfilter_get_by_name("buffer"), "in",
+			"width=1:height=1:pix_fmt=drm_prime:time_base=1/1", NULL,
+			self->filter_graph);
+	if (rc != 0)
+		return -1;
+
+	AVBufferSrcParameters *params = av_buffersrc_parameters_alloc();
+	if (!params)
+		return -1;
+
+	params->format = AV_PIX_FMT_DRM_PRIME;
+	params->width = self->width;
+	params->height = self->height;
+	params->sample_aspect_ratio = self->sample_aspect_ratio;
+	params->time_base = self->timebase;
+	params->hw_frames_ctx = self->hw_frames_ctx;
+
+	rc = av_buffersrc_parameters_set(self->filter_in, params);
+	assert(rc == 0);
+
+	av_free(params);
+	return 0;
+}
+
+static int h264_encoder__init_filters(struct h264_encoder_ffmpeg* self)
+{
+	int rc;
+
+	self->filter_graph = avfilter_graph_alloc();
+	if (!self->filter_graph)
+		return -1;
+
+	rc = h264_encoder__init_buffersrc(self);
+	if (rc != 0)
+		goto failure;
+
+	rc = avfilter_graph_create_filter(&self->filter_out,
+			avfilter_get_by_name("buffersink"), "out", NULL,
+			NULL, self->filter_graph);
+	if (rc != 0)
+		goto failure;
+
+	AVFilterInOut* inputs = avfilter_inout_alloc();
+	if (!inputs)
+		goto failure;
+
+	inputs->name = av_strdup("in");
+	inputs->filter_ctx = self->filter_in;
+	inputs->pad_idx = 0;
+	inputs->next = NULL;
+
+	AVFilterInOut* outputs = avfilter_inout_alloc();
+	if (!outputs) {
+		avfilter_inout_free(&inputs);
+		goto failure;
+	}
+
+	outputs->name = av_strdup("out");
+	outputs->filter_ctx = self->filter_out;
+	outputs->pad_idx = 0;
+	outputs->next = NULL;
+
+	rc = avfilter_graph_parse(self->filter_graph,
+			"hwmap=mode=direct:derive_device=vaapi"
+			",scale_vaapi=format=nv12:mode=fast",
+			outputs, inputs, NULL);
+	if (rc != 0)
+		goto failure;
+
+	assert(self->hw_device_ctx);
+
+	for (unsigned int i = 0; i < self->filter_graph->nb_filters; ++i) {
+		self->filter_graph->filters[i]->hw_device_ctx =
+			av_buffer_ref(self->hw_device_ctx);
+	}
+
+	rc = avfilter_graph_config(self->filter_graph, NULL);
+	if (rc != 0)
+		goto failure;
+
+	return 0;
+
+failure:
+	avfilter_graph_free(&self->filter_graph);
+	return -1;
+}
+
+static int h264_encoder__init_codec_context(struct h264_encoder_ffmpeg* self,
+		const AVCodec* codec, int quality)
+{
+	self->codec_ctx = avcodec_alloc_context3(codec);
+	if (!self->codec_ctx)
+		return -1;
+
+	struct AVCodecContext* c = self->codec_ctx;
+	c->width = self->width;
+	c->height = self->height;
+	c->time_base = self->timebase;
+	c->sample_aspect_ratio = self->sample_aspect_ratio;
+	c->pix_fmt = AV_PIX_FMT_VAAPI;
+	c->gop_size = INT32_MAX; /* We'll select key frames manually */
+	c->max_b_frames = 0; /* B-frames are bad for latency */
+	c->global_quality = quality;
+
+	/* open-h264 requires baseline profile, so we use constrained
+	 * baseline: AV_PROFILE_H264_BASELINE.
+	 * But that is not supported by many clients. So we use a "DEFAULT" profile.
+	 * 
+	 */
+	c->profile = AV_PROFILE_H264_MAIN;
+
+	return 0;
+}
+
+static int h264_encoder__init_hw_frames_context(struct h264_encoder_ffmpeg* self)
+{
+	self->hw_frames_ctx = av_hwframe_ctx_alloc(self->hw_device_ctx);
+	if (!self->hw_frames_ctx)
+		return -1;
+
+	AVHWFramesContext* c = (AVHWFramesContext*)self->hw_frames_ctx->data;
+	c->format = AV_PIX_FMT_DRM_PRIME;
+	c->sw_format = drm_to_av_pixel_format(self->format);
+	c->width = self->width;
+	c->height = self->height;
+
+	if (av_hwframe_ctx_init(self->hw_frames_ctx) < 0)
+		av_buffer_unref(&self->hw_frames_ctx);
+
+	return 0;
+}
+
+static int h264_encoder__schedule_work(struct h264_encoder_ffmpeg* self)
+{
+	if (self->current_fb)
+		return 0;
+
+	self->current_fb = fb_queue_dequeue(&self->fb_queue);
+	if (!self->current_fb)
+		return 0;
+
+	DTRACE_PROBE1(neatvnc, h264_encode_frame_begin, self->current_fb->pts);
+
+	self->current_frame_is_keyframe = self->base.next_frame_should_be_keyframe;
+	self->base.next_frame_should_be_keyframe = false;
+
+	return aml_start(aml_get_default(), self->work);
+}
+
+static int h264_encoder__encode(struct h264_encoder_ffmpeg* self,
+		AVFrame* frame_in)
+{
+	int rc;
+
+	rc = av_buffersrc_add_frame_flags(self->filter_in, frame_in,
+			AV_BUFFERSRC_FLAG_KEEP_REF);
+	if (rc != 0)
+		return -1;
+
+	AVFrame* filtered_frame = av_frame_alloc();
+	if (!filtered_frame)
+		return -1;
+
+	rc = av_buffersink_get_frame(self->filter_out, filtered_frame);
+	if (rc != 0)
+		goto get_frame_failure;
+
+	rc = avcodec_send_frame(self->codec_ctx, filtered_frame);
+	if (rc != 0)
+		goto send_frame_failure;
+
+	AVPacket* packet = av_packet_alloc();
+	assert(packet); // TODO
+
+	while (1) {
+		rc = avcodec_receive_packet(self->codec_ctx, packet);
+		if (rc != 0)
+			break;
+
+		vec_append(&self->current_packet, packet->data, packet->size);
+
+		packet->stream_index = 0;
+		av_packet_unref(packet);
+	}
+
+	// Frame should always start with a zero:
+	assert(self->current_packet.len == 0 ||
+			((char*)self->current_packet.data)[0] == 0);
+
+	av_packet_free(&packet);
+send_frame_failure:
+	av_frame_unref(filtered_frame);
+get_frame_failure:
+	av_frame_free(&filtered_frame);
+	return rc == AVERROR(EAGAIN) ? 0 : rc;
+}
+
+static void h264_encoder__do_work(void* handle)
+{
+	struct h264_encoder_ffmpeg* self = aml_get_userdata(handle);
+
+	AVFrame* frame = fb_to_avframe(self->current_fb);
+	assert(frame); // TODO
+
+	frame->hw_frames_ctx = av_buffer_ref(self->hw_frames_ctx);
+
+	if (self->current_frame_is_keyframe) {
+#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(58, 7, 100)
+		frame->flags |= AV_FRAME_FLAG_KEY;
+#else
+		frame->key_frame = 1;
+#endif
+		frame->pict_type = AV_PICTURE_TYPE_I;
+	} else {
+#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(58, 7, 100)
+		frame->flags &= ~AV_FRAME_FLAG_KEY;
+#else
+		frame->key_frame = 0;
+#endif
+		frame->pict_type = AV_PICTURE_TYPE_P;
+	}
+
+	int rc = h264_encoder__encode(self, frame);
+	if (rc != 0) {
+		char err[256];
+		av_strerror(rc, err, sizeof(err));
+		nvnc_log(NVNC_LOG_ERROR, "Failed to encode packet: %s", err);
+		goto failure;
+	}
+
+failure:
+	av_frame_unref(frame);
+	av_frame_free(&frame);
+}
+
+static void h264_encoder__on_work_done(void* handle)
+{
+	struct h264_encoder_ffmpeg* self = aml_get_userdata(handle);
+
+	uint64_t pts = nvnc_fb_get_pts(self->current_fb);
+	nvnc_fb_release(self->current_fb);
+	nvnc_fb_unref(self->current_fb);
+	self->current_fb = NULL;
+
+	DTRACE_PROBE1(neatvnc, h264_encode_frame_end, pts);
+
+	if (self->please_destroy) {
+		vec_destroy(&self->current_packet);
+		h264_encoder_destroy(&self->base);
+		return;
+	}
+
+	if (self->current_packet.len == 0) {
+		nvnc_log(NVNC_LOG_WARNING, "Whoops, encoded packet length is 0");
+		return;
+	}
+
+	void* userdata = self->base.userdata;
+
+	// Must make a copy of packet because the callback might destroy the
+	// encoder object.
+	struct vec packet;
+	vec_init(&packet, self->current_packet.len);
+	vec_append(&packet, self->current_packet.data,
+			self->current_packet.len);
+
+	vec_clear(&self->current_packet);
+	h264_encoder__schedule_work(self);
+
+	self->base.on_packet_ready(packet.data, packet.len, pts, userdata);
+	vec_destroy(&packet);
+}
+
+static int find_render_node(char *node, size_t maxlen) {
+	bool r = -1;
+	drmDevice *devices[64];
+
+	int n = drmGetDevices2(0, devices, sizeof(devices) / sizeof(devices[0]));
+	for (int i = 0; i < n; ++i) {
+		drmDevice *dev = devices[i];
+		if (!(dev->available_nodes & (1 << DRM_NODE_RENDER)))
+			continue;
+
+		strncpy(node, dev->nodes[DRM_NODE_RENDER], maxlen);
+		node[maxlen - 1] = '\0';
+		r = 0;
+		break;
+	}
+
+	drmFreeDevices(devices, n);
+	return r;
+}
+
+static struct h264_encoder* h264_encoder_ffmpeg_create(uint32_t width,
+		uint32_t height, uint32_t format, int quality)
+{
+	int rc;
+
+	struct h264_encoder_ffmpeg* self = calloc(1, sizeof(*self));
+	if (!self)
+		return NULL;
+
+	self->base.impl = &h264_encoder_ffmpeg_impl;
+
+	if (vec_init(&self->current_packet, 65536) < 0)
+		goto packet_failure;
+
+	self->work = aml_work_new(h264_encoder__do_work,
+			h264_encoder__on_work_done, self, NULL);
+	if (!self->work)
+		goto worker_failure;
+
+	char render_node[64];
+	if (find_render_node(render_node, sizeof(render_node)) < 0)
+		goto render_node_failure;
+
+	rc = av_hwdevice_ctx_create(&self->hw_device_ctx,
+			AV_HWDEVICE_TYPE_DRM, render_node, NULL, 0);
+	if (rc != 0)
+		goto hwdevice_ctx_failure;
+
+	self->base.next_frame_should_be_keyframe = true;
+	TAILQ_INIT(&self->fb_queue);
+
+	self->width = width;
+	self->height = height;
+	self->format = format;
+	self->timebase = (AVRational){1, 1000000};
+	self->sample_aspect_ratio = (AVRational){1, 1};
+	self->av_pixel_format = drm_to_av_pixel_format(format);
+	if (self->av_pixel_format == AV_PIX_FMT_NONE)
+		goto pix_fmt_failure;
+
+	const AVCodec* codec = avcodec_find_encoder_by_name("h264_vaapi");
+	if (!codec)
+		goto codec_failure;
+
+	if (h264_encoder__init_hw_frames_context(self) < 0)
+		goto hw_frames_context_failure;
+
+	if (h264_encoder__init_filters(self) < 0)
+		goto filter_failure;
+
+	if (h264_encoder__init_codec_context(self, codec, quality) < 0)
+		goto codec_context_failure;
+
+	self->codec_ctx->hw_frames_ctx =
+		av_buffer_ref(self->filter_out->inputs[0]->hw_frames_ctx);
+
+	AVDictionary *opts = NULL;
+	av_dict_set_int(&opts, "async_depth", 1, 0);
+
+	rc = avcodec_open2(self->codec_ctx, codec, &opts);
+	av_dict_free(&opts);
+
+	if (rc != 0)
+		goto avcodec_open_failure;
+
+	return &self->base;
+
+avcodec_open_failure:
+	avcodec_free_context(&self->codec_ctx);
+codec_context_failure:
+filter_failure:
+	av_buffer_unref(&self->hw_frames_ctx);
+hw_frames_context_failure:
+codec_failure:
+pix_fmt_failure:
+	av_buffer_unref(&self->hw_device_ctx);
+hwdevice_ctx_failure:
+render_node_failure:
+	aml_unref(self->work);
+worker_failure:
+	vec_destroy(&self->current_packet);
+packet_failure:
+	free(self);
+	return NULL;
+}
+
+static void h264_encoder_ffmpeg_destroy(struct h264_encoder* base)
+{
+	struct h264_encoder_ffmpeg* self = (struct h264_encoder_ffmpeg*)base;
+
+	if (self->current_fb) {
+		self->please_destroy = true;
+		return;
+	}
+
+	vec_destroy(&self->current_packet);
+	av_buffer_unref(&self->hw_frames_ctx);
+	avcodec_free_context(&self->codec_ctx);
+	av_buffer_unref(&self->hw_device_ctx);
+	avfilter_graph_free(&self->filter_graph);
+	aml_unref(self->work);
+	free(self);
+}
+
+static void h264_encoder_ffmpeg_feed(struct h264_encoder* base,
+		struct nvnc_fb* fb)
+{
+	struct h264_encoder_ffmpeg* self = (struct h264_encoder_ffmpeg*)base;
+	assert(fb->type == NVNC_FB_GBM_BO);
+
+	// TODO: Add transform filter
+	assert(fb->transform == NVNC_TRANSFORM_NORMAL);
+
+	int rc = fb_queue_enqueue(&self->fb_queue, fb);
+	assert(rc == 0); // TODO
+
+	nvnc_fb_hold(fb);
+
+	rc = h264_encoder__schedule_work(self);
+	assert(rc == 0); // TODO
+}
+
+struct h264_encoder_impl h264_encoder_ffmpeg_impl = {
+	.create = h264_encoder_ffmpeg_create,
+	.destroy = h264_encoder_ffmpeg_destroy,
+	.feed = h264_encoder_ffmpeg_feed,
+};

src/h264-encoder-v4l2m2m-impl.c

@@ -0,0 +1,741 @@
+/*
+ * Copyright (c) 2024 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "h264-encoder.h"
+#include "neatvnc.h"
+#include "fb.h"
+#include "pixels.h"
+
+#include <assert.h>
+#include <string.h>
+#include <stdio.h>
+#include <inttypes.h>
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <sys/ioctl.h>
+#include <linux/videodev2.h>
+#include <drm_fourcc.h>
+#include <gbm.h>
+#include <aml.h>
+#include <dirent.h>
+
+#define UDIV_UP(a, b) (((a) + (b) - 1) / (b))
+#define ALIGN_UP(a, b) ((b) * UDIV_UP((a), (b)))
+#define ARRAY_LENGTH(a) (sizeof(a) / sizeof((a)[0]))
+
+#define N_SRC_BUFS 3
+#define N_DST_BUFS 3
+
+struct h264_encoder_v4l2m2m_dst_buf {
+	struct v4l2_buffer buffer;
+	struct v4l2_plane plane;
+	void* payload;
+};
+
+struct h264_encoder_v4l2m2m_src_buf {
+	struct v4l2_buffer buffer;
+	struct v4l2_plane planes[4];
+	int fd;
+	bool is_taken;
+	struct nvnc_fb* fb;
+};
+
+struct h264_encoder_v4l2m2m {
+	struct h264_encoder base;
+
+	uint32_t width;
+	uint32_t height;
+	uint32_t format;
+	int quality; // TODO: Can we affect the quality?
+
+	char driver[16];
+
+	int fd;
+	struct aml_handler* handler;
+
+	struct h264_encoder_v4l2m2m_src_buf src_bufs[N_SRC_BUFS];
+	int src_buf_index;
+
+	struct h264_encoder_v4l2m2m_dst_buf dst_bufs[N_DST_BUFS];
+};
+
+struct h264_encoder_impl h264_encoder_v4l2m2m_impl;
+
+static int v4l2_qbuf(int fd, const struct v4l2_buffer* inbuf)
+{
+	assert(inbuf->length <= 4);
+	struct v4l2_plane planes[4];
+	struct v4l2_buffer outbuf;
+	outbuf = *inbuf;
+	memcpy(&planes, inbuf->m.planes, inbuf->length * sizeof(planes[0]));
+	outbuf.m.planes = planes;
+	return ioctl(fd, VIDIOC_QBUF, &outbuf);
+}
+
+static inline int v4l2_dqbuf(int fd, struct v4l2_buffer* buf)
+{
+	return ioctl(fd, VIDIOC_DQBUF, buf);
+}
+
+static struct h264_encoder_v4l2m2m_src_buf* take_src_buffer(
+		struct h264_encoder_v4l2m2m* self)
+{
+	unsigned int count = 0;
+	int i = self->src_buf_index;
+
+	struct h264_encoder_v4l2m2m_src_buf* buffer;
+	do {
+		buffer = &self->src_bufs[i++];
+		i %= ARRAY_LENGTH(self->src_bufs);
+	} while (++count < ARRAY_LENGTH(self->src_bufs) && buffer->is_taken);
+
+	if (buffer->is_taken)
+		return NULL;
+
+	self->src_buf_index = i;
+	buffer->is_taken = true;
+
+	return buffer;
+}
+
+static bool any_src_buf_is_taken(struct h264_encoder_v4l2m2m* self)
+{
+	bool result = false;
+	for (unsigned int i = 0; i < ARRAY_LENGTH(self->src_bufs); ++i)
+		if (self->src_bufs[i].is_taken)
+			result = true;
+	return result;
+}
+
+static int u32_cmp(const void* pa, const void* pb)
+{
+	const uint32_t *a = pa;
+	const uint32_t *b = pb;
+	return *a < *b ? -1 : *a > *b;
+}
+
+static size_t get_supported_formats(struct h264_encoder_v4l2m2m* self,
+		uint32_t* formats, size_t max_len)
+{
+	size_t i = 0;
+	for (;; ++i) {
+		struct v4l2_fmtdesc desc = {
+			.index = i,
+			.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
+		};
+		int rc = ioctl(self->fd, VIDIOC_ENUM_FMT, &desc);
+		if (rc < 0)
+			break;
+
+		nvnc_trace("Got pixel format: %s", desc.description);
+
+		formats[i] = desc.pixelformat;
+	}
+
+	qsort(formats, i, sizeof(*formats), u32_cmp);
+
+	return i;
+}
+
+static bool have_v4l2_format(const uint32_t* formats, size_t n_formats,
+		uint32_t format)
+{
+	return bsearch(&format, formats, n_formats, sizeof(format), u32_cmp);
+}
+
+static uint32_t v4l2_format_from_drm(const uint32_t* formats,
+		size_t n_formats, uint32_t drm_format)
+{
+#define TRY_FORMAT(f) \
+	if (have_v4l2_format(formats, n_formats, f)) \
+		return f
+
+	switch (drm_format) {
+	case DRM_FORMAT_RGBX8888:
+	case DRM_FORMAT_RGBA8888:
+		TRY_FORMAT(V4L2_PIX_FMT_RGBX32);
+		TRY_FORMAT(V4L2_PIX_FMT_RGBA32);
+		break;
+	case DRM_FORMAT_XRGB8888:
+	case DRM_FORMAT_ARGB8888:
+		TRY_FORMAT(V4L2_PIX_FMT_XRGB32);
+		TRY_FORMAT(V4L2_PIX_FMT_ARGB32);
+		TRY_FORMAT(V4L2_PIX_FMT_RGB32);
+		break;
+	case DRM_FORMAT_BGRX8888:
+	case DRM_FORMAT_BGRA8888:
+		TRY_FORMAT(V4L2_PIX_FMT_XBGR32);
+		TRY_FORMAT(V4L2_PIX_FMT_ABGR32);
+		TRY_FORMAT(V4L2_PIX_FMT_BGR32);
+		break;
+	case DRM_FORMAT_XBGR8888:
+	case DRM_FORMAT_ABGR8888:
+		TRY_FORMAT(V4L2_PIX_FMT_BGRX32);
+		TRY_FORMAT(V4L2_PIX_FMT_BGRA32);
+		break;
+	// TODO: More formats
+	}
+
+	return 0;
+#undef TRY_FORMAT
+}
+
+// This driver mixes up pixel formats...
+static uint32_t v4l2_format_from_drm_bcm2835(const uint32_t* formats,
+		size_t n_formats, uint32_t drm_format)
+{
+	switch (drm_format) {
+	case DRM_FORMAT_XRGB8888:
+	case DRM_FORMAT_ARGB8888:
+		return V4L2_PIX_FMT_RGBA32;
+	case DRM_FORMAT_BGRX8888:
+	case DRM_FORMAT_BGRA8888:
+		// TODO: This could also be ABGR, based on how this driver
+		// behaves
+		return V4L2_PIX_FMT_BGR32;
+	}
+	return 0;
+}
+
+static int set_src_fmt(struct h264_encoder_v4l2m2m* self)
+{
+	int rc;
+
+	uint32_t supported_formats[256];
+	size_t n_formats = get_supported_formats(self, supported_formats,
+			ARRAY_LENGTH(supported_formats));
+
+	uint32_t format;
+	if (strcmp(self->driver, "bcm2835-codec") == 0)
+		format = v4l2_format_from_drm_bcm2835(supported_formats,
+				n_formats, self->format);
+	else
+		format = v4l2_format_from_drm(supported_formats, n_formats,
+				self->format);
+	if (!format) {
+		nvnc_log(NVNC_LOG_DEBUG, "Failed to find a proper pixel format");
+		return -1;
+	}
+
+	struct v4l2_format fmt = {
+		.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
+	};
+	rc = ioctl(self->fd, VIDIOC_G_FMT, &fmt);
+	if (rc < 0) {
+		return -1;
+	}
+
+	struct v4l2_pix_format_mplane* pix_fmt = &fmt.fmt.pix_mp;
+	pix_fmt->pixelformat = format;
+	pix_fmt->width = ALIGN_UP(self->width, 16);
+	pix_fmt->height = ALIGN_UP(self->height, 16);
+
+	rc = ioctl(self->fd, VIDIOC_S_FMT, &fmt);
+	if (rc < 0) {
+		return -1;
+	}
+
+	return 0;
+}
+
+static int set_dst_fmt(struct h264_encoder_v4l2m2m* self)
+{
+	int rc;
+
+	struct v4l2_format fmt = {
+		.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE,
+	};
+	rc = ioctl(self->fd, VIDIOC_G_FMT, &fmt);
+	if (rc < 0) {
+		return -1;
+	}
+
+	struct v4l2_pix_format_mplane* pix_fmt = &fmt.fmt.pix_mp;
+	pix_fmt->pixelformat = V4L2_PIX_FMT_H264;
+	pix_fmt->width = self->width;
+	pix_fmt->height = self->height;
+
+	rc = ioctl(self->fd, VIDIOC_S_FMT, &fmt);
+	if (rc < 0) {
+		return -1;
+	}
+
+	return 0;
+}
+
+static int alloc_dst_buffers(struct h264_encoder_v4l2m2m* self)
+{
+	int n_bufs = ARRAY_LENGTH(self->dst_bufs);
+	int rc;
+
+	struct v4l2_requestbuffers req = {
+		.memory = V4L2_MEMORY_MMAP,
+		.count = n_bufs,
+		.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE,
+	};
+	rc = ioctl(self->fd, VIDIOC_REQBUFS, &req);
+	if (rc < 0)
+		return -1;
+
+	for (unsigned int i = 0; i < req.count; ++i) {
+		struct h264_encoder_v4l2m2m_dst_buf* buffer = &self->dst_bufs[i];
+		struct v4l2_buffer* buf = &buffer->buffer;
+
+		buf->index = i;
+		buf->type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+		buf->memory = V4L2_MEMORY_MMAP;
+		buf->length = 1;
+		buf->m.planes = &buffer->plane;
+
+		rc = ioctl(self->fd, VIDIOC_QUERYBUF, buf);
+		if (rc < 0)
+			return -1;
+
+		buffer->payload = mmap(0, buffer->plane.length,
+				PROT_READ | PROT_WRITE, MAP_SHARED, self->fd,
+				buffer->plane.m.mem_offset);
+		if (buffer->payload == MAP_FAILED) {
+			nvnc_log(NVNC_LOG_ERROR, "Whoops, mapping failed: %m");
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+static void enqueue_dst_buffers(struct h264_encoder_v4l2m2m* self)
+{
+	for (unsigned int i = 0; i < ARRAY_LENGTH(self->dst_bufs); ++i) {
+		int rc = v4l2_qbuf(self->fd, &self->dst_bufs[i].buffer);
+		assert(rc >= 0);
+	}
+}
+
+static void process_dst_bufs(struct h264_encoder_v4l2m2m* self)
+{
+	int rc;
+	struct v4l2_plane plane = { 0 };
+	struct v4l2_buffer buf = {
+		.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE,
+		.memory = V4L2_MEMORY_MMAP,
+		.length = 1,
+		.m.planes = &plane,
+	};
+
+	while (true) {
+		rc = v4l2_dqbuf(self->fd, &buf);
+		if (rc < 0)
+			break;
+
+		uint64_t pts = buf.timestamp.tv_sec * UINT64_C(1000000) +
+			buf.timestamp.tv_usec;
+		struct h264_encoder_v4l2m2m_dst_buf* dstbuf =
+			&self->dst_bufs[buf.index];
+		size_t size = buf.m.planes[0].bytesused;
+
+		static uint64_t last_pts;
+		if (last_pts && last_pts > pts) {
+			nvnc_log(NVNC_LOG_ERROR, "pts - last_pts = %"PRIi64,
+					(int64_t)pts - (int64_t)last_pts);
+		}
+		last_pts = pts;
+
+		nvnc_trace("Encoded frame (index %d) at %"PRIu64" µs with size: %zu",
+				buf.index, pts, size);
+
+		self->base.on_packet_ready(dstbuf->payload, size, pts,
+				self->base.userdata);
+
+		v4l2_qbuf(self->fd, &buf);
+	}
+}
+
+static void process_src_bufs(struct h264_encoder_v4l2m2m* self)
+{
+	int rc;
+	struct v4l2_plane planes[4] = { 0 };
+	struct v4l2_buffer buf = {
+		.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
+		.memory = V4L2_MEMORY_DMABUF,
+		.length = 1,
+		.m.planes = planes,
+	};
+
+	while (true) {
+		rc = v4l2_dqbuf(self->fd, &buf);
+		if (rc < 0)
+			break;
+
+		struct h264_encoder_v4l2m2m_src_buf* srcbuf =
+			&self->src_bufs[buf.index];
+		srcbuf->is_taken = false;
+
+		// TODO: This assumes that there's only one fd
+		close(srcbuf->planes[0].m.fd);
+
+		nvnc_fb_unmap(srcbuf->fb);
+		nvnc_fb_release(srcbuf->fb);
+		nvnc_fb_unref(srcbuf->fb);
+		srcbuf->fb = NULL;
+	}
+}
+
+static void stream_off(struct h264_encoder_v4l2m2m* self)
+{
+	int type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
+	ioctl(self->fd, VIDIOC_STREAMOFF, &type);
+
+	type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+	ioctl(self->fd, VIDIOC_STREAMOFF, &type);
+}
+
+static void free_dst_buffers(struct h264_encoder_v4l2m2m* self)
+{
+	for (unsigned int i = 0; i < ARRAY_LENGTH(self->dst_bufs); ++i) {
+		struct h264_encoder_v4l2m2m_dst_buf* buf = &self->dst_bufs[i];
+		munmap(buf->payload, buf->plane.length);
+	}
+}
+
+static int stream_on(struct h264_encoder_v4l2m2m* self)
+{
+	int type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
+	ioctl(self->fd, VIDIOC_STREAMON, &type);
+
+	type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+	return ioctl(self->fd, VIDIOC_STREAMON, &type);
+}
+
+static int alloc_src_buffers(struct h264_encoder_v4l2m2m* self)
+{
+	int rc;
+
+	struct v4l2_requestbuffers req = {
+		.memory = V4L2_MEMORY_DMABUF,
+		.count = N_SRC_BUFS,
+		.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
+	};
+	rc = ioctl(self->fd, VIDIOC_REQBUFS, &req);
+	if (rc < 0)
+		return -1;
+
+	for (int i = 0; i < N_SRC_BUFS; ++i) {
+		struct h264_encoder_v4l2m2m_src_buf* buffer = &self->src_bufs[i];
+		struct v4l2_buffer* buf = &buffer->buffer;
+
+		buf->index = i;
+		buf->type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
+		buf->memory = V4L2_MEMORY_DMABUF;
+		buf->length = 1;
+		buf->m.planes = buffer->planes;
+
+		rc = ioctl(self->fd, VIDIOC_QUERYBUF, buf);
+		if (rc < 0)
+			return -1;
+	}
+
+	return 0;
+}
+
+static void force_key_frame(struct h264_encoder_v4l2m2m* self)
+{
+	struct v4l2_control ctrl = { 0 };
+	ctrl.id = V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME;
+	ctrl.value = 0;
+	ioctl(self->fd, VIDIOC_S_CTRL, &ctrl);
+}
+
+static void encode_buffer(struct h264_encoder_v4l2m2m* self,
+		struct nvnc_fb* fb)
+{
+	struct h264_encoder_v4l2m2m_src_buf* srcbuf = take_src_buffer(self);
+	if (!srcbuf) {
+		nvnc_log(NVNC_LOG_ERROR, "Out of source buffers. Dropping frame...");
+		return;
+	}
+
+	assert(!srcbuf->fb);
+
+	nvnc_fb_ref(fb);
+	nvnc_fb_hold(fb);
+
+	/* For some reason the v4l2m2m h264 encoder in the Rapberry Pi 4 gets
+	 * really glitchy unless the buffer is mapped first.
+	 * This should probably be handled by the driver, but it's not.
+	 */
+	nvnc_fb_map(fb);
+
+	srcbuf->fb = fb;
+
+	struct gbm_bo* bo = nvnc_fb_get_gbm_bo(fb);
+
+	int n_planes = gbm_bo_get_plane_count(bo);
+	int fd = gbm_bo_get_fd(bo);
+	uint32_t height = ALIGN_UP(gbm_bo_get_height(bo), 16);
+
+	for (int i = 0; i < n_planes; ++i) {
+		uint32_t stride = gbm_bo_get_stride_for_plane(bo, i);
+		uint32_t offset = gbm_bo_get_offset(bo, i);
+		uint32_t size = stride * height;
+
+		srcbuf->buffer.m.planes[i].m.fd = fd;
+		srcbuf->buffer.m.planes[i].bytesused = size;
+		srcbuf->buffer.m.planes[i].length = size;
+		srcbuf->buffer.m.planes[i].data_offset = offset;
+	}
+
+	srcbuf->buffer.timestamp.tv_sec = fb->pts / UINT64_C(1000000);
+	srcbuf->buffer.timestamp.tv_usec = fb->pts % UINT64_C(1000000);
+
+	if (self->base.next_frame_should_be_keyframe)
+		force_key_frame(self);
+	self->base.next_frame_should_be_keyframe = false;
+
+	int rc = v4l2_qbuf(self->fd, &srcbuf->buffer);
+	if (rc < 0) {
+		nvnc_log(NVNC_LOG_PANIC, "Failed to enqueue buffer: %m");
+	}
+}
+
+static void process_fd_events(void* handle)
+{
+	struct h264_encoder_v4l2m2m* self = aml_get_userdata(handle);
+	process_dst_bufs(self);
+}
+
+static void h264_encoder_v4l2m2m_configure(struct h264_encoder_v4l2m2m* self)
+{
+	struct v4l2_control ctrl = { 0 };
+
+	ctrl.id = V4L2_CID_MPEG_VIDEO_H264_PROFILE;
+	ctrl.value = V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_BASELINE;
+	ioctl(self->fd, VIDIOC_S_CTRL, &ctrl);
+
+	ctrl.id = V4L2_CID_MPEG_VIDEO_H264_I_PERIOD;
+	ctrl.value = INT_MAX;
+	ioctl(self->fd, VIDIOC_S_CTRL, &ctrl);
+
+	ctrl.id = V4L2_CID_MPEG_VIDEO_BITRATE_MODE;
+	ctrl.value = V4L2_MPEG_VIDEO_BITRATE_MODE_CQ;
+	ioctl(self->fd, VIDIOC_S_CTRL, &ctrl);
+
+	ctrl.id = V4L2_CID_MPEG_VIDEO_CONSTANT_QUALITY;
+	ctrl.value = self->quality;
+	ioctl(self->fd, VIDIOC_S_CTRL, &ctrl);
+}
+
+static bool can_encode_to_h264(int fd)
+{
+	size_t i = 0;
+	for (;; ++i) {
+		struct v4l2_fmtdesc desc = {
+			.index = i,
+			.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE,
+		};
+		int rc = ioctl(fd, VIDIOC_ENUM_FMT, &desc);
+		if (rc < 0)
+			break;
+
+		if (desc.pixelformat == V4L2_PIX_FMT_H264)
+			return true;
+	}
+	return false;
+}
+
+static bool can_handle_frame_size(int fd, uint32_t width, uint32_t height)
+{
+	size_t i = 0;
+	for (;; ++i) {
+		struct v4l2_frmsizeenum size = {
+			.index = i,
+			.pixel_format = V4L2_PIX_FMT_H264,
+		};
+		int rc = ioctl(fd, VIDIOC_ENUM_FRAMESIZES, &size);
+		if (rc < 0)
+			break;
+
+		switch (size.type) {
+		case V4L2_FRMSIZE_TYPE_DISCRETE:
+			if (size.discrete.width == width &&
+					size.discrete.height == height)
+				return true;
+			break;
+		case V4L2_FRMSIZE_TYPE_CONTINUOUS:
+		case V4L2_FRMSIZE_TYPE_STEPWISE:
+			if (size.stepwise.min_width <= width &&
+					width <= size.stepwise.max_width &&
+					size.stepwise.min_height <= height &&
+					height <= size.stepwise.max_height &&
+					(16 % size.stepwise.step_width) == 0 &&
+					(16 % size.stepwise.step_height) == 0)
+				return true;
+			break;
+		}
+	}
+	return false;
+}
+
+static bool is_device_capable(int fd, uint32_t width, uint32_t height)
+{
+	struct v4l2_capability cap = { 0 };
+	int rc = ioctl(fd, VIDIOC_QUERYCAP, &cap);
+	if (rc < 0)
+		return false;
+
+	uint32_t required_caps = V4L2_CAP_VIDEO_M2M_MPLANE | V4L2_CAP_STREAMING;
+	if ((cap.capabilities & required_caps) != required_caps)
+		return false;
+
+	if (!can_encode_to_h264(fd))
+		return false;
+
+	if (!can_handle_frame_size(fd, width, height))
+		return false;
+
+	return true;
+}
+
+static int find_capable_device(uint32_t width, uint32_t height)
+{
+	int fd = -1;
+	DIR *dir = opendir("/dev");
+	assert(dir);
+
+	for (;;) {
+		struct dirent* entry = readdir(dir);
+		if (!entry)
+			break;
+
+		if (strncmp(entry->d_name, "video", 5) != 0)
+			continue;
+
+		char path[256];
+		snprintf(path, sizeof(path), "/dev/%s", entry->d_name);
+		fd = open(path, O_RDWR | O_CLOEXEC);
+		if (fd < 0) {
+			continue;
+		}
+
+		if (is_device_capable(fd, width, height)) {
+			nvnc_log(NVNC_LOG_DEBUG, "Using v4l2m2m device: %s",
+					path);
+			break;
+		}
+		close(fd);
+		fd = -1;
+	}
+
+	closedir(dir);
+	return fd;
+}
+
+static struct h264_encoder* h264_encoder_v4l2m2m_create(uint32_t width,
+		uint32_t height, uint32_t format, int quality)
+{
+	struct h264_encoder_v4l2m2m* self = calloc(1, sizeof(*self));
+	if (!self)
+		return NULL;
+
+	self->base.impl = &h264_encoder_v4l2m2m_impl;
+	self->fd = -1;
+	self->width = width;
+	self->height = height;
+	self->format = format;
+	self->quality = quality;
+
+	self->fd = find_capable_device(width, height);
+	if (self->fd < 0)
+		goto failure;
+
+	struct v4l2_capability cap = { 0 };
+	ioctl(self->fd, VIDIOC_QUERYCAP, &cap);
+	strncpy(self->driver, (const char*)cap.driver, sizeof(self->driver));
+
+	if (set_src_fmt(self) < 0)
+		goto failure;
+
+	if (set_dst_fmt(self) < 0)
+		goto failure;
+
+	h264_encoder_v4l2m2m_configure(self);
+
+	if (alloc_dst_buffers(self) < 0)
+		goto failure;
+
+	if (alloc_src_buffers(self) < 0)
+		goto failure;
+
+	enqueue_dst_buffers(self);
+
+	if (stream_on(self) < 0)
+		goto failure;
+
+	int flags = fcntl(self->fd, F_GETFL);
+	fcntl(self->fd, F_SETFL, flags | O_NONBLOCK);
+
+	self->handler = aml_handler_new(self->fd, process_fd_events, self, NULL);
+	aml_set_event_mask(self->handler, AML_EVENT_READ);
+
+	if (aml_start(aml_get_default(), self->handler) < 0) {
+		aml_unref(self->handler);
+		goto failure;
+	}
+
+	return &self->base;
+
+failure:
+	if (self->fd >= 0)
+		close(self->fd);
+	return NULL;
+}
+
+static void claim_all_src_bufs(
+		struct h264_encoder_v4l2m2m* self)
+{
+	for (;;) {
+		 process_src_bufs(self);
+		 if (!any_src_buf_is_taken(self))
+			 break;
+		 usleep(10000);
+	}
+}
+
+static void h264_encoder_v4l2m2m_destroy(struct h264_encoder* base)
+{
+	struct h264_encoder_v4l2m2m* self = (struct h264_encoder_v4l2m2m*)base;
+	claim_all_src_bufs(self);
+	aml_stop(aml_get_default(), self->handler);
+	aml_unref(self->handler);
+	stream_off(self);
+	free_dst_buffers(self);
+	if (self->fd >= 0)
+		close(self->fd);
+	free(self);
+}
+
+static void h264_encoder_v4l2m2m_feed(struct h264_encoder* base,
+		struct nvnc_fb* fb)
+{
+	struct h264_encoder_v4l2m2m* self = (struct h264_encoder_v4l2m2m*)base;
+	process_src_bufs(self);
+	encode_buffer(self, fb);
+}
+
+struct h264_encoder_impl h264_encoder_v4l2m2m_impl = {
+	.create = h264_encoder_v4l2m2m_create,
+	.destroy = h264_encoder_v4l2m2m_destroy,
+	.feed = h264_encoder_v4l2m2m_feed,
+};

src/h264-encoder.c

@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2024 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "h264-encoder.h"
+#include "config.h"
+
+#ifdef HAVE_FFMPEG
+extern struct h264_encoder_impl h264_encoder_ffmpeg_impl;
+#endif
+
+#ifdef HAVE_V4L2
+extern struct h264_encoder_impl h264_encoder_v4l2m2m_impl;
+#endif
+
+struct h264_encoder* h264_encoder_create(uint32_t width, uint32_t height,
+		uint32_t format, int quality)
+{
+	struct h264_encoder* encoder = NULL;
+
+#ifdef HAVE_V4L2
+	encoder = h264_encoder_v4l2m2m_impl.create(width, height, format, quality);
+	if (encoder) {
+		return encoder;
+	}
+#endif
+
+#ifdef HAVE_FFMPEG
+	encoder = h264_encoder_ffmpeg_impl.create(width, height, format, quality);
+	if (encoder) {
+		return encoder;
+	}
+#endif
+
+	return encoder;
+}
+
+void h264_encoder_destroy(struct h264_encoder* self)
+{
+	self->impl->destroy(self);
+}
+
+void h264_encoder_set_packet_handler_fn(struct h264_encoder* self,
+		h264_encoder_packet_handler_fn fn)
+{
+	self->on_packet_ready = fn;
+}
+
+void h264_encoder_set_userdata(struct h264_encoder* self, void* userdata)
+{
+	self->userdata = userdata;
+}
+
+void h264_encoder_feed(struct h264_encoder* self, struct nvnc_fb* fb)
+{
+	self->impl->feed(self, fb);
+}
+
+void h264_encoder_request_keyframe(struct h264_encoder* self)
+{
+	self->next_frame_should_be_keyframe = true;
+}

src/http.c

@@ -0,0 +1,483 @@
+/* Copyright (c) 2014-2016, Marel
+ * Copyright (c) 2023, Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "vec.h"
+#include "http.h"
+
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+
+enum httplex_token_type {
+	HTTPLEX_SOLIDUS,
+	HTTPLEX_CR,
+	HTTPLEX_LF,
+	HTTPLEX_WS,
+	HTTPLEX_LITERAL,
+	HTTPLEX_KEY,
+	HTTPLEX_VALUE,
+	HTTPLEX_QUERY,
+	HTTPLEX_AMPERSAND,
+	HTTPLEX_EQ,
+	HTTPLEX_END,
+};
+
+struct httplex_token {
+	enum httplex_token_type type;
+	const char* value;
+};
+
+enum httplex_state {
+	HTTPLEX_STATE_REQUEST = 0,
+	HTTPLEX_STATE_KEY,
+	HTTPLEX_STATE_VALUE,
+};
+
+struct httplex {
+	enum httplex_state state;
+	struct httplex_token current_token;
+	const char* input;
+	const char* pos;
+	const char* next_pos;
+	struct vec buffer;
+	int accepted;
+	int errno_;
+};
+
+static int httplex_init(struct httplex* self, const char* input)
+{
+	memset(self, 0, sizeof(*self));
+
+	self->input = input;
+	self->pos = input;
+	self->accepted = 1;
+
+	if (vec_reserve(&self->buffer, 256) < 0)
+		return -1;
+
+	return 0;
+}
+
+static void httplex_destroy(struct httplex* self)
+{
+	vec_destroy(&self->buffer);
+}
+
+static inline int httplex__is_literal(char c)
+{
+	switch (c) {
+	case '/': case '\r': case '\n': case ' ': case '\t':
+	case '?': case '&': case '=':
+		return 0;
+	}
+
+	return isprint(c);
+}
+
+static inline size_t httplex__literal_length(const char* str)
+{
+	size_t len = 0;
+	while (httplex__is_literal(*str++))
+		++len;
+	return len;
+}
+
+static int httplex__classify_request_token(struct httplex* self)
+{
+	switch (*self->pos) {
+	case '/':
+		self->current_token.type = HTTPLEX_SOLIDUS;
+		self->next_pos = self->pos + strspn(self->pos, "/");
+		return 0;
+	case '\r':
+		self->current_token.type = HTTPLEX_CR;
+		self->next_pos = self->pos + 1;
+		return 0;
+	case '\n':
+		self->current_token.type = HTTPLEX_LF;
+		self->next_pos = self->pos + 1;
+		return 0;
+	case '?':
+		self->current_token.type = HTTPLEX_QUERY;
+		self->next_pos = self->pos + 1;
+		return 0;
+	case '&':
+		self->current_token.type = HTTPLEX_AMPERSAND;
+		self->next_pos = self->pos + 1;
+		return 0;
+	case '=':
+		self->current_token.type = HTTPLEX_EQ;
+		self->next_pos = self->pos + 1;
+		return 0;
+	case ' ':
+	case '\t':
+		self->current_token.type = HTTPLEX_WS;
+		self->next_pos = self->pos + strspn(self->pos, " \t");
+		return 0;
+	}
+
+	if (httplex__is_literal(*self->pos)) {
+		self->current_token.type = HTTPLEX_LITERAL;
+		size_t len = httplex__literal_length(self->pos);
+		self->next_pos = self->pos + len;
+		vec_assign(&self->buffer, self->pos, len);
+		vec_append(&self->buffer, "", 1);
+		self->current_token.value = self->buffer.data;
+		return 0;
+	}
+
+	return -1;
+}
+
+static inline int httplex__is_key_char(char c)
+{
+	return isalnum(c) || c == '-';
+}
+
+static inline size_t httplex__key_length(const char* str)
+{
+	size_t len = 0;
+	while (httplex__is_key_char(*str++))
+		++len;
+	return len;
+}
+
+static int httplex__classify_key_token(struct httplex* self)
+{
+	switch (*self->pos) {
+	case '\r':
+		self->current_token.type = HTTPLEX_CR;
+		self->next_pos = self->pos + 1;
+		return 0;
+	case '\n':
+		self->current_token.type = HTTPLEX_LF;
+		self->next_pos = self->pos + 1;
+		return 0;
+	}
+
+	if (!httplex__is_key_char(*self->pos))
+		return -1;
+
+	size_t len = httplex__key_length(self->pos);
+
+	if (self->pos[len] != ':')
+		return -1;
+
+	len += 1;
+
+	self->next_pos = self->pos + len;
+	self->next_pos += strspn(self->next_pos, " \t");
+
+	vec_assign(&self->buffer, self->pos, len - 1);
+	vec_append(&self->buffer, "", 1);
+
+	self->current_token.type = HTTPLEX_KEY;
+	self->current_token.value = self->buffer.data;
+	return 0;
+}
+
+static int httplex__classify_value_token(struct httplex* self)
+{
+	size_t len = strcspn(self->pos, "\r");
+	if (strncmp(&self->pos[len], "\r\n", 2) != 0)
+		return -1;
+
+	self->next_pos = self->pos + len + 2;
+
+	vec_assign(&self->buffer, self->pos, len);
+	vec_append(&self->buffer, "", 1);
+
+	self->current_token.type = HTTPLEX_VALUE;
+	self->current_token.value = self->buffer.data;
+	return 0;
+}
+
+static int httplex__classify_token(struct httplex* self)
+{
+	switch (self->state) {
+	case HTTPLEX_STATE_REQUEST:
+		return httplex__classify_request_token(self);
+	case HTTPLEX_STATE_KEY:
+		return httplex__classify_key_token(self);
+	case HTTPLEX_STATE_VALUE:
+		return httplex__classify_value_token(self);
+	};
+
+	abort();
+	return -1;
+}
+
+static struct httplex_token* httplex_next_token(struct httplex* self)
+{
+	if (self->current_token.type == HTTPLEX_END)
+		return &self->current_token;
+
+	if (!self->accepted)
+		return &self->current_token;
+
+	if (self->next_pos)
+		self->pos = self->next_pos;
+
+	if (httplex__classify_token(self) < 0)
+		return NULL;
+
+	self->accepted = 0;
+
+	return &self->current_token;
+}
+
+static inline int httplex_accept_token(struct httplex* self)
+{
+	self->accepted = 1;
+	return 1;
+}
+
+static int http__literal(struct httplex* lex, const char* str)
+{
+	struct httplex_token* tok = httplex_next_token(lex);
+	if (!tok)
+		return 0;
+
+	if (tok->type != HTTPLEX_LITERAL)
+		return 0;
+
+	if (strcasecmp(str, tok->value) != 0)
+		return 0;
+
+	return httplex_accept_token(lex);
+}
+
+static int http__get(struct http_req* req, struct httplex* lex)
+{
+	return http__literal(lex, "GET");
+}
+
+static int http__method(struct http_req* req, struct httplex* lex)
+{
+	return http__get(req, lex);
+}
+
+static int http__peek(struct httplex* lex, enum httplex_token_type type)
+{
+	struct httplex_token* tok = httplex_next_token(lex);
+	if (!tok)
+		return 0;
+
+	if (tok->type != type)
+		return 0;
+
+	return 1;
+}
+
+static int http__expect(struct httplex* lex, enum httplex_token_type type)
+{
+	return http__peek(lex, type) && httplex_accept_token(lex);
+}
+
+static int http__version(struct httplex* lex)
+{
+	return http__literal(lex, "HTTP")
+	    && http__expect(lex, HTTPLEX_SOLIDUS)
+	    && http__literal(lex, "1.1");
+}
+
+static int http__url_path(struct http_req* req, struct httplex* lex)
+{
+	if (!http__expect(lex, HTTPLEX_SOLIDUS))
+		return 0;
+
+	struct httplex_token* tok = httplex_next_token(lex);
+	if (!tok)
+		return 0;
+
+	if (tok->type != HTTPLEX_LITERAL)
+		return tok->type == HTTPLEX_WS;
+
+	httplex_accept_token(lex);
+
+	return http__peek(lex, HTTPLEX_SOLIDUS)
+	     ? http__url_path(req, lex) : 1;
+}
+
+static int http__url_query(struct http_req* req, struct httplex* lex)
+{
+	return http__expect(lex, HTTPLEX_LITERAL)
+	    && http__expect(lex, HTTPLEX_EQ)
+	    && http__expect(lex, HTTPLEX_LITERAL)
+	    && http__expect(lex, HTTPLEX_AMPERSAND)
+	       ? http__url_query(req, lex) : 1;
+}
+
+static int http__url(struct http_req* req, struct httplex* lex)
+{
+	return http__url_path(req, lex)
+	    && http__expect(lex, HTTPLEX_QUERY) ? http__url_query(req, lex) : 1;
+}
+
+static int http__request(struct http_req* req, struct httplex* lex)
+{
+	return http__method(req, lex)
+	    && http__expect(lex, HTTPLEX_WS)
+	    && http__url(req, lex)
+	    && http__expect(lex, HTTPLEX_WS)
+	    && http__version(lex)
+	    && http__expect(lex, HTTPLEX_CR)
+	    && http__expect(lex, HTTPLEX_LF);
+}
+
+static int http__expect_key(struct httplex* lex, const char* key)
+{
+	struct httplex_token* tok = httplex_next_token(lex);
+	if (!tok)
+		return 0;
+
+	if (tok->type != HTTPLEX_KEY)
+		return 0;
+
+	if (key && strcasecmp(tok->value, key) != 0)
+		return 0;
+
+	return httplex_accept_token(lex);
+}
+
+static int http__content_length(struct http_req* req, struct httplex* lex)
+{
+	lex->state = HTTPLEX_STATE_KEY;
+	if (!http__expect_key(lex, "Content-Length"))
+		return 0;
+
+	lex->state = HTTPLEX_STATE_VALUE;
+	struct httplex_token* tok = httplex_next_token(lex);
+	if (!tok)
+		return 0;
+
+	if (tok->type != HTTPLEX_VALUE)
+		return 0;
+
+	req->content_length = atoi(tok->value);
+
+	return httplex_accept_token(lex);
+}
+
+static int http__content_type(struct http_req* req, struct httplex* lex)
+{
+	lex->state = HTTPLEX_STATE_KEY;
+	if (!http__expect_key(lex, "Content-Type"))
+		return 0;
+
+	lex->state = HTTPLEX_STATE_VALUE;
+	struct httplex_token* tok = httplex_next_token(lex);
+	if (!tok)
+		return 0;
+
+	if (tok->type != HTTPLEX_VALUE)
+		return 0;
+
+	req->content_type = strdup(tok->value);
+
+	return httplex_accept_token(lex);
+}
+
+static int http__field_key(struct http_req* req, struct httplex* lex)
+{
+	lex->state = HTTPLEX_STATE_KEY;
+
+	struct httplex_token* tok = httplex_next_token(lex);
+	if (!tok)
+		return 0;
+
+	if (tok->type != HTTPLEX_KEY)
+		return 0;
+
+	req->field[req->field_index].key = strdup(tok->value);
+
+	return httplex_accept_token(lex);
+}
+
+static int http__field_value(struct http_req* req, struct httplex* lex)
+{
+	lex->state = HTTPLEX_STATE_VALUE;
+
+	struct httplex_token* tok = httplex_next_token(lex);
+	if (!tok)
+		return 0;
+
+	if (tok->type != HTTPLEX_VALUE)
+		return 0;
+
+	req->field[req->field_index++].value = strdup(tok->value);
+
+	return httplex_accept_token(lex);
+}
+
+static int http__field_kv(struct http_req* req, struct httplex* lex)
+{
+	return http__field_key(req, lex)
+	    && http__field_value(req, lex);
+}
+
+static int http__header_kv(struct http_req* req, struct httplex* lex)
+{
+	return http__content_length(req, lex)
+	    || http__content_type(req, lex)
+	    || http__field_kv(req, lex);
+}
+
+static int http__header(struct http_req* req, struct httplex* lex)
+{
+	while (http__header_kv(req, lex));
+
+	lex->state = HTTPLEX_STATE_KEY;
+	if (http__expect(lex, HTTPLEX_CR))
+		return http__expect(lex, HTTPLEX_LF);
+
+	return 1;
+}
+
+int http_req_parse(struct http_req* req, const char* input)
+{
+	memset(req, 0, sizeof(*req));
+
+	struct httplex lex;
+	if (httplex_init(&lex, input) < 0)
+		return -1;
+
+	if (!http__request(req, &lex))
+		goto failure;
+
+	if (!http__header(req, &lex))
+		goto failure;
+
+	req->header_length = lex.next_pos - input;
+
+	httplex_destroy(&lex);
+	return 0;
+
+failure:
+	httplex_destroy(&lex);
+	http_req_free(req);
+	return -1;
+}
+
+void http_req_free(struct http_req* req)
+{
+	free(req->content_type);
+
+	for (size_t i = 0; i < HTTP_FIELD_INDEX_MAX && req->field[i].key; ++i) {
+		free(req->field[i].key);
+		free(req->field[i].value);
+	}
+}

src/logging.c

@@ -0,0 +1,215 @@
+/*
+ * Copyright (c) 2019 - 2022 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "neatvnc.h"
+#include "common.h"
+#include "logging.h"
+#include "config.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdbool.h>
+#include <stdarg.h>
+#include <string.h>
+#include <ctype.h>
+#include <threads.h>
+
+#ifdef HAVE_LIBAVUTIL
+#include <libavutil/avutil.h>
+#endif
+
+#define EXPORT __attribute__((visibility("default")))
+
+static nvnc_log_fn log_fn = nvnc_default_logger;
+static thread_local nvnc_log_fn thread_local_log_fn = NULL;
+
+#ifndef NDEBUG
+static enum nvnc_log_level log_level = NVNC_LOG_DEBUG;
+#else
+static enum nvnc_log_level log_level = NVNC_LOG_WARNING;
+#endif
+
+static bool is_initialised = false;
+
+static nvnc_log_fn get_log_fn(void)
+{
+	return thread_local_log_fn ? thread_local_log_fn : log_fn;
+}
+
+static char* trim_left(char* str)
+{
+	while (isspace(*str))
+		++str;
+	return str;
+}
+
+static char* trim_right(char* str)
+{
+	char* end = str + strlen(str) - 1;
+	while (str < end && isspace(*end))
+		*end-- = '\0';
+	return str;
+}
+
+static inline char* trim(char* str)
+{
+	return trim_right(trim_left(str));
+}
+
+static const char* log_level_to_string(enum nvnc_log_level level)
+{
+	switch (level) {
+	case NVNC_LOG_PANIC: return "PANIC";
+	case NVNC_LOG_ERROR: return "ERROR";
+	case NVNC_LOG_WARNING: return "Warning";
+	case NVNC_LOG_INFO: return "Info";
+	case NVNC_LOG_DEBUG: return "DEBUG";
+	case NVNC_LOG_TRACE: return "TRACE";
+	}
+
+	return "UNKNOWN";
+}
+
+static FILE* stream_for_log_level(enum nvnc_log_level level)
+{
+	switch (level) {
+	case NVNC_LOG_PANIC: return stderr;
+	case NVNC_LOG_ERROR: return stderr;
+	case NVNC_LOG_WARNING: return stderr;
+	case NVNC_LOG_INFO: return stdout;
+	case NVNC_LOG_DEBUG: return stdout;
+	case NVNC_LOG_TRACE: return stdout;
+	}
+
+	return stderr;
+}
+
+static void nvnc__vlog(const struct nvnc_log_data* meta, const char* fmt,
+		va_list args)
+{
+	char message[1024];
+
+	if (meta->level <= log_level) {
+		vsnprintf(message, sizeof(message), fmt, args);
+		get_log_fn()(meta, trim(message));
+	}
+
+	if (meta->level == NVNC_LOG_PANIC)
+		abort();
+}
+
+EXPORT
+void nvnc_default_logger(const struct nvnc_log_data* meta,
+		const char* message)
+{
+	const char* level = log_level_to_string(meta->level);
+	FILE* stream = stream_for_log_level(meta->level);
+
+	if (meta->level == NVNC_LOG_INFO)
+		fprintf(stream, "Info: %s\n", message);
+	else
+		fprintf(stream, "%s: %s: %d: %s\n", level, meta->file,
+				meta->line, message);
+
+	fflush(stream);
+}
+
+#ifdef HAVE_LIBAVUTIL
+static enum nvnc_log_level nvnc__log_level_from_av(int level)
+{
+	switch (level) {
+	case AV_LOG_PANIC: return NVNC_LOG_PANIC;
+	case AV_LOG_FATAL: return NVNC_LOG_ERROR;
+	case AV_LOG_ERROR: return NVNC_LOG_ERROR;
+	case AV_LOG_WARNING: return NVNC_LOG_WARNING;
+	case AV_LOG_INFO: return NVNC_LOG_INFO;
+	case AV_LOG_VERBOSE: return NVNC_LOG_INFO;
+	case AV_LOG_DEBUG: return NVNC_LOG_DEBUG;
+	case AV_LOG_TRACE: return NVNC_LOG_TRACE;
+	}
+
+	return NVNC_LOG_TRACE;
+}
+
+static int nvnc__log_level_to_av(enum nvnc_log_level level)
+{
+	switch (level) {
+	case NVNC_LOG_PANIC: return AV_LOG_PANIC;
+	case NVNC_LOG_ERROR: return AV_LOG_ERROR;
+	case NVNC_LOG_WARNING: return AV_LOG_WARNING;
+	case NVNC_LOG_INFO: return AV_LOG_INFO;
+	case NVNC_LOG_DEBUG: return AV_LOG_DEBUG;
+	case NVNC_LOG_TRACE: return AV_LOG_TRACE;
+	}
+
+	return AV_LOG_TRACE;
+}
+
+static void nvnc__av_log_callback(void* ptr, int level, const char* fmt,
+		va_list va)
+{
+	struct nvnc_log_data meta = {
+		.level = nvnc__log_level_from_av(level),
+		.file = "libav",
+		.line = 0,
+	};
+	nvnc__vlog(&meta, fmt, va);
+}
+#endif
+
+EXPORT
+void nvnc_set_log_level(enum nvnc_log_level level)
+{
+	log_level = level;
+
+#ifdef HAVE_LIBAVUTIL
+	av_log_set_level(nvnc__log_level_to_av(level));
+#endif
+}
+
+EXPORT
+void nvnc_set_log_fn(nvnc_log_fn fn)
+{
+	log_fn = fn ? fn : nvnc_default_logger;
+}
+
+EXPORT
+void nvnc_set_log_fn_thread_local(nvnc_log_fn fn)
+{
+	thread_local_log_fn = fn;
+}
+
+EXPORT
+void nvnc__log(const struct nvnc_log_data* meta,
+		const char* fmt, ...)
+{
+	va_list ap;
+	va_start(ap, fmt);
+	nvnc__vlog(meta, fmt, ap);
+	va_end(ap);
+}
+
+void nvnc__log_init(void)
+{
+	if (is_initialised)
+		return;
+
+#ifdef HAVE_LIBAVUTIL
+	av_log_set_callback(nvnc__av_log_callback);
+#endif
+
+	is_initialised = true;
+}

src/open-h264.c

@@ -0,0 +1,241 @@
+/*
+ * Copyright (c) 2021 - 2022 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "h264-encoder.h"
+#include "rfb-proto.h"
+#include "enc-util.h"
+#include "vec.h"
+#include "fb.h"
+#include "rcbuf.h"
+#include "encoder.h"
+#include "usdt.h"
+
+#include <stdlib.h>
+#include <math.h>
+
+typedef void (*open_h264_ready_fn)(void*);
+
+struct open_h264_header {
+	uint32_t length;
+	uint32_t flags;
+} RFB_PACKED;
+
+struct open_h264 {
+	struct encoder parent;
+
+	struct h264_encoder* encoder;
+
+	struct vec pending;
+	uint64_t pts;
+
+	uint32_t width;
+	uint32_t height;
+	uint32_t format;
+
+	bool needs_reset;
+
+	int quality;
+	bool quality_changed;
+};
+
+enum open_h264_flags {
+	OPEN_H264_FLAG_RESET_CONTEXT = 0,
+	OPEN_H264_FLAG_RESET_ALL_CONTEXTS = 1,
+};
+
+struct encoder* open_h264_new(void);
+static struct rcbuf* open_h264_pull(struct encoder* enc, uint64_t* pts);
+
+struct encoder_impl encoder_impl_open_h264;
+
+static inline struct open_h264* open_h264(struct encoder* enc)
+{
+	return (struct open_h264*)enc;
+}
+
+static void open_h264_handle_packet(const void* data, size_t size, uint64_t pts,
+		void* userdata)
+{
+	struct open_h264* self = userdata;
+
+	// Let's not deplete the RAM if the client isn't pulling
+	if (self->pending.len > 100000000) {
+		// TODO: Drop buffer and request a keyframe?
+		nvnc_log(NVNC_LOG_WARNING, "Pending buffer grew too large. Dropping packet...");
+		return;
+	}
+
+	vec_append(&self->pending, data, size);
+	self->pts = pts;
+
+	uint64_t rpts = NVNC_NO_PTS;
+	struct rcbuf* result = open_h264_pull(&self->parent, &rpts);
+
+	DTRACE_PROBE1(neatvnc, open_h264_finish_frame, rpts);
+
+	encoder_finish_frame(&self->parent, result, rpts);
+
+	rcbuf_unref(result);
+}
+
+static int open_h264_init_pending(struct open_h264* self)
+{
+	if (vec_init(&self->pending, 4096) < 0)
+		return -1;
+
+	vec_append_zero(&self->pending, sizeof(struct rfb_server_fb_rect));
+	vec_append_zero(&self->pending, sizeof(struct open_h264_header));
+
+	return 0;
+}
+
+struct encoder* open_h264_new(void)
+{
+	struct open_h264* self = calloc(1, sizeof(*self));
+	if (!self)
+		return NULL;
+
+	encoder_init(&self->parent, &encoder_impl_open_h264);
+
+	if (open_h264_init_pending(self) < 0) {
+		free(self);
+		return NULL;
+	}
+
+	self->pts = NVNC_NO_PTS;
+	self->quality = 6;
+
+	return (struct encoder*)self;
+}
+
+static void open_h264_destroy(struct encoder* enc)
+{
+	struct open_h264* self = open_h264(enc);
+
+	if (self->encoder)
+		h264_encoder_destroy(self->encoder);
+	vec_destroy(&self->pending);
+
+	free(self);
+}
+
+static int open_h264_resize(struct open_h264* self, struct nvnc_fb* fb)
+{
+	int quality = 51 - round((50.0 / 9.0) * (float)self->quality);
+
+	struct h264_encoder* encoder = h264_encoder_create(fb->width,
+			fb->height, fb->fourcc_format, quality);
+	if (!encoder)
+		return -1;
+
+	if (self->encoder)
+		h264_encoder_destroy(self->encoder);
+
+	h264_encoder_set_userdata(encoder, self);
+	h264_encoder_set_packet_handler_fn(encoder, open_h264_handle_packet);
+
+	self->encoder = encoder;
+
+	self->width = fb->width;
+	self->height = fb->height;
+	self->format = fb->fourcc_format;
+	self->needs_reset = true;
+	self->quality_changed = false;
+
+	return 0;
+}
+
+static int open_h264_encode(struct encoder* enc, struct nvnc_fb* fb,
+		struct pixman_region16* damage)
+{
+	DTRACE_PROBE1(neatvnc, open_h264_encode, fb->pts);
+
+	struct open_h264* self = open_h264(enc);
+	(void)damage;
+
+	if (fb->width != self->width || fb->height != self->height ||
+			fb->fourcc_format != self->format ||
+			self->quality_changed) {
+		if (open_h264_resize(self, fb) < 0)
+			return -1;
+	}
+
+	assert(self->width && self->height);
+
+	// TODO: encoder_feed should return an error code
+	h264_encoder_feed(self->encoder, fb);
+	return 0;
+}
+
+static struct rcbuf* open_h264_pull(struct encoder* enc, uint64_t* pts)
+{
+	struct open_h264* self = open_h264(enc);
+
+	size_t payload_size = self->pending.len
+		- sizeof(struct rfb_server_fb_rect)
+		- sizeof(struct open_h264_header);
+	if (payload_size == 0)
+		return NULL;
+
+	if (pts)
+		*pts = self->pts;
+	self->pts = NVNC_NO_PTS;
+
+	uint32_t flags = self->needs_reset ? OPEN_H264_FLAG_RESET_CONTEXT : 0;
+	self->needs_reset = false;
+
+	struct rfb_server_fb_rect* rect = self->pending.data;
+	rect->encoding = htonl(RFB_ENCODING_OPEN_H264);
+	rect->width = htons(self->width);
+	rect->height = htons(self->height);
+	rect->x = htons(self->parent.x_pos);
+	rect->y = htons(self->parent.y_pos);
+
+	struct open_h264_header* header =
+		(void*)(((uint8_t*)self->pending.data) + sizeof(*rect));
+	header->length = htonl(payload_size);
+	header->flags = htonl(flags);
+
+	enc->n_rects = 1;
+
+	struct rcbuf* payload = rcbuf_new(self->pending.data, self->pending.len);
+
+	open_h264_init_pending(self);
+	return payload;
+}
+
+static void open_h264_request_keyframe(struct encoder* enc)
+{
+	struct open_h264* self = open_h264(enc);
+	h264_encoder_request_keyframe(self->encoder);
+}
+
+static void open_h264_set_quality(struct encoder* enc, int value)
+{
+	struct open_h264* self = open_h264(enc);
+	if (value == 10)
+		value = 6;
+	self->quality_changed |= self->quality != value;
+	self->quality = value;
+}
+
+struct encoder_impl encoder_impl_open_h264 = {
+	.flags = ENCODER_IMPL_FLAG_IGNORES_DAMAGE,
+	.destroy = open_h264_destroy,
+	.encode = open_h264_encode,
+	.request_key_frame = open_h264_request_keyframe,
+	.set_quality = open_h264_set_quality,
+};

src/pixels.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019 - 2020 Andri Yngvason
+ * Copyright (c) 2019 - 2022 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -19,21 +19,24 @@
 #include <stdlib.h>
 #include <assert.h>
 #include <libdrm/drm_fourcc.h>
+#include <math.h>
 
 #define POPCOUNT(x) __builtin_popcount(x)
+#define UDIV_UP(a, b) (((a) + (b) - 1) / (b))
+#define XSTR(s) STR(s)
+#define STR(s) #s
 
-void pixel32_to_cpixel(uint8_t* restrict dst,
-                       const struct rfb_pixel_format* dst_fmt,
-                       const uint32_t* restrict src,
-                       const struct rfb_pixel_format* src_fmt,
-                       size_t bytes_per_cpixel, size_t len)
+static void pixel32_to_cpixel(uint8_t* restrict dst,
+		const struct rfb_pixel_format* dst_fmt,
+		const uint32_t* restrict src,
+		const struct rfb_pixel_format* src_fmt,
+		size_t bytes_per_cpixel, size_t len)
 {
 	assert(src_fmt->true_colour_flag);
-	assert(src_fmt->bits_per_pixel == 32);
 	assert(src_fmt->depth <= 32);
 	assert(dst_fmt->true_colour_flag);
 	assert(dst_fmt->bits_per_pixel <= 32);
-	assert(dst_fmt->depth <= 24);
+	assert(dst_fmt->depth <= 32);
 	assert(bytes_per_cpixel <= 4 && bytes_per_cpixel >= 1);
 
 	uint32_t src_red_shift = src_fmt->red_shift;
@@ -149,9 +152,181 @@ void pixel32_to_cpixel(uint8_t* restrict dst,
 #undef CONVERT_PIXELS
 }
 
+void pixel_to_cpixel(uint8_t* restrict dst,
+		const struct rfb_pixel_format* dst_fmt,
+		const uint8_t* restrict src,
+		const struct rfb_pixel_format* src_fmt,
+		size_t bytes_per_cpixel, size_t len)
+{
+	if (src_fmt->bits_per_pixel == 32) {
+		pixel32_to_cpixel(dst, dst_fmt, (uint32_t*)src, src_fmt, bytes_per_cpixel, len);
+		return;
+	}
+
+	assert(src_fmt->true_colour_flag);
+	assert(src_fmt->depth <= 32);
+	assert(dst_fmt->true_colour_flag);
+	assert(dst_fmt->bits_per_pixel <= 32);
+	assert(dst_fmt->depth <= 32);
+	assert(bytes_per_cpixel <= 4 && bytes_per_cpixel >= 1);
+
+	uint32_t src_bpp = src_fmt->bits_per_pixel / 8;
+	uint32_t src_red_shift = src_fmt->red_shift;
+	uint32_t src_green_shift = src_fmt->green_shift;
+	uint32_t src_blue_shift = src_fmt->blue_shift;
+
+	uint32_t dst_red_shift = dst_fmt->red_shift;
+	uint32_t dst_green_shift = dst_fmt->green_shift;
+	uint32_t dst_blue_shift = dst_fmt->blue_shift;
+
+	uint32_t src_red_max = src_fmt->red_max;
+	uint32_t src_green_max = src_fmt->green_max;
+	uint32_t src_blue_max = src_fmt->blue_max;
+
+	uint32_t src_red_bits = POPCOUNT(src_fmt->red_max);
+	uint32_t src_green_bits = POPCOUNT(src_fmt->green_max);
+	uint32_t src_blue_bits = POPCOUNT(src_fmt->blue_max);
+
+	uint32_t dst_red_bits = POPCOUNT(dst_fmt->red_max);
+	uint32_t dst_green_bits = POPCOUNT(dst_fmt->green_max);
+	uint32_t dst_blue_bits = POPCOUNT(dst_fmt->blue_max);
+
+	uint32_t dst_endian_correction;
+
+#define CONVERT_PIXELS(cpx, px)                                                \
+	{                                                                      \
+		uint32_t r, g, b;                                              \
+		r = ((px >> src_red_shift) & src_red_max) << dst_red_bits      \
+		        >> src_red_bits << dst_red_shift;                      \
+		g = ((px >> src_green_shift) & src_green_max) << dst_green_bits\
+		        >> src_green_bits << dst_green_shift;                  \
+		b = ((px >> src_blue_shift) & src_blue_max) << dst_blue_bits   \
+		        >> src_blue_bits << dst_blue_shift;                    \
+		cpx = r | g | b;                                               \
+	}
+
+	switch (bytes_per_cpixel) {
+	case 4:
+		if (dst_fmt->big_endian_flag) {
+			while (len--) {
+				uint32_t cpx, px = 0;
+				memcpy(&px, src, src_bpp);
+				src += src_bpp;
+
+				CONVERT_PIXELS(cpx, px)
+
+				*dst++ = (cpx >> 24) & 0xff;
+				*dst++ = (cpx >> 16) & 0xff;
+				*dst++ = (cpx >> 8) & 0xff;
+				*dst++ = (cpx >> 0) & 0xff;
+			}
+		} else {
+			while (len--) {
+				uint32_t cpx, px = 0;
+				memcpy(&px, src, src_bpp);
+				src += src_bpp;
+
+				CONVERT_PIXELS(cpx, px)
+
+				*dst++ = (cpx >> 0) & 0xff;
+				*dst++ = (cpx >> 8) & 0xff;
+				*dst++ = (cpx >> 16) & 0xff;
+				*dst++ = (cpx >> 24) & 0xff;
+			}
+		}
+		break;
+	case 3:
+		if (dst_fmt->bits_per_pixel == 32 && dst_fmt->depth <= 24) {
+			uint32_t min_dst_shift = dst_red_shift;
+			if (min_dst_shift > dst_green_shift)
+				min_dst_shift = dst_green_shift;
+			if (min_dst_shift > dst_blue_shift)
+				min_dst_shift = dst_blue_shift;
+
+			dst_red_shift -= min_dst_shift;
+			dst_green_shift -= min_dst_shift;
+			dst_blue_shift -= min_dst_shift;
+		}
+
+		dst_endian_correction = dst_fmt->big_endian_flag ? 16 : 0;
+
+		while (len--) {
+			uint32_t cpx, px = 0;
+			memcpy(&px, src, src_bpp);
+			src += src_bpp;
+
+			CONVERT_PIXELS(cpx, px)
+
+			*dst++ = (cpx >> (0 ^ dst_endian_correction)) & 0xff;
+			*dst++ = (cpx >> 8) & 0xff;
+			*dst++ = (cpx >> (16 ^ dst_endian_correction)) & 0xff;
+		}
+		break;
+	case 2:
+		dst_endian_correction = dst_fmt->big_endian_flag ? 8 : 0;
+
+		while (len--) {
+			uint32_t cpx, px = 0;
+			memcpy(&px, src, src_bpp);
+			src += src_bpp;
+
+			CONVERT_PIXELS(cpx, px)
+
+			*dst++ = (cpx >> (0 ^ dst_endian_correction)) & 0xff;
+			*dst++ = (cpx >> (8 ^ dst_endian_correction)) & 0xff;
+		}
+		break;
+	case 1:
+		while (len--) {
+			uint32_t cpx, px = 0;
+			memcpy(&px, src, src_bpp);
+			src += src_bpp;
+
+			CONVERT_PIXELS(cpx, px)
+
+			*dst++ = cpx & 0xff;
+		}
+		break;
+	default:
+		abort();
+	}
+
+#undef CONVERT_PIXELS
+}
+
 /* clang-format off */
 int rfb_pixfmt_from_fourcc(struct rfb_pixel_format *dst, uint32_t src) {
 	switch (src & ~DRM_FORMAT_BIG_ENDIAN) {
+	case DRM_FORMAT_RGBA1010102:
+	case DRM_FORMAT_RGBX1010102:
+		dst->red_shift = 22;
+		dst->green_shift = 12;
+		dst->blue_shift = 2;
+		goto bpp_32_10bit;
+	case DRM_FORMAT_BGRA1010102:
+	case DRM_FORMAT_BGRX1010102:
+		dst->red_shift = 2;
+		dst->green_shift = 12;
+		dst->blue_shift = 22;
+		goto bpp_32_10bit;
+	case DRM_FORMAT_ARGB2101010:
+	case DRM_FORMAT_XRGB2101010:
+		dst->red_shift = 20;
+		dst->green_shift = 10;
+		dst->blue_shift = 0;
+		goto bpp_32_10bit;
+	case DRM_FORMAT_ABGR2101010:
+	case DRM_FORMAT_XBGR2101010:
+		dst->red_shift = 0;
+		dst->green_shift = 10;
+		dst->blue_shift = 20;
+bpp_32_10bit:
+		dst->bits_per_pixel = 32;
+		dst->depth = 30;
+		dst->red_max = 0x3ff;
+		dst->green_max = 0x3ff;
+		dst->blue_max = 0x3ff;
+		break;
 	case DRM_FORMAT_RGBA8888:
 	case DRM_FORMAT_RGBX8888:
 		dst->red_shift = 24;
@@ -182,6 +357,22 @@ bpp_32:
 		dst->green_max = 0xff;
 		dst->blue_max = 0xff;
 		break;
+	case DRM_FORMAT_BGR888:
+		dst->red_shift = 0;
+		dst->green_shift = 8;
+		dst->blue_shift = 16;
+		goto bpp_24;
+	case DRM_FORMAT_RGB888:
+		dst->red_shift = 16;
+		dst->green_shift = 8;
+		dst->blue_shift = 0;
+bpp_24:
+		dst->bits_per_pixel = 24;
+		dst->depth = 24;
+		dst->red_max = 0xff;
+		dst->green_max = 0xff;
+		dst->blue_max = 0xff;
+		break;
 	case DRM_FORMAT_RGBA4444:
 	case DRM_FORMAT_RGBX4444:
 		dst->red_shift = 12;
@@ -220,4 +411,269 @@ bpp_16:
 	dst->true_colour_flag = 1;
 
 	return 0;
-};
+}
+
+int pixel_size_from_fourcc(uint32_t fourcc)
+{
+	switch (fourcc & ~DRM_FORMAT_BIG_ENDIAN) {
+	case DRM_FORMAT_RGBA1010102:
+	case DRM_FORMAT_RGBX1010102:
+	case DRM_FORMAT_BGRA1010102:
+	case DRM_FORMAT_BGRX1010102:
+	case DRM_FORMAT_ARGB2101010:
+	case DRM_FORMAT_XRGB2101010:
+	case DRM_FORMAT_ABGR2101010:
+	case DRM_FORMAT_XBGR2101010:
+	case DRM_FORMAT_RGBA8888:
+	case DRM_FORMAT_RGBX8888:
+	case DRM_FORMAT_BGRA8888:
+	case DRM_FORMAT_BGRX8888:
+	case DRM_FORMAT_ARGB8888:
+	case DRM_FORMAT_XRGB8888:
+	case DRM_FORMAT_ABGR8888:
+	case DRM_FORMAT_XBGR8888:
+		return 4;
+	case DRM_FORMAT_BGR888:
+	case DRM_FORMAT_RGB888:
+		return 3;
+	case DRM_FORMAT_RGBA4444:
+	case DRM_FORMAT_RGBX4444:
+	case DRM_FORMAT_BGRA4444:
+	case DRM_FORMAT_BGRX4444:
+	case DRM_FORMAT_ARGB4444:
+	case DRM_FORMAT_XRGB4444:
+	case DRM_FORMAT_ABGR4444:
+	case DRM_FORMAT_XBGR4444:
+		return 2;
+	}
+
+	return 0;
+}
+
+bool fourcc_to_pixman_fmt(pixman_format_code_t* dst, uint32_t src)
+{
+	assert(!(src & DRM_FORMAT_BIG_ENDIAN));
+
+#define LOWER_R r
+#define LOWER_G g
+#define LOWER_B b
+#define LOWER_A a
+#define LOWER_X x
+#define LOWER_
+#define LOWER(x) LOWER_##x
+
+#define CONCAT_(a, b) a ## b
+#define CONCAT(a, b) CONCAT_(a, b)
+
+#define FMT_DRM(x, y, z, v, a, b, c, d) DRM_FORMAT_##x##y##z##v##a##b##c##d
+
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+#define FMT_PIXMAN(x, y, z, v, a, b, c, d) \
+	CONCAT(CONCAT(CONCAT(CONCAT(CONCAT(CONCAT(CONCAT(CONCAT(\
+	PIXMAN_, LOWER(x)), a), LOWER(y)), b), LOWER(z)), c), LOWER(v)), d)
+#else
+#define FMT_PIXMAN(x, y, z, v, a, b, c, d) \
+	CONCAT(CONCAT(CONCAT(CONCAT(CONCAT(CONCAT(CONCAT(CONCAT(\
+	PIXMAN_, LOWER(v)), d), LOWER(z)), c), LOWER(y)), b), LOWER(x)), a)
+#endif
+
+	switch (src) {
+#define X(...) \
+	case FMT_DRM(__VA_ARGS__): *dst = FMT_PIXMAN(__VA_ARGS__); break
+
+	/* 32 bits */
+	X(A,R,G,B,8,8,8,8);
+	X(A,B,G,R,8,8,8,8);
+	X(X,R,G,B,8,8,8,8);
+	X(X,B,G,R,8,8,8,8);
+	X(R,G,B,A,8,8,8,8);
+	X(B,G,R,A,8,8,8,8);
+	X(R,G,B,X,8,8,8,8);
+	X(B,G,R,X,8,8,8,8);
+
+	/* 24 bits */
+	X(R,G,B,,8,8,8,);
+	X(B,G,R,,8,8,8,);
+
+	/* 16 bits */
+	X(R,G,B,,5,6,5,);
+	X(B,G,R,,5,6,5,);
+
+	/* These are incompatible on big endian */
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+	X(A,R,G,B,2,10,10,10);
+	X(X,R,G,B,2,10,10,10);
+	X(A,B,G,R,2,10,10,10);
+	X(X,B,G,R,2,10,10,10);
+	X(A,R,G,B,1,5,5,5);
+	X(A,B,G,R,1,5,5,5);
+	X(X,R,G,B,1,5,5,5);
+	X(X,B,G,R,1,5,5,5);
+	X(A,R,G,B,4,4,4,4);
+	X(A,B,G,R,4,4,4,4);
+	X(X,R,G,B,4,4,4,4);
+	X(X,B,G,R,4,4,4,4);
+#endif
+
+#undef X
+
+	default: return false;
+	}
+
+	return true;
+}
+
+static bool extract_alpha_mask_rgba32(uint8_t* dst, const uint32_t* src,
+		size_t len, int alpha_shift, uint32_t alpha_max)
+{
+	for (size_t i = 0; i < len; i++) {
+		uint8_t alpha = (src[i] >> alpha_shift) & alpha_max;
+		uint8_t binary = !!(alpha > alpha_max / 2);
+		dst[i / 8] |= binary << (7 - (i % 8));
+	}
+
+	return true;
+}
+
+static bool extract_alpha_mask_rgba16(uint8_t* dst, const uint16_t* src,
+		size_t len, int alpha_shift)
+{
+	for (size_t i = 0; i < len; i++) {
+		uint8_t alpha = (src[i] >> alpha_shift) & 0xf;
+		uint8_t binary = !!(alpha > 0xf / 2);
+		dst[i / 8] |= binary << (7 - (i % 8));
+	}
+
+	return true;
+}
+
+// Note: The destination buffer must be at least UDIV_UP(len, 8) long.
+bool extract_alpha_mask(uint8_t* dst, const void* src, uint32_t format,
+		size_t len)
+{
+	memset(dst, 0, UDIV_UP(len, 8));
+
+	switch (format & ~DRM_FORMAT_BIG_ENDIAN) {
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+	case DRM_FORMAT_RGBA1010102:
+	case DRM_FORMAT_BGRA1010102:
+		return extract_alpha_mask_rgba32(dst, src, len, 0, 3);
+	case DRM_FORMAT_ARGB2101010:
+	case DRM_FORMAT_ABGR2101010:
+		return extract_alpha_mask_rgba32(dst, src, len, 30, 3);
+	case DRM_FORMAT_RGBA8888:
+	case DRM_FORMAT_BGRA8888:
+		return extract_alpha_mask_rgba32(dst, src, len, 0, 0xff);
+	case DRM_FORMAT_ARGB8888:
+	case DRM_FORMAT_ABGR8888:
+		return extract_alpha_mask_rgba32(dst, src, len, 24, 0xff);
+	case DRM_FORMAT_RGBA4444:
+	case DRM_FORMAT_BGRA4444:
+		return extract_alpha_mask_rgba16(dst, src, len, 0);
+	case DRM_FORMAT_ARGB4444:
+	case DRM_FORMAT_ABGR4444:
+		return extract_alpha_mask_rgba16(dst, src, len, 12);
+#else
+	case DRM_FORMAT_RGBA1010102:
+	case DRM_FORMAT_BGRA1010102:
+		return extract_alpha_mask_rgba32(dst, src, len, 30, 3);
+	case DRM_FORMAT_ARGB2101010:
+	case DRM_FORMAT_ABGR2101010:
+		return extract_alpha_mask_rgba32(dst, src, len, 0, 3);
+	case DRM_FORMAT_RGBA8888:
+	case DRM_FORMAT_BGRA8888:
+		return extract_alpha_mask_rgba32(dst, src, len, 24, 0xff);
+	case DRM_FORMAT_ARGB8888:
+	case DRM_FORMAT_ABGR8888:
+		return extract_alpha_mask_rgba32(dst, src, len, 0, 0xff);
+	case DRM_FORMAT_RGBA4444:
+	case DRM_FORMAT_BGRA4444:
+		return extract_alpha_mask_rgba16(dst, src, len, 12);
+	case DRM_FORMAT_ARGB4444:
+	case DRM_FORMAT_ABGR4444:
+		return extract_alpha_mask_rgba16(dst, src, len, 0);
+#endif
+	}
+
+	return false;
+}
+
+const char* drm_format_to_string(uint32_t fmt)
+{
+	switch (fmt) {
+#define X(x) case DRM_FORMAT_ ## x: return XSTR(x);
+	X(RGBA1010102) \
+	X(RGBX1010102) \
+	X(BGRA1010102) \
+	X(BGRX1010102) \
+	X(ARGB2101010) \
+	X(XRGB2101010) \
+	X(ABGR2101010) \
+	X(XBGR2101010) \
+	X(RGBA8888) \
+	X(RGBX8888) \
+	X(BGRA8888) \
+	X(BGRX8888) \
+	X(ARGB8888) \
+	X(XRGB8888) \
+	X(ABGR8888) \
+	X(XBGR8888) \
+	X(RGB888) \
+	X(BGR888) \
+	X(RGBA4444) \
+	X(RGBX4444) \
+	X(BGRA4444) \
+	X(BGRX4444) \
+	X(ARGB4444) \
+	X(XRGB4444) \
+	X(ABGR4444) \
+	X(XBGR4444) \
+	X(RGB565)
+#undef X
+	}
+	return "UNKNOWN";
+}
+
+// Not exact, but close enough for debugging
+const char* rfb_pixfmt_to_string(const struct rfb_pixel_format* fmt)
+{
+	uint32_t profile = (fmt->red_shift << 16) | (fmt->green_shift << 8)
+		| (fmt->blue_shift);
+
+	switch (profile) {
+#define CASE(r, g, b) case ((r << 16) | (g << 8) | b)
+	CASE(22, 10, 2): return "RGBX1010102";
+	CASE(2, 12, 22): return "BGRX1010102";
+	CASE(20, 10, 0): return "XRGB2101010";
+	CASE(0, 10, 20): return "XBGR2101010";
+	CASE(24, 16, 8): return "RGBX8888";
+	CASE(8, 16, 24): return "BGRX8888";
+	CASE(16, 8, 0):  return "XRGB8888";
+	CASE(0, 8, 16):  return "XBGR8888";
+	CASE(12, 8, 4):  return "RGBX4444";
+	CASE(4, 8, 12):  return "BGRX4444";
+	CASE(8, 4, 0):   return "XRGB4444";
+	CASE(0, 4, 8):   return "XBGR4444";
+	CASE(11, 5, 0):  return "RGB565";
+	CASE(5, 2, 0):   return "RGB332";
+	CASE(0, 2, 5):   return "RGB332";
+	CASE(4, 2, 0):   return "RGB222";
+	CASE(0, 2, 4):   return "BGR222";
+#undef CASE
+	}
+	return "UNKNOWN";
+}
+
+void make_rgb332_pal8_map(struct rfb_set_colour_map_entries_msg* msg)
+{
+	msg->type = RFB_SERVER_TO_CLIENT_SET_COLOUR_MAP_ENTRIES;
+	msg->padding = 0;
+	msg->first_colour = htons(0);
+	msg->n_colours = htons(256);
+
+	for (unsigned int i = 0; i < 256; ++i) {
+		msg->colours[i].r = htons(round(65535.0 / 7.0 * ((i >> 5) & 7)));
+		msg->colours[i].g = htons(round(65535.0 / 7.0 * ((i >> 2) & 7)));
+		msg->colours[i].b = htons(round(65535.0 / 3.0 * (i & 3)));
+	}
+}

src/pngfb.c

@@ -73,8 +73,8 @@ struct nvnc_fb* read_png_file(const char* filename)
 	png_read_update_info(png, info);
 
 	size_t row_bytes = png_get_rowbytes(png, info);
-	assert(row_bytes == width * 4);
-	struct nvnc_fb* fb = nvnc_fb_new(width, height, DRM_FORMAT_ABGR8888);
+	struct nvnc_fb* fb = nvnc_fb_new(width, height, DRM_FORMAT_ABGR8888,
+			row_bytes / 4);
 	assert(fb);
 
 	uint8_t* addr = nvnc_fb_get_addr(fb);

src/raw-encoding.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019 - 2020 Andri Yngvason
+ * Copyright (c) 2019 - 2022 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -19,25 +19,60 @@
 #include "vec.h"
 #include "fb.h"
 #include "pixels.h"
-#include "raw-encoding.h"
 #include "enc-util.h"
+#include "encoder.h"
+#include "rcbuf.h"
 
+#include <stdlib.h>
 #include <pixman.h>
+#include <aml.h>
 
-static int raw_encode_box(struct vec* dst,
-                          const struct rfb_pixel_format* dst_fmt,
-                          const struct nvnc_fb* fb,
-                          const struct rfb_pixel_format* src_fmt, int x_start,
-                          int y_start, int stride, int width, int height)
+struct encoder* raw_encoder_new(void);
+
+struct raw_encoder {
+	struct encoder encoder;
+	struct rfb_pixel_format output_format;
+	struct aml_work* work;
+};
+
+struct raw_encoder_work {
+	struct raw_encoder* parent;
+	struct rfb_pixel_format output_format;
+	struct nvnc_fb* fb;
+	struct pixman_region16 damage;
+	int n_rects;
+	uint16_t x_pos, y_pos;
+	struct rcbuf *result;
+};
+
+struct encoder_impl encoder_impl_raw;
+
+static inline struct raw_encoder* raw_encoder(struct encoder* encoder)
 {
+	assert(encoder->impl == &encoder_impl_raw);
+	return (struct raw_encoder*)encoder;
+}
+
+static int raw_encode_box(struct raw_encoder_work* ctx, struct vec* dst,
+		const struct rfb_pixel_format* dst_fmt,
+		const struct nvnc_fb* fb,
+		const struct rfb_pixel_format* src_fmt, int x_start,
+		int y_start, int stride, int width, int height)
+{
+	uint16_t x_pos = ctx->x_pos;
+	uint16_t y_pos = ctx->y_pos;
+
 	int rc = -1;
 
-	rc = encode_rect_head(dst, RFB_ENCODING_RAW, x_start, y_start, width,
-			height);
+	rc = encode_rect_head(dst, RFB_ENCODING_RAW, x_pos + x_start,
+			y_pos + y_start, width, height);
 	if (rc < 0)
 		return -1;
 
-	uint32_t* b = fb->addr;
+	uint8_t* b = fb->addr;
+	int32_t src_bpp = src_fmt->bits_per_pixel / 8;
+	int32_t xoff = x_start * src_bpp;
+	int32_t src_stride = fb->stride * src_bpp;
 
 	int bpp = dst_fmt->bits_per_pixel / 8;
 
@@ -48,19 +83,19 @@ static int raw_encode_box(struct vec* dst,
 	uint8_t* d = dst->data;
 
 	for (int y = y_start; y < y_start + height; ++y) {
-		pixel32_to_cpixel(d + dst->len, dst_fmt,
-		                  b + x_start + y * stride, src_fmt,
-		                  bpp, width);
+		pixel_to_cpixel(d + dst->len, dst_fmt,
+				b + xoff + y * src_stride, src_fmt,
+				bpp, width);
 		dst->len += width * bpp;
 	}
 
 	return 0;
 }
 
-int raw_encode_frame(struct vec* dst, const struct rfb_pixel_format* dst_fmt,
-                     const struct nvnc_fb* src,
-                     const struct rfb_pixel_format* src_fmt,
-                     struct pixman_region16* region)
+static int raw_encode_frame(struct raw_encoder_work* ctx, struct vec* dst,
+		const struct rfb_pixel_format* dst_fmt, struct nvnc_fb* src,
+		const struct rfb_pixel_format* src_fmt,
+		struct pixman_region16* region)
 {
 	int rc = -1;
 
@@ -71,11 +106,7 @@ int raw_encode_frame(struct vec* dst, const struct rfb_pixel_format* dst_fmt,
 		n_rects = 1;
 	}
 
-	rc = vec_reserve(dst, src->width * src->height * 4);
-	if (rc < 0)
-		return -1;
-
-	rc = encode_rect_count(dst, n_rects);
+	rc = nvnc_fb_map(src);
 	if (rc < 0)
 		return -1;
 
@@ -85,11 +116,137 @@ int raw_encode_frame(struct vec* dst, const struct rfb_pixel_format* dst_fmt,
 		int box_width = box[i].x2 - x;
 		int box_height = box[i].y2 - y;
 
-		rc = raw_encode_box(dst, dst_fmt, src, src_fmt, x, y,
-		                    src->width, box_width, box_height);
+		rc = raw_encode_box(ctx, dst, dst_fmt, src, src_fmt, x, y,
+				src->stride, box_width, box_height);
 		if (rc < 0)
 			return -1;
 	}
 
+	ctx->n_rects = n_rects;
 	return 0;
 }
+
+static void raw_encoder_do_work(void* obj)
+{
+	struct raw_encoder_work* ctx = aml_get_userdata(obj);
+	int rc;
+
+	struct nvnc_fb* fb = ctx->fb;
+	assert(fb);
+
+	size_t bpp = ctx->output_format.bits_per_pixel / 8;
+	size_t n_rects = pixman_region_n_rects(&ctx->damage);
+	if (n_rects > UINT16_MAX)
+		n_rects = 1;
+	size_t buffer_size = calculate_region_area(&ctx->damage) * bpp
+		+ n_rects * sizeof(struct rfb_server_fb_rect);
+
+	struct vec dst;
+	rc = vec_init(&dst, buffer_size);
+	assert(rc == 0);
+
+	struct rfb_pixel_format src_fmt;
+	rc = rfb_pixfmt_from_fourcc(&src_fmt, nvnc_fb_get_fourcc_format(fb));
+	assert(rc == 0);
+
+	rc = raw_encode_frame(ctx, &dst, &ctx->output_format, fb, &src_fmt,
+			&ctx->damage);
+	assert(rc == 0);
+
+	ctx->result = rcbuf_new(dst.data, dst.len);
+	assert(ctx->result);
+}
+
+static void raw_encoder_on_done(void* obj)
+{
+	struct raw_encoder_work* ctx = aml_get_userdata(obj);
+	struct raw_encoder* self = ctx->parent;
+
+	assert(ctx->result);
+
+	self->encoder.n_rects = ctx->n_rects;
+
+	aml_unref(self->work);
+	self->work = NULL;
+
+	uint64_t pts = nvnc_fb_get_pts(ctx->fb);
+	encoder_finish_frame(&self->encoder, ctx->result, pts);
+}
+
+struct encoder* raw_encoder_new(void)
+{
+	struct raw_encoder* self = calloc(1, sizeof(*self));
+	if (!self)
+		return NULL;
+
+	encoder_init(&self->encoder, &encoder_impl_raw);
+
+	return (struct encoder*)self;
+}
+
+static void raw_encoder_destroy(struct encoder* encoder)
+{
+	struct raw_encoder* self = raw_encoder(encoder);
+	if (self->work) {
+		aml_stop(aml_get_default(), self->work);
+		aml_unref(self->work);
+	}
+	free(self);
+}
+
+static void raw_encoder_set_output_format(struct encoder* encoder,
+		const struct rfb_pixel_format* pixfmt)
+{
+	struct raw_encoder* self = raw_encoder(encoder);
+	memcpy(&self->output_format, pixfmt, sizeof(self->output_format));
+}
+
+static void raw_encoder_work_destroy(void* obj)
+{
+	struct raw_encoder_work* ctx = obj;
+	nvnc_fb_unref(ctx->fb);
+	pixman_region_fini(&ctx->damage);
+	if (ctx->result)
+		rcbuf_unref(ctx->result);
+	free(ctx);
+}
+
+static int raw_encoder_encode(struct encoder* encoder, struct nvnc_fb* fb,
+		struct pixman_region16* damage)
+{
+	struct raw_encoder* self = raw_encoder(encoder);
+
+	struct raw_encoder_work* ctx = calloc(1, sizeof(*ctx));
+	if (!ctx)
+		return -1;
+
+	self->work = aml_work_new(raw_encoder_do_work, raw_encoder_on_done,
+			ctx, raw_encoder_work_destroy);
+	if (!self->work) {
+		free(ctx);
+		return -1;
+	}
+
+	ctx->parent = self;
+	ctx->fb = fb;
+	memcpy(&ctx->output_format, &self->output_format,
+			sizeof(ctx->output_format));
+	ctx->x_pos = self->encoder.x_pos;
+	ctx->y_pos = self->encoder.y_pos;
+	nvnc_fb_ref(ctx->fb);
+	pixman_region_copy(&ctx->damage, damage);
+
+	int rc = aml_start(aml_get_default(), self->work);
+	if (rc < 0) {
+		aml_unref(self->work);
+		self->work = NULL;
+	}
+
+	return rc;
+}
+
+struct encoder_impl encoder_impl_raw = {
+	.destroy = raw_encoder_destroy,
+	.set_output_format = raw_encoder_set_output_format,
+	.encode = raw_encoder_encode,
+};

src/resampler.c

@@ -0,0 +1,243 @@
+/*
+ * Copyright (c) 2021 - 2022 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "resampler.h"
+#include "neatvnc.h"
+#include "fb.h"
+#include "transform-util.h"
+#include "pixels.h"
+#include "usdt.h"
+
+#include <stdlib.h>
+#include <aml.h>
+#include <pixman.h>
+#include <assert.h>
+#include <libdrm/drm_fourcc.h>
+
+struct fb_side_data {
+	struct pixman_region16 buffer_damage;
+	LIST_ENTRY(fb_side_data) link;
+};
+
+LIST_HEAD(fb_side_data_list, fb_side_data);
+
+struct resampler {
+	struct nvnc_fb_pool *pool;
+	struct fb_side_data_list fb_side_data_list;
+};
+
+struct resampler_work {
+	struct pixman_region16 frame_damage;
+	struct nvnc_fb* src;
+	struct nvnc_fb* dst;
+	resampler_fn on_done;
+	void* userdata;
+};
+
+static void fb_side_data_destroy(void* userdata)
+{
+	struct fb_side_data* fb_side_data = userdata;
+	LIST_REMOVE(fb_side_data, link);
+	pixman_region_fini(&fb_side_data->buffer_damage);
+	free(fb_side_data);
+}
+
+static void resampler_damage_all_buffers(struct resampler* self,
+		struct pixman_region16* region)
+{
+	struct fb_side_data *item;
+	LIST_FOREACH(item, &self->fb_side_data_list, link)
+		pixman_region_union(&item->buffer_damage, &item->buffer_damage,
+				region);
+}
+
+static void resampler_work_free(void* userdata)
+{
+	struct resampler_work* work = userdata;
+
+	nvnc_fb_release(work->src);
+	nvnc_fb_unref(work->src);
+
+	nvnc_fb_unref(work->dst);
+
+	pixman_region_fini(&work->frame_damage);
+
+	free(work);
+}
+
+struct resampler* resampler_create(void)
+{
+	struct resampler* self = calloc(1, sizeof(*self));
+	if (!self)
+		return NULL;
+
+	self->pool = nvnc_fb_pool_new(0, 0, DRM_FORMAT_INVALID, 0);
+	if (!self->pool) {
+		free(self);
+		return NULL;
+	}
+
+	LIST_INIT(&self->fb_side_data_list);
+
+	return self;
+}
+
+void resampler_destroy(struct resampler* self)
+{
+	nvnc_fb_pool_unref(self->pool);
+	free(self);
+}
+
+void resample_now(struct nvnc_fb* dst, struct nvnc_fb* src,
+		struct pixman_region16* damage)
+{
+	assert(dst->transform == NVNC_TRANSFORM_NORMAL);
+
+	bool ok __attribute__((unused));
+
+	pixman_format_code_t dst_fmt = 0;
+	ok = fourcc_to_pixman_fmt(&dst_fmt, dst->fourcc_format);
+	assert(ok);
+
+	pixman_image_t* dstimg = pixman_image_create_bits_no_clear(
+			dst_fmt, dst->width, dst->height, dst->addr,
+			nvnc_fb_get_pixel_size(dst) * dst->stride);
+
+	pixman_format_code_t src_fmt = 0;
+	ok = fourcc_to_pixman_fmt(&src_fmt, src->fourcc_format);
+	assert(ok);
+
+	pixman_image_t* srcimg = pixman_image_create_bits_no_clear(
+			src_fmt, src->width, src->height, src->addr,
+			nvnc_fb_get_pixel_size(src) * src->stride);
+
+	pixman_transform_t pxform;
+	nvnc_transform_to_pixman_transform(&pxform, src->transform,
+			src->width, src->height);
+
+	pixman_image_set_transform(srcimg, &pxform);
+
+	/* Side data contains the union of the buffer damage and the frame
+	 * damage.
+	 */
+	if (damage) {
+		pixman_image_set_clip_region(dstimg, damage);
+	}
+
+	pixman_image_composite(PIXMAN_OP_OVER, srcimg, NULL, dstimg,
+			0, 0,
+			0, 0,
+			0, 0,
+			dst->width, dst->height);
+
+	pixman_image_unref(srcimg);
+	pixman_image_unref(dstimg);
+}
+
+static void do_work(void* handle)
+{
+	struct aml_work* work = handle;
+	struct resampler_work* ctx = aml_get_userdata(work);
+
+	struct nvnc_fb* src = ctx->src;
+	struct nvnc_fb* dst = ctx->dst;
+	struct fb_side_data* dst_side_data = nvnc_get_userdata(dst);
+
+	resample_now(dst, src, &dst_side_data->buffer_damage);
+}
+
+static void on_work_done(void* handle)
+{
+	struct aml_work* work = handle;
+	struct resampler_work* ctx = aml_get_userdata(work);
+
+	ctx->on_done(ctx->dst, &ctx->frame_damage, ctx->userdata);
+}
+
+int resampler_feed(struct resampler* self, struct nvnc_fb* fb,
+		struct pixman_region16* damage, resampler_fn on_done,
+		void* userdata)
+{
+	DTRACE_PROBE2(neatvnc, resampler_feed, self, fb->pts);
+
+	if (fb->transform == NVNC_TRANSFORM_NORMAL) {
+		on_done(fb, damage, userdata);
+		return 0;
+	}
+
+	uint32_t width = fb->width;
+	uint32_t height = fb->height;
+
+	nvnc_transform_dimensions(fb->transform, &width, &height);
+	nvnc_fb_pool_resize(self->pool, width, height, fb->fourcc_format,
+			width);
+
+	struct aml* aml = aml_get_default();
+	assert(aml);
+
+	struct resampler_work* ctx = calloc(1, sizeof(*ctx));
+	if (!ctx)
+		return -1;
+
+	pixman_region_init(&ctx->frame_damage);
+	pixman_region_copy(&ctx->frame_damage, damage);
+
+	ctx->dst = nvnc_fb_pool_acquire(self->pool);
+	if (!ctx->dst)
+		goto acquire_failure;
+
+	struct fb_side_data* fb_side_data = nvnc_get_userdata(fb);
+	if (!fb_side_data) {
+		fb_side_data = calloc(1, sizeof(*fb_side_data));
+		if (!fb_side_data)
+			goto side_data_failure;
+
+		/* This is a new buffer, so the whole surface is damaged. */
+		pixman_region_init_rect(&fb_side_data->buffer_damage, 0, 0,
+				width, height);
+
+		nvnc_set_userdata(fb, fb_side_data, fb_side_data_destroy);
+		LIST_INSERT_HEAD(&self->fb_side_data_list, fb_side_data, link);
+	}
+
+	resampler_damage_all_buffers(self, damage);
+
+	ctx->src = fb;
+	nvnc_fb_ref(fb);
+	nvnc_fb_hold(fb);
+
+	ctx->on_done = on_done;
+	ctx->userdata = userdata;
+
+	struct aml_work* work = aml_work_new(do_work, on_work_done, ctx,
+			resampler_work_free);
+	if (!work) {
+		resampler_work_free(ctx);
+		return -1;
+	}
+
+	nvnc_fb_map(fb);
+
+	int rc = aml_start(aml, work);
+	aml_unref(work);
+	return rc;
+
+side_data_failure:
+	nvnc_fb_pool_release(self->pool, ctx->dst);
+acquire_failure:
+	free(ctx);
+	return -1;
+}

src/stream-common.c

@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2020 - 2023 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "stream.h"
+#include "stream-common.h"
+
+#include <stdlib.h>
+
+void stream_req__finish(struct stream_req* req, enum stream_req_status status)
+{
+	if (req->on_done)
+		req->on_done(req->userdata, status);
+
+	// exec userdata is heap allocated
+	if (req->exec && req->userdata)
+		free(req->userdata);
+
+	rcbuf_unref(req->payload);
+	free(req);
+}
+
+void stream__remote_closed(struct stream* self)
+{
+	stream_close(self);
+
+	if (self->on_event)
+		self->on_event(self, STREAM_EVENT_REMOTE_CLOSED);
+}

src/stream-gnutls.c

@@ -0,0 +1,292 @@
+/*
+ * Copyright (c) 2020 - 2023 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <string.h>
+#include <assert.h>
+#include <errno.h>
+#include <sys/uio.h>
+#include <limits.h>
+#include <aml.h>
+#include <fcntl.h>
+#include <poll.h>
+#include <sys/socket.h>
+
+#include <gnutls/gnutls.h>
+
+#include "rcbuf.h"
+#include "stream.h"
+#include "stream-common.h"
+#include "sys/queue.h"
+
+struct stream_gnutls {
+	struct stream base;
+
+	gnutls_session_t session;
+};
+
+static_assert(sizeof(struct stream_gnutls) <= STREAM_ALLOC_SIZE,
+		"struct stream_gnutls has grown too large, increase STREAM_ALLOC_SIZE");
+
+static int stream__try_tls_accept(struct stream* self);
+
+static int stream_gnutls_close(struct stream* base)
+{
+	struct stream_gnutls* self = (struct stream_gnutls*)base;
+
+	if (self->base.state == STREAM_STATE_CLOSED)
+		return -1;
+
+	self->base.state = STREAM_STATE_CLOSED;
+
+	while (!TAILQ_EMPTY(&self->base.send_queue)) {
+		struct stream_req* req = TAILQ_FIRST(&self->base.send_queue);
+		TAILQ_REMOVE(&self->base.send_queue, req, link);
+		stream_req__finish(req, STREAM_REQ_FAILED);
+	}
+
+	if (self->session)
+		gnutls_deinit(self->session);
+	self->session = NULL;
+
+	aml_stop(aml_get_default(), self->base.handler);
+	close(self->base.fd);
+	self->base.fd = -1;
+
+	return 0;
+}
+
+static void stream_gnutls_destroy(struct stream* self)
+{
+	stream_close(self);
+	aml_unref(self->handler);
+	free(self);
+}
+
+static int stream_gnutls__flush(struct stream* base)
+{
+	struct stream_gnutls* self = (struct stream_gnutls*)base;
+	while (!TAILQ_EMPTY(&self->base.send_queue)) {
+		assert(self->base.state != STREAM_STATE_CLOSED);
+
+		struct stream_req* req = TAILQ_FIRST(&self->base.send_queue);
+
+		ssize_t rc = gnutls_record_send(self->session,
+				req->payload->payload, req->payload->size);
+		if (rc < 0) {
+			if (gnutls_error_is_fatal(rc)) {
+				stream_close(base);
+				return -1;
+			}
+
+			stream__poll_rw(base);
+			return 0;
+		}
+
+		self->base.bytes_sent += rc;
+
+		ssize_t remaining = req->payload->size - rc;
+
+		if (remaining > 0) {
+			char* p = req->payload->payload;
+			size_t s = req->payload->size;
+			memmove(p, p + s - remaining, remaining);
+			req->payload->size = remaining;
+			stream__poll_rw(base);
+			return 1;
+		}
+
+		assert(remaining == 0);
+
+		TAILQ_REMOVE(&self->base.send_queue, req, link);
+		stream_req__finish(req, STREAM_REQ_DONE);
+	}
+
+	if (TAILQ_EMPTY(&base->send_queue) && base->state != STREAM_STATE_CLOSED)
+		stream__poll_r(base);
+
+	return 1;
+}
+
+static void stream_gnutls__on_readable(struct stream* self)
+{
+	switch (self->state) {
+	case STREAM_STATE_NORMAL:
+		/* fallthrough */
+	case STREAM_STATE_TLS_READY:
+		if (self->on_event)
+			self->on_event(self, STREAM_EVENT_READ);
+		break;
+	case STREAM_STATE_TLS_HANDSHAKE:
+		stream__try_tls_accept(self);
+		break;
+	case STREAM_STATE_CLOSED:
+		break;
+	}
+}
+
+static void stream_gnutls__on_writable(struct stream* self)
+{
+	switch (self->state) {
+	case STREAM_STATE_NORMAL:
+		/* fallthrough */
+	case STREAM_STATE_TLS_READY:
+		stream_gnutls__flush(self);
+		break;
+	case STREAM_STATE_TLS_HANDSHAKE:
+		stream__try_tls_accept(self);
+		break;
+	case STREAM_STATE_CLOSED:
+		break;
+	}
+}
+
+static void stream_gnutls__on_event(void* obj)
+{
+	struct stream* self = aml_get_userdata(obj);
+	uint32_t events = aml_get_revents(obj);
+
+	if (events & AML_EVENT_READ)
+		stream_gnutls__on_readable(self);
+
+	if (events & AML_EVENT_WRITE)
+		stream_gnutls__on_writable(self);
+}
+
+static int stream_gnutls_send(struct stream* self, struct rcbuf* payload,
+		stream_req_fn on_done, void* userdata)
+{
+	if (self->state == STREAM_STATE_CLOSED)
+		return -1;
+
+	struct stream_req* req = calloc(1, sizeof(*req));
+	if (!req)
+		return -1;
+
+	req->payload = payload;
+	req->on_done = on_done;
+	req->userdata = userdata;
+
+	TAILQ_INSERT_TAIL(&self->send_queue, req, link);
+
+	return stream_gnutls__flush(self);
+}
+
+static ssize_t stream_gnutls_read(struct stream* base, void* dst, size_t size)
+{
+	struct stream_gnutls* self = (struct stream_gnutls*)base;
+
+	ssize_t rc = gnutls_record_recv(self->session, dst, size);
+	if (rc == 0) {
+		stream__remote_closed(base);
+		return rc;
+	}
+	if (rc > 0) {
+		self->base.bytes_received += rc;
+		return rc;
+	}
+
+	switch (rc) {
+	case GNUTLS_E_INTERRUPTED:
+		errno = EINTR;
+		break;
+	case GNUTLS_E_AGAIN:
+		errno = EAGAIN;
+		break;
+	default:
+		errno = 0;
+		break;
+	}
+
+	// Make sure data wasn't being written.
+	assert(gnutls_record_get_direction(self->session) == 0);
+	return -1;
+}
+
+static int stream__try_tls_accept(struct stream* base)
+{
+	struct stream_gnutls* self = (struct stream_gnutls*)base;
+	int rc;
+
+	rc = gnutls_handshake(self->session);
+	if (rc == GNUTLS_E_SUCCESS) {
+		self->base.state = STREAM_STATE_TLS_READY;
+		stream__poll_r(base);
+		return 0;
+	}
+
+	if (gnutls_error_is_fatal(rc)) {
+		aml_stop(aml_get_default(), self->base.handler);
+		return -1;
+	}
+
+	int was_writing = gnutls_record_get_direction(self->session);
+	if (was_writing)
+		stream__poll_w(base);
+	else
+		stream__poll_r(base);
+
+	self->base.state = STREAM_STATE_TLS_HANDSHAKE;
+	return 0;
+}
+
+static struct stream_impl impl = {
+	.close = stream_gnutls_close,
+	.destroy = stream_gnutls_destroy,
+	.read = stream_gnutls_read,
+	.send = stream_gnutls_send,
+};
+
+int stream_upgrade_to_tls(struct stream* base, void* context)
+{
+	struct stream_gnutls* self = (struct stream_gnutls*)base;
+	int rc;
+
+	rc = gnutls_init(&self->session, GNUTLS_SERVER | GNUTLS_NONBLOCK);
+	if (rc != GNUTLS_E_SUCCESS)
+		return -1;
+
+	rc = gnutls_set_default_priority(self->session);
+	if (rc != GNUTLS_E_SUCCESS)
+		goto failure;
+
+	rc = gnutls_credentials_set(self->session, GNUTLS_CRD_CERTIFICATE,
+			context);
+	if (rc != GNUTLS_E_SUCCESS)
+		goto failure;
+
+	aml_stop(aml_get_default(), self->base.handler);
+	aml_unref(self->base.handler);
+
+	self->base.handler = aml_handler_new(self->base.fd,
+			stream_gnutls__on_event, self, NULL);
+	assert(self->base.handler);
+
+	rc = aml_start(aml_get_default(), self->base.handler);
+	assert(rc >= 0);
+
+	gnutls_transport_set_int(self->session, self->base.fd);
+
+	self->base.impl = &impl;
+
+	return stream__try_tls_accept(base);
+
+failure:
+	gnutls_deinit(self->session);
+	return -1;
+}

src/stream-rsa-aes.c

@@ -0,0 +1,211 @@
+/*
+ * Copyright (c) 2023 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <assert.h>
+#include <errno.h>
+#include <sys/param.h>
+#include <arpa/inet.h>
+
+#include "rcbuf.h"
+#include "stream.h"
+#include "stream-tcp.h"
+#include "stream-common.h"
+#include "crypto.h"
+#include "neatvnc.h"
+
+#define RSA_AES_BUFFER_SIZE 8192
+#define UDIV_UP(a, b) (((a) + (b) - 1) / (b))
+
+struct stream_rsa_aes {
+	struct stream base;
+
+	size_t read_index;
+	uint8_t* read_buffer;
+
+	struct crypto_cipher* cipher;
+};
+
+static_assert(sizeof(struct stream_rsa_aes) <= STREAM_ALLOC_SIZE,
+		"struct stream_rsa_aes has grown too large, increase STREAM_ALLOC_SIZE");
+
+static void stream_rsa_aes_destroy(struct stream* base)
+{
+	struct stream_rsa_aes* self = (struct stream_rsa_aes*)base;
+	crypto_cipher_del(self->cipher);
+	free(self->read_buffer);
+	stream_tcp_destroy(base);
+}
+
+static void stream_rsa_aes_read_into_buffer(struct stream_rsa_aes* self)
+{
+	ssize_t n_read = stream_tcp_read(&self->base,
+			self->read_buffer + self->read_index,
+			RSA_AES_BUFFER_SIZE - self->read_index);
+	if (n_read > 0)
+		self->read_index += n_read;
+}
+
+static ssize_t stream_rsa_aes_parse_header(struct stream_rsa_aes* self)
+{
+	if (self->read_index <= 2) {
+		return -1;
+	}
+
+	uint16_t len_be;
+	memcpy(&len_be, self->read_buffer, sizeof(len_be));
+	size_t len = ntohs(len_be);
+
+	if (self->read_index < 2 + 16 + len) {
+		return -1;
+	}
+
+	return len;
+}
+
+static ssize_t stream_rsa_aes_read_message(struct stream_rsa_aes* self,
+		uint8_t* dst, size_t size)
+{
+	ssize_t msg_len = stream_rsa_aes_parse_header(self);
+	if (msg_len < 0) {
+		return 0;
+	}
+
+	// The entire message must fit in dst
+	/* TODO: With this, stream_tcp__on_event won't run until network input
+	 * is received. We need to somehow schedule on_event or also buffer the
+	 * decrypted data here.
+	 * Another option would be to keep back the message counter in the
+	 * cipher until the message has been fully read.
+	 */
+	if ((size_t)msg_len > size)
+		return 0;
+
+	uint16_t msg_len_be = htons(msg_len);
+
+	uint8_t expected_mac[16];
+	ssize_t n = crypto_cipher_decrypt(self->cipher, dst, expected_mac,
+			self->read_buffer + 2, msg_len,
+			(uint8_t*)&msg_len_be, sizeof(msg_len_be));
+
+	uint8_t* actual_mac = self->read_buffer + 2 + msg_len;
+	if (memcmp(expected_mac, actual_mac, 16) != 0) {
+		nvnc_log(NVNC_LOG_DEBUG, "Message authentication failed");
+		errno = EBADMSG;
+		return -1;
+	}
+
+	self->read_index -= 2 + 16 + msg_len;
+	memmove(self->read_buffer, self->read_buffer + 2 + 16 + msg_len,
+			self->read_index);
+
+	return n;
+}
+
+static ssize_t stream_rsa_aes_read(struct stream* base, void* dst, size_t size)
+{
+	struct stream_rsa_aes* self = (struct stream_rsa_aes*)base;
+
+	stream_rsa_aes_read_into_buffer(self);
+	if (self->base.state == STREAM_STATE_CLOSED)
+		return 0;
+
+	size_t total_read = 0;
+
+	while (true) {
+		ssize_t n_read = stream_rsa_aes_read_message(self, dst, size);
+		if (n_read == 0)
+			break;
+
+		if (n_read < 0) {
+			if (errno == EAGAIN) {
+				break;
+			}
+			return -1;
+		}
+
+		total_read += n_read;
+		dst += n_read;
+		size -= n_read;
+	}
+
+	return total_read;
+}
+
+static int stream_rsa_aes_send(struct stream* base, struct rcbuf* payload,
+		stream_req_fn on_done, void* userdata)
+{
+	struct stream_rsa_aes* self = (struct stream_rsa_aes*)base;
+	size_t n_msg = UDIV_UP(payload->size, RSA_AES_BUFFER_SIZE);
+
+	struct vec buf;
+	vec_init(&buf, payload->size + n_msg * (2 + 16));
+
+	for (size_t i = 0; i < n_msg; ++i) {
+		size_t msglen = MIN(payload->size - i * RSA_AES_BUFFER_SIZE,
+				RSA_AES_BUFFER_SIZE);
+		uint16_t msglen_be = htons(msglen);
+
+		vec_append(&buf, &msglen_be, sizeof(msglen_be));
+
+		uint8_t mac[16];
+		crypto_cipher_encrypt(self->cipher, &buf, mac,
+				payload->payload + i * RSA_AES_BUFFER_SIZE,
+				msglen, (uint8_t*)&msglen_be, sizeof(msglen_be));
+		vec_append(&buf, mac, sizeof(mac));
+	}
+
+	size_t payload_size = payload->size;
+	rcbuf_unref(payload);
+
+	int r = stream_tcp_send(base, rcbuf_new(buf.data, buf.len), on_done,
+			userdata);
+	if (r < 0) {
+		return r;
+	}
+
+	return payload_size;
+}
+
+static struct stream_impl impl = {
+	.close = stream_tcp_close,
+	.destroy = stream_rsa_aes_destroy,
+	.read = stream_rsa_aes_read,
+	.send = stream_rsa_aes_send,
+};
+
+int stream_upgrade_to_rsa_eas(struct stream* base,
+		enum crypto_cipher_type cipher_type,
+		const uint8_t* enc_key, const uint8_t* dec_key)
+{
+	struct stream_rsa_aes* self = (struct stream_rsa_aes*)base;
+
+	self->read_index = 0;
+	self->read_buffer = malloc(RSA_AES_BUFFER_SIZE);
+	if (!self->read_buffer)
+		return -1;
+
+	self->cipher = crypto_cipher_new(enc_key, dec_key, cipher_type);
+	if (!self->cipher) {
+		free(self->read_buffer);
+		return -1;
+	}
+
+	self->base.impl = &impl;
+	return 0;
+}

src/stream-tcp.c

@@ -0,0 +1,312 @@
+/*
+ * Copyright (c) 2020 - 2023 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <string.h>
+#include <assert.h>
+#include <errno.h>
+#include <sys/uio.h>
+#include <limits.h>
+#include <aml.h>
+#include <fcntl.h>
+#include <poll.h>
+#include <sys/socket.h>
+
+#include "rcbuf.h"
+#include "stream.h"
+#include "stream-common.h"
+#include "stream-tcp.h"
+#include "sys/queue.h"
+#include "neatvnc.h"
+
+static_assert(sizeof(struct stream) <= STREAM_ALLOC_SIZE,
+		"struct stream has grown too large, increase STREAM_ALLOC_SIZE");
+
+int stream_tcp_close(struct stream* self)
+{
+	if (self->state == STREAM_STATE_CLOSED)
+		return -1;
+
+	self->state = STREAM_STATE_CLOSED;
+
+	while (!TAILQ_EMPTY(&self->send_queue)) {
+		struct stream_req* req = TAILQ_FIRST(&self->send_queue);
+		TAILQ_REMOVE(&self->send_queue, req, link);
+		stream_req__finish(req, STREAM_REQ_FAILED);
+	}
+
+	aml_stop(aml_get_default(), self->handler);
+	close(self->fd);
+	self->fd = -1;
+
+	return 0;
+}
+
+void stream_tcp_destroy(struct stream* self)
+{
+	vec_destroy(&self->tmp_buf);
+	stream_close(self);
+	aml_unref(self->handler);
+	free(self);
+}
+
+static int stream_tcp__flush(struct stream* self)
+{
+	if (self->cork)
+		return 0;
+
+	static struct iovec iov[IOV_MAX];
+	size_t n_msgs = 0;
+	ssize_t bytes_sent;
+
+	struct stream_req* req;
+	TAILQ_FOREACH(req, &self->send_queue, link) {
+		if (req->exec) {
+			if (req->payload)
+				rcbuf_unref(req->payload);
+			struct rcbuf* payload  = req->exec(self, req->userdata);
+			req->payload = payload;
+		}
+
+		iov[n_msgs].iov_base = req->payload->payload;
+		iov[n_msgs].iov_len = req->payload->size;
+
+		if (++n_msgs >= IOV_MAX)
+			break;
+	}
+
+	if (n_msgs == 0)
+		return 0;
+
+	struct msghdr msghdr = {
+		.msg_iov = iov,
+		.msg_iovlen = n_msgs,
+	};
+	bytes_sent = sendmsg(self->fd, &msghdr, MSG_NOSIGNAL);
+	if (bytes_sent < 0) {
+		if (errno == EAGAIN || errno == EWOULDBLOCK) {
+			stream__poll_rw(self);
+			errno = EAGAIN;
+			bytes_sent = 0;
+		} else if (errno == EPIPE) {
+			stream__remote_closed(self);
+			errno = EPIPE;
+		}
+
+		return bytes_sent;
+	}
+
+	self->bytes_sent += bytes_sent;
+
+	ssize_t bytes_left = bytes_sent;
+
+	struct stream_req* tmp;
+	TAILQ_FOREACH_SAFE(req, &self->send_queue, link, tmp) {
+		bytes_left -= req->payload->size;
+
+		if (bytes_left >= 0) {
+			TAILQ_REMOVE(&self->send_queue, req, link);
+			stream_req__finish(req, STREAM_REQ_DONE);
+		} else {
+			if (req->exec) {
+				free(req->userdata);
+				req->userdata = NULL;
+				req->exec = NULL;
+			}
+			char* p = req->payload->payload;
+			size_t s = req->payload->size;
+			memmove(p, p + s + bytes_left, -bytes_left);
+			req->payload->size = -bytes_left;
+			stream__poll_rw(self);
+		}
+
+		if (bytes_left <= 0)
+			break;
+	}
+
+	if (bytes_left == 0 && self->state != STREAM_STATE_CLOSED)
+		stream__poll_r(self);
+
+	assert(bytes_left <= 0);
+
+	return bytes_sent;
+}
+
+static void stream_tcp__on_readable(struct stream* self)
+{
+	switch (self->state) {
+	case STREAM_STATE_NORMAL:
+		/* fallthrough */
+		if (self->on_event)
+			self->on_event(self, STREAM_EVENT_READ);
+		break;
+	case STREAM_STATE_CLOSED:
+		break;
+	default:;
+	}
+}
+
+static void stream_tcp__on_writable(struct stream* self)
+{
+	switch (self->state) {
+	case STREAM_STATE_NORMAL:
+		/* fallthrough */
+		stream_tcp__flush(self);
+		break;
+	case STREAM_STATE_CLOSED:
+		break;
+	default:;
+	}
+}
+
+static void stream_tcp__on_event(void* obj)
+{
+	struct stream* self = aml_get_userdata(obj);
+	uint32_t events = aml_get_revents(obj);
+
+	if (events & AML_EVENT_READ)
+		stream_tcp__on_readable(self);
+
+	if (events & AML_EVENT_WRITE)
+		stream_tcp__on_writable(self);
+}
+
+ssize_t stream_tcp_read(struct stream* self, void* dst, size_t size)
+{
+	if (self->state != STREAM_STATE_NORMAL)
+		return -1;
+
+	uint8_t* read_buffer = dst;
+
+	if (self->cipher) {
+		vec_reserve(&self->tmp_buf, size);
+		read_buffer = self->tmp_buf.data;
+	}
+
+	ssize_t rc = read(self->fd, read_buffer, size);
+	if (rc == 0)
+		stream__remote_closed(self);
+	if (rc > 0)
+		self->bytes_received += rc;
+
+	return rc;
+}
+
+int stream_tcp_send(struct stream* self, struct rcbuf* payload,
+		stream_req_fn on_done, void* userdata)
+{
+	if (self->state == STREAM_STATE_CLOSED)
+		return -1;
+
+	struct stream_req* req = calloc(1, sizeof(*req));
+	if (!req)
+		return -1;
+
+	req->payload = payload;
+	req->on_done = on_done;
+	req->userdata = userdata;
+
+	TAILQ_INSERT_TAIL(&self->send_queue, req, link);
+
+	return stream_tcp__flush(self);
+}
+
+int stream_tcp_send_first(struct stream* self, struct rcbuf* payload)
+{
+	if (self->state == STREAM_STATE_CLOSED)
+		return -1;
+
+	struct stream_req* req = calloc(1, sizeof(*req));
+	if (!req)
+		return -1;
+
+	req->payload = payload;
+	TAILQ_INSERT_HEAD(&self->send_queue, req, link);
+
+	return stream_tcp__flush(self);
+}
+
+void stream_tcp_exec_and_send(struct stream* self,
+		stream_exec_fn exec_fn, void* userdata)
+{
+	if (self->state == STREAM_STATE_CLOSED)
+		return;
+
+	struct stream_req* req = calloc(1, sizeof(*req));
+	if (!req)
+		return;
+
+	req->exec = exec_fn;
+	req->userdata = userdata;
+
+	TAILQ_INSERT_TAIL(&self->send_queue, req, link);
+
+	stream_tcp__flush(self);
+}
+
+static struct stream_impl impl = {
+	.close = stream_tcp_close,
+	.destroy = stream_tcp_destroy,
+	.read = stream_tcp_read,
+	.send = stream_tcp_send,
+	.send_first = stream_tcp_send_first,
+	.exec_and_send = stream_tcp_exec_and_send,
+};
+
+int stream_tcp_init(struct stream* self, int fd, stream_event_fn on_event,
+		void* userdata)
+{
+	self->impl = &impl,
+	self->fd = fd;
+	self->on_event = on_event;
+	self->userdata = userdata;
+
+	TAILQ_INIT(&self->send_queue);
+
+	fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK);
+
+	self->handler = aml_handler_new(fd, stream_tcp__on_event, self, NULL);
+	if (!self->handler)
+		return -1;
+
+	if (aml_start(aml_get_default(), self->handler) < 0)
+		goto start_failure;
+
+	stream__poll_r(self);
+
+	return 0;
+
+start_failure:
+	aml_unref(self->handler);
+	return -1;
+}
+
+struct stream* stream_new(int fd, stream_event_fn on_event, void* userdata)
+{
+	struct stream* self = calloc(1, STREAM_ALLOC_SIZE);
+	if (!self)
+		return NULL;
+
+	if (stream_tcp_init(self, fd, on_event, userdata) < 0) {
+		free(self);
+		return NULL;
+	}
+
+	return self;
+
+}

src/stream-ws.c

@@ -0,0 +1,314 @@
+/*
+ * Copyright (c) 2023 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "stream.h"
+#include "stream-common.h"
+#include "stream-tcp.h"
+#include "websocket.h"
+#include "vec.h"
+#include "neatvnc.h"
+
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <sys/param.h>
+
+enum stream_ws_state {
+	STREAM_WS_STATE_HANDSHAKE = 0,
+	STREAM_WS_STATE_READY,
+};
+
+struct stream_ws_exec_ctx {
+	stream_exec_fn exec;
+	void* userdata;
+};
+
+struct stream_ws {
+	struct stream base;
+	stream_event_fn on_event;
+	enum stream_ws_state ws_state;
+	struct ws_frame_header header;
+	enum ws_opcode current_opcode;
+	size_t read_index;
+	uint8_t read_buffer[4096]; // TODO: Is this a reasonable size?
+};
+
+static void stream_ws_read_into_buffer(struct stream_ws* ws)
+{
+	ssize_t n_read = stream_tcp_read(&ws->base,
+			ws->read_buffer + ws->read_index,
+			sizeof(ws->read_buffer) - ws->read_index);
+	if (n_read > 0)
+		ws->read_index += n_read;
+}
+
+static void stream_ws_advance_read_buffer(struct stream_ws* ws, size_t size,
+		size_t offset)
+{
+	size_t payload_len = MIN(size, ws->read_index - offset);
+	payload_len = MIN(payload_len, ws->header.payload_length);
+
+	ws->read_index -= offset + payload_len;
+	memmove(ws->read_buffer, ws->read_buffer + offset + payload_len,
+			ws->read_index);
+	ws->header.payload_length -= payload_len;
+}
+
+static ssize_t stream_ws_copy_payload(struct stream_ws* ws, void* dst,
+		size_t size, size_t offset)
+{
+	size_t payload_len = MIN(size, ws->read_index - offset);
+	payload_len = MIN(payload_len, ws->header.payload_length);
+
+	ws_copy_payload(&ws->header, dst, ws->read_buffer + offset, payload_len);
+	stream_ws_advance_read_buffer(ws, size, offset);
+	return payload_len;
+}
+
+static ssize_t stream_ws_process_ping(struct stream_ws* ws, size_t offset)
+{
+	if (offset > 0) {
+		// This means we're at the start, so send a header
+		struct ws_frame_header reply = {
+			.fin = true,
+			.opcode = WS_OPCODE_PONG,
+			.payload_length = ws->header.payload_length,
+		};
+
+		uint8_t buf[WS_HEADER_MIN_SIZE];
+		int reply_len = ws_write_frame_header(buf, &reply);
+		stream_tcp_send(&ws->base, rcbuf_from_mem(buf, reply_len),
+				NULL, NULL);
+	}
+
+	int payload_len = MIN(ws->read_index, ws->header.payload_length);
+
+	// Feed back the payload:
+	stream_tcp_send(&ws->base, rcbuf_from_mem(ws->read_buffer + offset,
+			payload_len), NULL, NULL);
+
+	stream_ws_advance_read_buffer(ws, payload_len, offset);
+	return 0;
+}
+
+static ssize_t stream_ws_process_payload(struct stream_ws* ws, void* dst,
+		size_t size, size_t offset)
+{
+	switch (ws->current_opcode) {
+	case WS_OPCODE_CONT:
+		// Remote end started with a continuation frame. This is
+		// unexpected, so we'll just close.
+		stream__remote_closed(&ws->base);
+		return 0;
+	case WS_OPCODE_TEXT:
+		// This is unexpected, but let's just ignore it...
+		stream_ws_advance_read_buffer(ws, SIZE_MAX, offset);
+		return 0;
+	case WS_OPCODE_BIN:
+		return stream_ws_copy_payload(ws, dst, size, offset);
+	case WS_OPCODE_CLOSE:
+		stream__remote_closed(&ws->base);
+		return 0;
+	case WS_OPCODE_PING:
+		return stream_ws_process_ping(ws, offset);
+	case WS_OPCODE_PONG:
+		// Don't care
+		stream_ws_advance_read_buffer(ws, SIZE_MAX, offset);
+		return 0;
+	}
+	return -1;
+}
+
+/* We don't really care about framing. The binary data is just passed on as it
+ * arrives and it's not gathered into individual frames.
+ */
+static ssize_t stream_ws_read_frame(struct stream_ws* ws, void* dst,
+		size_t size)
+{
+	if (ws->header.payload_length > 0) {
+		nvnc_trace("Processing left-over payload chunk");
+		return stream_ws_process_payload(ws, dst, size, 0);
+	}
+
+	if (!ws_parse_frame_header(&ws->header, ws->read_buffer,
+			ws->read_index)) {
+		return 0;
+	}
+
+	if (ws->header.opcode != WS_OPCODE_CONT) {
+		ws->current_opcode = ws->header.opcode;
+	}
+
+	// The header is located at the start of the buffer, so an offset is
+	// needed.
+	return stream_ws_process_payload(ws, dst, size,
+			ws->header.header_length);
+}
+
+static ssize_t stream_ws_read_ready(struct stream_ws* ws, void* dst,
+		size_t size)
+{
+	size_t total_read = 0;
+
+	while (true) {
+		ssize_t n_read = stream_ws_read_frame(ws, dst, size);
+		if (n_read == 0)
+			break;
+
+		if (n_read < 0) {
+			if (errno == EAGAIN) {
+				break;
+			}
+			return -1;
+		}
+
+		total_read += n_read;
+		dst += n_read;
+		size -= n_read;
+	}
+
+	return total_read;
+}
+
+static ssize_t stream_ws_read_handshake(struct stream_ws* ws, void* dst,
+		size_t size)
+{
+	if (ws->read_index >= sizeof(ws->read_buffer)) {
+		// This header is suspiciously long
+		stream__remote_closed(&ws->base);
+		return -1;
+	}
+
+	ws->read_buffer[ws->read_index] = '\0';
+
+	char reply[512];
+	ssize_t header_len = ws_handshake(reply, sizeof(reply),
+			(const char*)ws->read_buffer);
+	if (header_len < 0)
+		return 0;
+
+	ws->base.cork = false;
+	stream_tcp_send_first(&ws->base, rcbuf_from_mem(reply, strlen(reply)));
+
+	ws->read_index -= header_len;
+	memmove(ws->read_buffer, ws->read_buffer + header_len, ws->read_index);
+
+	ws->ws_state = STREAM_WS_STATE_READY;
+	return stream_ws_read_ready(ws, dst, size);
+}
+
+static ssize_t stream_ws_read(struct stream* self, void* dst, size_t size)
+{
+	struct stream_ws* ws = (struct stream_ws*)self;
+
+	stream_ws_read_into_buffer(ws);
+	if (self->state == STREAM_STATE_CLOSED)
+		return 0;
+
+	switch (ws->ws_state) {
+	case STREAM_WS_STATE_HANDSHAKE:
+		return stream_ws_read_handshake(ws, dst, size);
+	case STREAM_WS_STATE_READY:
+		return stream_ws_read_ready(ws, dst, size);
+	}
+	abort();
+	return -1;
+}
+
+static int stream_ws_send(struct stream* self, struct rcbuf* payload,
+		stream_req_fn on_done, void* userdata)
+{
+	struct stream_ws* ws = (struct stream_ws*)self;
+
+	struct ws_frame_header head = {
+		.fin = true,
+		.opcode = WS_OPCODE_BIN,
+		.payload_length = payload->size,
+	};
+
+	uint8_t raw_head[WS_HEADER_MIN_SIZE];
+	int head_len = ws_write_frame_header(raw_head, &head);
+
+	stream_tcp_send(&ws->base, rcbuf_from_mem(&raw_head, head_len),
+			NULL, NULL);
+	return stream_tcp_send(&ws->base, payload, on_done, userdata);
+}
+
+static struct rcbuf* stream_ws_chained_exec(struct stream* tcp_stream,
+		void* userdata)
+{
+	struct stream_ws* ws = (struct stream_ws*)tcp_stream;
+	struct stream_ws_exec_ctx* ctx = userdata;
+
+	struct rcbuf* buf = ctx->exec(&ws->base, ctx->userdata);
+
+	// TODO: This also needs to be cleaned it it's left on the send queue
+	// when the stream is destroyed.
+	free(ctx->userdata);
+
+	struct vec out;
+	vec_init(&out, WS_HEADER_MIN_SIZE + buf->size + 1);
+
+	struct ws_frame_header head = {
+		.fin = true,
+		.opcode = WS_OPCODE_BIN,
+		.payload_length = buf->size,
+	};
+	int head_len = ws_write_frame_header(out.data, &head);
+	out.len += head_len;
+
+	vec_append(&out, buf->payload, buf->size);
+	rcbuf_unref(buf);
+	return rcbuf_new(out.data, out.len);
+}
+
+static void stream_ws_exec_and_send(struct stream* self, stream_exec_fn exec,
+		void* userdata)
+{
+	struct stream_ws* ws = (struct stream_ws*)self;
+
+	struct stream_ws_exec_ctx* ctx = calloc(1, sizeof(*ctx));
+	assert(ctx);
+
+	ctx->exec = exec;
+	ctx->userdata = userdata;
+
+	stream_tcp_exec_and_send(&ws->base, stream_ws_chained_exec, ctx);
+}
+
+static struct stream_impl impl = {
+	.close = stream_tcp_close,
+	.destroy = stream_tcp_destroy,
+	.read = stream_ws_read,
+	.send = stream_ws_send,
+	.exec_and_send = stream_ws_exec_and_send,
+};
+
+struct stream* stream_ws_new(int fd, stream_event_fn on_event, void* userdata)
+{
+	struct stream_ws *self = calloc(1, sizeof(*self));
+	if (!self)
+		return NULL;
+
+	stream_tcp_init(&self->base, fd, on_event, userdata);
+	self->base.impl =  &impl;
+
+	// Don't send anything until handshake is done:
+	self->base.cork = true;
+
+	return &self->base;
+}

src/stream.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020 Andri Yngvason
+ * Copyright (c) 2023 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -14,428 +14,54 @@
  * PERFORMANCE OF THIS SOFTWARE.
  */
 
-#include <stdlib.h>
-#include <stdio.h>
-#include <unistd.h>
-#include <string.h>
-#include <assert.h>
-#include <errno.h>
-#include <sys/uio.h>
-#include <limits.h>
-#include <aml.h>
-#include <fcntl.h>
-#include <poll.h>
-
-#ifdef ENABLE_TLS
-#include <gnutls/gnutls.h>
-#endif
-
-#include "rcbuf.h"
 #include "stream.h"
-#include "sys/queue.h"
 
-static void stream__on_event(void* obj);
-#ifdef ENABLE_TLS
-static int stream__try_tls_accept(struct stream* self);
-#endif
-
-static inline void stream__poll_r(struct stream* self)
-{
-	aml_set_event_mask(self->handler, AML_EVENT_READ);
-}
-
-static inline void stream__poll_w(struct stream* self)
-{
-	aml_set_event_mask(self->handler, AML_EVENT_WRITE);
-}
-
-static inline void stream__poll_rw(struct stream* self)
-{
-	aml_set_event_mask(self->handler, AML_EVENT_READ | AML_EVENT_WRITE);
-}
-
-static void stream_req__finish(struct stream_req* req,
-                               enum stream_req_status status)
-{
-	if (req->on_done)
-		req->on_done(req->userdata, status);
-
-	rcbuf_unref(req->payload);
-	free(req);
-}
+#include <assert.h>
 
 int stream_close(struct stream* self)
 {
-	if (self->state == STREAM_STATE_CLOSED)
-		return -1;
-
-	self->state = STREAM_STATE_CLOSED;
-
-	while (!TAILQ_EMPTY(&self->send_queue)) {
-		struct stream_req* req = TAILQ_FIRST(&self->send_queue);
-		TAILQ_REMOVE(&self->send_queue, req, link);
-		stream_req__finish(req, STREAM_REQ_FAILED);
-	}
-
-#ifdef ENABLE_TLS
-	if (self->tls_session)
-		gnutls_deinit(self->tls_session);
-	self->tls_session = NULL;
-#endif
-
-	// TODO: Maybe use explicit loop object instead of the default one?
-	aml_stop(aml_get_default(), self->handler);
-	close(self->fd);
-	self->fd = -1;
-
-	return 0;
+	assert(self->impl && self->impl->close);
+	return self->impl->close(self);
 }
 
 void stream_destroy(struct stream* self)
 {
-	stream_close(self);
-	aml_unref(self->handler);
-}
-
-static void stream__remote_closed(struct stream* self)
-{
-	stream_close(self);
-
-	if (self->on_event)
-		self->on_event(self, STREAM_EVENT_REMOTE_CLOSED);
-}
-
-static int stream__flush_plain(struct stream* self)
-{
-	static struct iovec iov[IOV_MAX];
-	size_t n_msgs = 0;
-	ssize_t bytes_sent;
-
-	struct stream_req* req;
-	TAILQ_FOREACH(req, &self->send_queue, link) {
-		iov[n_msgs].iov_base = req->payload->payload;
-		iov[n_msgs].iov_len = req->payload->size;
-
-		if (++n_msgs >= IOV_MAX)
-			break;
-	}
-
-	if (n_msgs == 0)
-		return 0;
-
-	bytes_sent = writev(self->fd, iov, n_msgs);
-	if (bytes_sent < 0) {
-		if (errno == EAGAIN || errno == EWOULDBLOCK) {
-			stream__poll_rw(self);
-			errno = EAGAIN;
-		} else if (errno == EPIPE) {
-			stream__remote_closed(self);
-			errno = EPIPE;
-		}
-
-		return bytes_sent;
-	}
-
-	self->bytes_sent += bytes_sent;
-
-	ssize_t bytes_left = bytes_sent;
-
-	struct stream_req* tmp;
-	TAILQ_FOREACH_SAFE(req, &self->send_queue, link, tmp) {
-		bytes_left -= req->payload->size;
-
-		if (bytes_left >= 0) {
-			TAILQ_REMOVE(&self->send_queue, req, link);
-			stream_req__finish(req, STREAM_REQ_DONE);
-		} else {
-			char* p = req->payload->payload;
-			size_t s = req->payload->size;
-			memmove(p, p + s + bytes_left, -bytes_left);
-			req->payload->size = -bytes_left;
-			stream__poll_rw(self);
-		}
-
-		if (bytes_left <= 0)
-			break;
-	}
-
-	if (bytes_left == 0 && self->state != STREAM_STATE_CLOSED)
-		stream__poll_r(self);
-
-	assert(bytes_left <= 0);
-
-	return bytes_sent;
-}
-
-#ifdef ENABLE_TLS
-static int stream__flush_tls(struct stream* self)
-{
-	while (!TAILQ_EMPTY(&self->send_queue)) {
-		struct stream_req* req = TAILQ_FIRST(&self->send_queue);
-
-		ssize_t rc = gnutls_record_send(
-			self->tls_session, req->payload->payload,
-			req->payload->size);
-		if (rc < 0) {
-			gnutls_record_discard_queued(self->tls_session);
-			if (gnutls_error_is_fatal(rc))
-				stream_close(self);
-			return -1;
-		}
-
-		self->bytes_sent += rc;
-
-		ssize_t remaining = req->payload->size - rc;
-
-		if (remaining > 0) {
-			char* p = req->payload->payload;
-			size_t s = req->payload->size;
-			memmove(p, p + s - remaining, remaining);
-			req->payload->size = remaining;
-			stream__poll_rw(self);
-			return 1;
-		}
-
-		assert(remaining == 0);
-
-		TAILQ_REMOVE(&self->send_queue, req, link);
-		stream_req__finish(req, STREAM_REQ_DONE);
-	}
-
-	if (TAILQ_EMPTY(&self->send_queue) && self->state != STREAM_STATE_CLOSED)
-		stream__poll_r(self);
-
-	return 1;
-}
-#endif
-
-static int stream__flush(struct stream* self)
-{
-	switch (self->state) {
-	case STREAM_STATE_NORMAL: return stream__flush_plain(self);
-#ifdef ENABLE_TLS
-	case STREAM_STATE_TLS_READY: return stream__flush_tls(self);
-#endif
-	default:
-		break;
-	}
-	abort();
-	return -1;
-}
-
-static void stream__on_readable(struct stream* self)
-{
-	switch (self->state) {
-	case STREAM_STATE_NORMAL:
-		/* fallthrough */
-#ifdef ENABLE_TLS
-	case STREAM_STATE_TLS_READY:
-#endif
-		if (self->on_event)
-			self->on_event(self, STREAM_EVENT_READ);
-		break;
-#ifdef ENABLE_TLS
-	case STREAM_STATE_TLS_HANDSHAKE:
-		stream__try_tls_accept(self);
-		break;
-#endif
-	case STREAM_STATE_CLOSED:
-		break;
-	}
-}
-
-static void stream__on_writable(struct stream* self)
-{
-	switch (self->state) {
-	case STREAM_STATE_NORMAL:
-		/* fallthrough */
-#ifdef ENABLE_TLS
-	case STREAM_STATE_TLS_READY:
-#endif
-		stream__flush(self);
-		break;
-#ifdef ENABLE_TLS
-	case STREAM_STATE_TLS_HANDSHAKE:
-		stream__try_tls_accept(self);
-		break;
-#endif
-	case STREAM_STATE_CLOSED:
-		break;
-	}
-}
-
-static void stream__on_event(void* obj)
-{
-	struct stream* self = aml_get_userdata(obj);
-	uint32_t events = aml_get_revents(obj);
-
-	if (events & AML_EVENT_READ)
-		stream__on_readable(self);
-
-	if (events & AML_EVENT_WRITE)
-		stream__on_writable(self);
-}
-
-struct stream* stream_new(int fd, stream_event_fn on_event, void* userdata)
-{
-	struct stream* self = calloc(1, sizeof(*self));
-	if (!self)
-		return NULL;
-
-	self->fd = fd;
-	self->on_event = on_event;
-	self->userdata = userdata;
-
-	TAILQ_INIT(&self->send_queue);
-
-	fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK);
-
-	self->handler = aml_handler_new(fd, stream__on_event, self, free);
-	if (!self->handler)
-		goto failure;
-
-	if (aml_start(aml_get_default(), self->handler) < 0)
-		goto start_failure;
-
-	stream__poll_r(self);
-
-	return self;
-
-start_failure:
-	aml_unref(self->handler);
-	self = NULL; /* Handled in unref */
-failure:
-	free(self);
-	return NULL;
+	assert(self->impl && self->impl->destroy);
+	return self->impl->destroy(self);
 }
 
 int stream_send(struct stream* self, struct rcbuf* payload,
-                stream_req_fn on_done, void* userdata)
+		stream_req_fn on_done, void* userdata)
 {
-	if (self->state == STREAM_STATE_CLOSED)
-		return -1;
+	assert(self->impl && self->impl->send);
+	return self->impl->send(self, payload, on_done, userdata);
+}
 
-	struct stream_req* req = calloc(1, sizeof(*req));
-	if (!req)
-		return -1;
-
-	req->payload = payload;
-	req->on_done = on_done;
-	req->userdata = userdata;
-
-	TAILQ_INSERT_TAIL(&self->send_queue, req, link);
-
-	return stream__flush(self);
+int stream_send_first(struct stream* self, struct rcbuf* payload)
+{
+	assert(self->impl && self->impl->send);
+	return self->impl->send_first(self, payload);
 }
 
 int stream_write(struct stream* self, const void* payload, size_t len,
-                 stream_req_fn on_done, void* userdata)
+		stream_req_fn on_done, void* userdata)
 {
 	struct rcbuf* buf = rcbuf_from_mem(payload, len);
 	return buf ? stream_send(self, buf, on_done, userdata) : -1;
 }
 
-static ssize_t stream__read_plain(struct stream* self, void* dst, size_t size)
-{
-	ssize_t rc = read(self->fd, dst, size);
-	if (rc == 0)
-		stream__remote_closed(self);
-	if (rc > 0)
-		self->bytes_received += rc;
-	return rc;
-}
-
-#ifdef ENABLE_TLS
-static ssize_t stream__read_tls(struct stream* self, void* dst, size_t size)
-{
-	ssize_t rc = gnutls_record_recv(self->tls_session, dst, size);
-	if (rc >= 0) {
-		self->bytes_received += rc;
-		return rc;
-	}
-
-	switch (rc) {
-	case GNUTLS_E_INTERRUPTED:
-		errno = EINTR;
-		break;
-	case GNUTLS_E_AGAIN:
-		errno = EAGAIN;
-		break;
-	default:
-		errno = 0;
-		break;
-	}
-
-	// Make sure data wasn't being written.
-	assert(gnutls_record_get_direction(self->tls_session) == 0);
-	return -1;
-}
-#endif
-
 ssize_t stream_read(struct stream* self, void* dst, size_t size)
 {
-	switch (self->state) {
-	case STREAM_STATE_NORMAL: return stream__read_plain(self, dst, size);
-#ifdef ENABLE_TLS
-	case STREAM_STATE_TLS_READY: return stream__read_tls(self, dst, size);
-#endif
-	default: break;
-	}
-
-	abort();
-	return -1;
+	assert(self->impl && self->impl->read);
+	return self->impl->read(self, dst, size);
 }
 
-#ifdef ENABLE_TLS
-static int stream__try_tls_accept(struct stream* self)
+void stream_exec_and_send(struct stream* self, stream_exec_fn exec_fn,
+		void* userdata)
 {
-	int rc;
-
-	rc = gnutls_handshake(self->tls_session);
-	if (rc == GNUTLS_E_SUCCESS) {
-		self->state = STREAM_STATE_TLS_READY;
-		stream__poll_r(self);
-		return 0;
-	}
-
-	if (gnutls_error_is_fatal(rc)) {
-		aml_stop(aml_get_default(), &self->handler);
-		return -1;
-	}
-
-	int was_writing = gnutls_record_get_direction(self->tls_session);
-	if (was_writing)
-		stream__poll_w(self);
+	assert(self->impl);
+	if (self->impl->exec_and_send)
+		self->impl->exec_and_send(self, exec_fn, userdata);
 	else
-		stream__poll_r(self);
-
-	self->state = STREAM_STATE_TLS_HANDSHAKE;
-	return 0;
+		stream_send(self, exec_fn(self, userdata), NULL, NULL);
 }
-
-int stream_upgrade_to_tls(struct stream* self, void* context)
-{
-	int rc;
-
-	rc = gnutls_init(&self->tls_session, GNUTLS_SERVER | GNUTLS_NONBLOCK);
-	if (rc != GNUTLS_E_SUCCESS)
-		return -1;
-
-	rc = gnutls_set_default_priority(self->tls_session);
-	if (rc != GNUTLS_E_SUCCESS)
-		goto failure;
-
-	rc = gnutls_credentials_set(self->tls_session, GNUTLS_CRD_CERTIFICATE,
-	                            context);
-	if (rc != GNUTLS_E_SUCCESS)
-		goto failure;
-
-	gnutls_transport_set_int(self->tls_session, self->fd);
-
-	return stream__try_tls_accept(self);
-
-failure:
-	gnutls_deinit(self->tls_session);
-	return -1;
-}
-#endif

src/tight.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019 - 2020 Andri Yngvason
+ * Copyright (c) 2019 - 2022 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -19,10 +19,11 @@
 #include "common.h"
 #include "pixels.h"
 #include "vec.h"
-#include "logging.h"
-#include "tight.h"
 #include "config.h"
 #include "enc-util.h"
+#include "fb.h"
+#include "rcbuf.h"
+#include "encoder.h"
 
 #include <stdlib.h>
 #include <unistd.h>
@@ -53,6 +54,38 @@
 
 #define MAX_TILE_SIZE (2 * TSL * TSL * 4)
 
+struct encoder* tight_encoder_new(uint16_t width, uint16_t height);
+
+typedef void (*tight_done_fn)(struct vec* frame, void*);
+
+struct tight_encoder {
+	struct encoder encoder;
+
+	uint32_t width;
+	uint32_t height;
+	uint32_t grid_width;
+	uint32_t grid_height;
+	int quality;
+
+	struct tight_tile* grid;
+
+	z_stream zs[4];
+	struct aml_work* zs_worker[4];
+
+	struct rfb_pixel_format dfmt;
+	struct rfb_pixel_format sfmt;
+	struct nvnc_fb* fb;
+	uint64_t pts;
+
+	uint32_t n_rects;
+	uint32_t n_jobs;
+
+	struct vec dst;
+
+	tight_done_fn on_frame_done;
+	void* userdata;
+};
+
 enum tight_tile_state {
 	TIGHT_TILE_READY = 0,
 	TIGHT_TILE_DAMAGED,
@@ -71,18 +104,26 @@ struct tight_zs_worker_ctx {
 	int index;
 };
 
+struct encoder_impl encoder_impl_tight;
+
 static void do_tight_zs_work(void*);
 static void on_tight_zs_work_done(void*);
 static int schedule_tight_finish(struct tight_encoder* self);
 
+static inline struct tight_encoder* tight_encoder(struct encoder* encoder)
+{
+	assert(encoder->impl == &encoder_impl_tight);
+	return (struct tight_encoder*)encoder;
+}
+
 static int tight_encoder_init_stream(z_stream* zs)
 {
 	int rc = deflateInit2(zs,
-	                      /* compression level: */ 1,
-	                      /*            method: */ Z_DEFLATED,
-	                      /*       window bits: */ 15,
-	                      /*         mem level: */ 9,
-	                      /*          strategy: */ Z_DEFAULT_STRATEGY);
+			/* compression level: */ 1,
+			/*            method: */ Z_DEFLATED,
+			/*       window bits: */ 15,
+			/*         mem level: */ 9,
+			/*          strategy: */ Z_DEFAULT_STRATEGY);
 	return rc == Z_OK ? 0 : -1;
 }
 
@@ -125,7 +166,7 @@ failure:
 	return -1;
 }
 
-int tight_encoder_resize(struct tight_encoder* self, uint32_t width,
+static int tight_encoder_resize(struct tight_encoder* self, uint32_t width,
 		uint32_t height)
 {
 	self->width = width;
@@ -142,7 +183,7 @@ int tight_encoder_resize(struct tight_encoder* self, uint32_t width,
 	return self->grid ? 0 : -1;
 }
 
-int tight_encoder_init(struct tight_encoder* self, uint32_t width,
+static int tight_encoder_init(struct tight_encoder* self, uint32_t width,
 		uint32_t height)
 {
 	memset(self, 0, sizeof(*self));
@@ -161,10 +202,12 @@ int tight_encoder_init(struct tight_encoder* self, uint32_t width,
 
 	aml_require_workers(aml_get_default(), 1);
 
+	self->pts = NVNC_NO_PTS;
+
 	return 0;
 }
 
-void tight_encoder_destroy(struct tight_encoder* self)
+static void tight_encoder_destroy(struct tight_encoder* self)
 {
 	aml_unref(self->zs_worker[3]);
 	aml_unref(self->zs_worker[2]);
@@ -218,7 +261,7 @@ static void tight_encode_size(struct vec* dst, size_t size)
 }
 
 static int tight_deflate(struct tight_tile* tile, void* src,
-			 size_t len, z_stream* zs, bool flush)
+		size_t len, z_stream* zs, bool flush)
 {
 	zs->next_in = src;
 	zs->avail_in = len;
@@ -259,13 +302,14 @@ static void tight_encode_tile_basic(struct tight_encoder* self,
 	else
 		memcpy(&cfmt, &self->dfmt, sizeof(cfmt));
 
-	uint32_t* addr = nvnc_fb_get_addr(self->fb);
-	uint32_t stride = nvnc_fb_get_width(self->fb);
-
+	uint8_t* addr = nvnc_fb_get_addr(self->fb);
+	int32_t bpp = self->sfmt.bits_per_pixel / 8;
+	int32_t byte_stride = nvnc_fb_get_stride(self->fb) * bpp;
+	int32_t xoff = x * bpp;
 	// TODO: Limit width and hight to the sides
 	for (uint32_t y = y_start; y < y_start + height; ++y) {
-		void* img = addr + x + y * stride;
-		pixel32_to_cpixel(row, &cfmt, img, &self->sfmt,
+		uint8_t* img = addr + xoff + y * byte_stride;
+		pixel_to_cpixel(row, &cfmt, img, &self->sfmt,
 				bytes_per_cpixel, width);
 
 		// TODO What to do if the buffer fills up?
@@ -292,6 +336,10 @@ static enum TJPF tight_get_jpeg_pixfmt(uint32_t fourcc)
 	case DRM_FORMAT_ABGR8888:
 	case DRM_FORMAT_XBGR8888:
 		return TJPF_RGBX;
+	case DRM_FORMAT_BGR888:
+		return TJPF_RGB;
+	case DRM_FORMAT_RGB888:
+		return TJPF_BGR;
 	}
 
 	return TJPF_UNKNOWN;
@@ -306,13 +354,7 @@ static int tight_encode_tile_jpeg(struct tight_encoder* self,
 	unsigned char* buffer = NULL;
 	unsigned long size = 0;
 
-	int quality; /* 1 - 100 */
-
-	switch (self->quality) {
-	case TIGHT_QUALITY_HIGH: quality = 66; break;
-	case TIGHT_QUALITY_LOW: quality = 33; break;
-	default: abort();
-	}
+	int quality = 11 * self->quality + 1;
 
 	uint32_t fourcc = nvnc_fb_get_fourcc_format(self->fb);
 	enum TJPF tjfmt = tight_get_jpeg_pixfmt(fourcc);
@@ -323,20 +365,25 @@ static int tight_encode_tile_jpeg(struct tight_encoder* self,
 	if (!handle)
 		return -1;
 
-	uint32_t* addr = nvnc_fb_get_addr(self->fb);
-	uint32_t stride = nvnc_fb_get_width(self->fb);
-	void* img = (uint32_t*)addr + x + y * stride;
+	uint8_t* addr = nvnc_fb_get_addr(self->fb);
+	int32_t bpp = self->sfmt.bits_per_pixel / 8;
+	int32_t byte_stride = nvnc_fb_get_stride(self->fb) * bpp;
+	int32_t xoff = x * bpp;
+	uint8_t* img = addr + xoff + y * byte_stride;
+
+	enum TJSAMP subsampling = (quality == 9) ? TJSAMP_444 : TJSAMP_420;
 
 	int rc = -1;
-	rc = tjCompress2(handle, img, width, stride * 4, height, tjfmt, &buffer,
-			&size, TJSAMP_422, quality, TJFLAG_FASTDCT);
+	rc = tjCompress2(handle, img, width, byte_stride, height, tjfmt, &buffer,
+			&size, subsampling, quality, TJFLAG_FASTDCT);
 	if (rc < 0) {
-		log_error("Failed to encode tight JPEG box: %s\n", tjGetErrorStr());
+		nvnc_log(NVNC_LOG_ERROR, "Failed to encode tight JPEG box: %s",
+				tjGetErrorStr());
 		goto failure;
 	}
 
 	if (size > MAX_TILE_SIZE) {
-		log_error("Whoops, encoded JPEG was too big for the buffer\n");
+		nvnc_log(NVNC_LOG_ERROR, "Whoops, encoded JPEG was too big for the buffer");
 		goto failure;
 	}
 
@@ -366,17 +413,10 @@ static void tight_encode_tile(struct tight_encoder* self,
 	tile->size = 0;
 
 #ifdef HAVE_JPEG
-	switch (self->quality) {
-	case TIGHT_QUALITY_LOSSLESS:
+	if (self->quality >= 10) {
 		tight_encode_tile_basic(self, tile, x, y, width, height, gx % 4);
-		break;
-	case TIGHT_QUALITY_HIGH:
-	case TIGHT_QUALITY_LOW:
-		// TODO: Use more workers for jpeg
+	} else {
 		tight_encode_tile_jpeg(self, tile, x, y, width, height);
-		break;
-	case TIGHT_QUALITY_UNSPEC:
-		abort();
 	}
 #else
 	tight_encode_tile_basic(self, tile, x, y, width, height, gx % 4);
@@ -406,13 +446,19 @@ static void on_tight_zs_work_done(void* obj)
 		nvnc_fb_unref(self->fb);
 		schedule_tight_finish(self);
 	}
+
+	encoder_unref(&self->encoder);
 }
 
 static int tight_schedule_zs_work(struct tight_encoder* self, int index)
 {
+	encoder_ref(&self->encoder);
+
 	int rc = aml_start(aml_get_default(), self->zs_worker[index]);
 	if (rc >= 0)
 		++self->n_jobs;
+	else
+		encoder_unref(&self->encoder);
 
 	return rc;
 }
@@ -431,13 +477,17 @@ static void tight_finish_tile(struct tight_encoder* self,
 {
 	struct tight_tile* tile = tight_tile(self, gx, gy);
 
+	uint16_t x_pos = self->encoder.x_pos;
+	uint16_t y_pos = self->encoder.y_pos;
+
 	uint32_t x = gx * TSL;
 	uint32_t y = gy * TSL;
 
 	uint32_t width = tight_tile_width(self, x);
 	uint32_t height = tight_tile_height(self, y);
 
-	encode_rect_head(&self->dst, RFB_ENCODING_TIGHT, x, y, width, height);
+	encode_rect_head(&self->dst, RFB_ENCODING_TIGHT, x_pos + x, y_pos + y,
+			width, height);
 
 	vec_append(&self->dst, &tile->type, sizeof(tile->type));
 	tight_encode_size(&self->dst, tile->size);
@@ -456,7 +506,6 @@ static void tight_finish(struct tight_encoder* self)
 
 static void do_tight_finish(void* obj)
 {
-	// TODO: Make sure there's no use-after-free here
 	struct tight_encoder* self = aml_get_userdata(obj);
 	tight_finish(self);
 }
@@ -464,47 +513,104 @@ static void do_tight_finish(void* obj)
 static void on_tight_finished(void* obj)
 {
 	struct tight_encoder* self = aml_get_userdata(obj);
-	self->on_frame_done(&self->dst, self->userdata);
+
+	struct rcbuf* result = rcbuf_new(self->dst.data, self->dst.len);
+	assert(result);
+
+	self->encoder.n_rects = self->n_rects;
+
+	encoder_finish_frame(&self->encoder, result, self->pts);
+
+	self->pts = NVNC_NO_PTS;
+	rcbuf_unref(result);
+	encoder_unref(&self->encoder);
 }
 
 static int schedule_tight_finish(struct tight_encoder* self)
 {
+	encoder_ref(&self->encoder);
+
 	struct aml_work* work = aml_work_new(do_tight_finish, on_tight_finished,
 			self, NULL);
-	if (!work)
+	if (!work) {
+		encoder_unref(&self->encoder);
 		return -1;
+	}
 
 	int rc = aml_start(aml_get_default(), work);
 	aml_unref(work);
 	return rc;
 }
 
-int tight_encode_frame(struct tight_encoder* self,
-		const struct rfb_pixel_format* dfmt,
-		struct nvnc_fb* src,
-		const struct rfb_pixel_format* sfmt,
-		struct pixman_region16* damage,
-		enum tight_quality quality,
-		tight_done_fn on_done, void* userdata)
+struct encoder* tight_encoder_new(uint16_t width, uint16_t height)
 {
-	memcpy(&self->dfmt, dfmt, sizeof(self->dfmt));
-	memcpy(&self->sfmt, sfmt, sizeof(self->sfmt));
-	self->fb = src;
-	self->quality = quality;
-	self->on_frame_done = on_done;
-	self->userdata = userdata;
+	struct tight_encoder* self = calloc(1, sizeof(*self));
+	if (!self)
+		return NULL;
 
-	uint32_t width = nvnc_fb_get_width(src);
-	uint32_t height = nvnc_fb_get_height(src);
-	int rc = vec_init(&self->dst, width * height * 4);
+	if (tight_encoder_init(self, width, height) < 0) {
+		free(self);
+		return NULL;
+	}
+
+	encoder_init(&self->encoder, &encoder_impl_tight);
+
+	return (struct encoder*)self;
+}
+
+static void tight_encoder_destroy_wrapper(struct encoder* encoder)
+{
+	tight_encoder_destroy(tight_encoder(encoder));
+	free(encoder);
+}
+
+static void tight_encoder_set_output_format(struct encoder* encoder,
+		const struct rfb_pixel_format* pixfmt)
+{
+	struct tight_encoder* self = tight_encoder(encoder);
+	memcpy(&self->dfmt, pixfmt, sizeof(self->dfmt));
+}
+
+static void tight_encoder_set_quality(struct encoder* encoder, int value)
+{
+	struct tight_encoder* self = tight_encoder(encoder);
+	self->quality = value;
+}
+
+static int tight_encoder_resize_wrapper(struct encoder* encoder, uint16_t width,
+		uint16_t height)
+{
+	struct tight_encoder* self = tight_encoder(encoder);
+	return tight_encoder_resize(self, width, height);
+}
+
+static int tight_encoder_encode(struct encoder* encoder, struct nvnc_fb* fb,
+		struct pixman_region16* damage)
+{
+	struct tight_encoder* self = tight_encoder(encoder);
+	int rc;
+
+	self->encoder.n_rects = 0;
+
+	rc = rfb_pixfmt_from_fourcc(&self->sfmt, nvnc_fb_get_fourcc_format(fb));
+	assert(rc == 0);
+
+	self->fb = fb;
+	self->pts = nvnc_fb_get_pts(fb);
+
+	rc = nvnc_fb_map(self->fb);
+	if (rc < 0)
+		return -1;
+
+	uint32_t width = nvnc_fb_get_width(fb);
+	uint32_t height = nvnc_fb_get_height(fb);
+	rc = vec_init(&self->dst, width * height * 4);
 	if (rc < 0)
 		return -1;
 
 	self->n_rects = tight_apply_damage(self, damage);
 	assert(self->n_rects > 0);
 
-	encode_rect_count(&self->dst, self->n_rects);
-
 	nvnc_fb_ref(self->fb);
 
 	if (tight_schedule_encoding_jobs(self) < 0) {
@@ -515,3 +621,11 @@ int tight_encode_frame(struct tight_encoder* self,
 
 	return 0;
 }
+
+struct encoder_impl encoder_impl_tight = {
+	.destroy = tight_encoder_destroy_wrapper,
+	.set_output_format = tight_encoder_set_output_format,
+	.set_quality = tight_encoder_set_quality,
+	.resize = tight_encoder_resize_wrapper,
+	.encode = tight_encoder_encode,
+};

src/transform-util.c

@@ -0,0 +1,238 @@
+/*
+ * Copyright (c) 2020 - 2021 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ *
+ * For code borrowed from wlroots:
+ * Copyright (c) 2017, 2018 Drew DeVault
+ * Copyright (c) 2014 Jari Vetoniemi
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "transform-util.h"
+#include "neatvnc.h"
+
+#include <stdlib.h>
+#include <pixman.h>
+
+/* Note: This function yields the inverse pixman transform of the
+ * nvnc_transform.
+ */
+void nvnc_transform_to_pixman_transform(pixman_transform_t* dst,
+		enum nvnc_transform src, int width, int height)
+{
+#define F1 pixman_fixed_1
+	switch (src) {
+	case NVNC_TRANSFORM_NORMAL:
+		{
+			pixman_transform_t t = {{
+				{ F1, 0, 0 },
+				{ 0, F1, 0 },
+				{ 0, 0, F1 },
+			}};
+			*dst = t;
+		}
+		return;
+	case NVNC_TRANSFORM_90:
+		{
+			pixman_transform_t t = {{
+				{ 0, F1, 0 },
+				{ -F1, 0, height * F1 },
+				{ 0, 0, F1 },
+			}};
+			*dst = t;
+		}
+		return;
+	case NVNC_TRANSFORM_180:
+		{
+			pixman_transform_t t = {{
+				{ -F1, 0, width * F1 },
+				{ 0, -F1, height * F1 },
+				{ 0, 0, F1 },
+			}};
+			*dst = t;
+		}
+		return;
+	case NVNC_TRANSFORM_270:
+		{
+			pixman_transform_t t = {{
+				{ 0, -F1, width * F1 },
+				{ F1, 0, 0 },
+				{ 0, 0, F1 },
+			}};
+			*dst = t;
+		}
+		return;
+	case NVNC_TRANSFORM_FLIPPED:
+		{
+			pixman_transform_t t = {{
+				{ -F1, 0, width * F1 },
+				{ 0, F1, 0 },
+				{ 0, 0, F1 },
+			}};
+			*dst = t;
+		}
+		return;
+	case NVNC_TRANSFORM_FLIPPED_90:
+		{
+			pixman_transform_t t = {{
+				{ 0, F1, 0 },
+				{ F1, 0, 0 },
+				{ 0, 0, F1 },
+			}};
+			*dst = t;
+		}
+		return;
+	case NVNC_TRANSFORM_FLIPPED_180:
+		{
+			pixman_transform_t t = {{
+				{ F1, 0, 0 },
+				{ 0, -F1, height * F1 },
+				{ 0, 0, F1 },
+			}};
+			*dst = t;
+		}
+		return;
+	case NVNC_TRANSFORM_FLIPPED_270:
+		{
+			pixman_transform_t t = {{
+				{ 0, -F1, width * F1 },
+				{ -F1, 0, height * F1 },
+				{ 0, 0, F1 },
+			}};
+			*dst = t;
+		}
+		return;
+	}
+#undef F1
+
+	abort();
+}
+
+static bool is_transform_90_degrees(enum nvnc_transform transform)
+{
+	switch (transform) {
+	case NVNC_TRANSFORM_90:
+	case NVNC_TRANSFORM_270:
+	case NVNC_TRANSFORM_FLIPPED_90:
+	case NVNC_TRANSFORM_FLIPPED_270:
+		return true;
+	default:
+		break;
+	}
+
+	return false;
+}
+
+void nvnc_transform_dimensions(enum nvnc_transform transform, uint32_t* width,
+		uint32_t* height)
+{
+	if (is_transform_90_degrees(transform)) {
+		uint32_t tmp = *width;
+		*width = *height;
+		*height = tmp;
+	}
+}
+
+/* Borrowed this from wlroots */
+void nvnc_transform_region(struct pixman_region16* dst,
+		struct pixman_region16* src, enum nvnc_transform transform,
+		int width, int height)
+{
+	if (transform == NVNC_TRANSFORM_NORMAL) {
+		pixman_region_copy(dst, src);
+		return;
+	}
+
+	int nrects = 0;
+	pixman_box16_t* src_rects = pixman_region_rectangles(src, &nrects);
+
+	pixman_box16_t* dst_rects = malloc(nrects * sizeof(*dst_rects));
+	if (dst_rects == NULL) {
+		return;
+	}
+
+	for (int i = 0; i < nrects; ++i) {
+		switch (transform) {
+		case NVNC_TRANSFORM_NORMAL:
+			dst_rects[i].x1 = src_rects[i].x1;
+			dst_rects[i].y1 = src_rects[i].y1;
+			dst_rects[i].x2 = src_rects[i].x2;
+			dst_rects[i].y2 = src_rects[i].y2;
+			break;
+		case NVNC_TRANSFORM_90:
+			dst_rects[i].x1 = height - src_rects[i].y2;
+			dst_rects[i].y1 = src_rects[i].x1;
+			dst_rects[i].x2 = height - src_rects[i].y1;
+			dst_rects[i].y2 = src_rects[i].x2;
+			break;
+		case NVNC_TRANSFORM_180:
+			dst_rects[i].x1 = width - src_rects[i].x2;
+			dst_rects[i].y1 = height - src_rects[i].y2;
+			dst_rects[i].x2 = width - src_rects[i].x1;
+			dst_rects[i].y2 = height - src_rects[i].y1;
+			break;
+		case NVNC_TRANSFORM_270:
+			dst_rects[i].x1 = src_rects[i].y1;
+			dst_rects[i].y1 = width - src_rects[i].x2;
+			dst_rects[i].x2 = src_rects[i].y2;
+			dst_rects[i].y2 = width - src_rects[i].x1;
+			break;
+		case NVNC_TRANSFORM_FLIPPED:
+			dst_rects[i].x1 = width - src_rects[i].x2;
+			dst_rects[i].y1 = src_rects[i].y1;
+			dst_rects[i].x2 = width - src_rects[i].x1;
+			dst_rects[i].y2 = src_rects[i].y2;
+			break;
+		case NVNC_TRANSFORM_FLIPPED_90:
+			dst_rects[i].x1 = src_rects[i].y1;
+			dst_rects[i].y1 = src_rects[i].x1;
+			dst_rects[i].x2 = src_rects[i].y2;
+			dst_rects[i].y2 = src_rects[i].x2;
+			break;
+		case NVNC_TRANSFORM_FLIPPED_180:
+			dst_rects[i].x1 = src_rects[i].x1;
+			dst_rects[i].y1 = height - src_rects[i].y2;
+			dst_rects[i].x2 = src_rects[i].x2;
+			dst_rects[i].y2 = height - src_rects[i].y1;
+			break;
+		case NVNC_TRANSFORM_FLIPPED_270:
+			dst_rects[i].x1 = height - src_rects[i].y2;
+			dst_rects[i].y1 = width - src_rects[i].x2;
+			dst_rects[i].x2 = height - src_rects[i].y1;
+			dst_rects[i].y2 = width - src_rects[i].x1;
+			break;
+		}
+	}
+
+	pixman_region_fini(dst);
+	pixman_region_init_rects(dst, dst_rects, nrects);
+	free(dst_rects);
+}

src/ws-framing.c

@@ -0,0 +1,153 @@
+/*
+ * Copyright (c) 2023 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "websocket.h"
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <unistd.h>
+#include <string.h>
+#include <assert.h>
+#include <arpa/inet.h>
+
+static inline uint64_t u64_from_network_order(uint64_t x)
+{
+#if __BYTE_ORDER__ == __BIG_ENDIAN__
+	return x;
+#else
+	return __builtin_bswap64(x);
+#endif
+}
+
+static inline uint64_t u64_to_network_order(uint64_t x)
+{
+#if __BYTE_ORDER__ == __BIG_ENDIAN__
+	return x;
+#else
+	return __builtin_bswap64(x);
+#endif
+}
+
+const char *ws_opcode_name(enum ws_opcode op)
+{
+	switch (op) {
+	case WS_OPCODE_CONT: return "cont";
+	case WS_OPCODE_TEXT: return "text";
+	case WS_OPCODE_BIN: return "bin";
+	case WS_OPCODE_CLOSE: return "close";
+	case WS_OPCODE_PING: return "ping";
+	case WS_OPCODE_PONG: return "pong";
+	}
+	return "INVALID";
+}
+
+bool ws_parse_frame_header(struct ws_frame_header* header,
+		const uint8_t* payload, size_t length)
+{
+	if (length < 2)
+		return false;
+
+	int i = 0;
+
+	header->fin = !!(payload[i] & 0x80);
+	header->opcode = (payload[i++] & 0x0f);
+	header->mask = !!(payload[i] & 0x80);
+	header->payload_length = payload[i++] & 0x7f;
+
+	if (header->payload_length == 126) {
+		if (length - i < 2)
+			return false;
+
+		uint16_t value = 0;
+		memcpy(&value, &payload[i], 2);
+		header->payload_length = ntohs(value);
+		i += 2;
+	} else if (header->payload_length == 127) {
+		if (length - i < 8)
+			return false;
+
+		uint64_t value = 0;
+		memcpy(&value, &payload[i], 8);
+		header->payload_length = u64_from_network_order(value);
+		i += 8;
+	}
+
+	if (header->mask) {
+		if (length - i < 4)
+			return false;
+
+		memcpy(header->masking_key, &payload[i], 4);
+		i += 4;
+	}
+
+	header->header_length = i;
+
+	return true;
+}
+
+void ws_apply_mask(const struct ws_frame_header* header,
+		uint8_t* restrict payload)
+{
+	assert(header->mask);
+
+	uint64_t len = header->payload_length;
+	const uint8_t* restrict key = header->masking_key;
+
+	for (uint64_t i = 0; i < len; ++i) {
+		payload[i] ^= key[i % 4];
+	}
+}
+
+void ws_copy_payload(const struct ws_frame_header* header,
+		uint8_t* restrict dst, const uint8_t* restrict src, size_t len)
+{
+	if (!header->mask) {
+		memcpy(dst, src, len);
+		return;
+	}
+
+	const uint8_t* restrict key = header->masking_key;
+	for (uint64_t i = 0; i < len; ++i) {
+		dst[i] = src[i] ^ key[i % 4];
+	}
+}
+
+int ws_write_frame_header(uint8_t* dst, const struct ws_frame_header* header)
+{
+	int i = 0;
+	dst[i++] = ((uint8_t)header->fin << 7) | (header->opcode);
+
+	if (header->payload_length <= 125) {
+		dst[i++] = ((uint8_t)header->mask << 7) | header->payload_length;
+	} else if (header->payload_length <= UINT16_MAX) {
+		dst[i++] = ((uint8_t)header->mask << 7) | 126;
+		uint16_t be = htons(header->payload_length);
+		memcpy(&dst[i], &be, 2);
+		i += 2;
+	} else {
+		dst[i++] = ((uint8_t)header->mask << 7) | 127;
+		uint64_t be = u64_to_network_order(header->payload_length);
+		memcpy(&dst[i], &be, 8);
+		i += 8;
+	}
+
+	if (header->mask) {
+		memcpy(dst, header->masking_key, 4);
+		i += 4;
+	}
+
+	return i;
+}

src/ws-handshake.c

@@ -0,0 +1,106 @@
+/*
+ * Copyright (c) 2023 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "websocket.h"
+#include "http.h"
+#include "crypto.h"
+#include "base64.h"
+
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdbool.h>
+#include <ctype.h>
+
+static const char magic_uuid[] = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11";
+
+static void tolower_and_remove_ws(char* dst, const char* src)
+{
+	while (*src)
+		if (!isspace(*src))
+			*dst++ = tolower(*src++);
+	*dst = '\0';
+}
+
+// TODO: Do some more sanity checks on the input
+ssize_t ws_handshake(char* output, size_t output_maxlen, const char* input)
+{
+	bool ok = false;
+	struct http_req req = {};
+	if (http_req_parse(&req, input) < 0)
+		return -1;
+
+	char protocols[256] = ",";
+	char versions[256] = ",";
+	char tmpstring[256];
+
+	const char *challenge = NULL;
+	for (size_t i = 0; i < req.field_index; ++i) {
+		if (strcasecmp(req.field[i].key, "Sec-WebSocket-Key") == 0) {
+			challenge = req.field[i].value;
+		}
+		if (strcasecmp(req.field[i].key, "Sec-WebSocket-Protocol") == 0) {
+			snprintf(tmpstring, sizeof(tmpstring), "%s%s,",
+					protocols, req.field[i].value);
+			tolower_and_remove_ws(protocols, tmpstring);
+		}
+		if (strcasecmp(req.field[i].key, "Sec-WebSocket-Version") == 0) {
+			snprintf(tmpstring, sizeof(tmpstring), "%s%s,",
+					versions, req.field[i].value);
+			tolower_and_remove_ws(versions, tmpstring);
+		}
+	}
+
+	if (!challenge)
+		goto failure;
+
+	bool have_protocols = strlen(protocols) != 1;
+	bool have_versions = strlen(versions) != 1;
+
+	if (have_protocols && !strstr(protocols, ",chat,"))
+		goto failure;
+
+	if (have_versions && !strstr(versions, ",13,"))
+		goto failure;
+
+	uint8_t hash[20];
+	crypto_hash_many(hash, sizeof(hash), CRYPTO_HASH_SHA1,
+			(struct crypto_data_entry[]){
+		{ (uint8_t*)challenge, strlen(challenge) },
+		{ (uint8_t*)magic_uuid, strlen(magic_uuid) },
+		{}
+	});
+
+	char response[BASE64_ENCODED_SIZE(sizeof(hash))] = {};
+	base64_encode(response, hash, sizeof(hash));
+
+	size_t len = snprintf(output, output_maxlen,
+			"HTTP/1.1 101 Switching Protocols\r\n"
+			"Upgrade: websocket\r\n"
+			"Connection: Upgrade\r\n"
+			"Sec-WebSocket-Accept: %s\r\n"
+			"%s%s"
+			"\r\n",
+			response,
+			have_protocols ? "Sec-WebSocket-Protocol: char\r\n" : "",
+			have_versions ? "Sec-WebSocket-Version: 13\r\n" : "");
+
+	ssize_t header_len = req.header_length;
+	ok = len < output_maxlen;
+failure:
+	http_req_free(&req);
+	return ok ? header_len : -1;
+}

src/zrle.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019 - 2020 Andri Yngvason
+ * Copyright (c) 2019 - 2022 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -16,11 +16,12 @@
 
 #include "rfb-proto.h"
 #include "vec.h"
-#include "zrle.h"
 #include "neatvnc.h"
 #include "pixels.h"
 #include "fb.h"
 #include "enc-util.h"
+#include "encoder.h"
+#include "rcbuf.h"
 
 #include <stdint.h>
 #include <unistd.h>
@@ -29,37 +30,63 @@
 #include <assert.h>
 #include <pixman.h>
 #include <zlib.h>
+#include <aml.h>
 
 #define TILE_LENGTH 64
 
 #define UDIV_UP(a, b) (((a) + (b) - 1) / (b))
 
-static inline int find_colour_in_palette(uint32_t* palette, int len,
-                                         uint32_t colour)
+struct encoder* zrle_encoder_new(void);
+
+struct zrle_encoder {
+	struct encoder encoder;
+
+	struct rfb_pixel_format output_format;
+
+	struct nvnc_fb* current_fb;
+	struct pixman_region16 current_damage;
+
+	struct rcbuf *current_result;
+
+	z_stream zs;
+
+	struct aml_work* work;
+};
+
+struct encoder_impl encoder_impl_zrle;
+
+static inline struct zrle_encoder* zrle_encoder(struct encoder* encoder)
+{
+	assert(encoder->impl == &encoder_impl_zrle);
+	return (struct zrle_encoder*)encoder;
+}
+
+static inline int find_colour_in_palette(uint8_t* palette, int len,
+		const uint8_t* colour, int bpp)
 {
 	for (int i = 0; i < len; ++i)
-		if (palette[i] == colour)
+		if (memcmp(palette + i * bpp, colour, bpp) == 0)
 			return i;
 
 	return -1;
 }
 
-static int zrle_get_tile_palette(uint32_t* palette, const uint32_t* src,
-                                 size_t length)
+static int zrle_get_tile_palette(uint8_t* palette, const uint8_t* src,
+		const int src_bpp, size_t length)
 {
 	int n = 0;
 
 	/* TODO: Maybe ignore the alpha channel */
-	palette[n++] = src[0];
+	memcpy(palette + (n++ * src_bpp), src, src_bpp);
 
 	for (size_t i = 0; i < length; ++i) {
-		uint32_t colour = src[i];
+		const uint8_t* colour_addr = src + i * src_bpp;
 
-		if (find_colour_in_palette(palette, n, colour) < 0) {
+		if (find_colour_in_palette(palette, n, colour_addr, src_bpp) < 0) {
 			if (n >= 16)
 				return -1;
 
-			palette[n++] = colour;
+			memcpy(palette + (n++ * src_bpp), colour_addr, src_bpp);
 		}
 	}
 
@@ -67,16 +94,16 @@ static int zrle_get_tile_palette(uint32_t* palette, const uint32_t* src,
 }
 
 static void zrle_encode_unichrome_tile(struct vec* dst,
-                                       const struct rfb_pixel_format* dst_fmt,
-                                       uint32_t colour,
-                                       const struct rfb_pixel_format* src_fmt)
+		const struct rfb_pixel_format* dst_fmt,
+		uint8_t* colour,
+		const struct rfb_pixel_format* src_fmt)
 {
 	int bytes_per_cpixel = calc_bytes_per_cpixel(dst_fmt);
 
 	vec_fast_append_8(dst, 1);
 
-	pixel32_to_cpixel(((uint8_t*)dst->data) + 1, dst_fmt, &colour, src_fmt,
-	                  bytes_per_cpixel, 1);
+	pixel_to_cpixel(((uint8_t*)dst->data) + 1, dst_fmt, colour, src_fmt,
+			bytes_per_cpixel, 1);
 
 	dst->len += bytes_per_cpixel;
 }
@@ -99,17 +126,18 @@ static void encode_run_length(struct vec* dst, uint8_t index, int run_length)
 }
 
 static void zrle_encode_packed_tile(struct vec* dst,
-                                    const struct rfb_pixel_format* dst_fmt,
-                                    const uint32_t* src,
-                                    const struct rfb_pixel_format* src_fmt,
-                                    size_t length, uint32_t* palette,
-                                    int palette_size)
+		const struct rfb_pixel_format* dst_fmt,
+		const uint8_t* src,
+		const struct rfb_pixel_format* src_fmt,
+		size_t length, uint8_t* palette,
+		int palette_size)
 {
 	int bytes_per_cpixel = calc_bytes_per_cpixel(dst_fmt);
+	int src_bpp = src_fmt->bits_per_pixel / 8;
 
 	uint8_t cpalette[16 * 3];
-	pixel32_to_cpixel((uint8_t*)cpalette, dst_fmt, palette, src_fmt,
-	                  bytes_per_cpixel, palette_size);
+	pixel_to_cpixel(cpalette, dst_fmt, palette, src_fmt,
+			bytes_per_cpixel, palette_size);
 
 	vec_fast_append_8(dst, 128 | palette_size);
 
@@ -119,64 +147,65 @@ static void zrle_encode_packed_tile(struct vec* dst,
 	int run_length = 1;
 
 	for (size_t i = 1; i < length; ++i) {
-		if (src[i] == src[i - 1]) {
+		if (memcmp(src + i * src_bpp, src + (i - 1) * src_bpp, src_bpp) == 0) {
 			run_length++;
 			continue;
 		}
 
-		index = find_colour_in_palette(palette, palette_size, src[i - 1]);
+		index = find_colour_in_palette(palette, palette_size, src + (i - 1) * src_bpp, src_bpp);
 		encode_run_length(dst, index, run_length);
 		run_length = 1;
 	}
 
 	if (run_length > 0) {
 		index = find_colour_in_palette(palette, palette_size,
-		                               src[length - 1]);
+				src + (length - 1) * src_bpp, src_bpp);
 		encode_run_length(dst, index, run_length);
 	}
 }
 
-static void zrle_copy_tile(uint32_t* dst, const uint32_t* src, int stride,
-                           int width, int height)
+static void zrle_copy_tile(uint8_t* tile, const uint8_t* src, int src_bpp,
+		int stride, int width, int height)
 {
+	int byte_stride = stride * src_bpp;
 	for (int y = 0; y < height; ++y)
-		memcpy(dst + y * width, src + y * stride, width * 4);
+		memcpy(tile + y * width * src_bpp, src + y * byte_stride, width * src_bpp);
 }
 
 static void zrle_encode_tile(struct vec* dst,
-                             const struct rfb_pixel_format* dst_fmt,
-                             const uint32_t* src,
-                             const struct rfb_pixel_format* src_fmt,
-                             size_t length)
+		const struct rfb_pixel_format* dst_fmt,
+		const uint8_t* src,
+		const struct rfb_pixel_format* src_fmt,
+		size_t length)
 {
 	int bytes_per_cpixel = calc_bytes_per_cpixel(dst_fmt);
-
+	int src_bpp = src_fmt->bits_per_pixel / 8;
 	vec_clear(dst);
 
-	uint32_t palette[16];
-	int palette_size = zrle_get_tile_palette(palette, src, length);
+	uint8_t palette[16 * 4];
+	int palette_size = zrle_get_tile_palette(palette, src, src_bpp, length);
 
 	if (palette_size == 1) {
-		zrle_encode_unichrome_tile(dst, dst_fmt, palette[0], src_fmt);
+		zrle_encode_unichrome_tile(dst, dst_fmt, &palette[0], src_fmt);
 		return;
 	}
 
 	if (palette_size > 1) {
 		zrle_encode_packed_tile(dst, dst_fmt, src, src_fmt, length,
-		                        palette, palette_size);
+				palette, palette_size);
 		return;
 	}
 
 	vec_fast_append_8(dst, 0);
 
-	pixel32_to_cpixel(((uint8_t*)dst->data) + 1, dst_fmt, src, src_fmt,
-	                  bytes_per_cpixel, length);
+	pixel_to_cpixel(((uint8_t*)dst->data) + 1, dst_fmt, (uint8_t*)src, src_fmt,
+			bytes_per_cpixel, length);
 
 	dst->len += bytes_per_cpixel * length;
 }
 
 static int zrle_deflate(struct vec* dst, const struct vec* src, z_stream* zs,
-                        bool flush)
+		bool flush)
 {
 	zs->next_in = src->data;
 	zs->avail_in = src->len;
@@ -200,24 +229,29 @@ static int zrle_deflate(struct vec* dst, const struct vec* src, z_stream* zs,
 	return 0;
 }
 
-static int zrle_encode_box(struct vec* out,
-                           const struct rfb_pixel_format* dst_fmt,
-                           const struct nvnc_fb* fb,
-                           const struct rfb_pixel_format* src_fmt, int x, int y,
-                           int stride, int width, int height, z_stream* zs)
+static int zrle_encode_box(struct zrle_encoder* self, struct vec* out,
+		const struct rfb_pixel_format* dst_fmt,
+		const struct nvnc_fb* fb,
+		const struct rfb_pixel_format* src_fmt, int x, int y,
+		int stride, int width, int height, z_stream* zs)
 {
 	int r = -1;
 	int bytes_per_cpixel = calc_bytes_per_cpixel(dst_fmt);
+	int src_bpp = src_fmt->bits_per_pixel / 8;
 	struct vec in;
 
-	uint32_t* tile = malloc(TILE_LENGTH * TILE_LENGTH * 4);
+	uint16_t x_pos = self->encoder.x_pos;
+	uint16_t y_pos = self->encoder.y_pos;
+
+	uint8_t* tile = malloc(TILE_LENGTH * TILE_LENGTH * 4);
 	if (!tile)
 		goto failure;
 
 	if (vec_init(&in, 1 + bytes_per_cpixel * TILE_LENGTH * TILE_LENGTH) < 0)
 		goto failure;
 
-	r = encode_rect_head(out, RFB_ENCODING_ZRLE, x, y, width, height);
+	r = encode_rect_head(out, RFB_ENCODING_ZRLE, x_pos + x, y_pos + y,
+			width, height);
 	if (r < 0)
 		goto failure;
 
@@ -237,14 +271,15 @@ static int zrle_encode_box(struct vec* out,
 		                          ? TILE_LENGTH
 		                          : height - tile_y;
 
-		int y_off = y + tile_y;
+		int y_off = (y + tile_y) * stride * src_bpp;
+		int x_off = (x + tile_x) * src_bpp;
 
 		zrle_copy_tile(tile,
-		               ((uint32_t*)fb->addr) + x + tile_x + y_off * stride,
-		               stride, tile_width, tile_height);
+				((uint8_t*)fb->addr) + x_off + y_off, src_bpp,
+				stride, tile_width, tile_height);
 
 		zrle_encode_tile(&in, dst_fmt, tile, src_fmt,
-		                 tile_width * tile_height);
+				tile_width * tile_height);
 
 		r = zrle_deflate(out, &in, zs, i == n_tiles - 1);
 		if (r < 0)
@@ -261,14 +296,15 @@ failure:
 #undef CHUNK
 }
 
-int zrle_encode_frame(z_stream* zs, struct vec* dst,
-                      const struct rfb_pixel_format* dst_fmt,
-                      const struct nvnc_fb* src,
-                      const struct rfb_pixel_format* src_fmt,
-                      struct pixman_region16* region)
+static int zrle_encode_frame(struct zrle_encoder* self, z_stream* zs,
+		struct vec* dst, const struct rfb_pixel_format* dst_fmt,
+		struct nvnc_fb* src, const struct rfb_pixel_format* src_fmt,
+		struct pixman_region16* region)
 {
 	int rc = -1;
 
+	self->encoder.n_rects = 0;
+
 	int n_rects = 0;
 	struct pixman_box16* box = pixman_region_rectangles(region, &n_rects);
 	if (n_rects > UINT16_MAX) {
@@ -276,7 +312,7 @@ int zrle_encode_frame(z_stream* zs, struct vec* dst,
 		n_rects = 1;
 	}
 
-	rc = encode_rect_count(dst, n_rects);
+	rc = nvnc_fb_map(src);
 	if (rc < 0)
 		return -1;
 
@@ -286,11 +322,147 @@ int zrle_encode_frame(z_stream* zs, struct vec* dst,
 		int box_width = box[i].x2 - x;
 		int box_height = box[i].y2 - y;
 
-		rc = zrle_encode_box(dst, dst_fmt, src, src_fmt, x, y,
-		                     src->width, box_width, box_height, zs);
+		rc = zrle_encode_box(self, dst, dst_fmt, src, src_fmt, x, y,
+				src->stride, box_width, box_height, zs);
 		if (rc < 0)
 			return -1;
 	}
 
+	self->encoder.n_rects = n_rects;
 	return 0;
 }
+
+static void zrle_encoder_do_work(void* obj)
+{
+	struct zrle_encoder* self = aml_get_userdata(obj);
+	int rc;
+
+	struct nvnc_fb* fb = self->current_fb;
+	assert(fb);
+
+	// TODO: Calculate the ideal buffer size based on the size of the
+	// damaged area.
+	size_t buffer_size = nvnc_fb_get_stride(fb) * nvnc_fb_get_height(fb) *
+		nvnc_fb_get_pixel_size(fb);
+
+	struct vec dst;
+	rc = vec_init(&dst, buffer_size);
+	assert(rc == 0);
+
+	struct rfb_pixel_format src_fmt;
+	rc = rfb_pixfmt_from_fourcc(&src_fmt, nvnc_fb_get_fourcc_format(fb));
+	assert(rc == 0);
+
+	rc = zrle_encode_frame(self, &self->zs, &dst, &self->output_format, fb,
+			&src_fmt, &self->current_damage);
+	assert(rc == 0);
+
+	self->current_result = rcbuf_new(dst.data, dst.len);
+	assert(self->current_result);
+}
+
+static void zrle_encoder_on_done(void* obj)
+{
+	struct zrle_encoder* self = aml_get_userdata(obj);
+
+	assert(self->current_result);
+
+	uint64_t pts = nvnc_fb_get_pts(self->current_fb);
+	nvnc_fb_unref(self->current_fb);
+	self->current_fb = NULL;
+
+	pixman_region_clear(&self->current_damage);
+
+	struct rcbuf* result = self->current_result;
+	self->current_result = NULL;
+
+	aml_unref(self->work);
+	self->work = NULL;
+
+	encoder_finish_frame(&self->encoder, result, pts);
+
+	rcbuf_unref(result);
+	encoder_unref(&self->encoder);
+}
+
+struct encoder* zrle_encoder_new(void)
+{
+	struct zrle_encoder* self = calloc(1, sizeof(*self));
+	if (!self)
+		return NULL;
+
+	encoder_init(&self->encoder, &encoder_impl_zrle);
+
+	int rc = deflateInit2(&self->zs,
+			/* compression level: */ 1,
+			/*            method: */ Z_DEFLATED,
+			/*       window bits: */ 15,
+			/*         mem level: */ 9,
+			/*          strategy: */ Z_DEFAULT_STRATEGY);
+	if (rc != Z_OK)
+		goto deflate_failure;
+
+	pixman_region_init(&self->current_damage);
+
+	return (struct encoder*)self;
+
+deflate_failure:
+	free(self);
+	return NULL;
+}
+
+static void zrle_encoder_destroy(struct encoder* encoder)
+{
+	struct zrle_encoder* self = zrle_encoder(encoder);
+	pixman_region_fini(&self->current_damage);
+	deflateEnd(&self->zs);
+	if (self->work)
+		aml_unref(self->work);
+	if (self->current_result)
+		rcbuf_unref(self->current_result);
+	free(self);
+}
+
+static void zrle_encoder_set_output_format(struct encoder* encoder,
+		const struct rfb_pixel_format* pixfmt)
+{
+	struct zrle_encoder* self = zrle_encoder(encoder);
+	memcpy(&self->output_format, pixfmt, sizeof(self->output_format));
+}
+
+static int zrle_encoder_encode(struct encoder* encoder, struct nvnc_fb* fb,
+		struct pixman_region16* damage)
+{
+	struct zrle_encoder* self = zrle_encoder(encoder);
+
+	assert(!self->current_fb);
+
+	self->work = aml_work_new(zrle_encoder_do_work, zrle_encoder_on_done,
+			self, NULL);
+	if (!self->work)
+		return -1;
+
+	self->current_fb = fb;
+	nvnc_fb_ref(self->current_fb);
+	pixman_region_copy(&self->current_damage, damage);
+
+	encoder_ref(&self->encoder);
+
+	int rc = aml_start(aml_get_default(), self->work);
+	if (rc < 0) {
+		encoder_unref(&self->encoder);
+		aml_unref(self->work);
+		self->work = NULL;
+		pixman_region_clear(&self->current_damage);
+		nvnc_fb_unref(self->current_fb);
+		self->current_fb = NULL;
+	}
+
+	return rc;
+}
+
+struct encoder_impl encoder_impl_zrle = {
+	.destroy = zrle_encoder_destroy,
+	.set_output_format = zrle_encoder_set_output_format,
+	.encode = zrle_encoder_encode,
+};

test/meson.build

@@ -0,0 +1,22 @@
+pixels = executable('pixels',
+	[
+		'test-pixels.c',
+		'../src/pixels.c',
+	],
+	include_directories: inc,
+	dependencies: [
+		pixman,
+		libdrm_inc,
+		libm,
+	],
+)
+test('pixels', pixels)
+
+base64 = executable('base64',
+	[
+		'test-base64.c',
+		'../src/base64.c',
+	],
+	include_directories: inc,
+)
+test('base64', base64)

test/test-base64.c

@@ -0,0 +1,129 @@
+#include "base64.h"
+
+#include <stdio.h>
+#include <stdbool.h>
+#include <string.h>
+
+static bool test_encode_0(void)
+{
+	char buf[1] = {};
+	base64_encode(buf, (const uint8_t*)"", 0);
+	return strlen(buf) == 0;
+}
+
+static bool test_encode_1(void)
+{
+	static const char input[] = "a";
+	char buf[BASE64_ENCODED_SIZE(sizeof(input) - 1)] = {};
+	base64_encode(buf, (const uint8_t*)input, sizeof(input) - 1);
+	return strcmp(buf, "YQ==") == 0;
+}
+
+static bool test_encode_2(void)
+{
+	static const char input[] = "ab";
+	char buf[BASE64_ENCODED_SIZE(sizeof(input) - 1)] = {};
+	base64_encode(buf, (const uint8_t*)input, sizeof(input) - 1);
+	return strcmp(buf, "YWI=") == 0;
+}
+
+static bool test_encode_3(void)
+{
+	static const char input[] = "abc";
+	char buf[BASE64_ENCODED_SIZE(sizeof(input) - 1)] = {};
+	base64_encode(buf, (const uint8_t*)input, sizeof(input) - 1);
+	return strcmp(buf, "YWJj") == 0;
+}
+
+static bool test_encode_4(void)
+{
+	static const char input[] = "abcd";
+	char buf[BASE64_ENCODED_SIZE(sizeof(input) - 1)] = {};
+	base64_encode(buf, (const uint8_t*)input, sizeof(input) - 1);
+	return strcmp(buf, "YWJjZA==") == 0;
+}
+
+static bool test_encode_5(void)
+{
+	static const char input[] = "abcde";
+	char buf[BASE64_ENCODED_SIZE(sizeof(input) - 1)] = {};
+	base64_encode(buf, (const uint8_t*)input, sizeof(input) - 1);
+	return strcmp(buf, "YWJjZGU=") == 0;
+}
+
+static bool test_decode_0(void)
+{
+	uint8_t buf[1] = {};
+	ssize_t r = base64_decode(buf, "");
+	return r == 0 && buf[0] == 0;
+}
+
+static bool test_decode_1(void)
+{
+	static const char input[] = "YQ==";
+	uint8_t buf[BASE64_DECODED_MAX_SIZE(sizeof(input) - 1)] = {};
+	ssize_t r = base64_decode(buf, input);
+	return r == 1 && memcmp(buf, "a", r) == 0;
+}
+
+static bool test_decode_2(void)
+{
+	static const char input[] = "YWI=";
+	uint8_t buf[BASE64_DECODED_MAX_SIZE(sizeof(input) - 1)] = {};
+	ssize_t r = base64_decode(buf, input);
+	return r == 2 && memcmp(buf, "ab", r) == 0;
+}
+
+static bool test_decode_3(void)
+{
+	static const char input[] = "YWJj";
+	uint8_t buf[BASE64_DECODED_MAX_SIZE(sizeof(input) - 1)] = {};
+	ssize_t r = base64_decode(buf, input);
+	return r == 3 && memcmp(buf, "abc", r) == 0;
+}
+
+static bool test_decode_4(void)
+{
+	static const char input[] = "YWJjZA==";
+	uint8_t buf[BASE64_DECODED_MAX_SIZE(sizeof(input) - 1)] = {};
+	ssize_t r = base64_decode(buf, input);
+	return r == 4 && memcmp(buf, "abcd", r) == 0;
+}
+
+static bool test_decode_5(void)
+{
+	static const char input[] = "YWJjZGU=";
+	uint8_t buf[BASE64_DECODED_MAX_SIZE(sizeof(input) - 1)] = {};
+	ssize_t r = base64_decode(buf, input);
+	return r == 5 && memcmp(buf, "abcde", r) == 0;
+}
+
+#define XSTR(s) STR(s)
+#define STR(s) #s
+
+#define RUN_TEST(name) ({ \
+	bool ok = test_ ## name(); \
+	printf("[%s] %s\n", ok ? " OK " : "FAIL", XSTR(name)); \
+	ok; \
+})
+
+int main()
+{
+	bool ok = true;
+
+	ok &= RUN_TEST(encode_0);
+	ok &= RUN_TEST(encode_1);
+	ok &= RUN_TEST(encode_2);
+	ok &= RUN_TEST(encode_3);
+	ok &= RUN_TEST(encode_4);
+	ok &= RUN_TEST(encode_5);
+
+	ok &= RUN_TEST(decode_0);
+	ok &= RUN_TEST(decode_1);
+	ok &= RUN_TEST(decode_2);
+	ok &= RUN_TEST(decode_3);
+	ok &= RUN_TEST(decode_4);
+	ok &= RUN_TEST(decode_5);
+
+	return ok ? 0 : 1;
+}

test/test-pixels.c

@@ -0,0 +1,222 @@
+#include "pixels.h"
+#include "rfb-proto.h"
+
+#include <stdio.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <string.h>
+#include <libdrm/drm_fourcc.h>
+
+#define swap32(x) (((x >> 24) & 0xff) | ((x << 8) & 0xff0000) | \
+		((x >> 8) & 0xff00) | ((x << 24) & 0xff000000))
+
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+#define u32_le(x) x
+#else
+#define u32_le(x) swap32(x)
+#endif
+
+#define XSTR(s) STR(s)
+#define STR(s) #s
+
+#define UDIV_UP(a, b) (((a) + (b) - 1) / (b))
+#define ARRAY_LEN(a) (sizeof(a) / (sizeof(a[0])))
+
+static bool test_pixel_to_cpixel_4bpp(void)
+{
+	uint32_t src = u32_le(0x11223344u);
+	uint32_t dst;
+	uint8_t* src_addr = (uint8_t*)&src;
+
+	struct rfb_pixel_format dstfmt = { 0 }, srcfmt = { 0 };
+
+	rfb_pixfmt_from_fourcc(&dstfmt, DRM_FORMAT_RGBA8888);
+
+	dst = 0;
+	rfb_pixfmt_from_fourcc(&srcfmt, DRM_FORMAT_RGBA8888);
+	pixel_to_cpixel((uint8_t*)&dst, &dstfmt, src_addr, &srcfmt, 4, 1);
+	if ((src & 0xffffff00u) != (dst & 0xffffff00u))
+		return false;
+
+	dst = 0;
+	rfb_pixfmt_from_fourcc(&dstfmt, DRM_FORMAT_ABGR8888);
+	pixel_to_cpixel((uint8_t*)&dst, &dstfmt, src_addr, &srcfmt, 4, 1);
+	if (dst != u32_le(0x00332211u))
+		return false;
+
+	dst = 0;
+	rfb_pixfmt_from_fourcc(&dstfmt, DRM_FORMAT_ARGB8888);
+	pixel_to_cpixel((uint8_t*)&dst, &dstfmt, src_addr, &srcfmt, 4, 1);
+	if (dst != u32_le(0x00112233u))
+		return false;
+
+	dst = 0;
+	rfb_pixfmt_from_fourcc(&dstfmt, DRM_FORMAT_BGRA8888);
+	pixel_to_cpixel((uint8_t*)&dst, &dstfmt, src_addr, &srcfmt, 4, 1);
+	if (dst != u32_le(0x33221100u))
+		return false;
+
+	return true;
+}
+
+static bool test_pixel_to_cpixel_3bpp(void)
+{
+	//44 is extra data that should not be copied anywhere below.
+	uint32_t src = u32_le(0x44112233u);
+	uint32_t dst;
+	uint8_t* src_addr = (uint8_t*)&src;
+
+	struct rfb_pixel_format dstfmt = { 0 }, srcfmt = { 0 };
+
+	rfb_pixfmt_from_fourcc(&srcfmt, DRM_FORMAT_RGB888);
+
+	dst = 0;
+	rfb_pixfmt_from_fourcc(&dstfmt, DRM_FORMAT_RGBA8888);
+	pixel_to_cpixel((uint8_t*)&dst, &dstfmt, src_addr, &srcfmt, 4, 1);
+	if (dst != u32_le(0x11223300u))
+		return false;
+
+	dst = 0;
+	rfb_pixfmt_from_fourcc(&dstfmt, DRM_FORMAT_ABGR8888);
+	pixel_to_cpixel((uint8_t*)&dst, &dstfmt, src_addr, &srcfmt, 4, 1);
+	if (dst != u32_le(0x00332211u))
+		return false;
+
+	dst = 0;
+	rfb_pixfmt_from_fourcc(&dstfmt, DRM_FORMAT_ARGB8888);
+	pixel_to_cpixel((uint8_t*)&dst, &dstfmt, src_addr, &srcfmt, 4, 1);
+	if (dst != u32_le(0x00112233u))
+		return false;
+
+	dst = 0;
+	rfb_pixfmt_from_fourcc(&dstfmt, DRM_FORMAT_BGRA8888);
+	pixel_to_cpixel((uint8_t*)&dst, &dstfmt, src_addr, &srcfmt, 4, 1);
+	if (dst != u32_le(0x33221100u))
+		return false;
+
+	return true;
+}
+
+static bool test_fourcc_to_pixman_fmt(void)
+{
+	pixman_format_code_t r;
+
+#define check(a, b) \
+	r = 0; \
+	if (!fourcc_to_pixman_fmt(&r, b) || a != r) { \
+		fprintf(stderr, "Failed check for %s\n", XSTR(a)); \
+		return false; \
+	} while(0)
+
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+	check(PIXMAN_a2r10g10b10, DRM_FORMAT_ARGB2101010);
+	check(PIXMAN_r8g8b8a8, DRM_FORMAT_RGBA8888);
+	check(PIXMAN_b8g8r8a8, DRM_FORMAT_BGRA8888);
+	check(PIXMAN_r5g6b5, DRM_FORMAT_RGB565);
+#else
+	check(PIXMAN_r8g8b8a8, DRM_FORMAT_ABGR8888);
+	check(PIXMAN_b8g8r8a8, DRM_FORMAT_ARGB8888);
+	check(PIXMAN_r5g6b5, DRM_FORMAT_BGR565);
+#endif
+
+#undef check
+	return true;
+}
+
+static bool test_extract_alpha_mask_rgba8888(void)
+{
+	uint32_t test_data[] = {
+		u32_le(0x00000000u), // Black, transparent
+		u32_le(0xff000000u), // Red, transparent
+		u32_le(0x00ff0000u), // Red, transparent
+		u32_le(0x0000ff00u), // Green, transparent
+		u32_le(0x000000ffu), // Black, opaque
+		u32_le(0xff0000ffu), // Red, opaque
+		u32_le(0x00ff00ffu), // Red, opaque
+		u32_le(0x0000ffffu), // Green, opaque
+	};
+
+	uint8_t mask[UDIV_UP(ARRAY_LEN(test_data), 8)] = { 0 };
+
+	bool ok = extract_alpha_mask(mask, test_data, DRM_FORMAT_RGBA8888,
+			ARRAY_LEN(test_data));
+	if (!ok) {
+		fprintf(stderr, "Failed to extract alpha mask");
+		return false;
+	}
+
+	if (mask[0] != 0x0f) {
+		fprintf(stderr, "Expected alpha mask to be 0b00001111 (0x0f), but it was %x",
+				mask[0]);
+		return false;
+	}
+
+	memset(mask, 0, sizeof(mask));
+
+	ok = extract_alpha_mask(mask, test_data, DRM_FORMAT_BGRA8888,
+			ARRAY_LEN(test_data));
+	if (!ok) {
+		fprintf(stderr, "Failed to extract alpha mask");
+		return false;
+	}
+
+	if (mask[0] != 0x0f) {
+		fprintf(stderr, "Expected alpha mask to be 0b00001111 (0x0f), but it was %x",
+				mask[0]);
+		return false;
+	}
+
+	return true;
+}
+
+static bool test_drm_format_to_string(void)
+{
+	if (strcmp(drm_format_to_string(DRM_FORMAT_RGBA8888), "RGBA8888") != 0)
+		return false;
+
+	if (strcmp(drm_format_to_string(DRM_FORMAT_RGBX8888), "RGBX8888") != 0)
+		return false;
+
+	if (strcmp(drm_format_to_string(DRM_FORMAT_RGB565), "RGB565") != 0)
+		return false;
+
+	return true;
+}
+
+static bool test_rfb_pixfmt_to_string(void)
+{
+	struct rfb_pixel_format rgbx8888;
+	struct rfb_pixel_format bgrx8888;
+	struct rfb_pixel_format xrgb8888;
+	struct rfb_pixel_format xbgr8888;
+
+	rfb_pixfmt_from_fourcc(&rgbx8888, DRM_FORMAT_RGBX8888);
+	rfb_pixfmt_from_fourcc(&bgrx8888, DRM_FORMAT_BGRX8888);
+	rfb_pixfmt_from_fourcc(&xrgb8888, DRM_FORMAT_XRGB8888);
+	rfb_pixfmt_from_fourcc(&xbgr8888, DRM_FORMAT_XBGR8888);
+
+	if (strcmp(rfb_pixfmt_to_string(&rgbx8888), "RGBX8888") != 0)
+		return false;
+
+	if (strcmp(rfb_pixfmt_to_string(&bgrx8888), "BGRX8888") != 0)
+		return false;
+
+	if (strcmp(rfb_pixfmt_to_string(&xrgb8888), "XRGB8888") != 0)
+		return false;
+
+	if (strcmp(rfb_pixfmt_to_string(&xbgr8888), "XBGR8888") != 0)
+		return false;
+
+	return true;
+}
+
+int main()
+{
+	bool ok = test_pixel_to_cpixel_4bpp() &&
+		test_pixel_to_cpixel_3bpp() &&
+		test_fourcc_to_pixman_fmt() &&
+		test_extract_alpha_mask_rgba8888() &&
+		test_drm_format_to_string() &&
+		test_rfb_pixfmt_to_string();
+	return ok ? 0 : 1;
+}