Create dedicated RSA-AES stream

The message format isn't really within the domain of the cipher, so it
doesn't belong to the crypto interface.

include/crypto.h

@@ -53,9 +53,11 @@ struct crypto_cipher* crypto_cipher_new(const uint8_t* enc_key,
 void crypto_cipher_del(struct crypto_cipher* self);
 
 bool crypto_cipher_encrypt(struct crypto_cipher* self, struct vec* dst,
-		const uint8_t* src, size_t len);
+		uint8_t* mac, const uint8_t* src, size_t len,
+		const uint8_t* ad, size_t ad_len);
 ssize_t crypto_cipher_decrypt(struct crypto_cipher* self, uint8_t* dst,
-		size_t dst_size, const uint8_t* src, size_t len);
+		uint8_t* mac, const uint8_t* src, size_t len,
+		const uint8_t* ad, size_t ad_len);
 
 void crypto_cipher_set_ad(struct crypto_cipher* self, const uint8_t* ad,
 		size_t len);

include/stream.h

@@ -70,7 +70,6 @@ struct stream_impl {
 			stream_req_fn on_done, void* userdata);
 	int (*send_first)(struct stream*, struct rcbuf* payload);
 	void (*exec_and_send)(struct stream*, stream_exec_fn, void* userdata);
-	int (*install_cipher)(struct stream*, struct crypto_cipher*);
 };
 
 struct stream {
@@ -115,4 +114,5 @@ void stream_exec_and_send(struct stream* self, stream_exec_fn, void* userdata);
 int stream_upgrade_to_tls(struct stream* self, void* context);
 #endif
 
-int stream_install_cipher(struct stream* self, struct crypto_cipher* cipher);
+int stream_upgrade_to_rsa_eas(struct stream* base, const uint8_t* enc_key,
+		const uint8_t* dec_key);

meson.build

@@ -124,7 +124,7 @@ if nettle.found() and hogweed.found() and gmp.found()
 	dependencies += [ nettle, hogweed, gmp ]
 	enable_websocket = true
 	config.set('HAVE_CRYPTO', true)
-	sources += 'src/crypto-nettle.c'
+	sources += ['src/crypto-nettle.c', 'src/stream-rsa-aes.c']
 endif
 
 if host_system == 'linux' and get_option('systemtap') and cc.has_header('sys/sdt.h')

src/crypto-nettle.c

@@ -55,10 +55,12 @@ struct crypto_cipher {
 	uint8_t read_buffer[65536];
 	uint8_t read_buffer_len;
 
-	bool (*encrypt)(struct crypto_cipher*, struct vec* dst,
+	bool (*encrypt)(struct crypto_cipher*, struct vec* dst, uint8_t* mac,
-			const uint8_t* src, size_t len);
+			const uint8_t* src, size_t src_len, const uint8_t* ad,
-	ssize_t (*decrypt)(struct crypto_cipher*, uint8_t* dst,
+			size_t ad_len);
-			size_t dst_size, const uint8_t* src, size_t len);
+	ssize_t (*decrypt)(struct crypto_cipher*, uint8_t* dst, uint8_t* mac,
+			const uint8_t* src, size_t src_len, const uint8_t* ad,
+			size_t ad_len);
 };
 
 struct crypto_hash {
@@ -283,18 +285,19 @@ struct crypto_key* crypto_derive_shared_secret(
 }
 
 static bool crypto_cipher_aes128_ecb_encrypt(struct crypto_cipher* self,
-		struct vec* dst, const uint8_t* src, size_t len)
+		struct vec* dst, uint8_t* mac, const uint8_t* src,
+		size_t len, const uint8_t* ad, size_t ad_len)
 {
-	vec_reserve(dst, len);
+	vec_reserve(dst, dst->len + len);
 	aes128_encrypt(&self->enc_ctx.aes128_ecb, len, dst->data, src);
 	dst->len = len;
 	return true;
 }
 
 static ssize_t crypto_cipher_aes128_ecb_decrypt(struct crypto_cipher* self,
-		uint8_t* dst, size_t dst_size, const uint8_t* src, size_t len)
+		uint8_t* dst, uint8_t* mac, const uint8_t* src, size_t len,
+		const uint8_t* ad, size_t ad_len)
 {
-	assert(dst_size <= len);
 	aes128_decrypt(&self->dec_ctx.aes128_ecb, len, dst, src);
 	return len;
 }
@@ -334,96 +337,32 @@ static void crypto_aes_eax_update_nonce(struct crypto_aes_eax* self)
 }
 
 static bool crypto_cipher_aes_eax_encrypt(struct crypto_cipher* self,
-		struct vec* dst, const uint8_t* src, size_t len)
+		struct vec* dst, uint8_t* mac, const uint8_t* src,
+		size_t src_len, const uint8_t* ad, size_t ad_len)
 {
-//	size_t msg_max_size = 65535;
+	vec_reserve(dst, dst->len + src_len);
-	size_t msg_max_size = 8192;
-	size_t n_msg = UDIV_UP(len, msg_max_size);
 
-	vec_clear(dst);
+	crypto_aes_eax_update_nonce(&self->enc_ctx.aes_eax);
-	vec_reserve(dst, len + n_msg * (2 + 16));
+	nettle_eax_aes128_update(&self->enc_ctx.aes_eax.ctx, ad_len,
+			(uint8_t*)ad);
+	nettle_eax_aes128_encrypt(&self->enc_ctx.aes_eax.ctx, src_len,
+			(uint8_t*)dst->data + dst->len, src);
+	dst->len += src_len;
 
-	for (size_t i = 0; i < n_msg; ++i) {
+	nettle_eax_aes128_digest(&self->enc_ctx.aes_eax.ctx, 16, mac);
-		size_t msglen = MIN(len - i * msg_max_size, msg_max_size);
-		uint16_t msglen_be = htons(msglen);
-		nvnc_trace("msglen %zu", msglen);
-
-		vec_append(dst, &msglen_be, sizeof(msglen_be));
-
-		crypto_aes_eax_update_nonce(&self->enc_ctx.aes_eax);
-		nettle_eax_aes128_update(&self->enc_ctx.aes_eax.ctx, 2,
-				(uint8_t*)&msglen_be);
-		nettle_eax_aes128_encrypt(&self->enc_ctx.aes_eax.ctx, msglen,
-				(uint8_t*)dst->data + dst->len, src + i * msg_max_size);
-		dst->len += msglen;
-
-		uint8_t mac[16];
-		nettle_eax_aes128_digest(&self->enc_ctx.aes_eax.ctx, sizeof(mac), mac);
-		vec_append(dst, &mac, sizeof(mac));
-	}
-
-	nvnc_trace("Encrypted buffer of size %zu", dst->len);
 
 	return true;
 }
 
-// TODO: Clean up this mess
 static ssize_t crypto_cipher_aes_eax_decrypt(struct crypto_cipher* self,
-		uint8_t* dst, size_t dst_size, const uint8_t* src, size_t len)
+		uint8_t* dst, uint8_t* mac, const uint8_t* src, size_t len,
+		const uint8_t* ad, size_t ad_len)
 {
-	size_t dst_index = 0;
+	crypto_aes_eax_update_nonce(&self->dec_ctx.aes_eax);
-	size_t rem = len;
+	nettle_eax_aes128_update(&self->dec_ctx.aes_eax.ctx, ad_len, ad);
-
+	nettle_eax_aes128_decrypt(&self->dec_ctx.aes_eax.ctx, len, dst, src);
-	while (rem) {
+	nettle_eax_aes128_digest(&self->dec_ctx.aes_eax.ctx, 16, mac);
-		size_t space = sizeof(self->read_buffer) - self->read_buffer_len;
+	return len;
-		memcpy(self->read_buffer, src + len - rem, MIN(space, rem));
-		self->read_buffer_len += len;
-
-		rem -= MIN(space, rem);
-
-		size_t index = 0;
-		for (;;) {
-			uint8_t* msg = &self->read_buffer[index];
-			uint16_t msglen_be;
-			memcpy(&msglen_be, msg, 2);
-			size_t msglen = ntohs(msglen_be);
-
-			if (self->read_buffer_len - index < msglen + 2) {
-				break;
-			}
-
-			if (msglen > dst_size - dst_index) {
-				break;
-			}
-
-			nvnc_trace("Got message of length: %zu", msglen);
-
-			crypto_aes_eax_update_nonce(&self->dec_ctx.aes_eax);
-			nettle_eax_aes128_update(&self->dec_ctx.aes_eax.ctx,
-					2, (uint8_t*)&msglen_be);
-
-			nettle_eax_aes128_decrypt(&self->dec_ctx.aes_eax.ctx,
-					msglen, dst + dst_index, msg + 2);
-			dst_index += msglen;
-			assert(dst_index <= len);
-
-			uint8_t expected_mac[16];
-			nettle_eax_aes128_digest(&self->dec_ctx.aes_eax.ctx, 16,
-					expected_mac);
-
-			uint8_t *mac = msg + 2 + msglen;
-			if (memcmp(expected_mac, mac, 16) != 0)
-				return -1; // Authentication failure
-
-			index += msglen + 2 + 16;
-		}
-
-		self->read_buffer_len -= index;
-		memmove(self->read_buffer, self->read_buffer + index,
-				self->read_buffer_len);
-	}
-
-	return dst_index;
 }
 
 static struct crypto_cipher* crypto_cipher_new_aes_eax(const uint8_t* enc_key,
@@ -464,15 +403,17 @@ void crypto_cipher_del(struct crypto_cipher* self)
 }
 
 bool crypto_cipher_encrypt(struct crypto_cipher* self, struct vec* dst,
-		const uint8_t* src, size_t len)
+		uint8_t* mac, const uint8_t* src, size_t src_len,
+		const uint8_t* ad, size_t ad_len)
 {
-	return self->encrypt(self, dst, src, len);
+	return self->encrypt(self, dst, mac, src, src_len, ad, ad_len);
 }
 
 ssize_t crypto_cipher_decrypt(struct crypto_cipher* self, uint8_t* dst,
-		size_t dst_size, const uint8_t* src, size_t len)
+		uint8_t* mac, const uint8_t* src, size_t src_len,
+		const uint8_t* ad, size_t ad_len)
 {
-	return self->decrypt(self, dst, dst_size, src, len);
+	return self->decrypt(self, dst, mac, src, src_len, ad, ad_len);
 }
 
 void crypto_cipher_set_ad(struct crypto_cipher* self, const uint8_t* ad,

src/server.c

@@ -465,8 +465,8 @@ static int on_apple_dh_response(struct nvnc_client* client)
 	char username[128] = {};
 	char* password = username + 64;
 
-	crypto_cipher_decrypt(cipher, (uint8_t*)username, sizeof(username),
+	crypto_cipher_decrypt(cipher, (uint8_t*)username, NULL,
-			msg->encrypted_credentials, sizeof(username));
+			msg->encrypted_credentials, sizeof(username), NULL, 0);
 	username[63] = '\0';
 	username[127] = '\0';
 	crypto_cipher_del(cipher);
@@ -626,10 +626,8 @@ static int on_rsa_aes_challenge(struct nvnc_client* client)
 	crypto_dump_base64("Server session key", server_session_key,
 			sizeof(server_session_key));
 
-	struct crypto_cipher* cipher = crypto_cipher_new(server_session_key,
+	stream_upgrade_to_rsa_eas(client->net_stream, server_session_key,
-			client_session_key, CRYPTO_CIPHER_AES_EAX);
+			client_session_key);
-	assert(cipher);
-	stream_install_cipher(client->net_stream, cipher);
 
 	uint8_t server_modulus[256];
 	uint8_t server_exponent[256];

src/stream-rsa-aes.c

@@ -0,0 +1,208 @@
+/*
+ * Copyright (c) 2023 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <assert.h>
+#include <errno.h>
+#include <sys/param.h>
+#include <arpa/inet.h>
+
+#include "rcbuf.h"
+#include "stream.h"
+#include "stream-tcp.h"
+#include "stream-common.h"
+#include "crypto.h"
+#include "neatvnc.h"
+
+#define RSA_AES_BUFFER_SIZE 8192
+#define UDIV_UP(a, b) (((a) + (b) - 1) / (b))
+
+struct stream_rsa_aes {
+	struct stream base;
+
+	size_t read_index;
+	uint8_t* read_buffer;
+
+	struct crypto_cipher* cipher;
+};
+
+static_assert(sizeof(struct stream_rsa_aes) <= STREAM_ALLOC_SIZE,
+		"struct stream_rsa_aes has grown too large, increase STREAM_ALLOC_SIZE");
+
+static void stream_rsa_aes_destroy(struct stream* base)
+{
+	struct stream_rsa_aes* self = (struct stream_rsa_aes*)base;
+	crypto_cipher_del(self->cipher);
+	free(self->read_buffer);
+	stream_tcp_destroy(base);
+}
+
+static void stream_rsa_aes_read_into_buffer(struct stream_rsa_aes* self)
+{
+	ssize_t n_read = stream_tcp_read(&self->base,
+			self->read_buffer + self->read_index,
+			RSA_AES_BUFFER_SIZE - self->read_index);
+	if (n_read > 0)
+		self->read_index += n_read;
+}
+
+static ssize_t stream_rsa_aes_parse_header(struct stream_rsa_aes* self)
+{
+	if (self->read_index <= 2) {
+		return -1;
+	}
+
+	uint16_t len_be;
+	memcpy(&len_be, self->read_buffer, sizeof(len_be));
+	size_t len = ntohs(len_be);
+
+	if (self->read_index < 2 + 16 + len) {
+		return -1;
+	}
+
+	return len;
+}
+
+static ssize_t stream_rsa_aes_read_message(struct stream_rsa_aes* self,
+		uint8_t* dst, size_t size)
+{
+	ssize_t msg_len = stream_rsa_aes_parse_header(self);
+	if (msg_len < 0) {
+		return 0;
+	}
+
+	// The entire message must fit in dst
+	/* TODO: With this, stream_tcp__on_event won't run until network input
+	 * is received. We need to somehow schedule on_event or also buffer the
+	 * decrypted data here.
+	 * Another option would be to keep back the message counter in the
+	 * cipher until the message has been fully read.
+	 */
+	if ((size_t)msg_len > size)
+		return 0;
+
+	uint16_t msg_len_be = htons(msg_len);
+
+	uint8_t expected_mac[16];
+	ssize_t n = crypto_cipher_decrypt(self->cipher, dst, expected_mac,
+			self->read_buffer + 2, msg_len,
+			(uint8_t*)&msg_len_be, sizeof(msg_len_be));
+
+	uint8_t* actual_mac = self->read_buffer + 2 + msg_len;
+	if (memcmp(expected_mac, actual_mac, 16) != 0) {
+		nvnc_log(NVNC_LOG_DEBUG, "Message authentication failed");
+		errno = EBADMSG;
+		return -1;
+	}
+
+	self->read_index -= 2 + 16 + msg_len;
+	memmove(self->read_buffer, self->read_buffer + 2 + 16 + msg_len,
+			self->read_index);
+
+	return n;
+}
+
+static ssize_t stream_rsa_aes_read(struct stream* base, void* dst, size_t size)
+{
+	struct stream_rsa_aes* self = (struct stream_rsa_aes*)base;
+
+	stream_rsa_aes_read_into_buffer(self);
+	if (self->base.state == STREAM_STATE_CLOSED)
+		return 0;
+
+	size_t total_read = 0;
+
+	while (true) {
+		ssize_t n_read = stream_rsa_aes_read_message(self, dst, size);
+		if (n_read == 0)
+			break;
+
+		if (n_read < 0) {
+			if (errno == EAGAIN) {
+				break;
+			}
+			return -1;
+		}
+
+		total_read += n_read;
+		dst += n_read;
+		size -= n_read;
+	}
+
+	return total_read;
+}
+
+static int stream_rsa_aes_send(struct stream* base, struct rcbuf* payload,
+                stream_req_fn on_done, void* userdata)
+{
+	struct stream_rsa_aes* self = (struct stream_rsa_aes*)base;
+	size_t n_msg = UDIV_UP(payload->size, RSA_AES_BUFFER_SIZE);
+
+	struct vec buf;
+	vec_init(&buf, payload->size + n_msg * (2 + 16));
+
+	for (size_t i = 0; i < n_msg; ++i) {
+		size_t msglen = MIN(payload->size - i * RSA_AES_BUFFER_SIZE,
+				RSA_AES_BUFFER_SIZE);
+		uint16_t msglen_be = htons(msglen);
+
+		vec_append(&buf, &msglen_be, sizeof(msglen_be));
+
+		uint8_t mac[16];
+		crypto_cipher_encrypt(self->cipher, &buf, mac,
+				payload->payload + i * RSA_AES_BUFFER_SIZE,
+				msglen, (uint8_t*)&msglen_be, sizeof(msglen_be));
+		vec_append(&buf, mac, sizeof(mac));
+	}
+
+	int r = stream_tcp_send(base, rcbuf_new(buf.data, buf.len), on_done,
+			userdata);
+	if (r < 0) {
+		return r;
+	}
+
+	return payload->size;
+}
+
+static struct stream_impl impl = {
+	.close = stream_tcp_close,
+	.destroy = stream_rsa_aes_destroy,
+	.read = stream_rsa_aes_read,
+	.send = stream_rsa_aes_send,
+};
+
+int stream_upgrade_to_rsa_eas(struct stream* base, const uint8_t* enc_key,
+		const uint8_t* dec_key)
+{
+	struct stream_rsa_aes* self = (struct stream_rsa_aes*)base;
+
+	self->read_index = 0;
+	self->read_buffer = malloc(RSA_AES_BUFFER_SIZE);
+	if (!self->read_buffer)
+		return -1;
+
+	self->cipher = crypto_cipher_new(enc_key, dec_key,
+			CRYPTO_CIPHER_AES_EAX);
+	if (!self->cipher) {
+		free(self->read_buffer);
+		return -1;
+	}
+
+	self->base.impl = &impl;
+	return 0;
+}

src/stream-tcp.c

@@ -32,22 +32,11 @@
 #include "stream-common.h"
 #include "stream-tcp.h"
 #include "sys/queue.h"
-#include "crypto.h"
 #include "neatvnc.h"
 
 static_assert(sizeof(struct stream) <= STREAM_ALLOC_SIZE,
 		"struct stream has grown too large, increase STREAM_ALLOC_SIZE");
 
-static struct rcbuf* encrypt_rcbuf(struct stream* self, struct rcbuf* payload)
-{
-	struct vec ciphertext = {};
-	crypto_cipher_encrypt(self->cipher, &ciphertext, payload->payload,
-			payload->size);
-	struct rcbuf* result = rcbuf_new(ciphertext.data, ciphertext.len);
-	rcbuf_unref(payload);
-	return result;
-}
-
 int stream_tcp_close(struct stream* self)
 {
 	if (self->state == STREAM_STATE_CLOSED)
@@ -71,7 +60,6 @@ int stream_tcp_close(struct stream* self)
 void stream_tcp_destroy(struct stream* self)
 {
 	vec_destroy(&self->tmp_buf);
-	crypto_cipher_del(self->cipher);
 	stream_close(self);
 	aml_unref(self->handler);
 	free(self);
@@ -92,8 +80,7 @@ static int stream_tcp__flush(struct stream* self)
 			if (req->payload)
 				rcbuf_unref(req->payload);
 			struct rcbuf* payload  = req->exec(self, req->userdata);
-			req->payload = self->cipher ?
+			req->payload = payload;
-				encrypt_rcbuf(self, payload) : payload;
 		}
 
 		iov[n_msgs].iov_base = req->payload->payload;
@@ -217,19 +204,6 @@ ssize_t stream_tcp_read(struct stream* self, void* dst, size_t size)
 	if (rc > 0)
 		self->bytes_received += rc;
 
-	if (rc > 0 && self->cipher) {
-		nvnc_trace("Got cipher text of length %zd", rc);
-		ssize_t len = crypto_cipher_decrypt(self->cipher, dst, size,
-				read_buffer, rc);
-		if (len < 0) {
-			nvnc_log(NVNC_LOG_ERROR, "Message authentication failed!");
-			stream__remote_closed(self);
-			errno = EPROTO;
-			return -1;
-		}
-		rc = len;
-	}
-
 	return rc;
 }
 
@@ -243,7 +217,7 @@ int stream_tcp_send(struct stream* self, struct rcbuf* payload,
 	if (!req)
 		return -1;
 
-	req->payload = self->cipher ? encrypt_rcbuf(self, payload) : payload;
+	req->payload = payload;
 	req->on_done = on_done;
 	req->userdata = userdata;
 
@@ -285,14 +259,6 @@ void stream_tcp_exec_and_send(struct stream* self,
 	stream_tcp__flush(self);
 }
 
-int stream_tcp_install_cipher(struct stream* self,
-		struct crypto_cipher* cipher)
-{
-	assert(!self->cipher);
-	self->cipher = cipher;
-	return 0;
-}
-
 static struct stream_impl impl = {
 	.close = stream_tcp_close,
 	.destroy = stream_tcp_destroy,
@@ -300,7 +266,6 @@ static struct stream_impl impl = {
 	.send = stream_tcp_send,
 	.send_first = stream_tcp_send_first,
 	.exec_and_send = stream_tcp_exec_and_send,
-	.install_cipher = stream_tcp_install_cipher,
 };
 
 int stream_tcp_init(struct stream* self, int fd, stream_event_fn on_event,

src/stream.c

@@ -65,11 +65,3 @@ void stream_exec_and_send(struct stream* self, stream_exec_fn exec_fn,
 	else
 		stream_send(self, exec_fn(self, userdata), NULL, NULL);
 }
-
-int stream_install_cipher(struct stream* self, struct crypto_cipher* cipher)
-{
-	if (!self->impl->install_cipher) {
-		return -1;
-	}
-	return self->impl->install_cipher(self, cipher);
-}