diff --git a/src/server.c b/src/server.c
index f12fdc8..14b9ad7 100644
--- a/src/server.c
+++ b/src/server.c
@@ -201,6 +201,12 @@ static int send_byte(struct nvnc_client* client, uint8_t value)
 	return stream_write(client->net_stream, &value, 1, NULL, NULL);
 }
 
+static int send_byte_and_close(struct nvnc_client* client, uint8_t value)
+{
+	return stream_write(client->net_stream, &value, 1, close_after_write,
+	                    client);
+}
+
 #ifdef ENABLE_TLS
 static int vencrypt_send_version(struct nvnc_client* client)
 {
@@ -247,15 +253,15 @@ static int on_vencrypt_subtype_message(struct nvnc_client* client)
 	enum rfb_vencrypt_subtype subtype = ntohl(*msg);
 
 	if (subtype != RFB_VENCRYPT_X509_PLAIN) {
-		send_byte(client, 0); // TODO Close after write
-		stream_close(client->net_stream);
-		client_unref(client);
+		client->state = VNC_CLIENT_STATE_ERROR;
+		send_byte_and_close(client, 0);
 		return sizeof(*msg);
 	}
 
 	send_byte(client, 1);
 
 	if (stream_upgrade_to_tls(client->net_stream, client->server->tls_creds) < 0) {
+		client->state = VNC_CLIENT_STATE_ERROR;
 		stream_close(client->net_stream);
 		client_unref(client);
 		return sizeof(*msg);