diff --git a/include/encoder.h b/include/encoder.h
index bd5544d..b460969 100644
--- a/include/encoder.h
+++ b/include/encoder.h
@@ -60,6 +60,8 @@ struct encoder_impl {
 struct encoder {
 	struct encoder_impl* impl;
 
+	int ref;
+
 	uint16_t x_pos;
 	uint16_t y_pos;
 
@@ -71,7 +73,10 @@ struct encoder {
 
 struct encoder* encoder_new(enum rfb_encodings type, uint16_t width,
 		uint16_t height);
-void encoder_destroy(struct encoder* self);
+void encoder_ref(struct encoder* self);
+void encoder_unref(struct encoder* self);
+
+void encoder_init(struct encoder* self, struct encoder_impl*);
 
 enum rfb_encodings encoder_get_type(const struct encoder* self);
 enum encoder_kind encoder_get_kind(const struct encoder* self);
diff --git a/src/encoder.c b/src/encoder.c
index 209c24c..1ed560d 100644
--- a/src/encoder.c
+++ b/src/encoder.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Andri Yngvason
+ * Copyright (c) 2021 - 2022 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -49,6 +49,12 @@ struct encoder* encoder_new(enum rfb_encodings type, uint16_t width,
 	return NULL;
 }
 
+void encoder_init(struct encoder* self, struct encoder_impl* impl)
+{
+	self->ref = 1;
+	self->impl = impl;
+}
+
 enum rfb_encodings encoder_get_type(const struct encoder* self)
 {
 	if (self->impl == &encoder_impl_raw)
@@ -77,11 +83,20 @@ enum encoder_kind encoder_get_kind(const struct encoder* self)
 	return ENCODER_KIND_INVALID;
 }
 
-void encoder_destroy(struct encoder* self)
+void encoder_ref(struct encoder* self)
+{
+	assert(self->ref > 0);
+	self->ref++;
+}
+
+void encoder_unref(struct encoder* self)
 {
 	if (!self)
 		return;
 
+	if (--self->ref != 0)
+		return;
+
 	if (self->impl->destroy)
 		self->impl->destroy(self);
 }
diff --git a/src/open-h264.c b/src/open-h264.c
index 296c4ef..b77d1c0 100644
--- a/src/open-h264.c
+++ b/src/open-h264.c
@@ -108,7 +108,7 @@ struct encoder* open_h264_new(void)
 	if (!self)
 		return NULL;
 
-	self->parent.impl = &encoder_impl_open_h264;
+	encoder_init(&self->parent, &encoder_impl_open_h264);
 
 	if (open_h264_init_pending(self) < 0) {
 		free(self);
diff --git a/src/raw-encoding.c b/src/raw-encoding.c
index 2138b90..5c6291d 100644
--- a/src/raw-encoding.c
+++ b/src/raw-encoding.c
@@ -184,7 +184,7 @@ struct encoder* raw_encoder_new(void)
 	if (!self)
 		return NULL;
 
-	self->encoder.impl = &encoder_impl_raw;
+	encoder_init(&self->encoder, &encoder_impl_raw);
 
 	pixman_region_init(&self->current_damage);
 
diff --git a/src/server.c b/src/server.c
index c88aa31..bec9432 100644
--- a/src/server.c
+++ b/src/server.c
@@ -122,7 +122,7 @@ static void client_close(struct nvnc_client* client)
 			!(client->encoder->impl->flags &
 					ENCODER_IMPL_FLAG_IGNORES_DAMAGE);
 	}
-	encoder_destroy(client->encoder);
+	encoder_unref(client->encoder);
 	pixman_region_fini(&client->damage);
 	free(client->cut_text.buffer);
 	free(client);
@@ -661,7 +661,7 @@ static void process_fb_update_requests(struct nvnc_client* client)
 				!(client->encoder->impl->flags &
 						ENCODER_IMPL_FLAG_IGNORES_DAMAGE);
 		}
-		encoder_destroy(client->encoder);
+		encoder_unref(client->encoder);
 		client->encoder = encoder_new(encoding, width, height);
 		if (!client->encoder) {
 			nvnc_log(NVNC_LOG_ERROR, "Failed to allocate new encoder");
diff --git a/src/tight.c b/src/tight.c
index 78b465d..43ce447 100644
--- a/src/tight.c
+++ b/src/tight.c
@@ -536,7 +536,7 @@ struct encoder* tight_encoder_new(uint16_t width, uint16_t height)
 		return NULL;
 	}
 
-	self->encoder.impl = &encoder_impl_tight;
+	encoder_init(&self->encoder, &encoder_impl_tight);
 
 	return (struct encoder*)self;
 }
diff --git a/src/zrle.c b/src/zrle.c
index 6c001c2..779c0dd 100644
--- a/src/zrle.c
+++ b/src/zrle.c
@@ -386,7 +386,7 @@ struct encoder* zrle_encoder_new(void)
 	if (!self)
 		return NULL;
 
-	self->encoder.impl = &encoder_impl_zrle;
+	encoder_init(&self->encoder, &encoder_impl_zrle);
 
 	int rc = deflateInit2(&self->zs,
 			/* compression level: */ 1,