diff --git a/include/common.h b/include/common.h
index b13dae9..47db985 100644
--- a/include/common.h
+++ b/include/common.h
@@ -113,6 +113,8 @@ struct nvnc {
 	nvnc_auth_fn auth_fn;
 	void* auth_ud;
 #endif
+
+	uint32_t n_damage_clients;
 };
 
 void nvnc__damage_region(struct nvnc* self,
diff --git a/src/server.c b/src/server.c
index 1a245e5..10d260c 100644
--- a/src/server.c
+++ b/src/server.c
@@ -105,6 +105,11 @@ static void client_close(struct nvnc_client* client)
 
 	LIST_REMOVE(client, link);
 	stream_destroy(client->net_stream);
+	if (client->encoder) {
+		client->server->n_damage_clients -=
+			!(client->encoder->impl->flags &
+					ENCODER_IMPL_FLAG_IGNORES_DAMAGE);
+	}
 	encoder_destroy(client->encoder);
 	pixman_region_fini(&client->damage);
 	free(client->cut_text.buffer);
@@ -560,6 +565,11 @@ static void process_fb_update_requests(struct nvnc_client* client)
 	if (!client->encoder || encoding != encoder_get_type(client->encoder)) {
 		int width = server->display->buffer->width;
 		int height = server->display->buffer->height;
+		if (client->encoder) {
+			server->n_damage_clients -=
+				!(client->encoder->impl->flags &
+						ENCODER_IMPL_FLAG_IGNORES_DAMAGE);
+		}
 		encoder_destroy(client->encoder);
 		client->encoder = encoder_new(encoding, width, height);
 		if (!client->encoder) {
@@ -572,6 +582,10 @@ static void process_fb_update_requests(struct nvnc_client* client)
 			client->encoder->on_done = on_encoder_push_done;
 			client->encoder->userdata = client;
 		}
+
+		server->n_damage_clients +=
+			!(client->encoder->impl->flags &
+					ENCODER_IMPL_FLAG_IGNORES_DAMAGE);
 	}
 
 	enum encoder_kind kind = encoder_get_kind(client->encoder);