Do tiled damage check in shader

include/render.h

@@ -59,6 +59,8 @@ struct renderer {
 		GLuint program;
 		GLint u_tex0;
 		GLint u_tex1;
+		GLint u_width;
+		GLint u_height;
 	} damage_shader;
 };
 

shaders/damage-fragment.glsl

@@ -3,10 +3,46 @@ precision mediump float;
 uniform sampler2D u_tex0;
 uniform sampler2D u_tex1;
 
+uniform int u_width;
+uniform int u_height;
+
 varying vec2 v_texture;
 
+bool is_pixel_damaged(vec2 pos)
+{
+	return texture2D(u_tex0, pos).rgb != texture2D(u_tex1, pos).rgb;
+}
+
+bool is_region_damaged(vec2 pos)
+{
+	int x_correction = u_width - (u_width / 32) * 32;
+	int y_correction = u_height - (u_height / 32) * 32;
+
+	float x_scaling = float(u_width) / float(u_width - x_correction);
+	float y_scaling = float(u_height) / float(u_height - y_correction);
+
+	float x_pixel = 1.0 / float(u_width);
+	float y_pixel = 1.0 / float(u_height);
+
+	bool r = false;
+
+	for (int y = -17; y < 17; ++y)
+		for (int x = -17; x < 17; ++x) {
+			float px = float(x) * x_pixel;
+			float py = float(y) * y_pixel;
+
+			float spx = x_scaling * (pos.x + px);
+			float spy = y_scaling * (pos.y + py);
+
+			if (is_pixel_damaged(vec2(spx, spy)))
+				r = true;
+		}
+
+	return r;
+}
+
 void main()
 {
-	float r = float(texture2D(u_tex0, v_texture).rgb != texture2D(u_tex1, v_texture).rgb);
+	float r = float(is_region_damaged(v_texture));
 	gl_FragColor = vec4(r);
 }

src/damage.c

@@ -90,6 +90,21 @@ void damage_check_tile_row(struct pixman_region16* damage,
 void damage_check(struct pixman_region16* damage, const uint8_t* buffer,
                   uint32_t width, uint32_t height, struct pixman_box16* hint)
 {
+	uint32_t h = UDIV_UP(height, 32);
+	uint32_t w = UDIV_UP(width, 32);
+
+	for (uint32_t y = 0; y < h; ++y)
+		for (uint32_t x = 0; x < w; ++x)
+			if (buffer[x + y * w])
+				pixman_region_union_rect(damage, damage,
+							 x * 32, y * 32, 32, 32);
+
+	pixman_region_intersect_rect(damage, damage, 0, 0, width, height);
+	return;
+
+
+
+
 	uint32_t tiled_width = UDIV_UP(width, 32);
 	uint8_t* row_buffer = malloc(tiled_width);
 	assert(row_buffer);

src/main.c

@@ -484,6 +484,10 @@ static void on_damage_check_done(struct pixman_region16* damage, void* userdata)
 {
 	struct wayvnc* self = userdata;
 
+	uint32_t width = output_get_transformed_width(self->selected_output);
+	uint32_t height = output_get_transformed_height(self->selected_output);
+	damage_dump(stdout, damage, width, height, 32);
+
 	if (pixman_region_not_empty(damage)) {
 		struct pixman_box16* ext = pixman_region_extents(damage);
 		uint32_t y = ext->y1;

src/render.c

@@ -44,6 +44,7 @@ enum {
 };
 
 #define ARRAY_LEN(a) (sizeof(a) / sizeof((a)[0]))
+#define UDIV_UP(a, b) (((a) + (b) - 1) / (b))
 
 #define XSTR(s) STR(s)
 #define STR(s) #s
@@ -393,9 +394,13 @@ void renderer_swap(struct renderer* self)
 
 void render_frame(struct renderer* self)
 {
+	uint32_t width = output_get_transformed_width(self->output);
+	uint32_t height = output_get_transformed_height(self->output);
+
 	renderer_swap(self);
 
 	glBindFramebuffer(GL_FRAMEBUFFER, self->frame_fbo[self->frame_index].fbo);
+	glViewport(0, 0, width, height);
 
 	glUseProgram(self->frame_shader.program);
 
@@ -411,7 +416,11 @@ void render_frame(struct renderer* self)
 
 void render_damage(struct renderer* self)
 {
+	uint32_t width = output_get_transformed_width(self->output);
+	uint32_t height = output_get_transformed_height(self->output);
+
 	glBindFramebuffer(GL_FRAMEBUFFER, self->damage_fbo.fbo);
+	glViewport(0, 0, UDIV_UP(width, 32), UDIV_UP(height, 32));
 
 	glActiveTexture(GL_TEXTURE0);
 	glBindTexture(GL_TEXTURE_2D, self->frame_fbo[self->frame_index].tex);
@@ -423,6 +432,9 @@ void render_damage(struct renderer* self)
 	glUniform1i(self->damage_shader.u_tex0, 0);
 	glUniform1i(self->damage_shader.u_tex1, 1);
 
+	glUniform1i(self->damage_shader.u_width, width);
+	glUniform1i(self->damage_shader.u_height, height);
+
 	gl_draw();
 
 	glBindFramebuffer(GL_FRAMEBUFFER, 0);
@@ -473,8 +485,10 @@ int create_textured_fbo(struct renderer_fbo* dst, GLint format, uint32_t width,
 	glBindTexture(GL_TEXTURE_2D, tex);
 	glTexImage2D(GL_TEXTURE_2D, 0, format, width, height, 0, format,
 	             GL_UNSIGNED_BYTE, NULL);
-	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
-	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
 	glBindTexture(GL_TEXTURE_2D, 0);
 
 	GLuint fbo = 0;
@@ -571,7 +585,7 @@ int renderer_init(struct renderer* self, const struct output* output,
 	if (create_textured_fbo(&self->frame_fbo[1], GL_RGBA, tf_width, tf_height) < 0)
 		goto frame_fbo_failure_1;
 
-	if (create_fbo(&self->damage_fbo, GL_R8_EXT, tf_width, tf_height) < 0)
+	if (create_fbo(&self->damage_fbo, GL_R8_EXT, UDIV_UP(tf_width, 32), UDIV_UP(tf_height, 32)) < 0)
 		goto damage_fbo_failure;
 
 	glBindFramebuffer(GL_FRAMEBUFFER, self->frame_fbo[0].fbo);
@@ -605,6 +619,10 @@ int renderer_init(struct renderer* self, const struct output* output,
 		glGetUniformLocation(self->damage_shader.program, "u_tex0");
 	self->damage_shader.u_tex1 =
 		glGetUniformLocation(self->damage_shader.program, "u_tex1");
+	self->damage_shader.u_width =
+		glGetUniformLocation(self->damage_shader.program, "u_width");
+	self->damage_shader.u_height =
+		glGetUniformLocation(self->damage_shader.program, "u_height");
 
 	self->output = output;
 	glGetIntegerv(GL_IMPLEMENTATION_COLOR_READ_FORMAT, &self->read_format);
@@ -751,6 +769,14 @@ void renderer_read_damage(struct renderer* self, void* dst, uint32_t y,
                           uint32_t height)
 {
 	glBindFramebuffer(GL_FRAMEBUFFER, self->damage_fbo.fbo);
-	renderer_read_pixels(self, dst, y, height);
+
+	uint32_t width = output_get_transformed_width(self->output);
+
+	GLint read_format, read_type;
+	glGetIntegerv(GL_IMPLEMENTATION_COLOR_READ_FORMAT, &read_format);
+	glGetIntegerv(GL_IMPLEMENTATION_COLOR_READ_TYPE, &read_type);
+
+	glReadPixels(0, y, UDIV_UP(width, 32), UDIV_UP(height, 32), read_format, read_type, dst);
+
 	glBindFramebuffer(GL_FRAMEBUFFER, 0);
 }