cursor: Handle rotated cursors

src/cursor.c

@@ -21,6 +21,8 @@
 #include "rfb-proto.h"
 #include "vec.h"
 #include "enc-util.h"
+#include "resampler.h"
+#include "transform-util.h"
 
 #include <stdlib.h>
 #include <unistd.h>
@@ -28,40 +30,66 @@
 
 #define UDIV_UP(a, b) (((a) + (b) - 1) / (b))
 
+static struct nvnc_fb* apply_transform(struct nvnc_fb* fb)
+{
+	if (fb->transform == NVNC_TRANSFORM_NORMAL) {
+		nvnc_fb_ref(fb);
+		return fb;
+	}
+
+	uint32_t width = fb->width;
+	uint32_t height = fb->height;
+
+	nvnc_transform_dimensions(fb->transform, &width, &height);
+	struct nvnc_fb* dst = nvnc_fb_new(width, height, fb->fourcc_format,
+			width);
+	assert(dst);
+
+	// TODO: Don't assume bpp
+	memset(dst->addr, 0, width * height * 4);
+
+	resample_now(dst, fb, NULL);
+
+	return dst;
+}
+
 int cursor_encode(struct vec* dst, struct rfb_pixel_format* pixfmt,
 		struct nvnc_fb* image, uint32_t width, uint32_t height,
 		uint32_t hotspot_x, uint32_t hotspot_y)
 {
-	// TODO: Handle rotated cursors
-
 	int rc = -1;
 
 	// Empty cursor
 	if (!image)
 		return encode_rect_head(dst, RFB_ENCODING_CURSOR, 0, 0, 0, 0);
 
-	assert(width <= image->width);
+	nvnc_transform_dimensions(image->transform, &width, &height);
-	assert(height <= image->height);
+	nvnc_transform_dimensions(image->transform, &hotspot_x, &hotspot_y);
 
 	if (nvnc_fb_map(image) < 0)
-		return -1;
+		goto failure;
+
+	image = apply_transform(image);
+
+	assert(width <= image->width);
+	assert(height <= image->height);
 
 	struct rfb_pixel_format srcfmt = { 0 };
 	rc = rfb_pixfmt_from_fourcc(&srcfmt, image->fourcc_format);
 	if (rc < 0)
-		return -1;
+		goto failure;
 
 	rc = encode_rect_head(dst, RFB_ENCODING_CURSOR, hotspot_x, hotspot_y,
 			width, height);
 	if (rc < 0)
-		return -1;
+		goto failure;
 
 	int bpp = pixfmt->bits_per_pixel / 8;
 	size_t size = width * height;
 
 	rc = vec_reserve(dst, dst->len + size * bpp + UDIV_UP(size, 8));
 	if (rc < 0)
-		return -1;
+		goto failure;
 
 	uint8_t* dstdata = dst->data;
 	dstdata += dst->len;
@@ -84,10 +112,13 @@ int cursor_encode(struct vec* dst, struct rfb_pixel_format* pixfmt,
 		if (!extract_alpha_mask(dstdata + y * UDIV_UP(width, 8),
 					(uint32_t*)image->addr + y * image->stride,
 					image->fourcc_format, width))
-			return -1;
+			goto failure;
 
 		dst->len += UDIV_UP(width, 8);
 	}
 
-	return 0;
+	rc = 0;
+failure:
+	nvnc_fb_unref(image);
+	return rc;
 }