neatvnc/src/cursor.c

/*
 * Copyright (c) 2022 Andri Yngvason
 *
 * Permission to use, copy, modify, and/or distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
 * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
 * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
 * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
 * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
 * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 * PERFORMANCE OF THIS SOFTWARE.
 */

#include "cursor.h"

#include "fb.h"
#include "pixels.h"
#include "rfb-proto.h"
#include "vec.h"
#include "enc-util.h"

#include <stdlib.h>
#include <unistd.h>
#include <assert.h>

#define UDIV_UP(a, b) (((a) + (b) - 1) / (b))

int cursor_encode(struct vec* dst, struct rfb_pixel_format* pixfmt,
		struct nvnc_fb* image, uint32_t width, uint32_t height,
		uint32_t hotspot_x, uint32_t hotspot_y)
{
	// TODO: Handle rotated cursors

	int rc = -1;

	assert(width <= image->width);
	assert(height <= image->height);

	if (nvnc_fb_map(image) < 0)
		return -1;

	struct rfb_pixel_format srcfmt = { 0 };
	rc = rfb_pixfmt_from_fourcc(&srcfmt, image->fourcc_format);
	if (rc < 0)
		return -1;

	rc = encode_rect_head(dst, RFB_ENCODING_CURSOR, hotspot_x, hotspot_y,
			width, height);
	if (rc < 0)
		return -1;

	int bpp = pixfmt->bits_per_pixel / 8;
	size_t size = width * height;

	rc = vec_reserve(dst, dst->len + size * bpp + UDIV_UP(size, 8));
	if (rc < 0)
		return -1;

	uint8_t* dstdata = dst->data;
	dstdata += dst->len;

	if((int32_t)width == image->stride) {
		pixel32_to_cpixel(dstdata, pixfmt, image->addr, &srcfmt, bpp, size);
	} else {
		for (int y = 0; y < height; ++y) {
			pixel32_to_cpixel(dstdata + y * bpp * width, pixfmt,
					(uint32_t*)image->addr + y * image->stride,
					&srcfmt, bpp, width);
		}
	}

	dst->len += size * bpp;
	dstdata = dst->data;
	dstdata += dst->len;

	for (int y = 0; y < height; ++y) {
		if (!extract_alpha_mask(dstdata + y * UDIV_UP(width, 8),
					(uint32_t*)image->addr + y * image->stride,
					image->fourcc_format, width))
			return -1;

		dst->len += UDIV_UP(width, 8);
	}

	return 0;
}