From 0cc6be091c040974d0b33a1b4305ce0447e9028c Mon Sep 17 00:00:00 2001
From: Andri Yngvason <andri@yngvason.is>
Date: Sun, 6 Feb 2022 15:01:33 +0000
Subject: [PATCH] pixels: Add function to extract alpha mask

---
 include/pixels.h |  3 +++
 src/pixels.c     | 55 ++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 58 insertions(+)

diff --git a/include/pixels.h b/include/pixels.h
index aa4c9ad..f20feb6 100644
--- a/include/pixels.h
+++ b/include/pixels.h
@@ -35,3 +35,6 @@ uint32_t rfb_pixfmt_to_fourcc(const struct rfb_pixel_format* fmt);
 int pixel_size_from_fourcc(uint32_t fourcc);
 
 bool fourcc_to_pixman_fmt(pixman_format_code_t* dst, uint32_t src);
+
+bool extract_alpha_mask(uint8_t* dst, const void* src, uint32_t format,
+		size_t len);
diff --git a/src/pixels.c b/src/pixels.c
index d8f9412..5223e26 100644
--- a/src/pixels.c
+++ b/src/pixels.c
@@ -21,6 +21,7 @@
 #include <libdrm/drm_fourcc.h>
 
 #define POPCOUNT(x) __builtin_popcount(x)
+#define UDIV_UP(a, b) (((a) + (b) - 1) / (b))
 
 void pixel32_to_cpixel(uint8_t* restrict dst,
                        const struct rfb_pixel_format* dst_fmt,
@@ -358,3 +359,57 @@ bool fourcc_to_pixman_fmt(pixman_format_code_t* dst, uint32_t src)
 
 	return true;
 }
+
+static bool extract_alpha_mask_rgba32(uint8_t* dst, const uint32_t* src,
+		size_t len, int alpha_shift, uint32_t alpha_max)
+{
+	for (size_t i = 0; i < len; i++) {
+		uint8_t alpha = (src[i] >> alpha_shift) & alpha_max;
+		uint8_t binary = !!(alpha > alpha_max / 2);
+		dst[i / 8] |= binary << (7 - (i % 8));
+	}
+
+	return true;
+}
+
+static bool extract_alpha_mask_rgba16(uint8_t* dst, const uint16_t* src,
+		size_t len, int alpha_shift)
+{
+	for (size_t i = 0; i < len; i++) {
+		uint8_t alpha = (src[i] >> alpha_shift) & 0xf;
+		uint8_t binary = !!(alpha > 0xf / 2);
+		dst[i / 8] |= binary << (7 - (i % 8));
+	}
+
+	return true;
+}
+
+// Note: The destination buffer must be at least UDIV_UP(len, 8) long.
+bool extract_alpha_mask(uint8_t* dst, const void* src, uint32_t format,
+		size_t len)
+{
+	memset(dst, 0, UDIV_UP(len, 8));
+
+	switch (format & ~DRM_FORMAT_BIG_ENDIAN) {
+	case DRM_FORMAT_RGBA1010102:
+	case DRM_FORMAT_BGRA1010102:
+		return extract_alpha_mask_rgba32(dst, src, len, 0, 3);
+	case DRM_FORMAT_ARGB2101010:
+	case DRM_FORMAT_ABGR2101010:
+		return extract_alpha_mask_rgba32(dst, src, len, 30, 3);
+	case DRM_FORMAT_RGBA8888:
+	case DRM_FORMAT_BGRA8888:
+		return extract_alpha_mask_rgba32(dst, src, len, 0, 0xff);
+	case DRM_FORMAT_ARGB8888:
+	case DRM_FORMAT_ABGR8888:
+		return extract_alpha_mask_rgba32(dst, src, len, 24, 0xff);
+	case DRM_FORMAT_RGBA4444:
+	case DRM_FORMAT_BGRA4444:
+		return extract_alpha_mask_rgba16(dst, src, len, 0);
+	case DRM_FORMAT_ARGB4444:
+	case DRM_FORMAT_ABGR4444:
+		return extract_alpha_mask_rgba16(dst, src, len, 12);
+	}
+
+	return false;
+}