Use drm format instead of wl_shm_format

Using the drm fourcc format as the main pixel format, helps when
adding linux-dmabuf support.

include/pixels.h

@@ -1,7 +1,25 @@
+/*
+ * Copyright (c) 2022 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
 #pragma once
 
 #include <wayland-client.h>
 #include <pixman.h>
 #include <stdbool.h>
 
-bool wl_shm_to_pixman_fmt(pixman_format_code_t* dst, enum wl_shm_format src);
+enum wl_shm_format drm_format_to_wl_shm(uint32_t in);
+uint32_t drm_format_from_wl_shm(enum wl_shm_format in);
+bool drm_format_to_pixman_fmt(pixman_format_code_t* dst, uint32_t src);

include/vnc.h

@@ -38,8 +38,7 @@ void vnc_client_destroy(struct vnc_client* self);
 
 int vnc_client_connect(struct vnc_client* self, const char* address, int port);
 
-int vnc_client_set_pixel_format(struct vnc_client* self,
+int vnc_client_set_pixel_format(struct vnc_client* self, uint32_t format);
-		enum wl_shm_format format);
 
 int vnc_client_get_fd(const struct vnc_client* self);
 int vnc_client_get_width(const struct vnc_client* self);

src/main.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020 Andri Yngvason
+ * Copyright (c) 2020 - 2022 Andri Yngvason
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -28,6 +28,7 @@
 #include <aml.h>
 #include <wayland-client.h>
 #include <xkbcommon/xkbcommon.h>
+#include <libdrm/drm_fourcc.h>
 
 #include "pixman.h"
 #include "xdg-shell.h"
@@ -42,7 +43,7 @@
 struct buffer {
 	int width, height, stride;
 	size_t size;
-	enum wl_shm_format format;
+	uint32_t format;
 	struct wl_buffer* wl_buffer;
 	void* pixels;
 	bool is_attached;
@@ -71,8 +72,7 @@ static struct wl_list seats;
 struct pointer_collection* pointers;
 struct keyboard_collection* keyboards;
 
-static enum wl_shm_format wl_shm_format;
+static uint32_t shm_format = DRM_FORMAT_INVALID;
-static bool have_format = false;
 
 static bool do_run = true;
 
@@ -196,7 +196,7 @@ static struct buffer* buffer_create(int width, int height, int stride,
 		goto pool_failure;
 
 	self->wl_buffer = wl_shm_pool_create_buffer(pool, 0, width, height,
-			stride, format);
+			stride, drm_format_to_wl_shm(format));
 	wl_shm_pool_destroy(pool);
 	if (!self->wl_buffer)
 		goto shm_failure;
@@ -217,25 +217,26 @@ failure:
 	return NULL;
 }
 
-static void shm_format(void* data, struct wl_shm* shm, uint32_t format)
+static void handle_shm_format(void* data, struct wl_shm* shm, uint32_t format)
 {
 	(void)data;
 	(void)wl_shm;
 
-	if (have_format)
+	if (shm_format != DRM_FORMAT_INVALID)
 		return;
 
-	switch (format) {
+	uint32_t drm_format = drm_format_from_wl_shm(format);
-	case WL_SHM_FORMAT_XRGB8888:
+
-		wl_shm_format = format;
+	switch (drm_format) {
-		have_format = true;
+	case DRM_FORMAT_XRGB8888:
+		shm_format = drm_format;
 	}
 
 	// TODO: Support more formats
 }
 
 static const struct wl_shm_listener shm_listener = {
-	.format = shm_format,
+	.format = handle_shm_format,
 };
 
 static void xdg_wm_base_ping(void* data, struct xdg_wm_base* shell,
@@ -341,11 +342,11 @@ static void window_transfer_pixels(struct window* w)
 	bool ok __attribute__((unused));
 
 	pixman_format_code_t dst_fmt = 0;
-	ok = wl_shm_to_pixman_fmt(&dst_fmt, w->back_buffer->format);
+	ok = drm_format_to_pixman_fmt(&dst_fmt, w->back_buffer->format);
 	assert(ok);
 
 	pixman_format_code_t src_fmt = 0;
-	ok = wl_shm_to_pixman_fmt(&src_fmt, w->back_buffer->format);
+	ok = drm_format_to_pixman_fmt(&src_fmt, w->back_buffer->format);
 	assert(ok);
 
 	pixman_image_t* dstimg = pixman_image_create_bits_no_clear(
@@ -423,8 +424,8 @@ static void window_resize(struct window* w, int width, int height)
 	buffer_destroy(w->front_buffer);
 	buffer_destroy(w->back_buffer);
 
-	w->front_buffer = buffer_create(width, height, 4 * width, wl_shm_format);
+	w->front_buffer = buffer_create(width, height, 4 * width, shm_format);
-	w->back_buffer = buffer_create(width, height, 4 * width, wl_shm_format);
+	w->back_buffer = buffer_create(width, height, 4 * width, shm_format);
 }
 
 static void xdg_toplevel_configure(void* data, struct xdg_toplevel* toplevel,
@@ -767,7 +768,7 @@ int main(int argc, char* argv[])
 	vnc->alloc_fb = on_vnc_client_alloc_fb;
 	vnc->update_fb = on_vnc_client_update_fb;
 
-	if (vnc_client_set_pixel_format(vnc, wl_shm_format) < 0) {
+	if (vnc_client_set_pixel_format(vnc, shm_format) < 0) {
 		fprintf(stderr, "Unsupported pixel format\n");
 		goto vnc_setup_failure;
 	}

src/pixels.c

@@ -1,8 +1,49 @@
+/*
+ * Copyright (c) 2022 Andri Yngvason
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <libdrm/drm_fourcc.h>
 #include <wayland-client.h>
 #include <pixman.h>
 #include <stdbool.h>
+#include <assert.h>
 
-bool wl_shm_to_pixman_fmt(pixman_format_code_t* dst, enum wl_shm_format src)
+enum wl_shm_format drm_format_to_wl_shm(uint32_t in)
+{
+	assert(!(in & DRM_FORMAT_BIG_ENDIAN));
+
+	switch (in) {
+	case DRM_FORMAT_ARGB8888: return WL_SHM_FORMAT_ARGB8888;
+	case DRM_FORMAT_XRGB8888: return WL_SHM_FORMAT_XRGB8888;
+	}
+
+	return in;
+}
+
+uint32_t drm_format_from_wl_shm(enum wl_shm_format in)
+{
+	switch (in) {
+	case WL_SHM_FORMAT_ARGB8888: return DRM_FORMAT_ARGB8888;
+	case WL_SHM_FORMAT_XRGB8888: return DRM_FORMAT_XRGB8888;
+	default:;
+	}
+
+	return in;
+}
+
+bool drm_format_to_pixman_fmt(pixman_format_code_t* dst, uint32_t src)
 {
 #define LOWER_R r
 #define LOWER_G g
@@ -15,7 +56,7 @@ bool wl_shm_to_pixman_fmt(pixman_format_code_t* dst, enum wl_shm_format src)
 #define CONCAT_(a, b) a ## b
 #define CONCAT(a, b) CONCAT_(a, b)
 
-#define FMT_WL_SHM(x, y, z, v, a, b, c, d) WL_SHM_FORMAT_##x##y##z##v##a##b##c##d
+#define FMT_DRM(x, y, z, v, a, b, c, d) DRM_FORMAT_##x##y##z##v##a##b##c##d
 
 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
 #define FMT_PIXMAN(x, y, z, v, a, b, c, d) \
@@ -29,7 +70,7 @@ bool wl_shm_to_pixman_fmt(pixman_format_code_t* dst, enum wl_shm_format src)
 
 	switch (src) {
 #define X(...) \
-	case FMT_WL_SHM(__VA_ARGS__): *dst = FMT_PIXMAN(__VA_ARGS__); break
+	case FMT_DRM(__VA_ARGS__): *dst = FMT_PIXMAN(__VA_ARGS__); break
 
 	/* 32 bits */
 	X(A,R,G,B,8,8,8,8);

src/vnc.c

@@ -21,7 +21,7 @@
 #include <stdbool.h>
 #include <pixman.h>
 #include <rfb/rfbclient.h>
-#include <wayland-client.h>
+#include <libdrm/drm_fourcc.h>
 
 #include "vnc.h"
 
@@ -139,15 +139,14 @@ int vnc_client_connect(struct vnc_client* self, const char* address, int port)
 	return 0;
 }
 
-int vnc_client_set_pixel_format(struct vnc_client* self,
+int vnc_client_set_pixel_format(struct vnc_client* self, uint32_t format)
-		enum wl_shm_format format)
 {
 	rfbPixelFormat* dst = &self->client->format;
 	int bpp = -1;
 
 	switch (format) {
-	case WL_SHM_FORMAT_ARGB8888:
+	case DRM_FORMAT_ARGB8888:
-	case WL_SHM_FORMAT_XRGB8888:
+	case DRM_FORMAT_XRGB8888:
 		dst->redShift = 16;
 		dst->greenShift = 8;
 		dst->blueShift = 0;