diff --git a/Makefile b/Makefile
index c65a4a0..a9bb47f 100644
--- a/Makefile
+++ b/Makefile
@@ -8,6 +8,7 @@ SOURCES := \
 	src/raw-encoding.c \
 	src/pixels.c \
 	src/damage.c \
+	src/fb.c \
 
 include common.mk
 
diff --git a/bench/zrle-bench.c b/bench/zrle-bench.c
index ffba945..a209a3a 100644
--- a/bench/zrle-bench.c
+++ b/bench/zrle-bench.c
@@ -42,39 +42,42 @@ void memcpy_unoptimized(void* dst, const void* src, size_t len)
 }
 #pragma pop_options
 
-int read_png_file(struct nvnc_fb* fb, const char *filename);
+struct nvnc_fb* read_png_file(const char *filename);
 
 int run_benchmark(const char *image)
 {
 	int rc = -1;
 
-	struct nvnc_fb fb = { 0 };
-	rc = read_png_file(&fb, image);
-	if (rc < 0)
+	struct nvnc_fb* fb = read_png_file(image);
+	if (!fb)
 		return -1;
 
+	void *addr = nvnc_fb_get_addr(fb);
+	int width = nvnc_fb_get_width(fb);
+	int height = nvnc_fb_get_height(fb);
+
 	struct rfb_pixel_format pixfmt;
 	rfb_pixfmt_from_fourcc(&pixfmt, DRM_FORMAT_ARGB8888);
 
 	struct pixman_region16 region;
 	pixman_region_init(&region);
 
-	pixman_region_union_rect(&region, &region, 0, 0, fb.width, fb.height);
+	pixman_region_union_rect(&region, &region, 0, 0, width, height);
 
 	struct vec frame;
-	vec_init(&frame, fb.width * fb.height * 3 / 2);
+	vec_init(&frame, width * height * 3 / 2);
 
 	z_stream zs = { 0 };
 
 	deflateInit(&zs, Z_DEFAULT_COMPRESSION);
 
-	void *dummy = malloc(fb.width * fb.height * 4);
+	void *dummy = malloc(width * height * 4);
 	if (!dummy)
 		goto failure;
 
 	uint64_t start_time = gettime_us(CLOCK_PROCESS_CPUTIME_ID);
 
-	memcpy_unoptimized(dummy, fb.addr, fb.width * fb.height * 4);
+	memcpy_unoptimized(dummy, addr, width * height * 4);
 
 	uint64_t end_time = gettime_us(CLOCK_PROCESS_CPUTIME_ID);
 	printf("memcpy baseline for %s took %"PRIu64" micro seconds\n", image,
@@ -83,7 +86,7 @@ int run_benchmark(const char *image)
 	free(dummy);
 
 	start_time = gettime_us(CLOCK_PROCESS_CPUTIME_ID);
-	rc = zrle_encode_frame(&zs, &frame, &pixfmt, &fb, &pixfmt, &region);
+	rc = zrle_encode_frame(&zs, &frame, &pixfmt, fb, &pixfmt, &region);
 
 	end_time = gettime_us(CLOCK_PROCESS_CPUTIME_ID);
 	printf("Encoding %s took %"PRIu64" micro seconds\n", image,
@@ -98,7 +101,7 @@ int run_benchmark(const char *image)
 failure:
 	pixman_region_fini(&region);
 	vec_destroy(&frame);
-	free(fb.addr);
+	nvnc_fb_unref(fb);
 	return 0;
 }
 
diff --git a/examples/draw.c b/examples/draw.c
index efb6a3b..e74dbe9 100644
--- a/examples/draw.c
+++ b/examples/draw.c
@@ -25,7 +25,7 @@
 #include <libdrm/drm_fourcc.h>
 
 struct draw {
-	struct nvnc_fb fb;
+	struct nvnc_fb* fb;
 };
 
 void on_fb_req(struct nvnc_client *client, bool incremental, uint16_t x, uint16_t y,
@@ -40,10 +40,12 @@ void on_fb_req(struct nvnc_client *client, bool incremental, uint16_t x, uint16_
 	struct draw *draw = nvnc_get_userdata(server);
 	assert(draw);
 
+	int fbwidth = nvnc_fb_get_width(draw->fb);
+	int fbheight = nvnc_fb_get_height(draw->fb);
+
 	struct pixman_region16 region;
-	pixman_region_init_rect(&region, 0, 0, draw->fb.width,
-				draw->fb.height);
-	nvnc_update_fb(server, &draw->fb, &region, NULL);
+	pixman_region_init_rect(&region, 0, 0, fbwidth, fbheight);
+	nvnc_update_fb(server, draw->fb, &region, NULL);
 	pixman_region_fini(&region);
 }
 
@@ -59,15 +61,16 @@ void on_pointer_event(struct nvnc_client *client, uint16_t x, uint16_t y,
 	struct draw *draw = nvnc_get_userdata(server);
 	assert(draw);
 
-	uint32_t *image = draw->fb.addr;
+	uint32_t *image = nvnc_fb_get_addr(draw->fb);
+	int width = nvnc_fb_get_width(draw->fb);
+	int height = nvnc_fb_get_height(draw->fb);
 
-	image[x + y * draw->fb.width] = 0;
+	image[x + y * width] = 0;
 
 	struct pixman_region16 region;
-	pixman_region_init_rect(&region, 0, 0, draw->fb.width,
-				draw->fb.height);
+	pixman_region_init_rect(&region, 0, 0, width, height);
 	pixman_region_intersect_rect(&region, &region, x, y, 1, 1);
-	nvnc_update_fb(server, &draw->fb, &region, NULL);
+	nvnc_update_fb(server, draw->fb, &region, NULL);
 	pixman_region_fini(&region);
 }
 
@@ -75,21 +78,18 @@ int main(int argc, char *argv[])
 {
 	struct draw draw; 
 
-	draw.fb.width = 500;
-	draw.fb.height = 500;
-	draw.fb.size = draw.fb.width * draw.fb.height * 4;
-	draw.fb.fourcc_format = DRM_FORMAT_RGBX8888;
-	draw.fb.fourcc_modifier = DRM_FORMAT_MOD_LINEAR;
+	int width = 500, height = 500;
+	uint32_t format = DRM_FORMAT_RGBX8888;
+	draw.fb = nvnc_fb_new(width, height, format);
+	assert(draw.fb);
 
-	draw.fb.addr = malloc(draw.fb.size);
-	assert(draw.fb.addr);
+	void* addr = nvnc_fb_get_addr(draw.fb);
 
-	memset(draw.fb.addr, 0xff, draw.fb.size);
+	memset(addr, 0xff, width * height * 4);
 
 	struct nvnc *server = nvnc_open("127.0.0.1", 5900);
 
-	nvnc_set_dimensions(server, draw.fb.width, draw.fb.height,
-			    draw.fb.fourcc_format);
+	nvnc_set_dimensions(server, width, height, format);
 	nvnc_set_name(server, "Draw");
 	nvnc_set_fb_req_fn(server, on_fb_req);
 	nvnc_set_pointer_fn(server, on_pointer_event);
diff --git a/examples/png-server.c b/examples/png-server.c
index 86a654c..8757eaa 100644
--- a/examples/png-server.c
+++ b/examples/png-server.c
@@ -21,7 +21,7 @@
 #include <assert.h>
 #include <pixman.h>
 
-int read_png_file(struct nvnc_fb* fb, const char *filename);
+struct nvnc_fb* read_png_file(const char *filename);
 
 void on_fb_req(struct nvnc_client *client, bool incremental,
 	       uint16_t x, uint16_t y, uint16_t width, uint16_t height)
@@ -36,7 +36,8 @@ void on_fb_req(struct nvnc_client *client, bool incremental,
 	assert(fb);
 
 	struct pixman_region16 region;
-	pixman_region_init_rect(&region, 0, 0, fb->width, fb->height);
+	pixman_region_init_rect(&region, 0, 0, nvnc_fb_get_width(fb),
+				nvnc_fb_get_height(fb));
 	nvnc_update_fb(server, fb, &region, NULL);
 	pixman_region_fini(&region);
 }
@@ -50,15 +51,19 @@ int main(int argc, char *argv[])
 		return 1;
 	}
 
-	struct nvnc_fb fb = { 0 };
-	if (read_png_file(&fb, file) < 0) {
+	struct nvnc_fb* fb = read_png_file(file);
+	if (!fb) {
 		printf("Failed to read png file\n");
 		return 1;
 	}
 
 	struct nvnc *server = nvnc_open("127.0.0.1", 5900);
 
-	nvnc_set_dimensions(server, fb.width, fb.height, fb.fourcc_format);
+	int width = nvnc_fb_get_width(fb);
+	int height = nvnc_fb_get_height(fb);
+	uint32_t fourcc_format = nvnc_fb_get_fourcc_format(fb);
+
+	nvnc_set_dimensions(server, width, height, fourcc_format);
 	nvnc_set_name(server, file);
 	nvnc_set_fb_req_fn(server, on_fb_req);
 	nvnc_set_userdata(server, &fb);
@@ -66,4 +71,5 @@ int main(int argc, char *argv[])
 	uv_run(uv_default_loop(), UV_RUN_DEFAULT);
 
 	nvnc_close(server);
+	nvnc_fb_unref(fb);
 }
diff --git a/include/fb.h b/include/fb.h
new file mode 100644
index 0000000..a725d5d
--- /dev/null
+++ b/include/fb.h
@@ -0,0 +1,15 @@
+#pragma once
+
+#include <unistd.h>
+#include <stdint.h>
+
+struct nvnc_fb {
+        int ref;
+        void *addr;
+        size_t size;
+        uint16_t width;
+        uint16_t height;
+        uint32_t fourcc_format;
+        uint64_t fourcc_modifier;
+};
+
diff --git a/include/neatvnc.h b/include/neatvnc.h
index 25a8071..3d21039 100644
--- a/include/neatvnc.h
+++ b/include/neatvnc.h
@@ -21,6 +21,7 @@
 
 struct nvnc;
 struct nvnc_client;
+struct nvnc_fb;
 struct pixman_region16;
 
 enum nvnc_button_mask {
@@ -35,17 +36,6 @@ enum nvnc_modifier {
         NVNC_MOD_Y_INVERT = 1 << 0,
 };
 
-struct nvnc_fb {
-        void *addr;
-        uint32_t size;
-        uint16_t width;
-        uint16_t height;
-        uint32_t fourcc_format;
-        uint64_t fourcc_modifier;
-        enum nvnc_modifier nvnc_modifier;
-        uint32_t reserved[4];
-};
-
 typedef void (*nvnc_key_fn)(struct nvnc_client*, uint32_t keysym, bool is_pressed);
 typedef void (*nvnc_pointer_fn)(struct nvnc_client*, uint16_t x, uint16_t y,
                                 enum nvnc_button_mask);
@@ -75,6 +65,17 @@ void nvnc_set_fb_req_fn(struct nvnc *self, nvnc_fb_req_fn);
 void nvnc_set_new_client_fn(struct nvnc *self, nvnc_client_fn);
 void nvnc_set_client_cleanup_fn(struct nvnc_client *self, nvnc_client_fn fn);
 
+struct nvnc_fb* nvnc_fb_new(uint16_t width, uint16_t height,
+                            uint32_t fourcc_format);
+
+void nvnc_fb_ref(struct nvnc_fb* fb);
+void nvnc_fb_unref(struct nvnc_fb* fb);
+
+void* nvnc_fb_get_addr(const struct nvnc_fb* fb);
+uint16_t nvnc_fb_get_width(const struct nvnc_fb* fb);
+uint16_t nvnc_fb_get_height(const struct nvnc_fb* fb);
+uint32_t nvnc_fb_get_fourcc_format(const struct nvnc_fb* fb);
+
 /*
  * Send an updated framebuffer to all clients with pending update requests.
  *
diff --git a/src/damage.c b/src/damage.c
index c529749..c94f9c4 100644
--- a/src/damage.c
+++ b/src/damage.c
@@ -1,4 +1,5 @@
 #include "neatvnc.h"
+#include "fb.h"
 
 #include <stdlib.h>
 #include <stdbool.h>
@@ -30,8 +31,6 @@ static bool fbs_are_compatible(const struct nvnc_fb *fb0,
 			       const struct nvnc_fb *fb1)
 {
 	return fb0->fourcc_format == fb1->fourcc_format
-	    && fb0->fourcc_modifier == fb1->fourcc_modifier
-	    && fb0->nvnc_modifier == fb1->nvnc_modifier
 	    && fb0->width == fb1->width
 	    && fb0->height == fb1->height;
 }
@@ -57,7 +56,7 @@ int check_damage_linear(struct pixman_region16 *damage,
 	int width = fb0->width;
 	int height = fb0->height;
 
-	bool y_invert = !!(fb0->nvnc_modifier & NVNC_MOD_Y_INVERT);
+	bool y_invert = false;
 
 	assert(x_hint + width_hint <= width);
 	assert(y_hint + height_hint <= height);
diff --git a/src/fb.c b/src/fb.c
new file mode 100644
index 0000000..894162e
--- /dev/null
+++ b/src/fb.c
@@ -0,0 +1,77 @@
+#include "fb.h"
+#include "neatvnc.h"
+
+#include <stdlib.h>
+#include <unistd.h>
+
+#define EXPORT __attribute__((visibility("default")))
+
+EXPORT
+struct nvnc_fb* nvnc_fb_new(uint16_t width, uint16_t height,
+                            uint32_t fourcc_format)
+{
+	struct nvnc_fb* fb = calloc(1, sizeof(*fb));
+	if (!fb)
+		return NULL;
+
+	fb->ref = 1;
+	fb->width = width;
+	fb->height = height;
+	fb->fourcc_format = fourcc_format;
+	fb->size = width * height * 4; /* Assume 4 byte format for now */
+
+	/* fb could be allocated in single allocation, but I want to reserve
+	 * the possiblity to create an fb with a pixel buffer passed from the
+	 * user.
+	 */
+	fb->addr = malloc(fb->size);
+	if (!fb->addr) {
+		free(fb);
+		fb = NULL;
+	}
+
+	return fb;
+}
+
+EXPORT
+void* nvnc_fb_get_addr(const struct nvnc_fb* fb)
+{
+	return fb->addr;
+}
+
+EXPORT
+uint16_t nvnc_fb_get_width(const struct nvnc_fb* fb)
+{
+	return fb->width;
+}
+
+EXPORT
+uint16_t nvnc_fb_get_height(const struct nvnc_fb* fb)
+{
+	return fb->height;
+}
+
+EXPORT
+uint32_t nvnc_fb_get_fourcc_format(const struct nvnc_fb* fb)
+{
+	return fb->fourcc_format;
+}
+
+void nvnc__fb_free(struct nvnc_fb* fb)
+{
+	free(fb->addr);
+	free(fb);
+}
+
+EXPORT
+void nvnc_fb_ref(struct nvnc_fb* fb)
+{
+	fb->ref++;
+}
+
+EXPORT
+void nvnc_fb_unref(struct nvnc_fb* fb)
+{
+	if (--fb->ref == 0)
+		nvnc__fb_free(fb);
+}
diff --git a/src/pngfb.c b/src/pngfb.c
index 7734c51..112e2c8 100644
--- a/src/pngfb.c
+++ b/src/pngfb.c
@@ -13,7 +13,7 @@
 #include <assert.h>
 #include <libdrm/drm_fourcc.h>
 
-int read_png_file(struct nvnc_fb* fb, const char *filename) {
+struct nvnc_fb* read_png_file(const char *filename) {
 	int width, height;
 	png_byte color_type;
 	png_byte bit_depth;
@@ -67,8 +67,11 @@ int read_png_file(struct nvnc_fb* fb, const char *filename) {
 	png_read_update_info(png, info);
 
 	size_t row_bytes = png_get_rowbytes(png, info);
-	uint8_t *addr = malloc(row_bytes * height);
-	assert(addr);
+	assert(row_bytes == width * 4);
+	struct nvnc_fb* fb = nvnc_fb_new(width, height, DRM_FORMAT_ARGB8888);
+	assert(fb);
+
+	uint8_t *addr = nvnc_fb_get_addr(fb);
 
 	row_pointers = malloc(sizeof(png_bytep) * height);
 	assert(row_pointers);
@@ -83,12 +86,5 @@ int read_png_file(struct nvnc_fb* fb, const char *filename) {
 
 	png_destroy_read_struct(&png, &info, NULL);
 
-	fb->addr = addr;
-	fb->size = width * height * 4;
-	fb->width = width;
-	fb->height = height;
-	fb->fourcc_format = DRM_FORMAT_ABGR8888;
-	fb->fourcc_modifier = DRM_FORMAT_MOD_LINEAR;
-
-	return 0;
+	return fb;
 }
diff --git a/src/raw-encoding.c b/src/raw-encoding.c
index 4fb3298..2ba06c9 100644
--- a/src/raw-encoding.c
+++ b/src/raw-encoding.c
@@ -1,6 +1,7 @@
 #include "neatvnc.h"
 #include "rfb-proto.h"
 #include "vec.h"
+#include "fb.h"
 
 #include <pixman.h>
 
@@ -25,7 +26,7 @@ int raw_encode_box(struct vec *dst, const struct rfb_pixel_format *dst_fmt,
 
 	uint32_t* b = fb->addr;
 
-	if (fb->nvnc_modifier & NVNC_MOD_Y_INVERT) {
+	if (false) {
 		for (int y = y_start; y < y_start + height; ++y)
 			for (int x = x_start; x < x_start + width; ++x)
 				vec_fast_append_32(dst, b[x + (fb->height - y - 1) * stride]);
diff --git a/src/server.c b/src/server.c
index ac77569..cc638db 100644
--- a/src/server.c
+++ b/src/server.c
@@ -20,6 +20,7 @@
 #include "raw-encoding.h"
 #include "vec.h"
 #include "type-macros.h"
+#include "fb.h"
 #include "neatvnc.h"
 
 #include <stdlib.h>
diff --git a/src/zrle.c b/src/zrle.c
index 8295455..a35274c 100644
--- a/src/zrle.c
+++ b/src/zrle.c
@@ -21,6 +21,7 @@
 #include "miniz.h"
 #include "neatvnc.h"
 #include "pixels.h"
+#include "fb.h"
 
 #include <stdint.h>
 #include <unistd.h>
@@ -250,14 +251,12 @@ int zrle_encode_box(struct vec* out, const struct rfb_pixel_format *dst_fmt,
 		int tile_width = width - tile_x >= TILE_LENGTH ? TILE_LENGTH : width - tile_x;
 		int tile_height = height - tile_y >= TILE_LENGTH ? TILE_LENGTH : height - tile_y;
 
-		int y_off = !(fb->nvnc_modifier & NVNC_MOD_Y_INVERT)
-			  ? y + tile_y
-			  : fb->height - y - tile_y - tile_height;
+		int y_off = y + tile_y;
 
 		zrle_copy_tile(tile,
 			       ((uint32_t*)fb->addr) + x + tile_x + y_off * stride,
 			       stride, tile_width, tile_height,
-			       fb->nvnc_modifier);
+			       0);
 
 		zrle_encode_tile(&in, dst_fmt, tile, src_fmt,
 				 tile_width * tile_height);