render/pixel-format: add support for block-based formats

Some formats like sub-sampled YCbCr use a block of bytes to
store the color values for more than one pixel. Update our format
table to be able to handle such formats.

7 files changed, 97 insertions, 53 deletions

diff --git a/render/allocator/drm_dumb.c b/render/allocator/drm_dumb.c
index 08ee0f0e..684ba2e3 100644
--- a/render/allocator/drm_dumb.c
+++ b/render/allocator/drm_dumb.c
@@ -44,6 +44,9 @@ static struct wlr_drm_dumb_buffer *create_buffer(
 		wlr_log(WLR_ERROR, "DRM format 0x%"PRIX32" not supported",
 			format->format);
 		return NULL;
+	} else if (pixel_format_info_pixels_per_block(info) != 1) {
+		wlr_log(WLR_ERROR, "Block formats are not supported");
+		return NULL;
 	}
 
 	struct wlr_drm_dumb_buffer *buffer = calloc(1, sizeof(*buffer));
@@ -55,7 +58,8 @@ static struct wlr_drm_dumb_buffer *create_buffer(
 
 	buffer->drm_fd = alloc->drm_fd;
 
-	if (drmModeCreateDumbBuffer(alloc->drm_fd, width, height, info->bpp, 0,
+	uint32_t bpp = 8 * info->bytes_per_block;
+	if (drmModeCreateDumbBuffer(alloc->drm_fd, width, height, bpp, 0,
 			&buffer->handle, &buffer->stride, &buffer->size) != 0) {
 		wlr_log_errno(WLR_ERROR, "Failed to create DRM dumb buffer");
 		goto create_destroy;
diff --git a/render/allocator/shm.c b/render/allocator/shm.c
index 9e49a144..55a8fab2 100644
--- a/render/allocator/shm.c
+++ b/render/allocator/shm.c
@@ -71,8 +71,7 @@ static struct wlr_buffer *allocator_create_buffer(
 	wlr_buffer_init(&buffer->base, &buffer_impl, width, height);
 
 	// TODO: consider using a single file for multiple buffers
-	int bytes_per_pixel = info->bpp / 8;
-	int stride = width * bytes_per_pixel; // TODO: align?
+	int stride = pixel_format_info_min_stride(info, width); // TODO: align?
 	buffer->size = stride * height;
 	buffer->shm.fd = allocate_shm_file(buffer->size);
 	if (buffer->shm.fd < 0) {
diff --git a/render/gles2/renderer.c b/render/gles2/renderer.c
index 1c8b4878..518068a3 100644
--- a/render/gles2/renderer.c
+++ b/render/gles2/renderer.c
@@ -464,6 +464,10 @@ static bool gles2_read_pixels(struct wlr_renderer *wlr_renderer,
 	const struct wlr_pixel_format_info *drm_fmt =
 		drm_get_pixel_format_info(fmt->drm_format);
 	assert(drm_fmt);
+	if (pixel_format_info_pixels_per_block(drm_fmt) != 1) {
+		wlr_log(WLR_ERROR, "Cannot read pixels: block formats are not supported");
+		return false;
+	}
 
 	push_gles2_debug(renderer);
 
@@ -474,7 +478,7 @@ static bool gles2_read_pixels(struct wlr_renderer *wlr_renderer,
 
 	unsigned char *p = (unsigned char *)data + dst_y * stride;
 	glPixelStorei(GL_PACK_ALIGNMENT, 1);
-	uint32_t pack_stride = width * drm_fmt->bpp / 8;
+	uint32_t pack_stride = pixel_format_info_min_stride(drm_fmt, width);
 	if (pack_stride == stride && dst_x == 0) {
 		// Under these particular conditions, we can read the pixels with only
 		// one glReadPixels call
@@ -486,7 +490,7 @@ static bool gles2_read_pixels(struct wlr_renderer *wlr_renderer,
 		for (size_t i = 0; i < height; ++i) {
 			uint32_t y = src_y + i;
 			glReadPixels(src_x, y, width, 1, fmt->gl_format,
-				fmt->gl_type, p + i * stride + dst_x * drm_fmt->bpp / 8);
+				fmt->gl_type, p + i * stride + dst_x * drm_fmt->bytes_per_block);
 		}
 	}
 
diff --git a/render/gles2/texture.c b/render/gles2/texture.c
index 63360d48..5faa54a2 100644
--- a/render/gles2/texture.c
+++ b/render/gles2/texture.c
@@ -56,6 +56,11 @@ static bool gles2_texture_update_from_buffer(struct wlr_texture *wlr_texture,
 	const struct wlr_pixel_format_info *drm_fmt =
 		drm_get_pixel_format_info(texture->drm_format);
 	assert(drm_fmt);
+	if (pixel_format_info_pixels_per_block(drm_fmt) != 1) {
+		wlr_buffer_end_data_ptr_access(buffer);
+		wlr_log(WLR_ERROR, "Cannot update texture: block formats are not supported");
+		return false;
+	}
 
 	if (!pixel_format_info_check_stride(drm_fmt, stride, buffer->width)) {
 		wlr_buffer_end_data_ptr_access(buffer);
@@ -76,7 +81,7 @@ static bool gles2_texture_update_from_buffer(struct wlr_texture *wlr_texture,
 	for (int i = 0; i < rects_len; i++) {
 		pixman_box32_t rect = rects[i];
 
-		glPixelStorei(GL_UNPACK_ROW_LENGTH_EXT, stride / (drm_fmt->bpp / 8));
+		glPixelStorei(GL_UNPACK_ROW_LENGTH_EXT, stride / drm_fmt->bytes_per_block);
 		glPixelStorei(GL_UNPACK_SKIP_PIXELS_EXT, rect.x1);
 		glPixelStorei(GL_UNPACK_SKIP_ROWS_EXT, rect.y1);
 
@@ -197,6 +202,10 @@ static struct wlr_texture *gles2_texture_from_pixels(
 	const struct wlr_pixel_format_info *drm_fmt =
 		drm_get_pixel_format_info(drm_format);
 	assert(drm_fmt);
+	if (pixel_format_info_pixels_per_block(drm_fmt) != 1) {
+		wlr_log(WLR_ERROR, "Cannot upload texture: block formats are not supported");
+		return NULL;
+	}
 
 	if (!pixel_format_info_check_stride(drm_fmt, stride, width)) {
 		return NULL;
@@ -227,7 +236,7 @@ static struct wlr_texture *gles2_texture_from_pixels(
 
 	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
 	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
-	glPixelStorei(GL_UNPACK_ROW_LENGTH_EXT, stride / (drm_fmt->bpp / 8));
+	glPixelStorei(GL_UNPACK_ROW_LENGTH_EXT, stride / drm_fmt->bytes_per_block);
 	glTexImage2D(GL_TEXTURE_2D, 0, internal_format, width, height, 0,
 		fmt->gl_format, fmt->gl_type, data);
 	glPixelStorei(GL_UNPACK_ROW_LENGTH_EXT, 0);
diff --git a/render/pixel_format.c b/render/pixel_format.c
index b83069b8..f3bec1d1 100644
--- a/render/pixel_format.c
+++ b/render/pixel_format.c
@@ -6,156 +6,156 @@
 static const struct wlr_pixel_format_info pixel_format_info[] = {
 	{
 		.drm_format = DRM_FORMAT_XRGB8888,
-		.bpp = 32,
+		.bytes_per_block = 4,
 	},
 	{
 		.drm_format = DRM_FORMAT_ARGB8888,
 		.opaque_substitute = DRM_FORMAT_XRGB8888,
-		.bpp = 32,
+		.bytes_per_block = 4,
 		.has_alpha = true,
 	},
 	{
 		.drm_format = DRM_FORMAT_XBGR8888,
-		.bpp = 32,
+		.bytes_per_block = 4,
 	},
 	{
 		.drm_format = DRM_FORMAT_ABGR8888,
 		.opaque_substitute = DRM_FORMAT_XBGR8888,
-		.bpp = 32,
+		.bytes_per_block = 4,
 		.has_alpha = true,
 	},
 	{
 		.drm_format = DRM_FORMAT_RGBX8888,
-		.bpp = 32,
+		.bytes_per_block = 4,
 	},
 	{
 		.drm_format = DRM_FORMAT_RGBA8888,
 		.opaque_substitute = DRM_FORMAT_RGBX8888,
-		.bpp = 32,
+		.bytes_per_block = 4,
 		.has_alpha = true,
 	},
 	{
 		.drm_format = DRM_FORMAT_BGRX8888,
-		.bpp = 32,
+		.bytes_per_block = 4,
 	},
 	{
 		.drm_format = DRM_FORMAT_BGRA8888,
 		.opaque_substitute = DRM_FORMAT_BGRX8888,
-		.bpp = 32,
+		.bytes_per_block = 4,
 		.has_alpha = true,
 	},
 	{
 		.drm_format = DRM_FORMAT_R8,
-		.bpp = 8,
+		.bytes_per_block = 1,
 	},
 	{
 		.drm_format = DRM_FORMAT_GR88,
-		.bpp = 16,
+		.bytes_per_block = 2,
 	},
 	{
 		.drm_format = DRM_FORMAT_RGB888,
-		.bpp = 24,
+		.bytes_per_block = 3,
 	},
 	{
 		.drm_format = DRM_FORMAT_BGR888,
-		.bpp = 24,
+		.bytes_per_block = 3,
 	},
 	{
 		.drm_format = DRM_FORMAT_RGBX4444,
-		.bpp = 16,
+		.bytes_per_block = 2,
 	},
 	{
 		.drm_format = DRM_FORMAT_RGBA4444,
 		.opaque_substitute = DRM_FORMAT_RGBX4444,
-		.bpp = 16,
+		.bytes_per_block = 2,
 		.has_alpha = true,
 	},
 	{
 		.drm_format = DRM_FORMAT_BGRX4444,
-		.bpp = 16,
+		.bytes_per_block = 2,
 	},
 	{
 		.drm_format = DRM_FORMAT_BGRA4444,
 		.opaque_substitute = DRM_FORMAT_BGRX4444,
-		.bpp = 16,
+		.bytes_per_block = 2,
 		.has_alpha = true,
 	},
 	{
 		.drm_format = DRM_FORMAT_RGBX5551,
-		.bpp = 16,
+		.bytes_per_block = 2,
 	},
 	{
 		.drm_format = DRM_FORMAT_RGBA5551,
 		.opaque_substitute = DRM_FORMAT_RGBX5551,
-		.bpp = 16,
+		.bytes_per_block = 2,
 		.has_alpha = true,
 	},
 	{
 		.drm_format = DRM_FORMAT_BGRX5551,
-		.bpp = 16,
+		.bytes_per_block = 2,
 	},
 	{
 		.drm_format = DRM_FORMAT_BGRA5551,
 		.opaque_substitute = DRM_FORMAT_BGRX5551,
-		.bpp = 16,
+		.bytes_per_block = 2,
 		.has_alpha = true,
 	},
 	{
 		.drm_format = DRM_FORMAT_XRGB1555,
-		.bpp = 16,
+		.bytes_per_block = 2,
 	},
 	{
 		.drm_format = DRM_FORMAT_ARGB1555,
 		.opaque_substitute = DRM_FORMAT_XRGB1555,
-		.bpp = 16,
+		.bytes_per_block = 2,
 		.has_alpha = true,
 	},
 	{
 		.drm_format = DRM_FORMAT_RGB565,
-		.bpp = 16,
+		.bytes_per_block = 2,
 	},
 	{
 		.drm_format = DRM_FORMAT_BGR565,
-		.bpp = 16,
+		.bytes_per_block = 2,
 	},
 	{
 		.drm_format = DRM_FORMAT_XRGB2101010,
-		.bpp = 32,
+		.bytes_per_block = 4,
 	},
 	{
 		.drm_format = DRM_FORMAT_ARGB2101010,
 		.opaque_substitute = DRM_FORMAT_XRGB2101010,
-		.bpp = 32,
+		.bytes_per_block = 4,
 		.has_alpha = true,
 	},
 	{
 		.drm_format = DRM_FORMAT_XBGR2101010,
-		.bpp = 32,
+		.bytes_per_block = 4,
 	},
 	{
 		.drm_format = DRM_FORMAT_ABGR2101010,
 		.opaque_substitute = DRM_FORMAT_XBGR2101010,
-		.bpp = 32,
+		.bytes_per_block = 4,
 		.has_alpha = true,
 	},
 	{
 		.drm_format = DRM_FORMAT_XBGR16161616F,
-		.bpp = 64,
+		.bytes_per_block = 8,
 	},
 	{
 		.drm_format = DRM_FORMAT_ABGR16161616F,
 		.opaque_substitute = DRM_FORMAT_XBGR16161616F,
-		.bpp = 64,
+		.bytes_per_block = 8,
 		.has_alpha = true,
 	},
 	{
 		.drm_format = DRM_FORMAT_XBGR16161616,
-		.bpp = 64,
+		.bytes_per_block = 8,
 	},
 	{
 		.drm_format = DRM_FORMAT_ABGR16161616,
 		.opaque_substitute = DRM_FORMAT_XBGR16161616,
-		.bpp = 64,
+		.bytes_per_block = 8,
 		.has_alpha = true,
 	},
 };
@@ -195,19 +195,46 @@ enum wl_shm_format convert_drm_format_to_wl_shm(uint32_t fmt) {
 	}
 }
 
+uint32_t pixel_format_info_pixels_per_block(const struct wlr_pixel_format_info *info) {
+	uint32_t pixels = info->block_width * info->block_height;
+	return pixels > 0 ? pixels : 1;
+}
+
+static int32_t div_round_up(int32_t dividend, int32_t divisor) {
+	int32_t quotient = dividend / divisor;
+	if (dividend % divisor != 0) {
+		quotient++;
+	}
+	return quotient;
+}
+
+int32_t pixel_format_info_min_stride(const struct wlr_pixel_format_info *fmt, int32_t width) {
+	int32_t pixels_per_block = (int32_t)pixel_format_info_pixels_per_block(fmt);
+	int32_t bytes_per_block = (int32_t)fmt->bytes_per_block;
+	if (width > INT32_MAX / bytes_per_block) {
+		wlr_log(WLR_DEBUG, "Invalid width %d (overflow)", width);
+		return 0;
+	}
+	return div_round_up(width * bytes_per_block, pixels_per_block);
+}
+
 bool pixel_format_info_check_stride(const struct wlr_pixel_format_info *fmt,
 		int32_t stride, int32_t width) {
-	assert(fmt->bpp > 0 && fmt->bpp % 8 == 0);
-	int32_t bytes_per_pixel = (int32_t)(fmt->bpp / 8);
-	if (stride % bytes_per_pixel != 0) {
+	int32_t bytes_per_block = (int32_t)fmt->bytes_per_block;
+	if (stride % bytes_per_block != 0) {
 		wlr_log(WLR_DEBUG, "Invalid stride %d (incompatible with %d "
-			"bytes-per-pixel)", stride, bytes_per_pixel);
+			"bytes-per-block)", stride, bytes_per_block);
 		return false;
 	}
-	if (stride / bytes_per_pixel < width) {
+
+	int32_t min_stride = pixel_format_info_min_stride(fmt, width);
+	if (min_stride <= 0) {
+		return false;
+	} else if (stride < min_stride) {
 		wlr_log(WLR_DEBUG, "Invalid stride %d (too small for %d "
-			"bytes-per-pixel and width %d)", stride, bytes_per_pixel, width);
+			"bytes-per-block and width %d)", stride, bytes_per_block, width);
 		return false;
 	}
+
 	return true;
 }
diff --git a/render/vulkan/renderer.c b/render/vulkan/renderer.c
index cf8c6a1f..dcb19c32 100644
--- a/render/vulkan/renderer.c
+++ b/render/vulkan/renderer.c
@@ -1671,6 +1671,9 @@ static bool vulkan_read_pixels(struct wlr_renderer *wlr_renderer,
 		wlr_log(WLR_ERROR, "vulkan_read_pixels: could not find pixel format info "
 				"for DRM format 0x%08x", drm_format);
 		return false;
+	} else if (pixel_format_info_pixels_per_block(pixel_format_info) != 1) {
+		wlr_log(WLR_ERROR, "vulkan_read_pixels: block formats are not supported");
+		return false;
 	}
 
 	const struct wlr_vk_format *wlr_vk_format = vulkan_get_format_from_drm(drm_format);
@@ -1864,13 +1867,13 @@ static bool vulkan_read_pixels(struct wlr_renderer *wlr_renderer,
 
 	const char *d = (const char *)v + img_sub_layout.offset;
 	unsigned char *p = (unsigned char *)data + dst_y * stride;
-	uint32_t bpp = pixel_format_info->bpp;
+	uint32_t bytes_per_pixel = pixel_format_info->bytes_per_block;
 	uint32_t pack_stride = img_sub_layout.rowPitch;
 	if (pack_stride == stride && dst_x == 0) {
 		memcpy(p, d, height * stride);
 	} else {
 		for (size_t i = 0; i < height; ++i) {
-			memcpy(p + i * stride + dst_x * bpp / 8, d + i * pack_stride, width * bpp / 8);
+			memcpy(p + i * stride + dst_x * bytes_per_pixel, d + i * pack_stride, width * bytes_per_pixel);
 		}
 	}
 
diff --git a/render/vulkan/texture.c b/render/vulkan/texture.c
index 8c602c2f..2518db05 100644
--- a/render/vulkan/texture.c
+++ b/render/vulkan/texture.c
@@ -49,7 +49,6 @@ static bool write_pixels(struct wlr_vk_texture *texture,
 	assert(format_info);
 
 	uint32_t bsize = 0;
-	unsigned bytespb = format_info->bpp / 8;
 
 	// deferred upload by transfer; using staging buffer
 	// calculate maximum side needed
@@ -64,7 +63,7 @@ static bool write_pixels(struct wlr_vk_texture *texture,
 		assert((uint32_t)rect.x2 <= texture->wlr_texture.width);
 		assert((uint32_t)rect.y2 <= texture->wlr_texture.height);
 
-		bsize += height * bytespb * width;
+		bsize += height * pixel_format_info_min_stride(format_info, width);
 	}
 
 	VkBufferImageCopy *copies = calloc((size_t)rects_len, sizeof(*copies));
@@ -74,7 +73,7 @@ static bool write_pixels(struct wlr_vk_texture *texture,
 	}
 
 	// get staging buffer
-	struct wlr_vk_buffer_span span = vulkan_get_stage_span(renderer, bsize, bytespb);
+	struct wlr_vk_buffer_span span = vulkan_get_stage_span(renderer, bsize, format_info->bytes_per_block);
 	if (!span.buffer || span.alloc.size != bsize) {
 		wlr_log(WLR_ERROR, "Failed to retrieve staging buffer");
 		free(copies);
@@ -100,13 +99,12 @@ static bool write_pixels(struct wlr_vk_texture *texture,
 		uint32_t height = rect.y2 - rect.y1;
 		uint32_t src_x = rect.x1;
 		uint32_t src_y = rect.y1;
-
-		uint32_t packed_stride = bytespb * width;
+		uint32_t packed_stride = (uint32_t)pixel_format_info_min_stride(format_info, width);
 
 		// write data into staging buffer span
 		const char *pdata = vdata; // data iterator
 		pdata += stride * src_y;
-		pdata += bytespb * src_x;
+		pdata += format_info->bytes_per_block * src_x;
 		if (src_x == 0 && width == texture->wlr_texture.width &&
 				stride == packed_stride) {
 			memcpy(map, pdata, packed_stride * height);