render/pixel-format: add support for block-based formats

Some formats like sub-sampled YCbCr use a block of bytes to
store the color values for more than one pixel. Update our format
table to be able to handle such formats.

2 files changed, 9 insertions, 8 deletions

diff --git a/render/vulkan/renderer.c b/render/vulkan/renderer.c
index cf8c6a1f..dcb19c32 100644
--- a/render/vulkan/renderer.c
+++ b/render/vulkan/renderer.c
@@ -1671,6 +1671,9 @@ static bool vulkan_read_pixels(struct wlr_renderer *wlr_renderer,
 		wlr_log(WLR_ERROR, "vulkan_read_pixels: could not find pixel format info "
 				"for DRM format 0x%08x", drm_format);
 		return false;
+	} else if (pixel_format_info_pixels_per_block(pixel_format_info) != 1) {
+		wlr_log(WLR_ERROR, "vulkan_read_pixels: block formats are not supported");
+		return false;
 	}
 
 	const struct wlr_vk_format *wlr_vk_format = vulkan_get_format_from_drm(drm_format);
@@ -1864,13 +1867,13 @@ static bool vulkan_read_pixels(struct wlr_renderer *wlr_renderer,
 
 	const char *d = (const char *)v + img_sub_layout.offset;
 	unsigned char *p = (unsigned char *)data + dst_y * stride;
-	uint32_t bpp = pixel_format_info->bpp;
+	uint32_t bytes_per_pixel = pixel_format_info->bytes_per_block;
 	uint32_t pack_stride = img_sub_layout.rowPitch;
 	if (pack_stride == stride && dst_x == 0) {
 		memcpy(p, d, height * stride);
 	} else {
 		for (size_t i = 0; i < height; ++i) {
-			memcpy(p + i * stride + dst_x * bpp / 8, d + i * pack_stride, width * bpp / 8);
+			memcpy(p + i * stride + dst_x * bytes_per_pixel, d + i * pack_stride, width * bytes_per_pixel);
 		}
 	}
 
diff --git a/render/vulkan/texture.c b/render/vulkan/texture.c
index 8c602c2f..2518db05 100644
--- a/render/vulkan/texture.c
+++ b/render/vulkan/texture.c
@@ -49,7 +49,6 @@ static bool write_pixels(struct wlr_vk_texture *texture,
 	assert(format_info);
 
 	uint32_t bsize = 0;
-	unsigned bytespb = format_info->bpp / 8;
 
 	// deferred upload by transfer; using staging buffer
 	// calculate maximum side needed
@@ -64,7 +63,7 @@ static bool write_pixels(struct wlr_vk_texture *texture,
 		assert((uint32_t)rect.x2 <= texture->wlr_texture.width);
 		assert((uint32_t)rect.y2 <= texture->wlr_texture.height);
 
-		bsize += height * bytespb * width;
+		bsize += height * pixel_format_info_min_stride(format_info, width);
 	}
 
 	VkBufferImageCopy *copies = calloc((size_t)rects_len, sizeof(*copies));
@@ -74,7 +73,7 @@ static bool write_pixels(struct wlr_vk_texture *texture,
 	}
 
 	// get staging buffer
-	struct wlr_vk_buffer_span span = vulkan_get_stage_span(renderer, bsize, bytespb);
+	struct wlr_vk_buffer_span span = vulkan_get_stage_span(renderer, bsize, format_info->bytes_per_block);
 	if (!span.buffer || span.alloc.size != bsize) {
 		wlr_log(WLR_ERROR, "Failed to retrieve staging buffer");
 		free(copies);
@@ -100,13 +99,12 @@ static bool write_pixels(struct wlr_vk_texture *texture,
 		uint32_t height = rect.y2 - rect.y1;
 		uint32_t src_x = rect.x1;
 		uint32_t src_y = rect.y1;
-
-		uint32_t packed_stride = bytespb * width;
+		uint32_t packed_stride = (uint32_t)pixel_format_info_min_stride(format_info, width);
 
 		// write data into staging buffer span
 		const char *pdata = vdata; // data iterator
 		pdata += stride * src_y;
-		pdata += bytespb * src_x;
+		pdata += format_info->bytes_per_block * src_x;
 		if (src_x == 0 && width == texture->wlr_texture.width &&
 				stride == packed_stride) {
 			memcpy(map, pdata, packed_stride * height);