render/pixel-format: add support for block-based formats

Some formats like sub-sampled YCbCr use a block of bytes to
store the color values for more than one pixel. Update our format
table to be able to handle such formats.

1 files changed, 65 insertions, 38 deletions

diff --git a/render/pixel_format.c b/render/pixel_format.c
index b83069b8..f3bec1d1 100644
--- a/render/pixel_format.c
+++ b/render/pixel_format.c
@@ -6,156 +6,156 @@
 static const struct wlr_pixel_format_info pixel_format_info[] = {
 	{
 		.drm_format = DRM_FORMAT_XRGB8888,
-		.bpp = 32,
+		.bytes_per_block = 4,
 	},
 	{
 		.drm_format = DRM_FORMAT_ARGB8888,
 		.opaque_substitute = DRM_FORMAT_XRGB8888,
-		.bpp = 32,
+		.bytes_per_block = 4,
 		.has_alpha = true,
 	},
 	{
 		.drm_format = DRM_FORMAT_XBGR8888,
-		.bpp = 32,
+		.bytes_per_block = 4,
 	},
 	{
 		.drm_format = DRM_FORMAT_ABGR8888,
 		.opaque_substitute = DRM_FORMAT_XBGR8888,
-		.bpp = 32,
+		.bytes_per_block = 4,
 		.has_alpha = true,
 	},
 	{
 		.drm_format = DRM_FORMAT_RGBX8888,
-		.bpp = 32,
+		.bytes_per_block = 4,
 	},
 	{
 		.drm_format = DRM_FORMAT_RGBA8888,
 		.opaque_substitute = DRM_FORMAT_RGBX8888,
-		.bpp = 32,
+		.bytes_per_block = 4,
 		.has_alpha = true,
 	},
 	{
 		.drm_format = DRM_FORMAT_BGRX8888,
-		.bpp = 32,
+		.bytes_per_block = 4,
 	},
 	{
 		.drm_format = DRM_FORMAT_BGRA8888,
 		.opaque_substitute = DRM_FORMAT_BGRX8888,
-		.bpp = 32,
+		.bytes_per_block = 4,
 		.has_alpha = true,
 	},
 	{
 		.drm_format = DRM_FORMAT_R8,
-		.bpp = 8,
+		.bytes_per_block = 1,
 	},
 	{
 		.drm_format = DRM_FORMAT_GR88,
-		.bpp = 16,
+		.bytes_per_block = 2,
 	},
 	{
 		.drm_format = DRM_FORMAT_RGB888,
-		.bpp = 24,
+		.bytes_per_block = 3,
 	},
 	{
 		.drm_format = DRM_FORMAT_BGR888,
-		.bpp = 24,
+		.bytes_per_block = 3,
 	},
 	{
 		.drm_format = DRM_FORMAT_RGBX4444,
-		.bpp = 16,
+		.bytes_per_block = 2,
 	},
 	{
 		.drm_format = DRM_FORMAT_RGBA4444,
 		.opaque_substitute = DRM_FORMAT_RGBX4444,
-		.bpp = 16,
+		.bytes_per_block = 2,
 		.has_alpha = true,
 	},
 	{
 		.drm_format = DRM_FORMAT_BGRX4444,
-		.bpp = 16,
+		.bytes_per_block = 2,
 	},
 	{
 		.drm_format = DRM_FORMAT_BGRA4444,
 		.opaque_substitute = DRM_FORMAT_BGRX4444,
-		.bpp = 16,
+		.bytes_per_block = 2,
 		.has_alpha = true,
 	},
 	{
 		.drm_format = DRM_FORMAT_RGBX5551,
-		.bpp = 16,
+		.bytes_per_block = 2,
 	},
 	{
 		.drm_format = DRM_FORMAT_RGBA5551,
 		.opaque_substitute = DRM_FORMAT_RGBX5551,
-		.bpp = 16,
+		.bytes_per_block = 2,
 		.has_alpha = true,
 	},
 	{
 		.drm_format = DRM_FORMAT_BGRX5551,
-		.bpp = 16,
+		.bytes_per_block = 2,
 	},
 	{
 		.drm_format = DRM_FORMAT_BGRA5551,
 		.opaque_substitute = DRM_FORMAT_BGRX5551,
-		.bpp = 16,
+		.bytes_per_block = 2,
 		.has_alpha = true,
 	},
 	{
 		.drm_format = DRM_FORMAT_XRGB1555,
-		.bpp = 16,
+		.bytes_per_block = 2,
 	},
 	{
 		.drm_format = DRM_FORMAT_ARGB1555,
 		.opaque_substitute = DRM_FORMAT_XRGB1555,
-		.bpp = 16,
+		.bytes_per_block = 2,
 		.has_alpha = true,
 	},
 	{
 		.drm_format = DRM_FORMAT_RGB565,
-		.bpp = 16,
+		.bytes_per_block = 2,
 	},
 	{
 		.drm_format = DRM_FORMAT_BGR565,
-		.bpp = 16,
+		.bytes_per_block = 2,
 	},
 	{
 		.drm_format = DRM_FORMAT_XRGB2101010,
-		.bpp = 32,
+		.bytes_per_block = 4,
 	},
 	{
 		.drm_format = DRM_FORMAT_ARGB2101010,
 		.opaque_substitute = DRM_FORMAT_XRGB2101010,
-		.bpp = 32,
+		.bytes_per_block = 4,
 		.has_alpha = true,
 	},
 	{
 		.drm_format = DRM_FORMAT_XBGR2101010,
-		.bpp = 32,
+		.bytes_per_block = 4,
 	},
 	{
 		.drm_format = DRM_FORMAT_ABGR2101010,
 		.opaque_substitute = DRM_FORMAT_XBGR2101010,
-		.bpp = 32,
+		.bytes_per_block = 4,
 		.has_alpha = true,
 	},
 	{
 		.drm_format = DRM_FORMAT_XBGR16161616F,
-		.bpp = 64,
+		.bytes_per_block = 8,
 	},
 	{
 		.drm_format = DRM_FORMAT_ABGR16161616F,
 		.opaque_substitute = DRM_FORMAT_XBGR16161616F,
-		.bpp = 64,
+		.bytes_per_block = 8,
 		.has_alpha = true,
 	},
 	{
 		.drm_format = DRM_FORMAT_XBGR16161616,
-		.bpp = 64,
+		.bytes_per_block = 8,
 	},
 	{
 		.drm_format = DRM_FORMAT_ABGR16161616,
 		.opaque_substitute = DRM_FORMAT_XBGR16161616,
-		.bpp = 64,
+		.bytes_per_block = 8,
 		.has_alpha = true,
 	},
 };
@@ -195,19 +195,46 @@ enum wl_shm_format convert_drm_format_to_wl_shm(uint32_t fmt) {
 	}
 }
 
+uint32_t pixel_format_info_pixels_per_block(const struct wlr_pixel_format_info *info) {
+	uint32_t pixels = info->block_width * info->block_height;
+	return pixels > 0 ? pixels : 1;
+}
+
+static int32_t div_round_up(int32_t dividend, int32_t divisor) {
+	int32_t quotient = dividend / divisor;
+	if (dividend % divisor != 0) {
+		quotient++;
+	}
+	return quotient;
+}
+
+int32_t pixel_format_info_min_stride(const struct wlr_pixel_format_info *fmt, int32_t width) {
+	int32_t pixels_per_block = (int32_t)pixel_format_info_pixels_per_block(fmt);
+	int32_t bytes_per_block = (int32_t)fmt->bytes_per_block;
+	if (width > INT32_MAX / bytes_per_block) {
+		wlr_log(WLR_DEBUG, "Invalid width %d (overflow)", width);
+		return 0;
+	}
+	return div_round_up(width * bytes_per_block, pixels_per_block);
+}
+
 bool pixel_format_info_check_stride(const struct wlr_pixel_format_info *fmt,
 		int32_t stride, int32_t width) {
-	assert(fmt->bpp > 0 && fmt->bpp % 8 == 0);
-	int32_t bytes_per_pixel = (int32_t)(fmt->bpp / 8);
-	if (stride % bytes_per_pixel != 0) {
+	int32_t bytes_per_block = (int32_t)fmt->bytes_per_block;
+	if (stride % bytes_per_block != 0) {
 		wlr_log(WLR_DEBUG, "Invalid stride %d (incompatible with %d "
-			"bytes-per-pixel)", stride, bytes_per_pixel);
+			"bytes-per-block)", stride, bytes_per_block);
 		return false;
 	}
-	if (stride / bytes_per_pixel < width) {
+
+	int32_t min_stride = pixel_format_info_min_stride(fmt, width);
+	if (min_stride <= 0) {
+		return false;
+	} else if (stride < min_stride) {
 		wlr_log(WLR_DEBUG, "Invalid stride %d (too small for %d "
-			"bytes-per-pixel and width %d)", stride, bytes_per_pixel, width);
+			"bytes-per-block and width %d)", stride, bytes_per_block, width);
 		return false;
 	}
+
 	return true;
 }