render/pixel-format: add support for block-based formats

Some formats like sub-sampled YCbCr use a block of bytes to
store the color values for more than one pixel. Update our format
table to be able to handle such formats.

2 files changed, 17 insertions, 4 deletions

diff --git a/render/gles2/renderer.c b/render/gles2/renderer.c
index 1c8b4878..518068a3 100644
--- a/render/gles2/renderer.c
+++ b/render/gles2/renderer.c
@@ -464,6 +464,10 @@ static bool gles2_read_pixels(struct wlr_renderer *wlr_renderer,
 	const struct wlr_pixel_format_info *drm_fmt =
 		drm_get_pixel_format_info(fmt->drm_format);
 	assert(drm_fmt);
+	if (pixel_format_info_pixels_per_block(drm_fmt) != 1) {
+		wlr_log(WLR_ERROR, "Cannot read pixels: block formats are not supported");
+		return false;
+	}
 
 	push_gles2_debug(renderer);
 
@@ -474,7 +478,7 @@ static bool gles2_read_pixels(struct wlr_renderer *wlr_renderer,
 
 	unsigned char *p = (unsigned char *)data + dst_y * stride;
 	glPixelStorei(GL_PACK_ALIGNMENT, 1);
-	uint32_t pack_stride = width * drm_fmt->bpp / 8;
+	uint32_t pack_stride = pixel_format_info_min_stride(drm_fmt, width);
 	if (pack_stride == stride && dst_x == 0) {
 		// Under these particular conditions, we can read the pixels with only
 		// one glReadPixels call
@@ -486,7 +490,7 @@ static bool gles2_read_pixels(struct wlr_renderer *wlr_renderer,
 		for (size_t i = 0; i < height; ++i) {
 			uint32_t y = src_y + i;
 			glReadPixels(src_x, y, width, 1, fmt->gl_format,
-				fmt->gl_type, p + i * stride + dst_x * drm_fmt->bpp / 8);
+				fmt->gl_type, p + i * stride + dst_x * drm_fmt->bytes_per_block);
 		}
 	}
 
diff --git a/render/gles2/texture.c b/render/gles2/texture.c
index 63360d48..5faa54a2 100644
--- a/render/gles2/texture.c
+++ b/render/gles2/texture.c
@@ -56,6 +56,11 @@ static bool gles2_texture_update_from_buffer(struct wlr_texture *wlr_texture,
 	const struct wlr_pixel_format_info *drm_fmt =
 		drm_get_pixel_format_info(texture->drm_format);
 	assert(drm_fmt);
+	if (pixel_format_info_pixels_per_block(drm_fmt) != 1) {
+		wlr_buffer_end_data_ptr_access(buffer);
+		wlr_log(WLR_ERROR, "Cannot update texture: block formats are not supported");
+		return false;
+	}
 
 	if (!pixel_format_info_check_stride(drm_fmt, stride, buffer->width)) {
 		wlr_buffer_end_data_ptr_access(buffer);
@@ -76,7 +81,7 @@ static bool gles2_texture_update_from_buffer(struct wlr_texture *wlr_texture,
 	for (int i = 0; i < rects_len; i++) {
 		pixman_box32_t rect = rects[i];
 
-		glPixelStorei(GL_UNPACK_ROW_LENGTH_EXT, stride / (drm_fmt->bpp / 8));
+		glPixelStorei(GL_UNPACK_ROW_LENGTH_EXT, stride / drm_fmt->bytes_per_block);
 		glPixelStorei(GL_UNPACK_SKIP_PIXELS_EXT, rect.x1);
 		glPixelStorei(GL_UNPACK_SKIP_ROWS_EXT, rect.y1);
 
@@ -197,6 +202,10 @@ static struct wlr_texture *gles2_texture_from_pixels(
 	const struct wlr_pixel_format_info *drm_fmt =
 		drm_get_pixel_format_info(drm_format);
 	assert(drm_fmt);
+	if (pixel_format_info_pixels_per_block(drm_fmt) != 1) {
+		wlr_log(WLR_ERROR, "Cannot upload texture: block formats are not supported");
+		return NULL;
+	}
 
 	if (!pixel_format_info_check_stride(drm_fmt, stride, width)) {
 		return NULL;
@@ -227,7 +236,7 @@ static struct wlr_texture *gles2_texture_from_pixels(
 
 	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
 	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
-	glPixelStorei(GL_UNPACK_ROW_LENGTH_EXT, stride / (drm_fmt->bpp / 8));
+	glPixelStorei(GL_UNPACK_ROW_LENGTH_EXT, stride / drm_fmt->bytes_per_block);
 	glTexImage2D(GL_TEXTURE_2D, 0, internal_format, width, height, 0,
 		fmt->gl_format, fmt->gl_type, data);
 	glPixelStorei(GL_UNPACK_ROW_LENGTH_EXT, 0);