render/gles2: hide shm formats without GL support

This change introduces a new function to check whether the renderer
has the needed GL extensions to read a given pixel format.

2 files changed, 21 insertions, 12 deletions

diff --git a/render/gles2/pixel_format.c b/render/gles2/pixel_format.c
index 9e671193..ceb8dadd 100644
--- a/render/gles2/pixel_format.c
+++ b/render/gles2/pixel_format.c
@@ -75,6 +75,15 @@ static const struct wlr_gles2_pixel_format formats[] = {
 
 // TODO: more pixel formats
 
+bool is_gles2_pixel_format_supported(const struct wlr_gles2_renderer *renderer,
+		const struct wlr_gles2_pixel_format *format) {
+	if (format->gl_format == GL_BGRA_EXT
+			&& !renderer->exts.EXT_read_format_bgra) {
+		return false;
+	}
+	return true;
+}
+
 const struct wlr_gles2_pixel_format *get_gles2_format_from_drm(uint32_t fmt) {
 	for (size_t i = 0; i < sizeof(formats) / sizeof(*formats); ++i) {
 		if (formats[i].drm_format == fmt) {
@@ -96,11 +105,16 @@ const struct wlr_gles2_pixel_format *get_gles2_format_from_gl(
 	return NULL;
 }
 
-const uint32_t *get_gles2_shm_formats(size_t *len) {
+const uint32_t *get_gles2_shm_formats(const struct wlr_gles2_renderer *renderer,
+		size_t *len) {
 	static uint32_t shm_formats[sizeof(formats) / sizeof(formats[0])];
-	*len = sizeof(formats) / sizeof(formats[0]);
+	size_t j = 0;
 	for (size_t i = 0; i < sizeof(formats) / sizeof(formats[0]); i++) {
-		shm_formats[i] = formats[i].drm_format;
+		if (!is_gles2_pixel_format_supported(renderer, &formats[i])) {
+			continue;
+		}
+		shm_formats[j++] = formats[i].drm_format;
 	}
+	*len = j;
 	return shm_formats;
 }
diff --git a/render/gles2/renderer.c b/render/gles2/renderer.c
index aa0e38f3..509d39d2 100644
--- a/render/gles2/renderer.c
+++ b/render/gles2/renderer.c
@@ -376,7 +376,8 @@ static void gles2_render_quad_with_matrix(struct wlr_renderer *wlr_renderer,
 
 static const uint32_t *gles2_get_shm_texture_formats(
 		struct wlr_renderer *wlr_renderer, size_t *len) {
-	return get_gles2_shm_formats(len);
+	struct wlr_gles2_renderer *renderer = gles2_get_renderer(wlr_renderer);
+	return get_gles2_shm_formats(renderer, len);
 }
 
 static const struct wlr_drm_format_set *gles2_get_dmabuf_texture_formats(
@@ -431,14 +432,8 @@ static bool gles2_read_pixels(struct wlr_renderer *wlr_renderer,
 
 	const struct wlr_gles2_pixel_format *fmt =
 		get_gles2_format_from_drm(drm_format);
-	if (fmt == NULL) {
-		wlr_log(WLR_ERROR, "Cannot read pixels: unsupported pixel format");
-		return false;
-	}
-
-	if (fmt->gl_format == GL_BGRA_EXT && !renderer->exts.EXT_read_format_bgra) {
-		wlr_log(WLR_ERROR,
-			"Cannot read pixels: missing GL_EXT_read_format_bgra extension");
+	if (fmt == NULL || !is_gles2_pixel_format_supported(renderer, fmt)) {
+		wlr_log(WLR_ERROR, "Cannot read pixels: unsupported pixel format 0x%"PRIX32, drm_format);
 		return false;
 	}