render/gles2: remove assumptions about supported formats

We were assuming GL_BGRA_EXT was always supported.

We now check that it's supported for rendering. We fail if it isn't because
this format is specified as "always supported" by the Wayland protocol.

We also check if it's supported for reading pixels. A new preferred_read_format
function returns the preferred format that can be used to read pixels. This is
used by the screencopy protocol.

2 files changed, 96 insertions, 18 deletions

diff --git a/render/gles2/pixel_format.c b/render/gles2/pixel_format.c
index 6f42e72e..a4b4c101 100644
--- a/render/gles2/pixel_format.c
+++ b/render/gles2/pixel_format.c
@@ -3,9 +3,9 @@
 #include "render/gles2.h"
 
 /*
-* The wayland formats are little endian while the GL formats are big endian,
-* so WL_SHM_FORMAT_ARGB8888 is actually compatible with GL_BGRA_EXT.
-*/
+ * The wayland formats are little endian while the GL formats are big endian,
+ * so WL_SHM_FORMAT_ARGB8888 is actually compatible with GL_BGRA_EXT.
+ */
 static const struct wlr_gles2_pixel_format formats[] = {
 	{
 		.wl_format = WL_SHM_FORMAT_ARGB8888,
@@ -60,7 +60,19 @@ const struct wlr_gles2_pixel_format *get_gles2_format_from_wl(
 	return NULL;
 }
 
-const enum wl_shm_format *get_gles2_formats(size_t *len) {
+const struct wlr_gles2_pixel_format *get_gles2_format_from_gl(
+		GLint gl_format, GLint gl_type, bool alpha) {
+	for (size_t i = 0; i < sizeof(formats) / sizeof(*formats); ++i) {
+		if (formats[i].gl_format == gl_format &&
+				formats[i].gl_type == gl_type &&
+				formats[i].has_alpha == alpha) {
+			return &formats[i];
+		}
+	}
+	return NULL;
+}
+
+const enum wl_shm_format *get_gles2_wl_formats(size_t *len) {
 	*len = sizeof(wl_formats) / sizeof(wl_formats[0]);
 	return wl_formats;
 }
diff --git a/render/gles2/renderer.c b/render/gles2/renderer.c
index b6e97041..ddcd219b 100644
--- a/render/gles2/renderer.c
+++ b/render/gles2/renderer.c
@@ -207,13 +207,17 @@ static void gles2_render_ellipse_with_matrix(struct wlr_renderer *wlr_renderer,
 
 static const enum wl_shm_format *gles2_renderer_formats(
 		struct wlr_renderer *wlr_renderer, size_t *len) {
-	return get_gles2_formats(len);
+	return get_gles2_wl_formats(len);
+}
+
+static bool gles2_format_supported(struct wlr_renderer *wlr_renderer,
+		enum wl_shm_format wl_fmt) {
+	return get_gles2_format_from_wl(wl_fmt) != NULL;
 }
 
 static bool gles2_resource_is_wl_drm_buffer(struct wlr_renderer *wlr_renderer,
 		struct wl_resource *resource) {
-	struct wlr_gles2_renderer *renderer =
-		gles2_get_renderer(wlr_renderer);
+	struct wlr_gles2_renderer *renderer = gles2_get_renderer(wlr_renderer);
 
 	if (!eglQueryWaylandBufferWL) {
 		return false;
@@ -249,6 +253,33 @@ static int gles2_get_dmabuf_modifiers(struct wlr_renderer *wlr_renderer,
 	return wlr_egl_get_dmabuf_modifiers(renderer->egl, format, modifiers);
 }
 
+static enum wl_shm_format gles2_preferred_read_format(
+		struct wlr_renderer *wlr_renderer) {
+	struct wlr_gles2_renderer *renderer =
+		gles2_get_renderer_in_context(wlr_renderer);
+
+	GLint gl_format = -1, gl_type = -1;
+	PUSH_GLES2_DEBUG;
+	glGetIntegerv(GL_IMPLEMENTATION_COLOR_READ_FORMAT, &gl_format);
+	glGetIntegerv(GL_IMPLEMENTATION_COLOR_READ_TYPE, &gl_type);
+	POP_GLES2_DEBUG;
+
+	EGLint alpha_size = -1;
+	eglGetConfigAttrib(renderer->egl->display, renderer->egl->config,
+		EGL_ALPHA_SIZE, &alpha_size);
+
+	const struct wlr_gles2_pixel_format *fmt =
+		get_gles2_format_from_gl(gl_format, gl_type, alpha_size > 0);
+	if (fmt != NULL) {
+		return fmt->wl_format;
+	}
+
+	if (renderer->exts.read_format_bgra_ext) {
+		return WL_SHM_FORMAT_XRGB8888;
+	}
+	return WL_SHM_FORMAT_XBGR8888;
+}
+
 static bool gles2_read_pixels(struct wlr_renderer *wlr_renderer,
 		enum wl_shm_format wl_fmt, uint32_t *flags, uint32_t stride,
 		uint32_t width, uint32_t height, uint32_t src_x, uint32_t src_y,
@@ -262,6 +293,12 @@ static bool gles2_read_pixels(struct wlr_renderer *wlr_renderer,
 		return false;
 	}
 
+	if (fmt->gl_format == GL_BGRA_EXT && !renderer->exts.read_format_bgra_ext) {
+		wlr_log(WLR_ERROR,
+			"Cannot read pixels: missing GL_EXT_read_format_bgra extension");
+		return false;
+	}
+
 	PUSH_GLES2_DEBUG;
 
 	// Make sure any pending drawing is finished before we try to read it
@@ -294,11 +331,6 @@ static bool gles2_read_pixels(struct wlr_renderer *wlr_renderer,
 	return glGetError() == GL_NO_ERROR;
 }
 
-static bool gles2_format_supported(struct wlr_renderer *wlr_renderer,
-		enum wl_shm_format wl_fmt) {
-	return get_gles2_format_from_wl(wl_fmt) != NULL;
-}
-
 static struct wlr_texture *gles2_texture_from_pixels(
 		struct wlr_renderer *wlr_renderer, enum wl_shm_format wl_fmt,
 		uint32_t stride, uint32_t width, uint32_t height, const void *data) {
@@ -342,7 +374,7 @@ static void gles2_destroy(struct wlr_renderer *wlr_renderer) {
 	glDeleteProgram(renderer->shaders.tex_ext.program);
 	POP_GLES2_DEBUG;
 
-	if (glDebugMessageCallbackKHR) {
+	if (renderer->exts.debug_khr) {
 		glDisable(GL_DEBUG_OUTPUT_KHR);
 		glDebugMessageCallbackKHR(NULL, NULL);
 	}
@@ -360,12 +392,13 @@ static const struct wlr_renderer_impl renderer_impl = {
 	.render_quad_with_matrix = gles2_render_quad_with_matrix,
 	.render_ellipse_with_matrix = gles2_render_ellipse_with_matrix,
 	.formats = gles2_renderer_formats,
+	.format_supported = gles2_format_supported,
 	.resource_is_wl_drm_buffer = gles2_resource_is_wl_drm_buffer,
 	.wl_drm_buffer_get_size = gles2_wl_drm_buffer_get_size,
 	.get_dmabuf_formats = gles2_get_dmabuf_formats,
 	.get_dmabuf_modifiers = gles2_get_dmabuf_modifiers,
+	.preferred_read_format = gles2_preferred_read_format,
 	.read_pixels = gles2_read_pixels,
-	.format_supported = gles2_format_supported,
 	.texture_from_pixels = gles2_texture_from_pixels,
 	.texture_from_wl_drm = gles2_texture_from_wl_drm,
 	.texture_from_dmabuf = gles2_texture_from_dmabuf,
@@ -466,6 +499,24 @@ error:
 	return 0;
 }
 
+static bool check_gl_ext(const char *exts, const char *ext) {
+	size_t extlen = strlen(ext);
+	const char *end = exts + strlen(exts);
+
+	while (exts < end) {
+		if (exts[0] == ' ') {
+			exts++;
+			continue;
+		}
+		size_t n = strcspn(exts, " ");
+		if (n == extlen && strncmp(ext, exts, n) == 0) {
+			return true;
+		}
+		exts += n;
+	}
+	return false;
+}
+
 extern const GLchar quad_vertex_src[];
 extern const GLchar quad_fragment_src[];
 extern const GLchar ellipse_fragment_src[];
@@ -487,14 +538,29 @@ struct wlr_renderer *wlr_gles2_renderer_create(struct wlr_egl *egl) {
 	wlr_renderer_init(&renderer->wlr_renderer, &renderer_impl);
 
 	renderer->egl = egl;
-	wlr_egl_make_current(renderer->egl, EGL_NO_SURFACE, NULL);
+	if (!wlr_egl_make_current(renderer->egl, EGL_NO_SURFACE, NULL)) {
+		free(renderer);
+		return NULL;
+	}
 
-	renderer->exts_str = (const char*) glGetString(GL_EXTENSIONS);
+	renderer->exts_str = (const char *)glGetString(GL_EXTENSIONS);
 	wlr_log(WLR_INFO, "Using %s", glGetString(GL_VERSION));
 	wlr_log(WLR_INFO, "GL vendor: %s", glGetString(GL_VENDOR));
 	wlr_log(WLR_INFO, "Supported GLES2 extensions: %s", renderer->exts_str);
 
-	if (glDebugMessageCallbackKHR && glDebugMessageControlKHR) {
+	if (!check_gl_ext(renderer->exts_str, "GL_EXT_texture_format_BGRA8888")) {
+		wlr_log(WLR_ERROR, "BGRA8888 format not supported by GLES2");
+		free(renderer);
+		return NULL;
+	}
+
+	renderer->exts.read_format_bgra_ext =
+		check_gl_ext(renderer->exts_str, "GL_EXT_read_format_bgra");
+	renderer->exts.debug_khr =
+		check_gl_ext(renderer->exts_str, "GL_KHR_debug") &&
+		glDebugMessageCallbackKHR && glDebugMessageControlKHR;
+
+	if (renderer->exts.debug_khr) {
 		glEnable(GL_DEBUG_OUTPUT_KHR);
 		glEnable(GL_DEBUG_OUTPUT_SYNCHRONOUS_KHR);
 		glDebugMessageCallbackKHR(gles2_log, NULL);
@@ -570,7 +636,7 @@ error:
 
 	POP_GLES2_DEBUG;
 
-	if (glDebugMessageCallbackKHR) {
+	if (renderer->exts.debug_khr) {
 		glDisable(GL_DEBUG_OUTPUT_KHR);
 		glDebugMessageCallbackKHR(NULL, NULL);
 	}