render/gles2: remove assumptions about supported formats

We were assuming GL_BGRA_EXT was always supported.

We now check that it's supported for rendering. We fail if it isn't because
this format is specified as "always supported" by the Wayland protocol.

We also check if it's supported for reading pixels. A new preferred_read_format
function returns the preferred format that can be used to read pixels. This is
used by the screencopy protocol.

7 files changed, 139 insertions, 23 deletions

diff --git a/include/render/gles2.h b/include/render/gles2.h
index 7ff2f174..063ed20a 100644
--- a/include/render/gles2.h
+++ b/include/render/gles2.h
@@ -40,6 +40,11 @@ struct wlr_gles2_renderer {
 	const char *exts_str;
 
 	struct {
+		bool read_format_bgra_ext;
+		bool debug_khr;
+	} exts;
+
+	struct {
 		struct {
 			GLuint program;
 			GLint proj;
@@ -87,7 +92,9 @@ struct wlr_gles2_texture {
 
 const struct wlr_gles2_pixel_format *get_gles2_format_from_wl(
 	enum wl_shm_format fmt);
-const enum wl_shm_format *get_gles2_formats(size_t *len);
+const struct wlr_gles2_pixel_format *get_gles2_format_from_gl(
+	GLint gl_format, GLint gl_type, bool alpha);
+const enum wl_shm_format *get_gles2_wl_formats(size_t *len);
 
 struct wlr_gles2_texture *gles2_get_texture(
 	struct wlr_texture *wlr_texture);
diff --git a/include/wlr/render/interface.h b/include/wlr/render/interface.h
index 905d419f..6b80a077 100644
--- a/include/wlr/render/interface.h
+++ b/include/wlr/render/interface.h
@@ -34,6 +34,8 @@ struct wlr_renderer_impl {
 		const float color[static 4], const float matrix[static 9]);
 	const enum wl_shm_format *(*formats)(
 		struct wlr_renderer *renderer, size_t *len);
+	bool (*format_supported)(struct wlr_renderer *renderer,
+		enum wl_shm_format fmt);
 	bool (*resource_is_wl_drm_buffer)(struct wlr_renderer *renderer,
 		struct wl_resource *resource);
 	void (*wl_drm_buffer_get_size)(struct wlr_renderer *renderer,
@@ -41,12 +43,11 @@ struct wlr_renderer_impl {
 	int (*get_dmabuf_formats)(struct wlr_renderer *renderer, int **formats);
 	int (*get_dmabuf_modifiers)(struct wlr_renderer *renderer, int format,
 		uint64_t **modifiers);
+	enum wl_shm_format (*preferred_read_format)(struct wlr_renderer *renderer);
 	bool (*read_pixels)(struct wlr_renderer *renderer, enum wl_shm_format fmt,
 		uint32_t *flags, uint32_t stride, uint32_t width, uint32_t height,
 		uint32_t src_x, uint32_t src_y, uint32_t dst_x, uint32_t dst_y,
 		void *data);
-	bool (*format_supported)(struct wlr_renderer *renderer,
-		enum wl_shm_format fmt);
 	struct wlr_texture *(*texture_from_pixels)(struct wlr_renderer *renderer,
 		enum wl_shm_format fmt, uint32_t stride, uint32_t width,
 		uint32_t height, const void *data);
diff --git a/include/wlr/render/wlr_renderer.h b/include/wlr/render/wlr_renderer.h
index 9c031b7f..02b4a11e 100644
--- a/include/wlr/render/wlr_renderer.h
+++ b/include/wlr/render/wlr_renderer.h
@@ -97,6 +97,11 @@ int wlr_renderer_get_dmabuf_formats(struct wlr_renderer *renderer,
 int wlr_renderer_get_dmabuf_modifiers(struct wlr_renderer *renderer, int format,
 	uint64_t **modifiers);
 /**
+ * Get the preferred format for reading pixels.
+ */
+bool wlr_renderer_preferred_read_format(struct wlr_renderer *renderer,
+	enum wl_shm_format *fmt);
+/**
  * Reads out of pixels of the currently bound surface into data. `stride` is in
  * bytes.
  *
diff --git a/render/gles2/pixel_format.c b/render/gles2/pixel_format.c
index 6f42e72e..a4b4c101 100644
--- a/render/gles2/pixel_format.c
+++ b/render/gles2/pixel_format.c
@@ -3,9 +3,9 @@
 #include "render/gles2.h"
 
 /*
-* The wayland formats are little endian while the GL formats are big endian,
-* so WL_SHM_FORMAT_ARGB8888 is actually compatible with GL_BGRA_EXT.
-*/
+ * The wayland formats are little endian while the GL formats are big endian,
+ * so WL_SHM_FORMAT_ARGB8888 is actually compatible with GL_BGRA_EXT.
+ */
 static const struct wlr_gles2_pixel_format formats[] = {
 	{
 		.wl_format = WL_SHM_FORMAT_ARGB8888,
@@ -60,7 +60,19 @@ const struct wlr_gles2_pixel_format *get_gles2_format_from_wl(
 	return NULL;
 }
 
-const enum wl_shm_format *get_gles2_formats(size_t *len) {
+const struct wlr_gles2_pixel_format *get_gles2_format_from_gl(
+		GLint gl_format, GLint gl_type, bool alpha) {
+	for (size_t i = 0; i < sizeof(formats) / sizeof(*formats); ++i) {
+		if (formats[i].gl_format == gl_format &&
+				formats[i].gl_type == gl_type &&
+				formats[i].has_alpha == alpha) {
+			return &formats[i];
+		}
+	}
+	return NULL;
+}
+
+const enum wl_shm_format *get_gles2_wl_formats(size_t *len) {
 	*len = sizeof(wl_formats) / sizeof(wl_formats[0]);
 	return wl_formats;
 }
diff --git a/render/gles2/renderer.c b/render/gles2/renderer.c
index b6e97041..ddcd219b 100644
--- a/render/gles2/renderer.c
+++ b/render/gles2/renderer.c
@@ -207,13 +207,17 @@ static void gles2_render_ellipse_with_matrix(struct wlr_renderer *wlr_renderer,
 
 static const enum wl_shm_format *gles2_renderer_formats(
 		struct wlr_renderer *wlr_renderer, size_t *len) {
-	return get_gles2_formats(len);
+	return get_gles2_wl_formats(len);
+}
+
+static bool gles2_format_supported(struct wlr_renderer *wlr_renderer,
+		enum wl_shm_format wl_fmt) {
+	return get_gles2_format_from_wl(wl_fmt) != NULL;
 }
 
 static bool gles2_resource_is_wl_drm_buffer(struct wlr_renderer *wlr_renderer,
 		struct wl_resource *resource) {
-	struct wlr_gles2_renderer *renderer =
-		gles2_get_renderer(wlr_renderer);
+	struct wlr_gles2_renderer *renderer = gles2_get_renderer(wlr_renderer);
 
 	if (!eglQueryWaylandBufferWL) {
 		return false;
@@ -249,6 +253,33 @@ static int gles2_get_dmabuf_modifiers(struct wlr_renderer *wlr_renderer,
 	return wlr_egl_get_dmabuf_modifiers(renderer->egl, format, modifiers);
 }
 
+static enum wl_shm_format gles2_preferred_read_format(
+		struct wlr_renderer *wlr_renderer) {
+	struct wlr_gles2_renderer *renderer =
+		gles2_get_renderer_in_context(wlr_renderer);
+
+	GLint gl_format = -1, gl_type = -1;
+	PUSH_GLES2_DEBUG;
+	glGetIntegerv(GL_IMPLEMENTATION_COLOR_READ_FORMAT, &gl_format);
+	glGetIntegerv(GL_IMPLEMENTATION_COLOR_READ_TYPE, &gl_type);
+	POP_GLES2_DEBUG;
+
+	EGLint alpha_size = -1;
+	eglGetConfigAttrib(renderer->egl->display, renderer->egl->config,
+		EGL_ALPHA_SIZE, &alpha_size);
+
+	const struct wlr_gles2_pixel_format *fmt =
+		get_gles2_format_from_gl(gl_format, gl_type, alpha_size > 0);
+	if (fmt != NULL) {
+		return fmt->wl_format;
+	}
+
+	if (renderer->exts.read_format_bgra_ext) {
+		return WL_SHM_FORMAT_XRGB8888;
+	}
+	return WL_SHM_FORMAT_XBGR8888;
+}
+
 static bool gles2_read_pixels(struct wlr_renderer *wlr_renderer,
 		enum wl_shm_format wl_fmt, uint32_t *flags, uint32_t stride,
 		uint32_t width, uint32_t height, uint32_t src_x, uint32_t src_y,
@@ -262,6 +293,12 @@ static bool gles2_read_pixels(struct wlr_renderer *wlr_renderer,
 		return false;
 	}
 
+	if (fmt->gl_format == GL_BGRA_EXT && !renderer->exts.read_format_bgra_ext) {
+		wlr_log(WLR_ERROR,
+			"Cannot read pixels: missing GL_EXT_read_format_bgra extension");
+		return false;
+	}
+
 	PUSH_GLES2_DEBUG;
 
 	// Make sure any pending drawing is finished before we try to read it
@@ -294,11 +331,6 @@ static bool gles2_read_pixels(struct wlr_renderer *wlr_renderer,
 	return glGetError() == GL_NO_ERROR;
 }
 
-static bool gles2_format_supported(struct wlr_renderer *wlr_renderer,
-		enum wl_shm_format wl_fmt) {
-	return get_gles2_format_from_wl(wl_fmt) != NULL;
-}
-
 static struct wlr_texture *gles2_texture_from_pixels(
 		struct wlr_renderer *wlr_renderer, enum wl_shm_format wl_fmt,
 		uint32_t stride, uint32_t width, uint32_t height, const void *data) {
@@ -342,7 +374,7 @@ static void gles2_destroy(struct wlr_renderer *wlr_renderer) {
 	glDeleteProgram(renderer->shaders.tex_ext.program);
 	POP_GLES2_DEBUG;
 
-	if (glDebugMessageCallbackKHR) {
+	if (renderer->exts.debug_khr) {
 		glDisable(GL_DEBUG_OUTPUT_KHR);
 		glDebugMessageCallbackKHR(NULL, NULL);
 	}
@@ -360,12 +392,13 @@ static const struct wlr_renderer_impl renderer_impl = {
 	.render_quad_with_matrix = gles2_render_quad_with_matrix,
 	.render_ellipse_with_matrix = gles2_render_ellipse_with_matrix,
 	.formats = gles2_renderer_formats,
+	.format_supported = gles2_format_supported,
 	.resource_is_wl_drm_buffer = gles2_resource_is_wl_drm_buffer,
 	.wl_drm_buffer_get_size = gles2_wl_drm_buffer_get_size,
 	.get_dmabuf_formats = gles2_get_dmabuf_formats,
 	.get_dmabuf_modifiers = gles2_get_dmabuf_modifiers,
+	.preferred_read_format = gles2_preferred_read_format,
 	.read_pixels = gles2_read_pixels,
-	.format_supported = gles2_format_supported,
 	.texture_from_pixels = gles2_texture_from_pixels,
 	.texture_from_wl_drm = gles2_texture_from_wl_drm,
 	.texture_from_dmabuf = gles2_texture_from_dmabuf,
@@ -466,6 +499,24 @@ error:
 	return 0;
 }
 
+static bool check_gl_ext(const char *exts, const char *ext) {
+	size_t extlen = strlen(ext);
+	const char *end = exts + strlen(exts);
+
+	while (exts < end) {
+		if (exts[0] == ' ') {
+			exts++;
+			continue;
+		}
+		size_t n = strcspn(exts, " ");
+		if (n == extlen && strncmp(ext, exts, n) == 0) {
+			return true;
+		}
+		exts += n;
+	}
+	return false;
+}
+
 extern const GLchar quad_vertex_src[];
 extern const GLchar quad_fragment_src[];
 extern const GLchar ellipse_fragment_src[];
@@ -487,14 +538,29 @@ struct wlr_renderer *wlr_gles2_renderer_create(struct wlr_egl *egl) {
 	wlr_renderer_init(&renderer->wlr_renderer, &renderer_impl);
 
 	renderer->egl = egl;
-	wlr_egl_make_current(renderer->egl, EGL_NO_SURFACE, NULL);
+	if (!wlr_egl_make_current(renderer->egl, EGL_NO_SURFACE, NULL)) {
+		free(renderer);
+		return NULL;
+	}
 
-	renderer->exts_str = (const char*) glGetString(GL_EXTENSIONS);
+	renderer->exts_str = (const char *)glGetString(GL_EXTENSIONS);
 	wlr_log(WLR_INFO, "Using %s", glGetString(GL_VERSION));
 	wlr_log(WLR_INFO, "GL vendor: %s", glGetString(GL_VENDOR));
 	wlr_log(WLR_INFO, "Supported GLES2 extensions: %s", renderer->exts_str);
 
-	if (glDebugMessageCallbackKHR && glDebugMessageControlKHR) {
+	if (!check_gl_ext(renderer->exts_str, "GL_EXT_texture_format_BGRA8888")) {
+		wlr_log(WLR_ERROR, "BGRA8888 format not supported by GLES2");
+		free(renderer);
+		return NULL;
+	}
+
+	renderer->exts.read_format_bgra_ext =
+		check_gl_ext(renderer->exts_str, "GL_EXT_read_format_bgra");
+	renderer->exts.debug_khr =
+		check_gl_ext(renderer->exts_str, "GL_KHR_debug") &&
+		glDebugMessageCallbackKHR && glDebugMessageControlKHR;
+
+	if (renderer->exts.debug_khr) {
 		glEnable(GL_DEBUG_OUTPUT_KHR);
 		glEnable(GL_DEBUG_OUTPUT_SYNCHRONOUS_KHR);
 		glDebugMessageCallbackKHR(gles2_log, NULL);
@@ -570,7 +636,7 @@ error:
 
 	POP_GLES2_DEBUG;
 
-	if (glDebugMessageCallbackKHR) {
+	if (renderer->exts.debug_khr) {
 		glDisable(GL_DEBUG_OUTPUT_KHR);
 		glDebugMessageCallbackKHR(NULL, NULL);
 	}
diff --git a/render/wlr_renderer.c b/render/wlr_renderer.c
index 31bf2b18..10dc6db2 100644
--- a/render/wlr_renderer.c
+++ b/render/wlr_renderer.c
@@ -139,6 +139,15 @@ int wlr_renderer_get_dmabuf_modifiers(struct wlr_renderer *r, int format,
 	return r->impl->get_dmabuf_modifiers(r, format, modifiers);
 }
 
+bool wlr_renderer_preferred_read_format(struct wlr_renderer *r,
+		enum wl_shm_format *fmt) {
+	if (!r->impl->preferred_read_format || !r->impl->read_pixels) {
+		return false;
+	}
+	*fmt = r->impl->preferred_read_format(r);
+	return true;
+}
+
 bool wlr_renderer_read_pixels(struct wlr_renderer *r, enum wl_shm_format fmt,
 		uint32_t *flags, uint32_t stride, uint32_t width, uint32_t height,
 		uint32_t src_x, uint32_t src_y, uint32_t dst_x, uint32_t dst_y,
diff --git a/types/wlr_screencopy_v1.c b/types/wlr_screencopy_v1.c
index 0819e48d..b186b89b 100644
--- a/types/wlr_screencopy_v1.c
+++ b/types/wlr_screencopy_v1.c
@@ -4,6 +4,7 @@
 #include <wlr/types/wlr_output.h>
 #include <wlr/types/wlr_screencopy_v1.h>
 #include <wlr/backend.h>
+#include <wlr/util/log.h>
 #include "wlr-screencopy-unstable-v1-protocol.h"
 #include "util/signal.h"
 
@@ -40,6 +41,7 @@ static void frame_handle_output_swap_buffers(struct wl_listener *listener,
 	struct wlr_output_event_swap_buffers *event = _data;
 	struct wlr_output *output = frame->output;
 	struct wlr_renderer *renderer = wlr_backend_get_renderer(output->backend);
+	assert(renderer);
 
 	wl_list_remove(&frame->output_swap_buffers.link);
 	wl_list_init(&frame->output_swap_buffers.link);
@@ -214,11 +216,25 @@ static void capture_output(struct wl_client *client,
 	wl_list_init(&frame->output_swap_buffers.link);
 	wl_list_init(&frame->buffer_destroy.link);
 
-	frame->format = WL_SHM_FORMAT_XRGB8888;
+	struct wlr_renderer *renderer = wlr_backend_get_renderer(output->backend);
+	assert(renderer);
+
+	if (!wlr_renderer_preferred_read_format(renderer, &frame->format)) {
+		wlr_log(WLR_ERROR,
+			"Failed to capture output: no read format supported by renderer");
+		goto error;
+	}
+
 	frame->box = buffer_box;
-	frame->stride = 4 * buffer_box.width;
+	frame->stride = 4 * buffer_box.width; // TODO: depends on read format
+
 	zwlr_screencopy_frame_v1_send_buffer(frame->resource, frame->format,
 		buffer_box.width, buffer_box.height, frame->stride);
+	return;
+
+error:
+	zwlr_screencopy_frame_v1_send_failed(frame->resource);
+	frame_destroy(frame);
 }
 
 static void manager_handle_capture_output(struct wl_client *client,