Merge pull request #1350 from emersion/check-gles2-bgra-ext

render/gles2: remove assumptions about supported formats

8 files changed, 160 insertions, 25 deletions

diff --git a/backend/drm/renderer.c b/backend/drm/renderer.c
index fa7d090e..70b1bcbe 100644
--- a/backend/drm/renderer.c
+++ b/backend/drm/renderer.c
@@ -30,8 +30,17 @@ bool init_drm_renderer(struct wlr_drm_backend *drm,
 		create_renderer_func = wlr_renderer_autocreate;
 	}
 
+	static EGLint config_attribs[] = {
+		EGL_RED_SIZE, 1,
+		EGL_GREEN_SIZE, 1,
+		EGL_BLUE_SIZE, 1,
+		EGL_ALPHA_SIZE, 1,
+		EGL_NONE,
+	};
+
 	renderer->wlr_rend = create_renderer_func(&renderer->egl,
-		EGL_PLATFORM_GBM_MESA, renderer->gbm, NULL, GBM_FORMAT_ARGB8888);
+		EGL_PLATFORM_GBM_MESA, renderer->gbm,
+		config_attribs, GBM_FORMAT_ARGB8888);
 
 	if (!renderer->wlr_rend) {
 		wlr_log(WLR_ERROR, "Failed to create EGL/WLR renderer");
diff --git a/include/render/gles2.h b/include/render/gles2.h
index 7ff2f174..1857383f 100644
--- a/include/render/gles2.h
+++ b/include/render/gles2.h
@@ -40,6 +40,12 @@ struct wlr_gles2_renderer {
 	const char *exts_str;
 
 	struct {
+		bool read_format_bgra_ext;
+		bool debug_khr;
+		bool egl_image_external_oes;
+	} exts;
+
+	struct {
 		struct {
 			GLuint program;
 			GLint proj;
@@ -87,7 +93,9 @@ struct wlr_gles2_texture {
 
 const struct wlr_gles2_pixel_format *get_gles2_format_from_wl(
 	enum wl_shm_format fmt);
-const enum wl_shm_format *get_gles2_formats(size_t *len);
+const struct wlr_gles2_pixel_format *get_gles2_format_from_gl(
+	GLint gl_format, GLint gl_type, bool alpha);
+const enum wl_shm_format *get_gles2_wl_formats(size_t *len);
 
 struct wlr_gles2_texture *gles2_get_texture(
 	struct wlr_texture *wlr_texture);
diff --git a/include/wlr/render/interface.h b/include/wlr/render/interface.h
index 905d419f..6b80a077 100644
--- a/include/wlr/render/interface.h
+++ b/include/wlr/render/interface.h
@@ -34,6 +34,8 @@ struct wlr_renderer_impl {
 		const float color[static 4], const float matrix[static 9]);
 	const enum wl_shm_format *(*formats)(
 		struct wlr_renderer *renderer, size_t *len);
+	bool (*format_supported)(struct wlr_renderer *renderer,
+		enum wl_shm_format fmt);
 	bool (*resource_is_wl_drm_buffer)(struct wlr_renderer *renderer,
 		struct wl_resource *resource);
 	void (*wl_drm_buffer_get_size)(struct wlr_renderer *renderer,
@@ -41,12 +43,11 @@ struct wlr_renderer_impl {
 	int (*get_dmabuf_formats)(struct wlr_renderer *renderer, int **formats);
 	int (*get_dmabuf_modifiers)(struct wlr_renderer *renderer, int format,
 		uint64_t **modifiers);
+	enum wl_shm_format (*preferred_read_format)(struct wlr_renderer *renderer);
 	bool (*read_pixels)(struct wlr_renderer *renderer, enum wl_shm_format fmt,
 		uint32_t *flags, uint32_t stride, uint32_t width, uint32_t height,
 		uint32_t src_x, uint32_t src_y, uint32_t dst_x, uint32_t dst_y,
 		void *data);
-	bool (*format_supported)(struct wlr_renderer *renderer,
-		enum wl_shm_format fmt);
 	struct wlr_texture *(*texture_from_pixels)(struct wlr_renderer *renderer,
 		enum wl_shm_format fmt, uint32_t stride, uint32_t width,
 		uint32_t height, const void *data);
diff --git a/include/wlr/render/wlr_renderer.h b/include/wlr/render/wlr_renderer.h
index 9c031b7f..02b4a11e 100644
--- a/include/wlr/render/wlr_renderer.h
+++ b/include/wlr/render/wlr_renderer.h
@@ -97,6 +97,11 @@ int wlr_renderer_get_dmabuf_formats(struct wlr_renderer *renderer,
 int wlr_renderer_get_dmabuf_modifiers(struct wlr_renderer *renderer, int format,
 	uint64_t **modifiers);
 /**
+ * Get the preferred format for reading pixels.
+ */
+bool wlr_renderer_preferred_read_format(struct wlr_renderer *renderer,
+	enum wl_shm_format *fmt);
+/**
  * Reads out of pixels of the currently bound surface into data. `stride` is in
  * bytes.
  *
diff --git a/render/gles2/pixel_format.c b/render/gles2/pixel_format.c
index 6f42e72e..a4b4c101 100644
--- a/render/gles2/pixel_format.c
+++ b/render/gles2/pixel_format.c
@@ -3,9 +3,9 @@
 #include "render/gles2.h"
 
 /*
-* The wayland formats are little endian while the GL formats are big endian,
-* so WL_SHM_FORMAT_ARGB8888 is actually compatible with GL_BGRA_EXT.
-*/
+ * The wayland formats are little endian while the GL formats are big endian,
+ * so WL_SHM_FORMAT_ARGB8888 is actually compatible with GL_BGRA_EXT.
+ */
 static const struct wlr_gles2_pixel_format formats[] = {
 	{
 		.wl_format = WL_SHM_FORMAT_ARGB8888,
@@ -60,7 +60,19 @@ const struct wlr_gles2_pixel_format *get_gles2_format_from_wl(
 	return NULL;
 }
 
-const enum wl_shm_format *get_gles2_formats(size_t *len) {
+const struct wlr_gles2_pixel_format *get_gles2_format_from_gl(
+		GLint gl_format, GLint gl_type, bool alpha) {
+	for (size_t i = 0; i < sizeof(formats) / sizeof(*formats); ++i) {
+		if (formats[i].gl_format == gl_format &&
+				formats[i].gl_type == gl_type &&
+				formats[i].has_alpha == alpha) {
+			return &formats[i];
+		}
+	}
+	return NULL;
+}
+
+const enum wl_shm_format *get_gles2_wl_formats(size_t *len) {
 	*len = sizeof(wl_formats) / sizeof(wl_formats[0]);
 	return wl_formats;
 }
diff --git a/render/gles2/renderer.c b/render/gles2/renderer.c
index b6e97041..a5521909 100644
--- a/render/gles2/renderer.c
+++ b/render/gles2/renderer.c
@@ -135,6 +135,12 @@ static bool gles2_render_texture_with_matrix(struct wlr_renderer *wlr_renderer,
 	case WLR_GLES2_TEXTURE_DMABUF:
 		shader = &renderer->shaders.tex_ext;
 		target = GL_TEXTURE_EXTERNAL_OES;
+
+		if (!renderer->exts.egl_image_external_oes) {
+			wlr_log(WLR_ERROR, "Failed to render texture: "
+				"GL_TEXTURE_EXTERNAL_OES not supported");
+			return false;
+		}
 		break;
 	}
 
@@ -207,13 +213,17 @@ static void gles2_render_ellipse_with_matrix(struct wlr_renderer *wlr_renderer,
 
 static const enum wl_shm_format *gles2_renderer_formats(
 		struct wlr_renderer *wlr_renderer, size_t *len) {
-	return get_gles2_formats(len);
+	return get_gles2_wl_formats(len);
+}
+
+static bool gles2_format_supported(struct wlr_renderer *wlr_renderer,
+		enum wl_shm_format wl_fmt) {
+	return get_gles2_format_from_wl(wl_fmt) != NULL;
 }
 
 static bool gles2_resource_is_wl_drm_buffer(struct wlr_renderer *wlr_renderer,
 		struct wl_resource *resource) {
-	struct wlr_gles2_renderer *renderer =
-		gles2_get_renderer(wlr_renderer);
+	struct wlr_gles2_renderer *renderer = gles2_get_renderer(wlr_renderer);
 
 	if (!eglQueryWaylandBufferWL) {
 		return false;
@@ -249,6 +259,33 @@ static int gles2_get_dmabuf_modifiers(struct wlr_renderer *wlr_renderer,
 	return wlr_egl_get_dmabuf_modifiers(renderer->egl, format, modifiers);
 }
 
+static enum wl_shm_format gles2_preferred_read_format(
+		struct wlr_renderer *wlr_renderer) {
+	struct wlr_gles2_renderer *renderer =
+		gles2_get_renderer_in_context(wlr_renderer);
+
+	GLint gl_format = -1, gl_type = -1;
+	PUSH_GLES2_DEBUG;
+	glGetIntegerv(GL_IMPLEMENTATION_COLOR_READ_FORMAT, &gl_format);
+	glGetIntegerv(GL_IMPLEMENTATION_COLOR_READ_TYPE, &gl_type);
+	POP_GLES2_DEBUG;
+
+	EGLint alpha_size = -1;
+	eglGetConfigAttrib(renderer->egl->display, renderer->egl->config,
+		EGL_ALPHA_SIZE, &alpha_size);
+
+	const struct wlr_gles2_pixel_format *fmt =
+		get_gles2_format_from_gl(gl_format, gl_type, alpha_size > 0);
+	if (fmt != NULL) {
+		return fmt->wl_format;
+	}
+
+	if (renderer->exts.read_format_bgra_ext) {
+		return WL_SHM_FORMAT_XRGB8888;
+	}
+	return WL_SHM_FORMAT_XBGR8888;
+}
+
 static bool gles2_read_pixels(struct wlr_renderer *wlr_renderer,
 		enum wl_shm_format wl_fmt, uint32_t *flags, uint32_t stride,
 		uint32_t width, uint32_t height, uint32_t src_x, uint32_t src_y,
@@ -262,6 +299,12 @@ static bool gles2_read_pixels(struct wlr_renderer *wlr_renderer,
 		return false;
 	}
 
+	if (fmt->gl_format == GL_BGRA_EXT && !renderer->exts.read_format_bgra_ext) {
+		wlr_log(WLR_ERROR,
+			"Cannot read pixels: missing GL_EXT_read_format_bgra extension");
+		return false;
+	}
+
 	PUSH_GLES2_DEBUG;
 
 	// Make sure any pending drawing is finished before we try to read it
@@ -294,11 +337,6 @@ static bool gles2_read_pixels(struct wlr_renderer *wlr_renderer,
 	return glGetError() == GL_NO_ERROR;
 }
 
-static bool gles2_format_supported(struct wlr_renderer *wlr_renderer,
-		enum wl_shm_format wl_fmt) {
-	return get_gles2_format_from_wl(wl_fmt) != NULL;
-}
-
 static struct wlr_texture *gles2_texture_from_pixels(
 		struct wlr_renderer *wlr_renderer, enum wl_shm_format wl_fmt,
 		uint32_t stride, uint32_t width, uint32_t height, const void *data) {
@@ -342,7 +380,7 @@ static void gles2_destroy(struct wlr_renderer *wlr_renderer) {
 	glDeleteProgram(renderer->shaders.tex_ext.program);
 	POP_GLES2_DEBUG;
 
-	if (glDebugMessageCallbackKHR) {
+	if (renderer->exts.debug_khr) {
 		glDisable(GL_DEBUG_OUTPUT_KHR);
 		glDebugMessageCallbackKHR(NULL, NULL);
 	}
@@ -360,12 +398,13 @@ static const struct wlr_renderer_impl renderer_impl = {
 	.render_quad_with_matrix = gles2_render_quad_with_matrix,
 	.render_ellipse_with_matrix = gles2_render_ellipse_with_matrix,
 	.formats = gles2_renderer_formats,
+	.format_supported = gles2_format_supported,
 	.resource_is_wl_drm_buffer = gles2_resource_is_wl_drm_buffer,
 	.wl_drm_buffer_get_size = gles2_wl_drm_buffer_get_size,
 	.get_dmabuf_formats = gles2_get_dmabuf_formats,
 	.get_dmabuf_modifiers = gles2_get_dmabuf_modifiers,
+	.preferred_read_format = gles2_preferred_read_format,
 	.read_pixels = gles2_read_pixels,
-	.format_supported = gles2_format_supported,
 	.texture_from_pixels = gles2_texture_from_pixels,
 	.texture_from_wl_drm = gles2_texture_from_wl_drm,
 	.texture_from_dmabuf = gles2_texture_from_dmabuf,
@@ -466,6 +505,24 @@ error:
 	return 0;
 }
 
+static bool check_gl_ext(const char *exts, const char *ext) {
+	size_t extlen = strlen(ext);
+	const char *end = exts + strlen(exts);
+
+	while (exts < end) {
+		if (exts[0] == ' ') {
+			exts++;
+			continue;
+		}
+		size_t n = strcspn(exts, " ");
+		if (n == extlen && strncmp(ext, exts, n) == 0) {
+			return true;
+		}
+		exts += n;
+	}
+	return false;
+}
+
 extern const GLchar quad_vertex_src[];
 extern const GLchar quad_fragment_src[];
 extern const GLchar ellipse_fragment_src[];
@@ -487,14 +544,32 @@ struct wlr_renderer *wlr_gles2_renderer_create(struct wlr_egl *egl) {
 	wlr_renderer_init(&renderer->wlr_renderer, &renderer_impl);
 
 	renderer->egl = egl;
-	wlr_egl_make_current(renderer->egl, EGL_NO_SURFACE, NULL);
+	if (!wlr_egl_make_current(renderer->egl, EGL_NO_SURFACE, NULL)) {
+		free(renderer);
+		return NULL;
+	}
 
-	renderer->exts_str = (const char*) glGetString(GL_EXTENSIONS);
+	renderer->exts_str = (const char *)glGetString(GL_EXTENSIONS);
 	wlr_log(WLR_INFO, "Using %s", glGetString(GL_VERSION));
 	wlr_log(WLR_INFO, "GL vendor: %s", glGetString(GL_VENDOR));
 	wlr_log(WLR_INFO, "Supported GLES2 extensions: %s", renderer->exts_str);
 
-	if (glDebugMessageCallbackKHR && glDebugMessageControlKHR) {
+	if (!check_gl_ext(renderer->exts_str, "GL_EXT_texture_format_BGRA8888")) {
+		wlr_log(WLR_ERROR, "BGRA8888 format not supported by GLES2");
+		free(renderer);
+		return NULL;
+	}
+
+	renderer->exts.read_format_bgra_ext =
+		check_gl_ext(renderer->exts_str, "GL_EXT_read_format_bgra");
+	renderer->exts.debug_khr =
+		check_gl_ext(renderer->exts_str, "GL_KHR_debug") &&
+		glDebugMessageCallbackKHR && glDebugMessageControlKHR;
+	renderer->exts.egl_image_external_oes =
+		check_gl_ext(renderer->exts_str, "GL_OES_EGL_image_external") &&
+		glEGLImageTargetTexture2DOES;
+
+	if (renderer->exts.debug_khr) {
 		glEnable(GL_DEBUG_OUTPUT_KHR);
 		glEnable(GL_DEBUG_OUTPUT_SYNCHRONOUS_KHR);
 		glDebugMessageCallbackKHR(gles2_log, NULL);
@@ -545,7 +620,7 @@ struct wlr_renderer *wlr_gles2_renderer_create(struct wlr_egl *egl) {
 	renderer->shaders.tex_rgbx.tex = glGetUniformLocation(prog, "tex");
 	renderer->shaders.tex_rgbx.alpha = glGetUniformLocation(prog, "alpha");
 
-	if (glEGLImageTargetTexture2DOES) {
+	if (renderer->exts.egl_image_external_oes) {
 		renderer->shaders.tex_ext.program = prog =
 			link_program(tex_vertex_src, tex_fragment_src_external);
 		if (!renderer->shaders.tex_ext.program) {
@@ -570,7 +645,7 @@ error:
 
 	POP_GLES2_DEBUG;
 
-	if (glDebugMessageCallbackKHR) {
+	if (renderer->exts.debug_khr) {
 		glDisable(GL_DEBUG_OUTPUT_KHR);
 		glDebugMessageCallbackKHR(NULL, NULL);
 	}
diff --git a/render/wlr_renderer.c b/render/wlr_renderer.c
index 31bf2b18..10dc6db2 100644
--- a/render/wlr_renderer.c
+++ b/render/wlr_renderer.c
@@ -139,6 +139,15 @@ int wlr_renderer_get_dmabuf_modifiers(struct wlr_renderer *r, int format,
 	return r->impl->get_dmabuf_modifiers(r, format, modifiers);
 }
 
+bool wlr_renderer_preferred_read_format(struct wlr_renderer *r,
+		enum wl_shm_format *fmt) {
+	if (!r->impl->preferred_read_format || !r->impl->read_pixels) {
+		return false;
+	}
+	*fmt = r->impl->preferred_read_format(r);
+	return true;
+}
+
 bool wlr_renderer_read_pixels(struct wlr_renderer *r, enum wl_shm_format fmt,
 		uint32_t *flags, uint32_t stride, uint32_t width, uint32_t height,
 		uint32_t src_x, uint32_t src_y, uint32_t dst_x, uint32_t dst_y,
diff --git a/types/wlr_screencopy_v1.c b/types/wlr_screencopy_v1.c
index 0819e48d..b186b89b 100644
--- a/types/wlr_screencopy_v1.c
+++ b/types/wlr_screencopy_v1.c
@@ -4,6 +4,7 @@
 #include <wlr/types/wlr_output.h>
 #include <wlr/types/wlr_screencopy_v1.h>
 #include <wlr/backend.h>
+#include <wlr/util/log.h>
 #include "wlr-screencopy-unstable-v1-protocol.h"
 #include "util/signal.h"
 
@@ -40,6 +41,7 @@ static void frame_handle_output_swap_buffers(struct wl_listener *listener,
 	struct wlr_output_event_swap_buffers *event = _data;
 	struct wlr_output *output = frame->output;
 	struct wlr_renderer *renderer = wlr_backend_get_renderer(output->backend);
+	assert(renderer);
 
 	wl_list_remove(&frame->output_swap_buffers.link);
 	wl_list_init(&frame->output_swap_buffers.link);
@@ -214,11 +216,25 @@ static void capture_output(struct wl_client *client,
 	wl_list_init(&frame->output_swap_buffers.link);
 	wl_list_init(&frame->buffer_destroy.link);
 
-	frame->format = WL_SHM_FORMAT_XRGB8888;
+	struct wlr_renderer *renderer = wlr_backend_get_renderer(output->backend);
+	assert(renderer);
+
+	if (!wlr_renderer_preferred_read_format(renderer, &frame->format)) {
+		wlr_log(WLR_ERROR,
+			"Failed to capture output: no read format supported by renderer");
+		goto error;
+	}
+
 	frame->box = buffer_box;
-	frame->stride = 4 * buffer_box.width;
+	frame->stride = 4 * buffer_box.width; // TODO: depends on read format
+
 	zwlr_screencopy_frame_v1_send_buffer(frame->resource, frame->format,
 		buffer_box.width, buffer_box.height, frame->stride);
+	return;
+
+error:
+	zwlr_screencopy_frame_v1_send_failed(frame->resource);
+	frame_destroy(frame);
 }
 
 static void manager_handle_capture_output(struct wl_client *client,