Merge pull request #1350 from emersion/check-gles2-bgra-ext

render/gles2: remove assumptions about supported formats

3 files changed, 115 insertions, 19 deletions

diff --git a/render/gles2/pixel_format.c b/render/gles2/pixel_format.c
index 6f42e72e..a4b4c101 100644
--- a/render/gles2/pixel_format.c
+++ b/render/gles2/pixel_format.c
@@ -3,9 +3,9 @@
 #include "render/gles2.h"
 
 /*
-* The wayland formats are little endian while the GL formats are big endian,
-* so WL_SHM_FORMAT_ARGB8888 is actually compatible with GL_BGRA_EXT.
-*/
+ * The wayland formats are little endian while the GL formats are big endian,
+ * so WL_SHM_FORMAT_ARGB8888 is actually compatible with GL_BGRA_EXT.
+ */
 static const struct wlr_gles2_pixel_format formats[] = {
 	{
 		.wl_format = WL_SHM_FORMAT_ARGB8888,
@@ -60,7 +60,19 @@ const struct wlr_gles2_pixel_format *get_gles2_format_from_wl(
 	return NULL;
 }
 
-const enum wl_shm_format *get_gles2_formats(size_t *len) {
+const struct wlr_gles2_pixel_format *get_gles2_format_from_gl(
+		GLint gl_format, GLint gl_type, bool alpha) {
+	for (size_t i = 0; i < sizeof(formats) / sizeof(*formats); ++i) {
+		if (formats[i].gl_format == gl_format &&
+				formats[i].gl_type == gl_type &&
+				formats[i].has_alpha == alpha) {
+			return &formats[i];
+		}
+	}
+	return NULL;
+}
+
+const enum wl_shm_format *get_gles2_wl_formats(size_t *len) {
 	*len = sizeof(wl_formats) / sizeof(wl_formats[0]);
 	return wl_formats;
 }
diff --git a/render/gles2/renderer.c b/render/gles2/renderer.c
index b6e97041..a5521909 100644
--- a/render/gles2/renderer.c
+++ b/render/gles2/renderer.c
@@ -135,6 +135,12 @@ static bool gles2_render_texture_with_matrix(struct wlr_renderer *wlr_renderer,
 	case WLR_GLES2_TEXTURE_DMABUF:
 		shader = &renderer->shaders.tex_ext;
 		target = GL_TEXTURE_EXTERNAL_OES;
+
+		if (!renderer->exts.egl_image_external_oes) {
+			wlr_log(WLR_ERROR, "Failed to render texture: "
+				"GL_TEXTURE_EXTERNAL_OES not supported");
+			return false;
+		}
 		break;
 	}
 
@@ -207,13 +213,17 @@ static void gles2_render_ellipse_with_matrix(struct wlr_renderer *wlr_renderer,
 
 static const enum wl_shm_format *gles2_renderer_formats(
 		struct wlr_renderer *wlr_renderer, size_t *len) {
-	return get_gles2_formats(len);
+	return get_gles2_wl_formats(len);
+}
+
+static bool gles2_format_supported(struct wlr_renderer *wlr_renderer,
+		enum wl_shm_format wl_fmt) {
+	return get_gles2_format_from_wl(wl_fmt) != NULL;
 }
 
 static bool gles2_resource_is_wl_drm_buffer(struct wlr_renderer *wlr_renderer,
 		struct wl_resource *resource) {
-	struct wlr_gles2_renderer *renderer =
-		gles2_get_renderer(wlr_renderer);
+	struct wlr_gles2_renderer *renderer = gles2_get_renderer(wlr_renderer);
 
 	if (!eglQueryWaylandBufferWL) {
 		return false;
@@ -249,6 +259,33 @@ static int gles2_get_dmabuf_modifiers(struct wlr_renderer *wlr_renderer,
 	return wlr_egl_get_dmabuf_modifiers(renderer->egl, format, modifiers);
 }
 
+static enum wl_shm_format gles2_preferred_read_format(
+		struct wlr_renderer *wlr_renderer) {
+	struct wlr_gles2_renderer *renderer =
+		gles2_get_renderer_in_context(wlr_renderer);
+
+	GLint gl_format = -1, gl_type = -1;
+	PUSH_GLES2_DEBUG;
+	glGetIntegerv(GL_IMPLEMENTATION_COLOR_READ_FORMAT, &gl_format);
+	glGetIntegerv(GL_IMPLEMENTATION_COLOR_READ_TYPE, &gl_type);
+	POP_GLES2_DEBUG;
+
+	EGLint alpha_size = -1;
+	eglGetConfigAttrib(renderer->egl->display, renderer->egl->config,
+		EGL_ALPHA_SIZE, &alpha_size);
+
+	const struct wlr_gles2_pixel_format *fmt =
+		get_gles2_format_from_gl(gl_format, gl_type, alpha_size > 0);
+	if (fmt != NULL) {
+		return fmt->wl_format;
+	}
+
+	if (renderer->exts.read_format_bgra_ext) {
+		return WL_SHM_FORMAT_XRGB8888;
+	}
+	return WL_SHM_FORMAT_XBGR8888;
+}
+
 static bool gles2_read_pixels(struct wlr_renderer *wlr_renderer,
 		enum wl_shm_format wl_fmt, uint32_t *flags, uint32_t stride,
 		uint32_t width, uint32_t height, uint32_t src_x, uint32_t src_y,
@@ -262,6 +299,12 @@ static bool gles2_read_pixels(struct wlr_renderer *wlr_renderer,
 		return false;
 	}
 
+	if (fmt->gl_format == GL_BGRA_EXT && !renderer->exts.read_format_bgra_ext) {
+		wlr_log(WLR_ERROR,
+			"Cannot read pixels: missing GL_EXT_read_format_bgra extension");
+		return false;
+	}
+
 	PUSH_GLES2_DEBUG;
 
 	// Make sure any pending drawing is finished before we try to read it
@@ -294,11 +337,6 @@ static bool gles2_read_pixels(struct wlr_renderer *wlr_renderer,
 	return glGetError() == GL_NO_ERROR;
 }
 
-static bool gles2_format_supported(struct wlr_renderer *wlr_renderer,
-		enum wl_shm_format wl_fmt) {
-	return get_gles2_format_from_wl(wl_fmt) != NULL;
-}
-
 static struct wlr_texture *gles2_texture_from_pixels(
 		struct wlr_renderer *wlr_renderer, enum wl_shm_format wl_fmt,
 		uint32_t stride, uint32_t width, uint32_t height, const void *data) {
@@ -342,7 +380,7 @@ static void gles2_destroy(struct wlr_renderer *wlr_renderer) {
 	glDeleteProgram(renderer->shaders.tex_ext.program);
 	POP_GLES2_DEBUG;
 
-	if (glDebugMessageCallbackKHR) {
+	if (renderer->exts.debug_khr) {
 		glDisable(GL_DEBUG_OUTPUT_KHR);
 		glDebugMessageCallbackKHR(NULL, NULL);
 	}
@@ -360,12 +398,13 @@ static const struct wlr_renderer_impl renderer_impl = {
 	.render_quad_with_matrix = gles2_render_quad_with_matrix,
 	.render_ellipse_with_matrix = gles2_render_ellipse_with_matrix,
 	.formats = gles2_renderer_formats,
+	.format_supported = gles2_format_supported,
 	.resource_is_wl_drm_buffer = gles2_resource_is_wl_drm_buffer,
 	.wl_drm_buffer_get_size = gles2_wl_drm_buffer_get_size,
 	.get_dmabuf_formats = gles2_get_dmabuf_formats,
 	.get_dmabuf_modifiers = gles2_get_dmabuf_modifiers,
+	.preferred_read_format = gles2_preferred_read_format,
 	.read_pixels = gles2_read_pixels,
-	.format_supported = gles2_format_supported,
 	.texture_from_pixels = gles2_texture_from_pixels,
 	.texture_from_wl_drm = gles2_texture_from_wl_drm,
 	.texture_from_dmabuf = gles2_texture_from_dmabuf,
@@ -466,6 +505,24 @@ error:
 	return 0;
 }
 
+static bool check_gl_ext(const char *exts, const char *ext) {
+	size_t extlen = strlen(ext);
+	const char *end = exts + strlen(exts);
+
+	while (exts < end) {
+		if (exts[0] == ' ') {
+			exts++;
+			continue;
+		}
+		size_t n = strcspn(exts, " ");
+		if (n == extlen && strncmp(ext, exts, n) == 0) {
+			return true;
+		}
+		exts += n;
+	}
+	return false;
+}
+
 extern const GLchar quad_vertex_src[];
 extern const GLchar quad_fragment_src[];
 extern const GLchar ellipse_fragment_src[];
@@ -487,14 +544,32 @@ struct wlr_renderer *wlr_gles2_renderer_create(struct wlr_egl *egl) {
 	wlr_renderer_init(&renderer->wlr_renderer, &renderer_impl);
 
 	renderer->egl = egl;
-	wlr_egl_make_current(renderer->egl, EGL_NO_SURFACE, NULL);
+	if (!wlr_egl_make_current(renderer->egl, EGL_NO_SURFACE, NULL)) {
+		free(renderer);
+		return NULL;
+	}
 
-	renderer->exts_str = (const char*) glGetString(GL_EXTENSIONS);
+	renderer->exts_str = (const char *)glGetString(GL_EXTENSIONS);
 	wlr_log(WLR_INFO, "Using %s", glGetString(GL_VERSION));
 	wlr_log(WLR_INFO, "GL vendor: %s", glGetString(GL_VENDOR));
 	wlr_log(WLR_INFO, "Supported GLES2 extensions: %s", renderer->exts_str);
 
-	if (glDebugMessageCallbackKHR && glDebugMessageControlKHR) {
+	if (!check_gl_ext(renderer->exts_str, "GL_EXT_texture_format_BGRA8888")) {
+		wlr_log(WLR_ERROR, "BGRA8888 format not supported by GLES2");
+		free(renderer);
+		return NULL;
+	}
+
+	renderer->exts.read_format_bgra_ext =
+		check_gl_ext(renderer->exts_str, "GL_EXT_read_format_bgra");
+	renderer->exts.debug_khr =
+		check_gl_ext(renderer->exts_str, "GL_KHR_debug") &&
+		glDebugMessageCallbackKHR && glDebugMessageControlKHR;
+	renderer->exts.egl_image_external_oes =
+		check_gl_ext(renderer->exts_str, "GL_OES_EGL_image_external") &&
+		glEGLImageTargetTexture2DOES;
+
+	if (renderer->exts.debug_khr) {
 		glEnable(GL_DEBUG_OUTPUT_KHR);
 		glEnable(GL_DEBUG_OUTPUT_SYNCHRONOUS_KHR);
 		glDebugMessageCallbackKHR(gles2_log, NULL);
@@ -545,7 +620,7 @@ struct wlr_renderer *wlr_gles2_renderer_create(struct wlr_egl *egl) {
 	renderer->shaders.tex_rgbx.tex = glGetUniformLocation(prog, "tex");
 	renderer->shaders.tex_rgbx.alpha = glGetUniformLocation(prog, "alpha");
 
-	if (glEGLImageTargetTexture2DOES) {
+	if (renderer->exts.egl_image_external_oes) {
 		renderer->shaders.tex_ext.program = prog =
 			link_program(tex_vertex_src, tex_fragment_src_external);
 		if (!renderer->shaders.tex_ext.program) {
@@ -570,7 +645,7 @@ error:
 
 	POP_GLES2_DEBUG;
 
-	if (glDebugMessageCallbackKHR) {
+	if (renderer->exts.debug_khr) {
 		glDisable(GL_DEBUG_OUTPUT_KHR);
 		glDebugMessageCallbackKHR(NULL, NULL);
 	}
diff --git a/render/wlr_renderer.c b/render/wlr_renderer.c
index 31bf2b18..10dc6db2 100644
--- a/render/wlr_renderer.c
+++ b/render/wlr_renderer.c
@@ -139,6 +139,15 @@ int wlr_renderer_get_dmabuf_modifiers(struct wlr_renderer *r, int format,
 	return r->impl->get_dmabuf_modifiers(r, format, modifiers);
 }
 
+bool wlr_renderer_preferred_read_format(struct wlr_renderer *r,
+		enum wl_shm_format *fmt) {
+	if (!r->impl->preferred_read_format || !r->impl->read_pixels) {
+		return false;
+	}
+	*fmt = r->impl->preferred_read_format(r);
+	return true;
+}
+
 bool wlr_renderer_read_pixels(struct wlr_renderer *r, enum wl_shm_format fmt,
 		uint32_t *flags, uint32_t stride, uint32_t width, uint32_t height,
 		uint32_t src_x, uint32_t src_y, uint32_t dst_x, uint32_t dst_y,