backend/drm: fix GBM format mismatch

We create the EGL config with GBM_FORMAT_ARGB8888, but then initialize GBM BOs
with GBM_FORMAT_XRGB8888. This mismatch confuses Mesa.

Instead, we can always use GBM_FORMAT_ARGB8888, and use DRM_FORMAT_XRGB8888
when calling drmModeAddFB2.

Fixes https://github.com/swaywm/wlroots/issues/1438

4 files changed, 41 insertions, 14 deletions

diff --git a/backend/drm/atomic.c b/backend/drm/atomic.c
index fc649d68..c9fb1f45 100644
--- a/backend/drm/atomic.c
+++ b/backend/drm/atomic.c
@@ -172,7 +172,8 @@ static bool atomic_crtc_set_cursor(struct wlr_drm_backend *drm,
 	atomic_begin(crtc, &atom);
 
 	if (bo) {
-		set_plane_props(&atom, plane, crtc->id, get_fb_for_bo(bo), false);
+		uint32_t fb_id = get_fb_for_bo(bo, plane->drm_format);
+		set_plane_props(&atom, plane, crtc->id, fb_id, false);
 	} else {
 		atomic_add(&atom, plane->id, plane->props.fb_id, 0);
 		atomic_add(&atom, plane->id, plane->props.crtc_id, 0);
diff --git a/backend/drm/drm.c b/backend/drm/drm.c
index 28ee41a5..999ca494 100644
--- a/backend/drm/drm.c
+++ b/backend/drm/drm.c
@@ -1,5 +1,6 @@
 #define _POSIX_C_SOURCE 200112L
 #include <assert.h>
+#include <drm_fourcc.h>
 #include <drm_mode.h>
 #include <EGL/egl.h>
 #include <EGL/eglext.h>
@@ -109,8 +110,8 @@ static bool init_planes(struct wlr_drm_backend *drm) {
 
 		p->id = plane->plane_id;
 		p->possible_crtcs = plane->possible_crtcs;
-		uint64_t type;
 
+		uint64_t type;
 		if (!get_drm_plane_props(drm->fd, p->id, &p->props) ||
 				!get_drm_prop(drm->fd, p->id, p->props.type, &type)) {
 			drmModeFreePlane(plane);
@@ -120,6 +121,25 @@ static bool init_planes(struct wlr_drm_backend *drm) {
 		p->type = type;
 		drm->num_type_planes[type]++;
 
+		// Choose an RGB format for the plane
+		uint32_t rgb_format = DRM_FORMAT_INVALID;
+		for (size_t j = 0; j < plane->count_formats; ++j) {
+			uint32_t fmt = plane->formats[j];
+			if (fmt == DRM_FORMAT_ARGB8888) {
+				// Prefer formats with alpha channel
+				rgb_format = fmt;
+				break;
+			} else if (fmt == DRM_FORMAT_XRGB8888) {
+				rgb_format = fmt;
+			}
+		}
+		if (rgb_format == DRM_FORMAT_INVALID) {
+			wlr_log(WLR_ERROR, "Failed to find an RGB format for plane %zu", i);
+			drmModeFreePlane(plane);
+			goto error_planes;
+		}
+		p->drm_format = rgb_format;
+
 		drmModeFreePlane(plane);
 	}
 
@@ -247,7 +267,7 @@ static bool drm_connector_swap_buffers(struct wlr_output *output,
 	if (drm->parent) {
 		bo = copy_drm_surface_mgpu(&plane->mgpu_surf, bo);
 	}
-	uint32_t fb_id = get_fb_for_bo(bo);
+	uint32_t fb_id = get_fb_for_bo(bo, plane->drm_format);
 
 	if (conn->pageflip_pending) {
 		wlr_log(WLR_ERROR, "Skipping pageflip on output '%s'", conn->output.name);
@@ -366,7 +386,7 @@ static void drm_connector_start_renderer(struct wlr_drm_connector *conn) {
 
 	struct gbm_bo *bo = get_drm_surface_front(
 		drm->parent ? &plane->mgpu_surf : &plane->surf);
-	uint32_t fb_id = get_fb_for_bo(bo);
+	uint32_t fb_id = get_fb_for_bo(bo, plane->drm_format);
 
 	struct wlr_drm_mode *mode = (struct wlr_drm_mode *)conn->output.current_mode;
 	if (drm->iface->crtc_pageflip(drm, conn, crtc, fb_id, &mode->drm_mode)) {
@@ -528,7 +548,7 @@ static bool drm_connector_set_mode(struct wlr_output *output,
 		conn->output.name, mode->width, mode->height, mode->refresh);
 
 	if (!init_drm_plane_surfaces(conn->crtc->primary, drm,
-			mode->width, mode->height, GBM_FORMAT_XRGB8888)) {
+			mode->width, mode->height, drm->renderer.gbm_format)) {
 		wlr_log(WLR_ERROR, "Failed to initialize renderer for plane");
 		return false;
 	}
@@ -622,13 +642,13 @@ static bool drm_connector_set_cursor(struct wlr_output *output,
 			drm->parent ? &drm->parent->renderer : &drm->renderer;
 
 		if (!init_drm_surface(&plane->surf, renderer, w, h,
-				GBM_FORMAT_ARGB8888, 0)) {
+				renderer->gbm_format, 0)) {
 			wlr_log(WLR_ERROR, "Cannot allocate cursor resources");
 			return false;
 		}
 
 		plane->cursor_bo = gbm_bo_create(drm->renderer.gbm, w, h,
-			GBM_FORMAT_ARGB8888, GBM_BO_USE_CURSOR | GBM_BO_USE_WRITE);
+			renderer->gbm_format, GBM_BO_USE_CURSOR | GBM_BO_USE_WRITE);
 		if (!plane->cursor_bo) {
 			wlr_log_errno(WLR_ERROR, "Failed to create cursor bo");
 			return false;
@@ -785,7 +805,6 @@ static bool drm_connector_schedule_frame(struct wlr_output *output) {
 	if (drm->parent) {
 		bo = copy_drm_surface_mgpu(&plane->mgpu_surf, bo);
 	}
-	uint32_t fb_id = get_fb_for_bo(bo);
 
 	if (conn->pageflip_pending) {
 		wlr_log(WLR_ERROR, "Skipping pageflip on output '%s'",
@@ -793,6 +812,7 @@ static bool drm_connector_schedule_frame(struct wlr_output *output) {
 		return true;
 	}
 
+	uint32_t fb_id = get_fb_for_bo(bo, plane->drm_format);
 	if (!drm->iface->crtc_pageflip(drm, conn, crtc, fb_id, NULL)) {
 		return false;
 	}
@@ -998,7 +1018,7 @@ static void realloc_crtcs(struct wlr_drm_backend *drm, bool *changed_outputs) {
 		}
 
 		if (!init_drm_plane_surfaces(conn->crtc->primary, drm,
-				mode->width, mode->height, GBM_FORMAT_XRGB8888)) {
+				mode->width, mode->height, drm->renderer.gbm_format)) {
 			wlr_log(WLR_ERROR, "Failed to initialize renderer for plane");
 			drm_connector_cleanup(conn);
 			break;
diff --git a/backend/drm/renderer.c b/backend/drm/renderer.c
index 70b1bcbe..fa235af9 100644
--- a/backend/drm/renderer.c
+++ b/backend/drm/renderer.c
@@ -38,10 +38,10 @@ bool init_drm_renderer(struct wlr_drm_backend *drm,
 		EGL_NONE,
 	};
 
+	renderer->gbm_format = GBM_FORMAT_ARGB8888;
 	renderer->wlr_rend = create_renderer_func(&renderer->egl,
 		EGL_PLATFORM_GBM_MESA, renderer->gbm,
-		config_attribs, GBM_FORMAT_ARGB8888);
-
+		config_attribs, renderer->gbm_format);
 	if (!renderer->wlr_rend) {
 		wlr_log(WLR_ERROR, "Failed to create EGL/WLR renderer");
 		goto error_gbm;
diff --git a/backend/drm/util.c b/backend/drm/util.c
index 6f2dd5be..c2aa0fd8 100644
--- a/backend/drm/util.c
+++ b/backend/drm/util.c
@@ -1,3 +1,5 @@
+#include <assert.h>
+#include <drm_fourcc.h>
 #include <drm_mode.h>
 #include <drm.h>
 #include <gbm.h>
@@ -176,12 +178,16 @@ static void free_fb(struct gbm_bo *bo, void *data) {
 	}
 }
 
-uint32_t get_fb_for_bo(struct gbm_bo *bo) {
+uint32_t get_fb_for_bo(struct gbm_bo *bo, uint32_t drm_format) {
 	uint32_t id = (uintptr_t)gbm_bo_get_user_data(bo);
 	if (id) {
 		return id;
 	}
 
+	assert(gbm_bo_get_format(bo) == GBM_FORMAT_ARGB8888);
+	assert(drm_format == DRM_FORMAT_ARGB8888 ||
+		drm_format == DRM_FORMAT_XRGB8888);
+
 	struct gbm_device *gbm = gbm_bo_get_device(bo);
 
 	int fd = gbm_device_get_fd(gbm);
@@ -190,9 +196,9 @@ uint32_t get_fb_for_bo(struct gbm_bo *bo) {
 	uint32_t handles[4] = {gbm_bo_get_handle(bo).u32};
 	uint32_t pitches[4] = {gbm_bo_get_stride(bo)};
 	uint32_t offsets[4] = {gbm_bo_get_offset(bo, 0)};
-	uint32_t format = gbm_bo_get_format(bo);
 
-	if (drmModeAddFB2(fd, width, height, format, handles, pitches, offsets, &id, 0)) {
+	if (drmModeAddFB2(fd, width, height, drm_format,
+			handles, pitches, offsets, &id, 0)) {
 		wlr_log_errno(WLR_ERROR, "Unable to add DRM framebuffer");
 	}