wlr_drm_format: Rework wlr_drm_format_intersect

Now it takes a reference to a destination format

2 files changed, 25 insertions, 31 deletions

diff --git a/backend/drm/drm.c b/backend/drm/drm.c
index 0ac41afb..d2a4f074 100644
--- a/backend/drm/drm.c
+++ b/backend/drm/drm.c
@@ -503,18 +503,16 @@ static bool drm_connector_state_update_primary_fb(struct wlr_drm_connector *conn
 
 	struct wlr_buffer *local_buf;
 	if (drm->parent) {
-		struct wlr_drm_format *format =
-			drm_plane_pick_render_format(plane, &drm->mgpu_renderer);
-		if (format == NULL) {
+		struct wlr_drm_format format = {0};
+		if (!drm_plane_pick_render_format(plane, &format, &drm->mgpu_renderer)) {
 			wlr_log(WLR_ERROR, "Failed to pick primary plane format");
 			return false;
 		}
 
 		// TODO: fallback to modifier-less buffer allocation
 		bool ok = init_drm_surface(&plane->mgpu_surf, &drm->mgpu_renderer,
-			source_buf->width, source_buf->height, format);
-		wlr_drm_format_finish(format);
-		free(format);
+			source_buf->width, source_buf->height, &format);
+		wlr_drm_format_finish(&format);
 		if (!ok) {
 			return false;
 		}
@@ -945,17 +943,15 @@ static bool drm_connector_set_cursor(struct wlr_output *output,
 
 		struct wlr_buffer *local_buf;
 		if (drm->parent) {
-			struct wlr_drm_format *format =
-				drm_plane_pick_render_format(plane, &drm->mgpu_renderer);
-			if (format == NULL) {
+			struct wlr_drm_format format = {0};
+			if (!drm_plane_pick_render_format(plane, &format, &drm->mgpu_renderer)) {
 				wlr_log(WLR_ERROR, "Failed to pick cursor plane format");
 				return false;
 			}
 
 			bool ok = init_drm_surface(&plane->mgpu_surf, &drm->mgpu_renderer,
-				buffer->width, buffer->height, format);
-			wlr_drm_format_finish(format);
-			free(format);
+				buffer->width, buffer->height, &format);
+			wlr_drm_format_finish(&format);
 			if (!ok) {
 				return false;
 			}
diff --git a/backend/drm/renderer.c b/backend/drm/renderer.c
index c24ff73d..8d137ae1 100644
--- a/backend/drm/renderer.c
+++ b/backend/drm/renderer.c
@@ -136,50 +136,48 @@ void drm_plane_finish_surface(struct wlr_drm_plane *plane) {
 	finish_drm_surface(&plane->mgpu_surf);
 }
 
-struct wlr_drm_format *drm_plane_pick_render_format(
-		struct wlr_drm_plane *plane, struct wlr_drm_renderer *renderer) {
+bool drm_plane_pick_render_format(struct wlr_drm_plane *plane,
+		struct wlr_drm_format *fmt, struct wlr_drm_renderer *renderer) {
 	const struct wlr_drm_format_set *render_formats =
 		wlr_renderer_get_render_formats(renderer->wlr_rend);
 	if (render_formats == NULL) {
 		wlr_log(WLR_ERROR, "Failed to get render formats");
-		return NULL;
+		return false;
 	}
 
 	const struct wlr_drm_format_set *plane_formats = &plane->formats;
 
-	uint32_t fmt = DRM_FORMAT_ARGB8888;
-	if (!wlr_drm_format_set_get(&plane->formats, fmt)) {
+	uint32_t format = DRM_FORMAT_ARGB8888;
+	if (!wlr_drm_format_set_get(&plane->formats, format)) {
 		const struct wlr_pixel_format_info *format_info =
-			drm_get_pixel_format_info(fmt);
+			drm_get_pixel_format_info(format);
 		assert(format_info != NULL &&
 			format_info->opaque_substitute != DRM_FORMAT_INVALID);
-		fmt = format_info->opaque_substitute;
+		format = format_info->opaque_substitute;
 	}
 
 	const struct wlr_drm_format *render_format =
-		wlr_drm_format_set_get(render_formats, fmt);
+		wlr_drm_format_set_get(render_formats, format);
 	if (render_format == NULL) {
-		wlr_log(WLR_DEBUG, "Renderer doesn't support format 0x%"PRIX32, fmt);
-		return NULL;
+		wlr_log(WLR_DEBUG, "Renderer doesn't support format 0x%"PRIX32, format);
+		return false;
 	}
 
 	const struct wlr_drm_format *plane_format =
-		wlr_drm_format_set_get(plane_formats, fmt);
+		wlr_drm_format_set_get(plane_formats, format);
 	if (plane_format == NULL) {
 		wlr_log(WLR_DEBUG, "Plane %"PRIu32" doesn't support format 0x%"PRIX32,
-			plane->id, fmt);
-		return NULL;
+			plane->id, format);
+		return false;
 	}
 
-	struct wlr_drm_format *format =
-		wlr_drm_format_intersect(plane_format, render_format);
-	if (format == NULL) {
+	if (!wlr_drm_format_intersect(fmt, plane_format, render_format)) {
 		wlr_log(WLR_DEBUG, "Failed to intersect plane and render "
-			"modifiers for format 0x%"PRIX32, fmt);
-		return NULL;
+			"modifiers for format 0x%"PRIX32, format);
+		return false;
 	}
 
-	return format;
+	return true;
 }
 
 void drm_fb_clear(struct wlr_drm_fb **fb_ptr) {