output: take a wlr_buffer in set_cursor

Instead of passing a wlr_texture to the backend, directly pass a
wlr_buffer. Use get_cursor_size and get_cursor_formats to create
a wlr_buffer that can be used as a cursor.

We don't want to pass a wlr_texture because we want to remove as
many rendering bits from the backend as possible.

4 files changed, 67 insertions, 240 deletions

diff --git a/backend/drm/drm.c b/backend/drm/drm.c
index c9d3b54a..9efff464 100644
--- a/backend/drm/drm.c
+++ b/backend/drm/drm.c
@@ -25,7 +25,9 @@
 #include "backend/drm/iface.h"
 #include "backend/drm/util.h"
 #include "render/pixel_format.h"
+#include "render/drm_format_set.h"
 #include "render/swapchain.h"
+#include "render/wlr_renderer.h"
 #include "types/wlr_buffer.h"
 #include "util/signal.h"
 
@@ -329,7 +331,7 @@ static bool drm_connector_attach_render(struct wlr_output *output,
 static void drm_plane_set_committed(struct wlr_drm_plane *plane) {
 	drm_fb_move(&plane->queued_fb, &plane->pending_fb);
 
-	if (plane->queued_fb) {
+	if (plane->queued_fb && plane->surf.swapchain) {
 		wlr_swapchain_set_buffer_submitted(plane->surf.swapchain,
 			plane->queued_fb->wlr_buf);
 	}
@@ -834,9 +836,7 @@ struct wlr_output_mode *wlr_drm_connector_add_mode(struct wlr_output *output,
 }
 
 static bool drm_connector_set_cursor(struct wlr_output *output,
-		struct wlr_texture *texture, float scale,
-		enum wl_output_transform transform,
-		int32_t hotspot_x, int32_t hotspot_y, bool update_texture) {
+		struct wlr_buffer *buffer, int hotspot_x, int hotspot_y) {
 	struct wlr_drm_connector *conn = get_drm_connector_from_output(output);
 	struct wlr_drm_backend *drm = conn->backend;
 	struct wlr_drm_crtc *crtc = conn->crtc;
@@ -850,90 +850,59 @@ static bool drm_connector_set_cursor(struct wlr_output *output,
 		return false;
 	}
 
-	if (!plane->surf.swapchain) {
-		int ret;
-		uint64_t w, h;
-		ret = drmGetCap(drm->fd, DRM_CAP_CURSOR_WIDTH, &w);
-		w = ret ? 64 : w;
-		ret = drmGetCap(drm->fd, DRM_CAP_CURSOR_HEIGHT, &h);
-		h = ret ? 64 : h;
-
-		if (!drm_plane_init_surface(plane, drm, w, h, true)) {
-			wlr_drm_conn_log(conn, WLR_ERROR, "Cannot allocate cursor resources");
-			return false;
-		}
-	}
-
-	struct wlr_box hotspot = { .x = hotspot_x, .y = hotspot_y };
-	wlr_box_transform(&hotspot, &hotspot,
-		wlr_output_transform_invert(output->transform),
-		plane->surf.width, plane->surf.height);
-
-	if (plane->cursor_hotspot_x != hotspot.x ||
-			plane->cursor_hotspot_y != hotspot.y) {
+	if (plane->cursor_hotspot_x != hotspot_x ||
+			plane->cursor_hotspot_y != hotspot_y) {
 		// Update cursor hotspot
-		conn->cursor_x -= hotspot.x - plane->cursor_hotspot_x;
-		conn->cursor_y -= hotspot.y - plane->cursor_hotspot_y;
-		plane->cursor_hotspot_x = hotspot.x;
-		plane->cursor_hotspot_y = hotspot.y;
+		conn->cursor_x -= hotspot_x - plane->cursor_hotspot_x;
+		conn->cursor_y -= hotspot_y - plane->cursor_hotspot_y;
+		plane->cursor_hotspot_x = hotspot_x;
+		plane->cursor_hotspot_y = hotspot_y;
 
 		wlr_output_update_needs_frame(output);
 	}
 
-	if (!update_texture) {
-		// Don't update cursor image
-		return true;
-	}
-
 	plane->cursor_enabled = false;
-	if (texture != NULL) {
-		int width = texture->width * output->scale / scale;
-		int height = texture->height * output->scale / scale;
-
-		if (width > (int)plane->surf.width || height > (int)plane->surf.height) {
-			wlr_drm_conn_log(conn, WLR_ERROR, "Cursor too large (max %dx%d)",
-				(int)plane->surf.width, (int)plane->surf.height);
-			return false;
-		}
-
-		if (!drm_surface_make_current(&plane->surf, NULL)) {
+	if (buffer != NULL) {
+		if ((uint64_t)buffer->width != drm->cursor_width ||
+				(uint64_t)buffer->height != drm->cursor_height) {
+			wlr_drm_conn_log(conn, WLR_DEBUG, "Cursor buffer size mismatch");
 			return false;
 		}
 
-		struct wlr_renderer *rend = plane->surf.renderer->wlr_rend;
-
-		struct wlr_box cursor_box = { .width = width, .height = height };
+		struct wlr_buffer *local_buf;
+		if (drm->parent) {
+			struct wlr_drm_format *format =
+				drm_plane_pick_render_format(plane, &drm->renderer);
+			if (format == NULL) {
+				wlr_log(WLR_ERROR, "Failed to pick cursor plane format");
+				return false;
+			}
 
-		float output_matrix[9];
-		wlr_matrix_identity(output_matrix);
-		if (output->transform != WL_OUTPUT_TRANSFORM_NORMAL) {
-			struct wlr_box tr_size = {
-				.width = plane->surf.width,
-				.height = plane->surf.height,
-			};
-			wlr_box_transform(&tr_size, &tr_size, output->transform, 0, 0);
+			bool ok = init_drm_surface(&plane->mgpu_surf, &drm->renderer,
+				buffer->width, buffer->height, format);
+			free(format);
+			if (!ok) {
+				return false;
+			}
 
-			wlr_matrix_translate(output_matrix, plane->surf.width / 2.0,
-					plane->surf.height / 2.0);
-			wlr_matrix_transform(output_matrix, output->transform);
-			wlr_matrix_translate(output_matrix, - tr_size.width / 2.0,
-					- tr_size.height / 2.0);
+			local_buf = drm_surface_blit(&plane->mgpu_surf, buffer);
+			if (local_buf == NULL) {
+				return false;
+			}
+		} else {
+			local_buf = wlr_buffer_lock(buffer);
 		}
 
-		float matrix[9];
-		wlr_matrix_project_box(matrix, &cursor_box, transform, 0,
-				output_matrix);
-
-		wlr_renderer_begin(rend, plane->surf.width, plane->surf.height);
-		wlr_renderer_clear(rend, (float[]){ 0.0, 0.0, 0.0, 0.0 });
-		wlr_render_texture_with_matrix(rend, texture, matrix, 1.0);
-		wlr_renderer_end(rend);
-
-		if (!drm_plane_lock_surface(plane, drm)) {
+		bool ok = drm_fb_import(&plane->pending_fb, drm, local_buf,
+			&plane->formats);
+		wlr_buffer_unlock(local_buf);
+		if (!ok) {
 			return false;
 		}
 
 		plane->cursor_enabled = true;
+		plane->cursor_width = buffer->width;
+		plane->cursor_height = buffer->height;
 	}
 
 	wlr_output_update_needs_frame(output);
@@ -977,8 +946,8 @@ bool drm_connector_is_cursor_visible(struct wlr_drm_connector *conn) {
 	return plane->cursor_enabled &&
 		conn->cursor_x < conn->output.width &&
 		conn->cursor_y < conn->output.height &&
-		conn->cursor_x + (int)plane->surf.width >= 0 &&
-		conn->cursor_y + (int)plane->surf.height >= 0;
+		conn->cursor_x + plane->cursor_width >= 0 &&
+		conn->cursor_y + plane->cursor_height >= 0;
 }
 
 static void dealloc_crtc(struct wlr_drm_connector *conn);
diff --git a/backend/drm/renderer.c b/backend/drm/renderer.c
index 7d6c155a..e1e35e63 100644
--- a/backend/drm/renderer.c
+++ b/backend/drm/renderer.c
@@ -62,7 +62,7 @@ void finish_drm_renderer(struct wlr_drm_renderer *renderer) {
 	gbm_device_destroy(renderer->gbm);
 }
 
-static bool init_drm_surface(struct wlr_drm_surface *surf,
+bool init_drm_surface(struct wlr_drm_surface *surf,
 		struct wlr_drm_renderer *renderer, uint32_t width, uint32_t height,
 		const struct wlr_drm_format *drm_format) {
 	if (surf->width == width && surf->height == height) {
@@ -126,7 +126,7 @@ void drm_surface_unset_current(struct wlr_drm_surface *surf) {
 	surf->back_buffer = NULL;
 }
 
-static struct wlr_buffer *drm_surface_blit(struct wlr_drm_surface *surf,
+struct wlr_buffer *drm_surface_blit(struct wlr_drm_surface *surf,
 		struct wlr_buffer *buffer) {
 	struct wlr_renderer *renderer = surf->renderer->wlr_rend;
 
@@ -191,7 +191,7 @@ static struct wlr_drm_format *create_linear_format(uint32_t format) {
 	return fmt;
 }
 
-static struct wlr_drm_format *drm_plane_pick_render_format(
+struct wlr_drm_format *drm_plane_pick_render_format(
 		struct wlr_drm_plane *plane, struct wlr_drm_renderer *renderer) {
 	const struct wlr_drm_format_set *render_formats =
 		wlr_renderer_get_render_formats(renderer->wlr_rend);
diff --git a/backend/wayland/output.c b/backend/wayland/output.c
index cdf78b2a..62e9b5af 100644
--- a/backend/wayland/output.c
+++ b/backend/wayland/output.c
@@ -406,28 +406,12 @@ static void output_rollback_render(struct wlr_output *wlr_output) {
 }
 
 static bool output_set_cursor(struct wlr_output *wlr_output,
-		struct wlr_texture *texture, float scale,
-		enum wl_output_transform transform,
-		int32_t hotspot_x, int32_t hotspot_y, bool update_texture) {
+		struct wlr_buffer *wlr_buffer, int hotspot_x, int hotspot_y) {
 	struct wlr_wl_output *output = get_wl_output_from_output(wlr_output);
 	struct wlr_wl_backend *backend = output->backend;
-	struct wlr_renderer *renderer = wlr_backend_get_renderer(&backend->backend);
-	struct wlr_allocator *allocator = backend_get_allocator(&backend->backend);
-
-	struct wlr_box hotspot = { .x = hotspot_x, .y = hotspot_y };
-	wlr_box_transform(&hotspot, &hotspot,
-		wlr_output_transform_invert(wlr_output->transform),
-		output->cursor.width, output->cursor.height);
-
-	// TODO: use output->wlr_output.transform to transform pixels and hotpot
-	output->cursor.hotspot_x = hotspot.x;
-	output->cursor.hotspot_y = hotspot.y;
 
-	if (!update_texture) {
-		// Update hotspot without changing cursor image
-		update_wl_output_cursor(output);
-		return true;
-	}
+	output->cursor.hotspot_x = hotspot_x;
+	output->cursor.hotspot_y = hotspot_y;
 
 	if (output->cursor.surface == NULL) {
 		output->cursor.surface =
@@ -435,58 +419,7 @@ static bool output_set_cursor(struct wlr_output *wlr_output,
 	}
 	struct wl_surface *surface = output->cursor.surface;
 
-	if (texture != NULL) {
-		int width = texture->width * wlr_output->scale / scale;
-		int height = texture->height * wlr_output->scale / scale;
-
-		if (output->cursor.swapchain == NULL ||
-				output->cursor.swapchain->width != width ||
-				output->cursor.swapchain->height != height) {
-			wlr_swapchain_destroy(output->cursor.swapchain);
-			output->cursor.swapchain = wlr_swapchain_create(allocator,
-				width, height, output->backend->format);
-			if (output->cursor.swapchain == NULL) {
-				return false;
-			}
-		}
-
-		struct wlr_buffer *wlr_buffer =
-			wlr_swapchain_acquire(output->cursor.swapchain, NULL);
-		if (wlr_buffer == NULL) {
-			return false;
-		}
-
-		if (!wlr_renderer_bind_buffer(renderer, wlr_buffer)) {
-			return false;
-		}
-
-		struct wlr_box cursor_box = {
-			.width = width,
-			.height = height,
-		};
-
-		float output_matrix[9];
-		wlr_matrix_identity(output_matrix);
-		if (wlr_output->transform != WL_OUTPUT_TRANSFORM_NORMAL) {
-			struct wlr_box tr_size = { .width = width, .height = height };
-			wlr_box_transform(&tr_size, &tr_size, wlr_output->transform, 0, 0);
-
-			wlr_matrix_translate(output_matrix, width / 2.0, height / 2.0);
-			wlr_matrix_transform(output_matrix, wlr_output->transform);
-			wlr_matrix_translate(output_matrix,
-				- tr_size.width / 2.0, - tr_size.height / 2.0);
-		}
-
-		float matrix[9];
-		wlr_matrix_project_box(matrix, &cursor_box, transform, 0, output_matrix);
-
-		wlr_renderer_begin(renderer, width, height);
-		wlr_renderer_clear(renderer, (float[]){ 0.0, 0.0, 0.0, 0.0 });
-		wlr_render_texture_with_matrix(renderer, texture, matrix, 1.0);
-		wlr_renderer_end(renderer);
-
-		wlr_renderer_bind_buffer(renderer, NULL);
-
+	if (wlr_buffer != NULL) {
 		struct wlr_wl_buffer *buffer =
 			get_or_create_wl_buffer(output->backend, wlr_buffer);
 		if (buffer == NULL) {
@@ -496,11 +429,6 @@ static bool output_set_cursor(struct wlr_output *wlr_output,
 		wl_surface_attach(surface, buffer->wl_buffer, 0, 0);
 		wl_surface_damage_buffer(surface, 0, 0, INT32_MAX, INT32_MAX);
 		wl_surface_commit(surface);
-
-		wlr_buffer_unlock(wlr_buffer);
-
-		output->cursor.width = width;
-		output->cursor.height = height;
 	} else {
 		wl_surface_attach(surface, NULL, 0, 0);
 		wl_surface_commit(surface);
@@ -530,7 +458,6 @@ static void output_destroy(struct wlr_output *wlr_output) {
 
 	wl_list_remove(&output->link);
 
-	wlr_swapchain_destroy(output->cursor.swapchain);
 	if (output->cursor.surface) {
 		wl_surface_destroy(output->cursor.surface);
 	}
diff --git a/backend/x11/output.c b/backend/x11/output.c
index bba938d2..56161dee 100644
--- a/backend/x11/output.c
+++ b/backend/x11/output.c
@@ -420,83 +420,38 @@ static void update_x11_output_cursor(struct wlr_x11_output *output,
 }
 
 static bool output_cursor_to_picture(struct wlr_x11_output *output,
-		struct wlr_texture *texture, enum wl_output_transform transform,
-		int width, int height) {
+		struct wlr_buffer *buffer) {
 	struct wlr_x11_backend *x11 = output->x11;
-	struct wlr_allocator *allocator = backend_get_allocator(&x11->backend);
 	struct wlr_renderer *renderer = wlr_backend_get_renderer(&x11->backend);
-	int depth = 32;
-	int stride = width * 4;
 
 	if (output->cursor.pic != XCB_NONE) {
 		xcb_render_free_picture(x11->xcb, output->cursor.pic);
 	}
 	output->cursor.pic = XCB_NONE;
 
-	if (texture == NULL) {
+	if (buffer == NULL) {
 		return true;
 	}
 
-	if (output->cursor.swapchain == NULL ||
-			output->cursor.swapchain->width != width ||
-			output->cursor.swapchain->height != height) {
-		wlr_swapchain_destroy(output->cursor.swapchain);
-		output->cursor.swapchain = wlr_swapchain_create(allocator,
-			width, height, x11->drm_format);
-		if (output->cursor.swapchain == NULL) {
-			return false;
-		}
-	}
-
-	struct wlr_buffer *wlr_buffer =
-		wlr_swapchain_acquire(output->cursor.swapchain, NULL);
-	if (wlr_buffer == NULL) {
-		return false;
-	}
+	int depth = 32;
+	int stride = buffer->width * 4;
 
-	if (!wlr_renderer_bind_buffer(renderer, wlr_buffer)) {
+	if (!wlr_renderer_bind_buffer(renderer, buffer)) {
 		return false;
 	}
 
-	uint8_t *data = malloc(width * height * 4);
+	uint8_t *data = malloc(buffer->height * stride);
 	if (data == NULL) {
 		return false;
 	}
 
-	struct wlr_box cursor_box = {
-		.width = width,
-		.height = height,
-	};
-
-	float output_matrix[9];
-	wlr_matrix_identity(output_matrix);
-	if (output->wlr_output.transform != WL_OUTPUT_TRANSFORM_NORMAL) {
-		struct wlr_box tr_size = { .width = width, .height = height };
-		wlr_box_transform(&tr_size, &tr_size, output->wlr_output.transform, 0, 0);
-
-		wlr_matrix_translate(output_matrix, width / 2.0, height / 2.0);
-		wlr_matrix_transform(output_matrix, output->wlr_output.transform);
-		wlr_matrix_translate(output_matrix,
-			- tr_size.width / 2.0, - tr_size.height / 2.0);
-	}
-
-	float matrix[9];
-	wlr_matrix_project_box(matrix, &cursor_box, transform, 0, output_matrix);
-
-	wlr_renderer_begin(renderer, width, height);
-	wlr_renderer_clear(renderer, (float[]){ 0.0, 0.0, 0.0, 0.0 });
-	wlr_render_texture_with_matrix(renderer, texture, matrix, 1.0);
-	wlr_renderer_end(renderer);
-
 	bool result = wlr_renderer_read_pixels(
 		renderer, DRM_FORMAT_ARGB8888, NULL,
-		width * 4, width, height, 0, 0, 0, 0,
+		stride, buffer->width, buffer->height, 0, 0, 0, 0,
 		data);
 
 	wlr_renderer_bind_buffer(renderer, NULL);
 
-	wlr_buffer_unlock(wlr_buffer);
-
 	if (!result) {
 		free(data);
 		return false;
@@ -504,7 +459,7 @@ static bool output_cursor_to_picture(struct wlr_x11_output *output,
 
 	xcb_pixmap_t pix = xcb_generate_id(x11->xcb);
 	xcb_create_pixmap(x11->xcb, depth, pix, output->win,
-		width, height);
+		buffer->width, buffer->height);
 
 	output->cursor.pic = xcb_generate_id(x11->xcb);
 	xcb_render_create_picture(x11->xcb, output->cursor.pic,
@@ -514,9 +469,8 @@ static bool output_cursor_to_picture(struct wlr_x11_output *output,
 	xcb_create_gc(x11->xcb, gc, pix, 0, NULL);
 
 	xcb_put_image(x11->xcb, XCB_IMAGE_FORMAT_Z_PIXMAP,
-		pix, gc, width, height, 0, 0, 0, depth,
-		stride * height * sizeof(uint8_t),
-		data);
+		pix, gc, buffer->width, buffer->height, 0, 0, 0, depth,
+		stride * buffer->height * sizeof(uint8_t), data);
 	free(data);
 	xcb_free_gc(x11->xcb, gc);
 	xcb_free_pixmap(x11->xcb, pix);
@@ -525,55 +479,32 @@ static bool output_cursor_to_picture(struct wlr_x11_output *output,
 }
 
 static bool output_set_cursor(struct wlr_output *wlr_output,
-		struct wlr_texture *texture, float scale,
-		enum wl_output_transform transform,
-		int32_t hotspot_x, int32_t hotspot_y, bool update_texture) {
+		struct wlr_buffer *buffer, int32_t hotspot_x, int32_t hotspot_y) {
 	struct wlr_x11_output *output = get_x11_output_from_output(wlr_output);
 	struct wlr_x11_backend *x11 = output->x11;
-	int width = 0, height = 0;
 
 	if (x11->argb32 == XCB_NONE) {
 		return false;
 	}
 
-	if (texture != NULL) {
-		width = texture->width * wlr_output->scale / scale;
-		height = texture->height * wlr_output->scale / scale;
-
+	if (buffer != NULL) {
 		if (hotspot_x < 0) {
 			hotspot_x = 0;
 		}
-		if ((uint32_t)hotspot_x > texture->width) {
-			hotspot_x = texture->width;
+		if (hotspot_x > buffer->width) {
+			hotspot_x = buffer->width;
 		}
 		if (hotspot_y < 0) {
 			hotspot_y = 0;
 		}
-		if ((uint32_t)hotspot_y > texture->height) {
-			hotspot_y = texture->height;
-		}
-	}
-
-	struct wlr_box hotspot = { .x = hotspot_x, .y = hotspot_y };
-	wlr_box_transform(&hotspot, &hotspot,
-		wlr_output_transform_invert(wlr_output->transform),
-		width, height);
-
-	if (!update_texture) {
-		// This means we previously had a failure of some sort.
-		if (texture != NULL && output->cursor.pic == XCB_NONE) {
-			return false;
+		if (hotspot_y > buffer->height) {
+			hotspot_y = buffer->height;
 		}
-
-		// Update hotspot without changing cursor image
-		update_x11_output_cursor(output, hotspot.x, hotspot.y);
-		return true;
 	}
 
-	bool success = output_cursor_to_picture(output, texture, transform,
-		width, height);
+	bool success = output_cursor_to_picture(output, buffer);
 
-	update_x11_output_cursor(output, hotspot.x, hotspot.y);
+	update_x11_output_cursor(output, hotspot_x, hotspot_y);
 
 	return success;
 }