output: always use hardware cursors if available

This changes the `wlr_output_impl.set_cursor` function to take a
`wlr_texture` instead of a byte buffer. This simplifies the
DRM and Wayland backends since they were creating textures from
the byte buffer anyway.

With this commit, performance should be improved when moving the
cursor since outputs don't need to be re-rendered anymore.

6 files changed, 112 insertions, 139 deletions

diff --git a/backend/drm/drm.c b/backend/drm/drm.c
index dc512151..70089e06 100644
--- a/backend/drm/drm.c
+++ b/backend/drm/drm.c
@@ -537,8 +537,8 @@ static void drm_connector_transform(struct wlr_output *output,
 }
 
 static bool drm_connector_set_cursor(struct wlr_output *output,
-		const uint8_t *buf, int32_t stride, uint32_t width, uint32_t height,
-		int32_t hotspot_x, int32_t hotspot_y, bool update_pixels) {
+		struct wlr_texture *texture, int32_t hotspot_x, int32_t hotspot_y,
+		bool update_texture) {
 	struct wlr_drm_connector *conn = (struct wlr_drm_connector *)output;
 	struct wlr_drm_backend *drm = (struct wlr_drm_backend *)output->backend;
 	struct wlr_drm_renderer *renderer = &drm->renderer;
@@ -567,11 +567,6 @@ static bool drm_connector_set_cursor(struct wlr_output *output,
 		ret = drmGetCap(drm->fd, DRM_CAP_CURSOR_HEIGHT, &h);
 		h = ret ? 64 : h;
 
-		if (width > w || height > h) {
-			wlr_log(L_INFO, "Cursor too large (max %dx%d)", (int)w, (int)h);
-			return false;
-		}
-
 		if (!init_drm_surface(&plane->surf, renderer, w, h,
 				GBM_FORMAT_ARGB8888, 0)) {
 			wlr_log(L_ERROR, "Cannot allocate cursor resources");
@@ -612,14 +607,22 @@ static bool drm_connector_set_cursor(struct wlr_output *output,
 		wlr_output_update_needs_swap(output);
 	}
 
-	if (!update_pixels) {
+	if (!update_texture) {
 		// Don't update cursor image
 		return true;
 	}
 
-	plane->cursor_enabled = buf != NULL;
+	plane->cursor_enabled = false;
+	if (texture != NULL) {
+		int width, height;
+		wlr_texture_get_size(texture, &width, &height);
+
+		if (width > (int)plane->surf.width || height > (int)plane->surf.height) {
+			wlr_log(L_ERROR, "Cursor too large (max %dx%d)",
+				(int)plane->surf.width, (int)plane->surf.height);
+			return false;
+		}
 
-	if (buf != NULL) {
 		uint32_t bo_width = gbm_bo_get_width(plane->cursor_bo);
 		uint32_t bo_height = gbm_bo_get_height(plane->cursor_bo);
 
@@ -635,13 +638,6 @@ static bool drm_connector_set_cursor(struct wlr_output *output,
 
 		struct wlr_renderer *rend = plane->surf.renderer->wlr_rend;
 
-		struct wlr_texture *texture = wlr_texture_from_pixels(rend,
-			WL_SHM_FORMAT_ARGB8888, stride, width, height, buf);
-		if (texture == NULL) {
-			wlr_log(L_ERROR, "Unable to create texture");
-			return false;
-		}
-
 		wlr_renderer_begin(rend, plane->surf.width, plane->surf.height);
 		wlr_renderer_clear(rend, (float[]){ 0.0, 0.0, 0.0, 0.0 });
 		wlr_render_texture(rend, texture, plane->matrix, 0, 0, 1.0f);
@@ -652,8 +648,9 @@ static bool drm_connector_set_cursor(struct wlr_output *output,
 
 		swap_drm_surface_buffers(&plane->surf, NULL);
 
-		wlr_texture_destroy(texture);
 		gbm_bo_unmap(plane->cursor_bo, bo_data);
+
+		plane->cursor_enabled = true;
 	}
 
 	if (!drm->session->active) {
diff --git a/backend/wayland/backend.c b/backend/wayland/backend.c
index 15d8feab..dc50cf23 100644
--- a/backend/wayland/backend.c
+++ b/backend/wayland/backend.c
@@ -164,8 +164,16 @@ struct wlr_backend *wlr_wl_backend_create(struct wl_display *display,
 		goto error_registry;
 	}
 
+	static EGLint config_attribs[] = {
+		EGL_SURFACE_TYPE, EGL_WINDOW_BIT,
+		EGL_RED_SIZE, 1,
+		EGL_GREEN_SIZE, 1,
+		EGL_BLUE_SIZE, 1,
+		EGL_ALPHA_SIZE, 1,
+		EGL_NONE,
+	};
 	if (!wlr_egl_init(&backend->egl, EGL_PLATFORM_WAYLAND_EXT,
-			backend->remote_display, NULL, WL_SHM_FORMAT_ARGB8888)) {
+			backend->remote_display, config_attribs, WL_SHM_FORMAT_ARGB8888)) {
 		wlr_log(L_ERROR, "Could not initialize EGL");
 		goto error_egl;
 	}
diff --git a/backend/wayland/output.c b/backend/wayland/output.c
index 209b060f..c498fc4e 100644
--- a/backend/wayland/output.c
+++ b/backend/wayland/output.c
@@ -9,6 +9,7 @@
 #include <wayland-client.h>
 #include <wlr/interfaces/wlr_output.h>
 #include <wlr/render/wlr_renderer.h>
+#include <wlr/types/wlr_matrix.h>
 #include <wlr/util/log.h>
 #include "backend/wayland.h"
 #include "util/signal.h"
@@ -72,8 +73,8 @@ static void output_transform(struct wlr_output *_output,
 }
 
 static bool output_set_cursor(struct wlr_output *_output,
-		const uint8_t *buf, int32_t stride, uint32_t width, uint32_t height,
-		int32_t hotspot_x, int32_t hotspot_y, bool update_pixels) {
+		struct wlr_texture *texture, int32_t hotspot_x, int32_t hotspot_y,
+		bool update_texture) {
 	struct wlr_wl_output *output =
 		(struct wlr_wl_output *)_output;
 	struct wlr_wl_backend *backend = output->backend;
@@ -82,76 +83,47 @@ static bool output_set_cursor(struct wlr_output *_output,
 	output->cursor.hotspot_x = hotspot_x;
 	output->cursor.hotspot_y = hotspot_y;
 
-	if (!update_pixels) {
+	if (!update_texture) {
 		// Update hotspot without changing cursor image
 		update_wl_output_cursor(output);
 		return true;
 	}
-	if (!buf) {
-		// Hide cursor
-		if (output->cursor.surface) {
-			wl_surface_destroy(output->cursor.surface);
-			munmap(output->cursor.data, output->cursor.buf_size);
-			output->cursor.surface = NULL;
-			output->cursor.buf_size = 0;
-		}
-		update_wl_output_cursor(output);
-		return true;
-	}
 
-	if (!backend->shm || !backend->pointer) {
-		wlr_log(L_INFO, "cannot set cursor: no wl_shm or wl_pointer");
-		return false;
-	}
-
-	if (!output->cursor.surface) {
+	if (output->cursor.surface == NULL) {
 		output->cursor.surface =
-			wl_compositor_create_surface(output->backend->compositor);
+			wl_compositor_create_surface(backend->compositor);
 	}
+	struct wl_surface *surface = output->cursor.surface;
 
-	uint32_t size = stride * height;
-	if (output->cursor.buf_size != size) {
-		if (output->cursor.buffer) {
-			wl_buffer_destroy(output->cursor.buffer);
-		}
+	if (texture != NULL) {
+		int width, height;
+		wlr_texture_get_size(texture, &width, &height);
 
-		if (size > output->cursor.buf_size) {
-			if (output->cursor.pool) {
-				wl_shm_pool_destroy(output->cursor.pool);
-				output->cursor.pool = NULL;
-				munmap(output->cursor.data, output->cursor.buf_size);
-			}
+		if (output->cursor.egl_window == NULL) {
+			output->cursor.egl_window =
+				wl_egl_window_create(surface, width, height);
 		}
+		wl_egl_window_resize(output->cursor.egl_window, width, height, 0, 0);
 
-		if (!output->cursor.pool) {
-			int fd = os_create_anonymous_file(size);
-			if (fd < 0) {
-				wlr_log_errno(L_INFO,
-					"creating anonymous file for cursor buffer failed");
-				return false;
-			}
-
-			output->cursor.data = mmap(NULL, size, PROT_READ | PROT_WRITE,
-				MAP_SHARED, fd, 0);
-			if (output->cursor.data == MAP_FAILED) {
-				close(fd);
-				wlr_log_errno(L_INFO, "mmap failed");
-				return false;
-			}
-
-			output->cursor.pool = wl_shm_create_pool(backend->shm, fd, size);
-			close(fd);
-		}
+		EGLSurface egl_surface =
+			wlr_egl_create_surface(&backend->egl, output->cursor.egl_window);
 
-		output->cursor.buffer = wl_shm_pool_create_buffer(output->cursor.pool,
-			0, width, height, stride, WL_SHM_FORMAT_ARGB8888);
-		output->cursor.buf_size = size;
-	}
+		wlr_egl_make_current(&backend->egl, egl_surface, NULL);
 
-	memcpy(output->cursor.data, buf, size);
-	wl_surface_attach(output->cursor.surface, output->cursor.buffer, 0, 0);
-	wl_surface_damage(output->cursor.surface, 0, 0, width, height);
-	wl_surface_commit(output->cursor.surface);
+		float matrix[9];
+		wlr_matrix_projection(matrix, width, height, WL_OUTPUT_TRANSFORM_NORMAL);
+
+		wlr_renderer_begin(backend->renderer, width, height);
+		wlr_renderer_clear(backend->renderer, (float[]){ 0.0, 0.0, 0.0, 0.0 });
+		wlr_render_texture(backend->renderer, texture, matrix, 0, 0, 1.0);
+		wlr_renderer_end(backend->renderer);
+
+		wlr_egl_swap_buffers(&backend->egl, egl_surface, NULL);
+		wlr_egl_destroy_surface(&backend->egl, egl_surface);
+	} else {
+		wl_surface_attach(surface, NULL, 0, 0);
+		wl_surface_commit(surface);
+	}
 
 	update_wl_output_cursor(output);
 	return true;
@@ -166,16 +138,9 @@ static void output_destroy(struct wlr_output *wlr_output) {
 
 	wl_list_remove(&output->link);
 
-	if (output->cursor.buf_size != 0) {
-		assert(output->cursor.data);
-		assert(output->cursor.buffer);
-		assert(output->cursor.pool);
-
-		wl_buffer_destroy(output->cursor.buffer);
-		munmap(output->cursor.data, output->cursor.buf_size);
-		wl_shm_pool_destroy(output->cursor.pool);
+	if (output->cursor.egl_window != NULL) {
+		wl_egl_window_destroy(output->cursor.egl_window);
 	}
-
 	if (output->cursor.surface) {
 		wl_surface_destroy(output->cursor.surface);
 	}
diff --git a/include/backend/wayland.h b/include/backend/wayland.h
index 38c7ee32..fec328ba 100644
--- a/include/backend/wayland.h
+++ b/include/backend/wayland.h
@@ -40,25 +40,22 @@ struct wlr_wl_output {
 	struct wlr_output wlr_output;
 
 	struct wlr_wl_backend *backend;
+	struct wl_list link;
+
 	struct wl_surface *surface;
+	struct wl_callback *frame_callback;
 	struct zxdg_surface_v6 *xdg_surface;
 	struct zxdg_toplevel_v6 *xdg_toplevel;
 	struct wl_egl_window *egl_window;
-	struct wl_callback *frame_callback;
+	EGLSurface egl_surface;
+
+	uint32_t enter_serial;
 
 	struct {
-		struct wl_shm_pool *pool;
-		void *buffer; // actually a (client-side) struct wl_buffer *
-		uint32_t buf_size;
-		uint8_t *data;
 		struct wl_surface *surface;
+		struct wl_egl_window *egl_window;
 		int32_t hotspot_x, hotspot_y;
 	} cursor;
-
-	uint32_t enter_serial;
-
-	void *egl_surface;
-	struct wl_list link;
 };
 
 struct wlr_wl_input_device {
diff --git a/include/wlr/interfaces/wlr_output.h b/include/wlr/interfaces/wlr_output.h
index c9ddf615..d8c54067 100644
--- a/include/wlr/interfaces/wlr_output.h
+++ b/include/wlr/interfaces/wlr_output.h
@@ -13,9 +13,8 @@ struct wlr_output_impl {
 		int32_t height, int32_t refresh);
 	void (*transform)(struct wlr_output *output,
 		enum wl_output_transform transform);
-	bool (*set_cursor)(struct wlr_output *output, const uint8_t *buf,
-		int32_t stride, uint32_t width, uint32_t height,
-		int32_t hotspot_x, int32_t hotspot_y, bool update_pixels);
+	bool (*set_cursor)(struct wlr_output *output, struct wlr_texture *texture,
+		int32_t hotspot_x, int32_t hotspot_y, bool update_texture);
 	bool (*move_cursor)(struct wlr_output *output, int x, int y);
 	void (*destroy)(struct wlr_output *output);
 	bool (*make_current)(struct wlr_output *output, int *buffer_age);
diff --git a/types/wlr_output.c b/types/wlr_output.c
index a8361ead..4ba7c9db 100644
--- a/types/wlr_output.c
+++ b/types/wlr_output.c
@@ -698,6 +698,27 @@ static void output_cursor_update_visible(struct wlr_output_cursor *cursor) {
 	cursor->visible = visible;
 }
 
+static bool output_cursor_attempt_hardware(struct wlr_output_cursor *cursor) {
+	struct wlr_texture *texture = cursor->texture;
+	if (cursor->surface != NULL) {
+		texture = cursor->surface->texture;
+	}
+
+	struct wlr_output_cursor *hwcur = cursor->output->hardware_cursor;
+	if (cursor->output->impl->set_cursor && (hwcur == NULL || hwcur == cursor)) {
+		if (cursor->output->impl->move_cursor && hwcur != cursor) {
+			cursor->output->impl->move_cursor(cursor->output,
+				(int)cursor->x, (int)cursor->y);
+		}
+		if (cursor->output->impl->set_cursor(cursor->output, texture,
+				cursor->hotspot_x, cursor->hotspot_y, true)) {
+			cursor->output->hardware_cursor = cursor;
+			return true;
+		}
+	}
+	return false;
+}
+
 bool wlr_output_cursor_set_image(struct wlr_output_cursor *cursor,
 		const uint8_t *pixels, int32_t stride, uint32_t width, uint32_t height,
 		int32_t hotspot_x, int32_t hotspot_y) {
@@ -713,33 +734,26 @@ bool wlr_output_cursor_set_image(struct wlr_output_cursor *cursor,
 	cursor->hotspot_y = hotspot_y;
 	output_cursor_update_visible(cursor);
 
-	struct wlr_output_cursor *hwcur = cursor->output->hardware_cursor;
-	if (cursor->output->impl->set_cursor && (hwcur == NULL || hwcur == cursor)) {
-		if (cursor->output->impl->move_cursor && hwcur != cursor) {
-			cursor->output->impl->move_cursor(cursor->output,
-				(int)cursor->x, (int)cursor->y);
-		}
-		int ok = cursor->output->impl->set_cursor(cursor->output, pixels,
-			stride, width, height, hotspot_x, hotspot_y, true);
-		if (ok) {
-			cursor->output->hardware_cursor = cursor;
-			return true;
+	wlr_texture_destroy(cursor->texture);
+	cursor->texture = NULL;
+
+	cursor->enabled = false;
+	if (pixels != NULL) {
+		cursor->texture = wlr_texture_from_pixels(renderer,
+			WL_SHM_FORMAT_ARGB8888, stride, width, height, pixels);
+		if (cursor->texture == NULL) {
+			return false;
 		}
+		cursor->enabled = true;
 	}
 
-	wlr_log(L_DEBUG, "Falling back to software cursor");
-	output_cursor_damage_whole(cursor);
-
-	cursor->enabled = pixels != NULL;
-	if (!cursor->enabled) {
+	if (output_cursor_attempt_hardware(cursor)) {
 		return true;
 	}
 
-	wlr_texture_destroy(cursor->texture);
-
-	cursor->texture = wlr_texture_from_pixels(renderer, WL_SHM_FORMAT_ARGB8888,
-		stride, width, height, pixels);
-	return cursor->texture != NULL;
+	wlr_log(L_DEBUG, "Falling back to software cursor");
+	output_cursor_damage_whole(cursor);
+	return true;
 }
 
 static void output_cursor_commit(struct wlr_output_cursor *cursor) {
@@ -752,15 +766,15 @@ static void output_cursor_commit(struct wlr_output_cursor *cursor) {
 	cursor->width = cursor->surface->current->width * cursor->output->scale;
 	cursor->height = cursor->surface->current->height * cursor->output->scale;
 
-	if (cursor->output->hardware_cursor != cursor) {
-		output_cursor_damage_whole(cursor);
-	} else {
-		// TODO: upload pixels
-
+	if (output_cursor_attempt_hardware(cursor)) {
 		struct timespec now;
 		clock_gettime(CLOCK_MONOTONIC, &now);
 		wlr_surface_send_frame_done(cursor->surface, &now);
+		return;
 	}
+
+	// Fallback to software cursor
+	output_cursor_damage_whole(cursor);
 }
 
 static void output_cursor_handle_commit(struct wl_listener *listener,
@@ -800,7 +814,7 @@ void wlr_output_cursor_set_surface(struct wlr_output_cursor *cursor,
 
 		if (cursor->output->hardware_cursor == cursor &&
 				cursor->output->impl->set_cursor) {
-			cursor->output->impl->set_cursor(cursor->output, NULL, 0, 0, 0,
+			cursor->output->impl->set_cursor(cursor->output, NULL,
 				hotspot_x, hotspot_y, false);
 		}
 		return;
@@ -808,15 +822,6 @@ void wlr_output_cursor_set_surface(struct wlr_output_cursor *cursor,
 
 	output_cursor_reset(cursor);
 
-	// Disable hardware cursor for surfaces
-	// TODO: support hardware cursors
-	if (cursor->output->hardware_cursor == cursor &&
-			cursor->output->impl->set_cursor) {
-		cursor->output->impl->set_cursor(cursor->output, NULL, 0, 0, 0, 0, 0,
-			true);
-		cursor->output->hardware_cursor = NULL;
-	}
-
 	cursor->surface = surface;
 	cursor->hotspot_x = hotspot_x;
 	cursor->hotspot_y = hotspot_y;
@@ -833,7 +838,10 @@ void wlr_output_cursor_set_surface(struct wlr_output_cursor *cursor,
 		cursor->width = 0;
 		cursor->height = 0;
 
-		// TODO: if hardware cursor, disable cursor
+		if (cursor->output->hardware_cursor == cursor &&
+				cursor->output->impl->set_cursor) {
+			cursor->output->impl->set_cursor(cursor->output, NULL, 0, 0, false);
+		}
 	}
 }
 
@@ -896,8 +904,7 @@ void wlr_output_cursor_destroy(struct wlr_output_cursor *cursor) {
 	if (cursor->output->hardware_cursor == cursor) {
 		// If this cursor was the hardware cursor, disable it
 		if (cursor->output->impl->set_cursor) {
-			cursor->output->impl->set_cursor(cursor->output, NULL, 0, 0, 0, 0,
-				0, true);
+			cursor->output->impl->set_cursor(cursor->output, NULL, 0, 0, true);
 		}
 		cursor->output->hardware_cursor = NULL;
 	}