Merge pull request #1245 from emersion/capture-disable-hw-cursors

Disable hardware cursors when capturing output

7 files changed, 90 insertions, 14 deletions

diff --git a/examples/dmabuf-capture.c b/examples/dmabuf-capture.c
index de3567f4..2c510b5a 100644
--- a/examples/dmabuf-capture.c
+++ b/examples/dmabuf-capture.c
@@ -45,6 +45,7 @@ struct capture_context {
 
 	/* Target */
 	struct wl_output *target_output;
+	bool with_cursor;
 
 	/* Main frame callback */
 	struct zwlr_export_dmabuf_frame_v1 *frame_callback;
@@ -454,7 +455,7 @@ static const struct zwlr_export_dmabuf_frame_v1_listener frame_listener = {
 
 static void register_cb(struct capture_context *ctx) {
 	ctx->frame_callback = zwlr_export_dmabuf_manager_v1_capture_output(
-			ctx->export_manager, 0, ctx->target_output);
+			ctx->export_manager, ctx->with_cursor, ctx->target_output);
 
 	zwlr_export_dmabuf_frame_v1_add_listener(ctx->frame_callback,
 			&frame_listener, ctx);
@@ -802,6 +803,7 @@ int main(int argc, char *argv[]) {
 	}
 
 	ctx.target_output = o->output;
+	ctx.with_cursor = true;
 	ctx.hw_device_type = av_hwdevice_find_type_by_name(argv[2]);
 	ctx.hardware_device = argv[3];
 
diff --git a/include/wlr/types/wlr_export_dmabuf_v1.h b/include/wlr/types/wlr_export_dmabuf_v1.h
index 56767540..8669574d 100644
--- a/include/wlr/types/wlr_export_dmabuf_v1.h
+++ b/include/wlr/types/wlr_export_dmabuf_v1.h
@@ -9,6 +9,7 @@
 #ifndef WLR_TYPES_WLR_EXPORT_DMABUF_V1_H
 #define WLR_TYPES_WLR_EXPORT_DMABUF_V1_H
 
+#include <stdbool.h>
 #include <wayland-server.h>
 #include <wlr/render/dmabuf.h>
 
@@ -22,6 +23,8 @@ struct wlr_export_dmabuf_frame_v1 {
 	struct wlr_dmabuf_attributes attribs;
 	struct wlr_output *output;
 
+	bool cursor_locked;
+
 	struct wl_listener output_swap_buffers;
 };
 
diff --git a/include/wlr/types/wlr_output.h b/include/wlr/types/wlr_output.h
index ded57959..ecd4f759 100644
--- a/include/wlr/types/wlr_output.h
+++ b/include/wlr/types/wlr_output.h
@@ -107,6 +107,7 @@ struct wlr_output {
 
 	struct wl_list cursors; // wlr_output_cursor::link
 	struct wlr_output_cursor *hardware_cursor;
+	int software_cursor_locks; // number of locks forcing software cursors
 
 	// the output position in layout space reported to clients
 	int32_t lx, ly;
@@ -196,6 +197,14 @@ bool wlr_output_export_dmabuf(struct wlr_output *output,
 void wlr_output_set_fullscreen_surface(struct wlr_output *output,
 	struct wlr_surface *surface);
 struct wlr_output *wlr_output_from_resource(struct wl_resource *resource);
+/**
+ * Locks the output to only use software cursors instead of hardware cursors.
+ * This is useful if hardware cursors need to be temporarily disabled (e.g.
+ * during screen capture). There must be as many unlocks as there have been
+ * locks to restore the original state. There should never be an unlock before
+ * a lock.
+ */
+void wlr_output_lock_software_cursors(struct wlr_output *output, bool lock);
 
 
 struct wlr_output_cursor *wlr_output_cursor_create(struct wlr_output *output);
diff --git a/include/wlr/types/wlr_screencopy_v1.h b/include/wlr/types/wlr_screencopy_v1.h
index aba32a45..c7197bab 100644
--- a/include/wlr/types/wlr_screencopy_v1.h
+++ b/include/wlr/types/wlr_screencopy_v1.h
@@ -9,6 +9,7 @@
 #ifndef WLR_TYPES_WLR_SCREENCOPY_V1_H
 #define WLR_TYPES_WLR_SCREENCOPY_V1_H
 
+#include <stdbool.h>
 #include <wayland-server.h>
 #include <wlr/types/wlr_box.h>
 
@@ -35,6 +36,8 @@ struct wlr_screencopy_frame_v1 {
 	struct wlr_box box;
 	int stride;
 
+	bool overlay_cursor, cursor_locked;
+
 	struct wl_shm_buffer *buffer;
 	struct wl_listener buffer_destroy;
 
diff --git a/types/wlr_export_dmabuf_v1.c b/types/wlr_export_dmabuf_v1.c
index 72c9bcec..85ccf1eb 100644
--- a/types/wlr_export_dmabuf_v1.c
+++ b/types/wlr_export_dmabuf_v1.c
@@ -29,14 +29,26 @@ static const struct zwlr_export_dmabuf_frame_v1_interface frame_impl = {
 	.destroy = frame_handle_destroy,
 };
 
-static void frame_handle_resource_destroy(struct wl_resource *resource) {
-	struct wlr_export_dmabuf_frame_v1 *frame = frame_from_resource(resource);
+static void frame_destroy(struct wlr_export_dmabuf_frame_v1 *frame) {
+	if (frame == NULL) {
+		return;
+	}
+	if (frame->cursor_locked) {
+		wlr_output_lock_software_cursors(frame->output, false);
+	}
 	wl_list_remove(&frame->link);
 	wl_list_remove(&frame->output_swap_buffers.link);
 	wlr_dmabuf_attributes_finish(&frame->attribs);
+	// Make the frame resource inert
+	wl_resource_set_user_data(frame->resource, NULL);
 	free(frame);
 }
 
+static void frame_handle_resource_destroy(struct wl_resource *resource) {
+	struct wlr_export_dmabuf_frame_v1 *frame = frame_from_resource(resource);
+	frame_destroy(frame);
+}
+
 static void frame_output_handle_swap_buffers(struct wl_listener *listener,
 		void *data) {
 	struct wlr_export_dmabuf_frame_v1 *frame =
@@ -51,6 +63,7 @@ static void frame_output_handle_swap_buffers(struct wl_listener *listener,
 	uint32_t tv_sec_lo = tv_sec & 0xFFFFFFFF;
 	zwlr_export_dmabuf_frame_v1_send_ready(frame->resource,
 		tv_sec_hi, tv_sec_lo, event->when->tv_nsec);
+	frame_destroy(frame);
 }
 
 
@@ -96,6 +109,7 @@ static void manager_handle_capture_output(struct wl_client *client,
 	if (!output->impl->export_dmabuf) {
 		zwlr_export_dmabuf_frame_v1_send_cancel(frame->resource,
 			ZWLR_EXPORT_DMABUF_FRAME_V1_CANCEL_REASON_PERMANENT);
+		frame_destroy(frame);
 		return;
 	}
 
@@ -103,9 +117,15 @@ static void manager_handle_capture_output(struct wl_client *client,
 	if (!wlr_output_export_dmabuf(output, attribs)) {
 		zwlr_export_dmabuf_frame_v1_send_cancel(frame->resource,
 			ZWLR_EXPORT_DMABUF_FRAME_V1_CANCEL_REASON_TEMPORARY);
+		frame_destroy(frame);
 		return;
 	}
 
+	if (overlay_cursor) {
+		wlr_output_lock_software_cursors(frame->output, true);
+		frame->cursor_locked = true;
+	}
+
 	uint32_t frame_flags = ZWLR_EXPORT_DMABUF_FRAME_V1_FLAGS_TRANSIENT;
 	uint32_t mod_high = attribs->modifier >> 32;
 	uint32_t mod_low = attribs->modifier & 0xFFFFFFFF;
diff --git a/types/wlr_output.c b/types/wlr_output.c
index 4e564534..62fd9fb7 100644
--- a/types/wlr_output.c
+++ b/types/wlr_output.c
@@ -273,6 +273,13 @@ void wlr_output_init(struct wlr_output *output, struct wlr_backend *backend,
 	wl_signal_init(&output->events.destroy);
 	pixman_region32_init(&output->damage);
 
+	const char *no_hardware_cursors = getenv("WLR_NO_HARDWARE_CURSORS");
+	if (no_hardware_cursors != NULL && strcmp(no_hardware_cursors, "1") == 0) {
+		wlr_log(WLR_DEBUG,
+			"WLR_NO_HARDWARE_CURSORS set, forcing software cursors");
+		output->software_cursor_locks = 1;
+	}
+
 	output->display_destroy.notify = handle_display_destroy;
 	wl_display_add_destroy_listener(display, &output->display_destroy);
 
@@ -469,13 +476,6 @@ bool wlr_output_swap_buffers(struct wlr_output *output, struct timespec *when,
 		output->idle_frame = NULL;
 	}
 
-	struct wlr_output_event_swap_buffers event = {
-		.output = output,
-		.when = when,
-		.damage = damage,
-	};
-	wlr_signal_emit_safe(&output->events.swap_buffers, &event);
-
 	int width, height;
 	wlr_output_transformed_resolution(output, &width, &height);
 
@@ -510,6 +510,13 @@ bool wlr_output_swap_buffers(struct wlr_output *output, struct timespec *when,
 		}
 	}
 
+	struct wlr_output_event_swap_buffers event = {
+		.output = output,
+		.when = when,
+		.damage = damage,
+	};
+	wlr_signal_emit_safe(&output->events.swap_buffers, &event);
+
 	// Transform damage into renderer coordinates, ie. upside down
 	enum wl_output_transform transform = wlr_output_transform_compose(
 		wlr_output_transform_invert(output->transform),
@@ -666,6 +673,32 @@ struct wlr_output *wlr_output_from_resource(struct wl_resource *resource) {
 	return wl_resource_get_user_data(resource);
 }
 
+static void output_cursor_damage_whole(struct wlr_output_cursor *cursor);
+
+void wlr_output_lock_software_cursors(struct wlr_output *output, bool lock) {
+	if (lock) {
+		++output->software_cursor_locks;
+	} else {
+		assert(output->software_cursor_locks > 0);
+		--output->software_cursor_locks;
+	}
+	wlr_log(WLR_DEBUG, "%s hardware cursors on output '%s' (locks: %d)",
+		lock ? "Disabling" : "Enabling", output->name,
+		output->software_cursor_locks);
+
+	if (output->software_cursor_locks > 0 && output->hardware_cursor != NULL) {
+		assert(output->impl->set_cursor);
+		output->impl->set_cursor(output, NULL, 1,
+			WL_OUTPUT_TRANSFORM_NORMAL, 0, 0, true);
+		output_cursor_damage_whole(output->hardware_cursor);
+		output->hardware_cursor = NULL;
+	}
+
+	// If it's possible to use hardware cursors again, don't switch immediately
+	// since a recorder is likely to lock software cursors for the next frame
+	// again.
+}
+
 
 static void output_cursor_damage_whole(struct wlr_output_cursor *cursor) {
 	struct wlr_box box;
@@ -721,10 +754,7 @@ static bool output_cursor_attempt_hardware(struct wlr_output_cursor *cursor) {
 		transform = cursor->surface->current.transform;
 	}
 
-	const char *no_hardware_cursors = getenv("WLR_NO_HARDWARE_CURSORS");
-	if (no_hardware_cursors != NULL && strcmp(no_hardware_cursors, "1") == 0) {
-		wlr_log(WLR_DEBUG,
-			"WLR_NO_HARDWARE_CURSORS set, forcing software cursors");
+	if (cursor->output->software_cursor_locks > 0) {
 		return false;
 	}
 
diff --git a/types/wlr_screencopy_v1.c b/types/wlr_screencopy_v1.c
index 4f044eba..0819e48d 100644
--- a/types/wlr_screencopy_v1.c
+++ b/types/wlr_screencopy_v1.c
@@ -22,6 +22,9 @@ static void frame_destroy(struct wlr_screencopy_frame_v1 *frame) {
 	if (frame == NULL) {
 		return;
 	}
+	if (frame->cursor_locked) {
+		wlr_output_lock_software_cursors(frame->output, false);
+	}
 	wl_list_remove(&frame->link);
 	wl_list_remove(&frame->output_swap_buffers.link);
 	wl_list_remove(&frame->buffer_destroy.link);
@@ -133,6 +136,11 @@ static void frame_handle_copy(struct wl_client *client,
 	// Schedule a buffer swap
 	output->needs_swap = true;
 	wlr_output_schedule_frame(output);
+
+	if (frame->overlay_cursor) {
+		wlr_output_lock_software_cursors(output, true);
+		frame->cursor_locked = true;
+	}
 }
 
 static void frame_handle_destroy(struct wl_client *client,
@@ -189,6 +197,7 @@ static void capture_output(struct wl_client *client,
 	}
 	frame->manager = manager;
 	frame->output = output;
+	frame->overlay_cursor = !!overlay_cursor;
 
 	frame->resource = wl_resource_create(client,
 		&zwlr_screencopy_frame_v1_interface, version, id);