Add wlr_output_layer

This is based on previous work [1] [2].

This new API allows compositors to display buffers without needing to
perform rendering operations. This API can be implemented on Wayland
using subsurfaces and on DRM using KMS planes.

Compared to [1], this approach leverages wlr_addon_set to let backends
attach their own private state to layers, removes the pending
state (necessary for interop with wlr_output_commit_state()) and
enum wlr_output_layer_state_field.

[1]: https://gitlab.freedesktop.org/wlroots/wlroots/-/merge_requests/1985
[2]: https://gitlab.freedesktop.org/wlroots/wlroots/-/merge_requests/3447

6 files changed, 124 insertions, 1 deletions

diff --git a/include/wlr/interfaces/wlr_output.h b/include/wlr/interfaces/wlr_output.h
index 073ed023..6c0bbd6c 100644
--- a/include/wlr/interfaces/wlr_output.h
+++ b/include/wlr/interfaces/wlr_output.h
@@ -22,7 +22,8 @@
 	WLR_OUTPUT_STATE_SCALE | \
 	WLR_OUTPUT_STATE_TRANSFORM | \
 	WLR_OUTPUT_STATE_RENDER_FORMAT | \
-	WLR_OUTPUT_STATE_SUBPIXEL)
+	WLR_OUTPUT_STATE_SUBPIXEL | \
+	WLR_OUTPUT_STATE_LAYERS)
 
 /**
  * A backend implementation of struct wlr_output.
diff --git a/include/wlr/types/wlr_output.h b/include/wlr/types/wlr_output.h
index 45a93d47..5731eb91 100644
--- a/include/wlr/types/wlr_output.h
+++ b/include/wlr/types/wlr_output.h
@@ -67,6 +67,7 @@ enum wlr_output_state_field {
 	WLR_OUTPUT_STATE_GAMMA_LUT = 1 << 7,
 	WLR_OUTPUT_STATE_RENDER_FORMAT = 1 << 8,
 	WLR_OUTPUT_STATE_SUBPIXEL = 1 << 9,
+	WLR_OUTPUT_STATE_LAYERS = 1 << 10,
 };
 
 enum wlr_output_state_mode_type {
@@ -104,6 +105,10 @@ struct wlr_output_state {
 	// only valid if WLR_OUTPUT_STATE_GAMMA_LUT
 	uint16_t *gamma_lut;
 	size_t gamma_lut_size;
+
+	// only valid if WLR_OUTPUT_STATE_LAYERS
+	struct wlr_output_layer_state *layers;
+	size_t layers_len;
 };
 
 struct wlr_output_impl;
@@ -191,6 +196,8 @@ struct wlr_output {
 	struct wlr_buffer *cursor_front_buffer;
 	int software_cursor_locks; // number of locks forcing software cursors
 
+	struct wl_list layers; // wlr_output_layer.link
+
 	struct wlr_allocator *allocator;
 	struct wlr_renderer *renderer;
 	struct wlr_swapchain *swapchain;
diff --git a/include/wlr/types/wlr_output_layer.h b/include/wlr/types/wlr_output_layer.h
new file mode 100644
index 00000000..92980108
--- /dev/null
+++ b/include/wlr/types/wlr_output_layer.h
@@ -0,0 +1,68 @@
+/*
+ * This an unstable interface of wlroots. No guarantees are made regarding the
+ * future consistency of this API.
+ */
+#ifndef WLR_USE_UNSTABLE
+#error "Add -DWLR_USE_UNSTABLE to enable unstable wlroots features"
+#endif
+
+#ifndef WLR_TYPES_WLR_OUTPUT_LAYER_H
+#define WLR_TYPES_WLR_OUTPUT_LAYER_H
+
+#include <wlr/types/wlr_output.h>
+#include <wlr/util/addon.h>
+
+/**
+ * An output layer.
+ *
+ * Output layers are displayed between the output primary buffer (see
+ * wlr_output_attach_buffer() and wlr_output_attach_render()) and the cursor
+ * buffer. They can offload some rendering work to the backend.
+ *
+ * To configure output layers, callers should call wlr_output_layer_create() to
+ * create layers, attach struct wlr_output_layer_state onto
+ * struct wlr_output_state to describe their new state, and commit the output.
+ *
+ * Backends may have arbitrary limitations when it comes to displaying output
+ * layers. Backends indicate whether or not a layer can be displayed via
+ * wlr_output_layer_state.accepted after wlr_output_test() or
+ * wlr_output_commit() is called. Compositors using the output layers API
+ * directly are expected to setup layers, call wlr_output_test(), paint the
+ * layers that the backend rejected with the renderer, then call
+ * wlr_output_commit().
+ */
+struct wlr_output_layer {
+	struct wl_list link; // wlr_output.layers
+	struct wlr_addon_set addons;
+
+	void *data;
+};
+
+/**
+ * State for an output layer.
+ */
+struct wlr_output_layer_state {
+	struct wlr_output_layer *layer;
+
+	// Buffer to display, or NULL to disable the layer
+	struct wlr_buffer *buffer;
+	// Position in output-buffer-local coordinates
+	int x, y;
+
+	// Populated by the backend after wlr_output_test() and wlr_output_commit(),
+	// indicates whether the backend has acknowledged and will take care of
+	// displaying the layer
+	bool accepted;
+};
+
+/**
+ * Create a new output layer.
+ */
+struct wlr_output_layer *wlr_output_layer_create(struct wlr_output *output);
+
+/**
+ * Destroy an output layer.
+ */
+void wlr_output_layer_destroy(struct wlr_output_layer *layer);
+
+#endif
diff --git a/types/meson.build b/types/meson.build
index bcf1073b..eb8aa158 100644
--- a/types/meson.build
+++ b/types/meson.build
@@ -56,6 +56,7 @@ wlr_files += files(
 	'wlr_linux_dmabuf_v1.c',
 	'wlr_matrix.c',
 	'wlr_output_damage.c',
+	'wlr_output_layer.c',
 	'wlr_output_layout.c',
 	'wlr_output_management_v1.c',
 	'wlr_output_power_management_v1.c',
diff --git a/types/output/output.c b/types/output/output.c
index dc2335cd..c0d5719e 100644
--- a/types/output/output.c
+++ b/types/output/output.c
@@ -6,6 +6,7 @@
 #include <wlr/interfaces/wlr_output.h>
 #include <wlr/types/wlr_compositor.h>
 #include <wlr/types/wlr_matrix.h>
+#include <wlr/types/wlr_output_layer.h>
 #include <wlr/util/log.h>
 #include "render/allocator/allocator.h"
 #include "render/swapchain.h"
@@ -356,6 +357,7 @@ void wlr_output_init(struct wlr_output *output, struct wlr_backend *backend,
 	output->scale = 1;
 	output->commit_seq = 0;
 	wl_list_init(&output->cursors);
+	wl_list_init(&output->layers);
 	wl_list_init(&output->resources);
 	wl_signal_init(&output->events.frame);
 	wl_signal_init(&output->events.damage);
@@ -399,6 +401,11 @@ void wlr_output_destroy(struct wlr_output *output) {
 		wlr_output_cursor_destroy(cursor);
 	}
 
+	struct wlr_output_layer *layer, *tmp_layer;
+	wl_list_for_each_safe(layer, tmp_layer, &output->layers, link) {
+		wlr_output_layer_destroy(layer);
+	}
+
 	wlr_swapchain_destroy(output->cursor_swapchain);
 	wlr_buffer_unlock(output->cursor_front_buffer);
 
@@ -665,6 +672,12 @@ static bool output_basic_test(struct wlr_output *output,
 		return false;
 	}
 
+	if (state->committed & WLR_OUTPUT_STATE_LAYERS) {
+		for (size_t i = 0; i < state->layers_len; i++) {
+			state->layers[i].accepted = false;
+		}
+	}
+
 	return true;
 }
 
@@ -823,6 +836,15 @@ bool wlr_output_commit_state(struct wlr_output *output,
 		output->needs_frame = false;
 	}
 
+	if (pending.committed & WLR_OUTPUT_STATE_LAYERS) {
+		// Commit layer ordering
+		for (size_t i = 0; i < pending.layers_len; i++) {
+			struct wlr_output_layer *layer = pending.layers[i].layer;
+			wl_list_remove(&layer->link);
+			wl_list_insert(output->layers.prev, &layer->link);
+		}
+	}
+
 	if ((pending.committed & WLR_OUTPUT_STATE_BUFFER) &&
 			output->swapchain != NULL) {
 		wlr_swapchain_set_buffer_submitted(output->swapchain, pending.buffer);
diff --git a/types/wlr_output_layer.c b/types/wlr_output_layer.c
new file mode 100644
index 00000000..f50e6854
--- /dev/null
+++ b/types/wlr_output_layer.c
@@ -0,0 +1,24 @@
+#include <stdlib.h>
+#include <wlr/types/wlr_output_layer.h>
+
+struct wlr_output_layer *wlr_output_layer_create(struct wlr_output *output) {
+	struct wlr_output_layer *layer = calloc(1, sizeof(*layer));
+	if (layer == NULL) {
+		return NULL;
+	}
+
+	wl_list_insert(&output->layers, &layer->link);
+	wlr_addon_set_init(&layer->addons);
+
+	return layer;
+}
+
+void wlr_output_layer_destroy(struct wlr_output_layer *layer) {
+	if (layer == NULL) {
+		return;
+	}
+
+	wlr_addon_set_finish(&layer->addons);
+	wl_list_remove(&layer->link);
+	free(layer);
+}