output: re-introduce atomic mode, enabled, scale and transform

This reverts commit 01f903874b7e27539488fad7f31476d5bcbc6ac9 and re-applies
commit ee5f98ad49fed0439f3313ec685307831d1d1d05.

Updates: https://github.com/swaywm/wlroots/issues/1640 (Atomic output updates issue)
See also: https://github.com/swaywm/wlroots/pull/1762 (Atomic output updates original PR)
See also: https://github.com/swaywm/wlroots/issues/1780 (Issue caused by atomic output updates)
See also: https://github.com/swaywm/sway/issues/4419 (Issue caused by atomic output updates)
See also: https://github.com/swaywm/wlroots/pull/1781 (Revert PR)

6 files changed, 203 insertions, 89 deletions

diff --git a/backend/drm/drm.c b/backend/drm/drm.c
index a4f22590..60c9e44c 100644
--- a/backend/drm/drm.c
+++ b/backend/drm/drm.c
@@ -334,12 +334,9 @@ static bool drm_connector_attach_render(struct wlr_output *output,
 	return make_drm_surface_current(&conn->crtc->primary->surf, buffer_age);
 }
 
-static bool drm_connector_commit(struct wlr_output *output) {
+static bool drm_connector_commit_buffer(struct wlr_output *output) {
 	struct wlr_drm_connector *conn = get_drm_connector_from_output(output);
 	struct wlr_drm_backend *drm = get_drm_backend_from_backend(output->backend);
-	if (!drm->session->active) {
-		return false;
-	}
 
 	struct wlr_drm_crtc *crtc = conn->crtc;
 	if (!crtc) {
@@ -415,6 +412,49 @@ static bool drm_connector_commit(struct wlr_output *output) {
 	return true;
 }
 
+static bool drm_connector_set_custom_mode(struct wlr_output *output,
+	int32_t width, int32_t height, int32_t refresh);
+
+static bool drm_connector_commit(struct wlr_output *output) {
+	struct wlr_drm_backend *drm = get_drm_backend_from_backend(output->backend);
+
+	if (!drm->session->active) {
+		return false;
+	}
+
+	if (output->pending.committed & WLR_OUTPUT_STATE_ENABLED) {
+		if (!enable_drm_connector(output, output->pending.enabled)) {
+			return false;
+		}
+	}
+
+	if (output->pending.committed & WLR_OUTPUT_STATE_MODE) {
+		switch (output->pending.mode_type) {
+		case WLR_OUTPUT_STATE_MODE_FIXED:
+			if (!drm_connector_set_mode(output, output->pending.mode)) {
+				return false;
+			}
+			break;
+		case WLR_OUTPUT_STATE_MODE_CUSTOM:
+			if (!drm_connector_set_custom_mode(output,
+					output->pending.custom_mode.width,
+					output->pending.custom_mode.height,
+					output->pending.custom_mode.refresh)) {
+				return false;
+			}
+			break;
+		}
+	}
+
+	if (output->pending.committed & WLR_OUTPUT_STATE_BUFFER) {
+		if (!drm_connector_commit_buffer(output)) {
+			return false;
+		}
+	}
+
+	return true;
+}
+
 static void fill_empty_gamma_table(size_t size,
 		uint16_t *r, uint16_t *g, uint16_t *b) {
 	for (uint32_t i = 0; i < size; ++i) {
@@ -1002,9 +1042,6 @@ static void drm_connector_destroy(struct wlr_output *output) {
 }
 
 static const struct wlr_output_impl output_impl = {
-	.enable = enable_drm_connector,
-	.set_mode = drm_connector_set_mode,
-	.set_custom_mode = drm_connector_set_custom_mode,
 	.set_cursor = drm_connector_set_cursor,
 	.move_cursor = drm_connector_move_cursor,
 	.destroy = drm_connector_destroy,
diff --git a/backend/headless/output.c b/backend/headless/output.c
index add00f76..a2836159 100644
--- a/backend/headless/output.c
+++ b/backend/headless/output.c
@@ -58,8 +58,26 @@ static bool output_attach_render(struct wlr_output *wlr_output,
 }
 
 static bool output_commit(struct wlr_output *wlr_output) {
-	// Nothing needs to be done for pbuffers
-	wlr_output_send_present(wlr_output, NULL);
+	if (wlr_output->pending.committed & WLR_OUTPUT_STATE_ENABLED) {
+		wlr_log(WLR_DEBUG, "Cannot disable a headless output");
+		return false;
+	}
+
+	if (wlr_output->pending.committed & WLR_OUTPUT_STATE_MODE) {
+		assert(wlr_output->pending.mode_type == WLR_OUTPUT_STATE_MODE_CUSTOM);
+		if (!output_set_custom_mode(wlr_output,
+				wlr_output->pending.custom_mode.width,
+				wlr_output->pending.custom_mode.height,
+				wlr_output->pending.custom_mode.refresh)) {
+			return false;
+		}
+	}
+
+	if (wlr_output->pending.committed & WLR_OUTPUT_STATE_BUFFER) {
+		// Nothing needs to be done for pbuffers
+		wlr_output_send_present(wlr_output, NULL);
+	}
+
 	return true;
 }
 
@@ -76,7 +94,6 @@ static void output_destroy(struct wlr_output *wlr_output) {
 }
 
 static const struct wlr_output_impl output_impl = {
-	.set_custom_mode = output_set_custom_mode,
 	.destroy = output_destroy,
 	.attach_render = output_attach_render,
 	.commit = output_commit,
diff --git a/backend/noop/output.c b/backend/noop/output.c
index 1022ac6f..15c84941 100644
--- a/backend/noop/output.c
+++ b/backend/noop/output.c
@@ -12,19 +12,30 @@ static struct wlr_noop_output *noop_output_from_output(
 	return (struct wlr_noop_output *)wlr_output;
 }
 
-static bool output_set_custom_mode(struct wlr_output *wlr_output,
-		int32_t width, int32_t height, int32_t refresh) {
-	wlr_output_update_custom_mode(wlr_output, width, height, refresh);
-	return true;
-}
-
 static bool output_attach_render(struct wlr_output *wlr_output,
 		int *buffer_age) {
 	return false;
 }
 
 static bool output_commit(struct wlr_output *wlr_output) {
-	return false;
+	if (wlr_output->pending.committed & WLR_OUTPUT_STATE_ENABLED) {
+		wlr_log(WLR_DEBUG, "Cannot disable a noop output");
+		return false;
+	}
+
+	if (wlr_output->pending.committed & WLR_OUTPUT_STATE_MODE) {
+		assert(wlr_output->pending.mode_type == WLR_OUTPUT_STATE_MODE_CUSTOM);
+		wlr_output_update_custom_mode(wlr_output,
+			wlr_output->pending.custom_mode.width,
+			wlr_output->pending.custom_mode.height,
+			wlr_output->pending.custom_mode.refresh);
+	}
+
+	if (wlr_output->pending.committed & WLR_OUTPUT_STATE_BUFFER) {
+		return false;
+	}
+
+	return true;
 }
 
 static void output_destroy(struct wlr_output *wlr_output) {
@@ -37,7 +48,6 @@ static void output_destroy(struct wlr_output *wlr_output) {
 }
 
 static const struct wlr_output_impl output_impl = {
-	.set_custom_mode = output_set_custom_mode,
 	.destroy = output_destroy,
 	.attach_render = output_attach_render,
 	.commit = output_commit,
diff --git a/backend/rdp/output.c b/backend/rdp/output.c
index 302a99de..3043d2c8 100644
--- a/backend/rdp/output.c
+++ b/backend/rdp/output.c
@@ -166,9 +166,9 @@ static bool nsc_swap_buffers(
 	return true;
 }
 
-static bool output_commit(struct wlr_output *wlr_output) {
-	struct wlr_rdp_output *output =
-		rdp_output_from_output(wlr_output);
+static bool output_commit_buffer(struct wlr_rdp_output *output) {
+	struct wlr_output *wlr_output = &output->wlr_output;
+
 	bool ret = false;
 
 	pixman_region32_t output_region;
@@ -220,6 +220,33 @@ out:
 	return ret;
 }
 
+static bool output_commit(struct wlr_output *wlr_output) {
+	struct wlr_rdp_output *output = rdp_output_from_output(wlr_output);
+
+	if (wlr_output->pending.committed & WLR_OUTPUT_STATE_ENABLED) {
+		wlr_log(WLR_DEBUG, "Cannot disable an RDP output");
+		return false;
+	}
+
+	if (wlr_output->pending.committed & WLR_OUTPUT_STATE_MODE) {
+		assert(wlr_output->pending.mode_type == WLR_OUTPUT_STATE_MODE_CUSTOM);
+		if (!output_set_custom_mode(wlr_output,
+				wlr_output->pending.custom_mode.width,
+				wlr_output->pending.custom_mode.height,
+				wlr_output->pending.custom_mode.refresh)) {
+			return false;
+		}
+	}
+
+	if (wlr_output->pending.committed & WLR_OUTPUT_STATE_BUFFER) {
+		if (!output_commit_buffer(output)) {
+			return false;
+		}
+	}
+
+	return true;
+}
+
 static void output_destroy(struct wlr_output *wlr_output) {
 	struct wlr_rdp_output *output =
 		rdp_output_from_output(wlr_output);
@@ -234,7 +261,6 @@ static void output_destroy(struct wlr_output *wlr_output) {
 }
 
 static const struct wlr_output_impl output_impl = {
-	.set_custom_mode = output_set_custom_mode,
 	.destroy = output_destroy,
 	.attach_render = output_attach_render,
 	.commit = output_commit,
diff --git a/backend/wayland/output.c b/backend/wayland/output.c
index adff43af..967011e8 100644
--- a/backend/wayland/output.c
+++ b/backend/wayland/output.c
@@ -166,78 +166,86 @@ static bool output_commit(struct wlr_output *wlr_output) {
 	struct wlr_wl_output *output =
 		get_wl_output_from_output(wlr_output);
 
-	if (output->frame_callback != NULL) {
-		wlr_log(WLR_ERROR, "Skipping buffer swap");
+	if (wlr_output->pending.committed & WLR_OUTPUT_STATE_ENABLED) {
+		wlr_log(WLR_DEBUG, "Cannot disable a Wayland output");
 		return false;
 	}
 
-	output->frame_callback = wl_surface_frame(output->surface);
-	wl_callback_add_listener(output->frame_callback, &frame_listener, output);
-
-	pixman_region32_t *damage = NULL;
-	if (wlr_output->pending.committed & WLR_OUTPUT_STATE_DAMAGE) {
-		damage = &wlr_output->pending.damage;
+	if (wlr_output->pending.committed & WLR_OUTPUT_STATE_MODE) {
+		assert(wlr_output->pending.mode_type == WLR_OUTPUT_STATE_MODE_CUSTOM);
+		if (!output_set_custom_mode(wlr_output,
+				wlr_output->pending.custom_mode.width,
+				wlr_output->pending.custom_mode.height,
+				wlr_output->pending.custom_mode.refresh)) {
+			return false;
+		}
 	}
 
-	wlr_buffer_unref(output->current_buffer);
-	output->current_buffer = NULL;
-	if (output->current_wl_buffer != NULL) {
-		wl_buffer_destroy(output->current_wl_buffer);
-		output->current_wl_buffer = NULL;
-	}
+	if (wlr_output->pending.committed & WLR_OUTPUT_STATE_BUFFER) {
+		wlr_buffer_unref(output->current_buffer);
+		output->current_buffer = NULL;
+		if (output->current_wl_buffer != NULL) {
+			wl_buffer_destroy(output->current_wl_buffer);
+			output->current_wl_buffer = NULL;
+		}
 
-	struct wp_presentation_feedback *wp_feedback = NULL;
-	if (output->backend->presentation != NULL) {
-		wp_feedback = wp_presentation_feedback(output->backend->presentation,
-			output->surface);
-	}
+		struct wp_presentation_feedback *wp_feedback = NULL;
+		if (output->backend->presentation != NULL) {
+			wp_feedback = wp_presentation_feedback(output->backend->presentation,
+				output->surface);
+		}
 
-	assert(wlr_output->pending.committed & WLR_OUTPUT_STATE_BUFFER);
-	switch (wlr_output->pending.buffer_type) {
-	case WLR_OUTPUT_STATE_BUFFER_RENDER:
-		if (!wlr_egl_swap_buffers(&output->backend->egl,
-				output->egl_surface, damage)) {
-			return false;
+		pixman_region32_t *damage = NULL;
+		if (wlr_output->pending.committed & WLR_OUTPUT_STATE_DAMAGE) {
+			damage = &wlr_output->pending.damage;
 		}
-		break;
-	case WLR_OUTPUT_STATE_BUFFER_SCANOUT:
-		if (damage == NULL) {
-			wl_surface_damage_buffer(output->surface,
-				0, 0, INT32_MAX, INT32_MAX);
-		} else {
-			int rects_len;
-			pixman_box32_t *rects =
-				pixman_region32_rectangles(damage, &rects_len);
-			for (int i = 0; i < rects_len; i++) {
-				pixman_box32_t *r = &rects[i];
-				wl_surface_damage_buffer(output->surface, r->x1, r->y1,
-					r->x2 - r->x1, r->y2 - r->y1);
+
+		switch (wlr_output->pending.buffer_type) {
+		case WLR_OUTPUT_STATE_BUFFER_RENDER:
+			if (!wlr_egl_swap_buffers(&output->backend->egl,
+					output->egl_surface, damage)) {
+				return false;
+			}
+			break;
+		case WLR_OUTPUT_STATE_BUFFER_SCANOUT:
+			if (damage == NULL) {
+				wl_surface_damage_buffer(output->surface,
+					0, 0, INT32_MAX, INT32_MAX);
+			} else {
+				int rects_len;
+				pixman_box32_t *rects =
+					pixman_region32_rectangles(damage, &rects_len);
+				for (int i = 0; i < rects_len; i++) {
+					pixman_box32_t *r = &rects[i];
+					wl_surface_damage_buffer(output->surface, r->x1, r->y1,
+						r->x2 - r->x1, r->y2 - r->y1);
+				}
 			}
+			wl_surface_commit(output->surface);
+
+			output->current_buffer = wlr_buffer_ref(wlr_output->pending.buffer);
+			output->current_wl_buffer = output->pending_wl_buffer;
+			output->pending_wl_buffer = NULL;
+			break;
 		}
-		wl_surface_commit(output->surface);
 
-		output->current_buffer = wlr_buffer_ref(wlr_output->pending.buffer);
-		output->current_wl_buffer = output->pending_wl_buffer;
-		output->pending_wl_buffer = NULL;
-		break;
-	}
+		if (wp_feedback != NULL) {
+			struct wlr_wl_presentation_feedback *feedback =
+				calloc(1, sizeof(*feedback));
+			if (feedback == NULL) {
+				wp_presentation_feedback_destroy(wp_feedback);
+				return false;
+			}
+			feedback->output = output;
+			feedback->feedback = wp_feedback;
+			feedback->commit_seq = output->wlr_output.commit_seq + 1;
+			wl_list_insert(&output->presentation_feedbacks, &feedback->link);
 
-	if (wp_feedback != NULL) {
-		struct wlr_wl_presentation_feedback *feedback =
-			calloc(1, sizeof(*feedback));
-		if (feedback == NULL) {
-			wp_presentation_feedback_destroy(wp_feedback);
-			return false;
+			wp_presentation_feedback_add_listener(wp_feedback,
+				&presentation_feedback_listener, feedback);
+		} else {
+			wlr_output_send_present(wlr_output, NULL);
 		}
-		feedback->output = output;
-		feedback->feedback = wp_feedback;
-		feedback->commit_seq = output->wlr_output.commit_seq + 1;
-		wl_list_insert(&output->presentation_feedbacks, &feedback->link);
-
-		wp_presentation_feedback_add_listener(wp_feedback,
-			&presentation_feedback_listener, feedback);
-	} else {
-		wlr_output_send_present(wlr_output, NULL);
 	}
 
 	return true;
@@ -382,7 +390,6 @@ static bool output_schedule_frame(struct wlr_output *wlr_output) {
 }
 
 static const struct wlr_output_impl output_impl = {
-	.set_custom_mode = output_set_custom_mode,
 	.destroy = output_destroy,
 	.attach_render = output_attach_render,
 	.attach_buffer  = output_attach_buffer,
diff --git a/backend/x11/output.c b/backend/x11/output.c
index b6d00852..c827b9d5 100644
--- a/backend/x11/output.c
+++ b/backend/x11/output.c
@@ -103,21 +103,38 @@ static bool output_commit(struct wlr_output *wlr_output) {
 	struct wlr_x11_output *output = get_x11_output_from_output(wlr_output);
 	struct wlr_x11_backend *x11 = output->x11;
 
-	pixman_region32_t *damage = NULL;
-	if (wlr_output->pending.committed & WLR_OUTPUT_STATE_DAMAGE) {
-		damage = &wlr_output->pending.damage;
+	if (wlr_output->pending.committed & WLR_OUTPUT_STATE_ENABLED) {
+		wlr_log(WLR_DEBUG, "Cannot disable an X11 output");
+		return false;
 	}
 
-	if (!wlr_egl_swap_buffers(&x11->egl, output->surf, damage)) {
-		return false;
+	if (wlr_output->pending.committed & WLR_OUTPUT_STATE_MODE) {
+		assert(wlr_output->pending.mode_type == WLR_OUTPUT_STATE_MODE_CUSTOM);
+		if (!output_set_custom_mode(wlr_output,
+				wlr_output->pending.custom_mode.width,
+				wlr_output->pending.custom_mode.height,
+				wlr_output->pending.custom_mode.refresh)) {
+			return false;
+		}
+	}
+
+	if (wlr_output->pending.committed & WLR_OUTPUT_STATE_BUFFER) {
+		pixman_region32_t *damage = NULL;
+		if (wlr_output->pending.committed & WLR_OUTPUT_STATE_DAMAGE) {
+			damage = &wlr_output->pending.damage;
+		}
+
+		if (!wlr_egl_swap_buffers(&x11->egl, output->surf, damage)) {
+			return false;
+		}
+
+		wlr_output_send_present(wlr_output, NULL);
 	}
 
-	wlr_output_send_present(wlr_output, NULL);
 	return true;
 }
 
 static const struct wlr_output_impl output_impl = {
-	.set_custom_mode = output_set_custom_mode,
 	.destroy = output_destroy,
 	.attach_render = output_attach_render,
 	.commit = output_commit,