output: improve transform matrix calculation

Compute only the transform matrix in the output. The projection matrix
will be calculated inside the gles2 renderer when we start rendering.

The goal is to help the pixman rendering process.

4 files changed, 29 insertions, 12 deletions

diff --git a/backend/drm/drm.c b/backend/drm/drm.c
index b3f52cd5..e06213fe 100644
--- a/backend/drm/drm.c
+++ b/backend/drm/drm.c
@@ -895,10 +895,6 @@ static bool drm_connector_set_cursor(struct wlr_output *output,
 		}
 	}
 
-	float hotspot_proj[9];
-	wlr_matrix_projection(hotspot_proj, plane->surf.width,
-		plane->surf.height, output->transform);
-
 	struct wlr_box hotspot = { .x = hotspot_x, .y = hotspot_y };
 	wlr_box_transform(&hotspot, &hotspot,
 		wlr_output_transform_invert(output->transform),
@@ -939,8 +935,19 @@ static bool drm_connector_set_cursor(struct wlr_output *output,
 
 		struct wlr_box cursor_box = { .width = width, .height = height };
 
+		float output_matrix[9];
+		wlr_matrix_identity(output_matrix);
+		if (output->transform != WL_OUTPUT_TRANSFORM_NORMAL) {
+			wlr_matrix_translate(output_matrix, plane->surf.width / 2.0,
+					plane->surf.height / 2.0);
+			wlr_matrix_transform(output_matrix, output->transform);
+			wlr_matrix_translate(output_matrix, - plane->surf.width / 2.0,
+					- plane->surf.height / 2.0);
+		}
+
 		float matrix[9];
-		wlr_matrix_project_box(matrix, &cursor_box, transform, 0, hotspot_proj);
+		wlr_matrix_project_box(matrix, &cursor_box, transform, 0,
+				output_matrix);
 
 		wlr_renderer_begin(rend, plane->surf.width, plane->surf.height);
 		wlr_renderer_clear(rend, (float[]){ 0.0, 0.0, 0.0, 0.0 });
diff --git a/backend/drm/renderer.c b/backend/drm/renderer.c
index ad823398..260d041f 100644
--- a/backend/drm/renderer.c
+++ b/backend/drm/renderer.c
@@ -156,7 +156,7 @@ static struct wlr_buffer *drm_surface_blit(struct wlr_drm_surface *surf,
 	}
 
 	float mat[9];
-	wlr_matrix_projection(mat, 1, 1, WL_OUTPUT_TRANSFORM_NORMAL);
+	wlr_matrix_identity(mat);
 
 	wlr_renderer_begin(renderer, surf->width, surf->height);
 	wlr_renderer_clear(renderer, (float[]){ 0.0, 0.0, 0.0, 0.0 });
diff --git a/backend/wayland/output.c b/backend/wayland/output.c
index d5410e0d..1e13f455 100644
--- a/backend/wayland/output.c
+++ b/backend/wayland/output.c
@@ -431,11 +431,16 @@ static bool output_set_cursor(struct wlr_output *wlr_output,
 			.height = height,
 		};
 
-		float projection[9];
-		wlr_matrix_projection(projection, width, height, wlr_output->transform);
+		float output_matrix[9];
+		wlr_matrix_identity(output_matrix);
+		if (wlr_output->transform != WL_OUTPUT_TRANSFORM_NORMAL) {
+			wlr_matrix_translate(output_matrix, width / 2.0, height / 2.0);
+			wlr_matrix_transform(output_matrix, wlr_output->transform);
+			wlr_matrix_translate(output_matrix, - width / 2.0, - height / 2.0);
+		}
 
 		float matrix[9];
-		wlr_matrix_project_box(matrix, &cursor_box, transform, 0, projection);
+		wlr_matrix_project_box(matrix, &cursor_box, transform, 0, output_matrix);
 
 		wlr_renderer_begin(backend->renderer, width, height);
 		wlr_renderer_clear(backend->renderer, (float[]){ 0.0, 0.0, 0.0, 0.0 });
diff --git a/backend/x11/output.c b/backend/x11/output.c
index ef2d1c13..406185e3 100644
--- a/backend/x11/output.c
+++ b/backend/x11/output.c
@@ -415,11 +415,16 @@ static bool output_cursor_to_picture(struct wlr_x11_output *output,
 		.height = height,
 	};
 
-	float projection[9];
-	wlr_matrix_projection(projection, width, height, output->wlr_output.transform);
+	float output_matrix[9];
+	wlr_matrix_identity(output_matrix);
+	if (output->wlr_output.transform != WL_OUTPUT_TRANSFORM_NORMAL) {
+		wlr_matrix_translate(output_matrix, width / 2.0, height / 2.0);
+		wlr_matrix_transform(output_matrix, output->wlr_output.transform);
+		wlr_matrix_translate(output_matrix, - width / 2.0, - height / 2.0);
+	}
 
 	float matrix[9];
-	wlr_matrix_project_box(matrix, &cursor_box, transform, 0, projection);
+	wlr_matrix_project_box(matrix, &cursor_box, transform, 0, output_matrix);
 
 	wlr_renderer_begin(x11->renderer, width, height);
 	wlr_renderer_clear(x11->renderer, (float[]){ 0.0, 0.0, 0.0, 0.0 });