output: improve transform matrix calculation

Compute only the transform matrix in the output. The projection matrix
will be calculated inside the gles2 renderer when we start rendering.

The goal is to help the pixman rendering process.

7 files changed, 50 insertions, 18 deletions

diff --git a/backend/drm/drm.c b/backend/drm/drm.c
index b3f52cd5..e06213fe 100644
--- a/backend/drm/drm.c
+++ b/backend/drm/drm.c
@@ -895,10 +895,6 @@ static bool drm_connector_set_cursor(struct wlr_output *output,
 		}
 	}
 
-	float hotspot_proj[9];
-	wlr_matrix_projection(hotspot_proj, plane->surf.width,
-		plane->surf.height, output->transform);
-
 	struct wlr_box hotspot = { .x = hotspot_x, .y = hotspot_y };
 	wlr_box_transform(&hotspot, &hotspot,
 		wlr_output_transform_invert(output->transform),
@@ -939,8 +935,19 @@ static bool drm_connector_set_cursor(struct wlr_output *output,
 
 		struct wlr_box cursor_box = { .width = width, .height = height };
 
+		float output_matrix[9];
+		wlr_matrix_identity(output_matrix);
+		if (output->transform != WL_OUTPUT_TRANSFORM_NORMAL) {
+			wlr_matrix_translate(output_matrix, plane->surf.width / 2.0,
+					plane->surf.height / 2.0);
+			wlr_matrix_transform(output_matrix, output->transform);
+			wlr_matrix_translate(output_matrix, - plane->surf.width / 2.0,
+					- plane->surf.height / 2.0);
+		}
+
 		float matrix[9];
-		wlr_matrix_project_box(matrix, &cursor_box, transform, 0, hotspot_proj);
+		wlr_matrix_project_box(matrix, &cursor_box, transform, 0,
+				output_matrix);
 
 		wlr_renderer_begin(rend, plane->surf.width, plane->surf.height);
 		wlr_renderer_clear(rend, (float[]){ 0.0, 0.0, 0.0, 0.0 });
diff --git a/backend/drm/renderer.c b/backend/drm/renderer.c
index ad823398..260d041f 100644
--- a/backend/drm/renderer.c
+++ b/backend/drm/renderer.c
@@ -156,7 +156,7 @@ static struct wlr_buffer *drm_surface_blit(struct wlr_drm_surface *surf,
 	}
 
 	float mat[9];
-	wlr_matrix_projection(mat, 1, 1, WL_OUTPUT_TRANSFORM_NORMAL);
+	wlr_matrix_identity(mat);
 
 	wlr_renderer_begin(renderer, surf->width, surf->height);
 	wlr_renderer_clear(renderer, (float[]){ 0.0, 0.0, 0.0, 0.0 });
diff --git a/backend/wayland/output.c b/backend/wayland/output.c
index d5410e0d..1e13f455 100644
--- a/backend/wayland/output.c
+++ b/backend/wayland/output.c
@@ -431,11 +431,16 @@ static bool output_set_cursor(struct wlr_output *wlr_output,
 			.height = height,
 		};
 
-		float projection[9];
-		wlr_matrix_projection(projection, width, height, wlr_output->transform);
+		float output_matrix[9];
+		wlr_matrix_identity(output_matrix);
+		if (wlr_output->transform != WL_OUTPUT_TRANSFORM_NORMAL) {
+			wlr_matrix_translate(output_matrix, width / 2.0, height / 2.0);
+			wlr_matrix_transform(output_matrix, wlr_output->transform);
+			wlr_matrix_translate(output_matrix, - width / 2.0, - height / 2.0);
+		}
 
 		float matrix[9];
-		wlr_matrix_project_box(matrix, &cursor_box, transform, 0, projection);
+		wlr_matrix_project_box(matrix, &cursor_box, transform, 0, output_matrix);
 
 		wlr_renderer_begin(backend->renderer, width, height);
 		wlr_renderer_clear(backend->renderer, (float[]){ 0.0, 0.0, 0.0, 0.0 });
diff --git a/backend/x11/output.c b/backend/x11/output.c
index ef2d1c13..406185e3 100644
--- a/backend/x11/output.c
+++ b/backend/x11/output.c
@@ -415,11 +415,16 @@ static bool output_cursor_to_picture(struct wlr_x11_output *output,
 		.height = height,
 	};
 
-	float projection[9];
-	wlr_matrix_projection(projection, width, height, output->wlr_output.transform);
+	float output_matrix[9];
+	wlr_matrix_identity(output_matrix);
+	if (output->wlr_output.transform != WL_OUTPUT_TRANSFORM_NORMAL) {
+		wlr_matrix_translate(output_matrix, width / 2.0, height / 2.0);
+		wlr_matrix_transform(output_matrix, output->wlr_output.transform);
+		wlr_matrix_translate(output_matrix, - width / 2.0, - height / 2.0);
+	}
 
 	float matrix[9];
-	wlr_matrix_project_box(matrix, &cursor_box, transform, 0, projection);
+	wlr_matrix_project_box(matrix, &cursor_box, transform, 0, output_matrix);
 
 	wlr_renderer_begin(x11->renderer, width, height);
 	wlr_renderer_clear(x11->renderer, (float[]){ 0.0, 0.0, 0.0, 0.0 });
diff --git a/include/render/gles2.h b/include/render/gles2.h
index c152b944..8b4de5a2 100644
--- a/include/render/gles2.h
+++ b/include/render/gles2.h
@@ -34,6 +34,7 @@ struct wlr_gles2_tex_shader {
 struct wlr_gles2_renderer {
 	struct wlr_renderer wlr_renderer;
 
+	float projection[9];
 	struct wlr_egl *egl;
 	int drm_fd;
 
diff --git a/render/gles2/renderer.c b/render/gles2/renderer.c
index 196d0573..504af574 100644
--- a/render/gles2/renderer.c
+++ b/render/gles2/renderer.c
@@ -191,6 +191,10 @@ static void gles2_begin(struct wlr_renderer *wlr_renderer, uint32_t width,
 	renderer->viewport_width = width;
 	renderer->viewport_height = height;
 
+	// refresh projection matrix
+	wlr_matrix_projection(renderer->projection, width, height,
+			WL_OUTPUT_TRANSFORM_NORMAL);
+
 	// enable transparency
 	glEnable(GL_BLEND);
 	glBlendFunc(GL_ONE, GL_ONE_MINUS_SRC_ALPHA);
@@ -271,7 +275,8 @@ static bool gles2_render_subtexture_with_matrix(
 	}
 
 	float gl_matrix[9];
-	wlr_matrix_multiply(gl_matrix, flip_180, matrix);
+	wlr_matrix_multiply(gl_matrix, renderer->projection, matrix);
+	wlr_matrix_multiply(gl_matrix, flip_180, gl_matrix);
 
 	// OpenGL ES 2 requires the glUniformMatrix3fv transpose parameter to be set
 	// to GL_FALSE
@@ -325,7 +330,8 @@ static void gles2_render_quad_with_matrix(struct wlr_renderer *wlr_renderer,
 		gles2_get_renderer_in_context(wlr_renderer);
 
 	float gl_matrix[9];
-	wlr_matrix_multiply(gl_matrix, flip_180, matrix);
+	wlr_matrix_multiply(gl_matrix, renderer->projection, matrix);
+	wlr_matrix_multiply(gl_matrix, flip_180, gl_matrix);
 
 	// OpenGL ES 2 requires the glUniformMatrix3fv transpose parameter to be set
 	// to GL_FALSE
@@ -355,7 +361,8 @@ static void gles2_render_ellipse_with_matrix(struct wlr_renderer *wlr_renderer,
 		gles2_get_renderer_in_context(wlr_renderer);
 
 	float gl_matrix[9];
-	wlr_matrix_multiply(gl_matrix, flip_180, matrix);
+	wlr_matrix_multiply(gl_matrix, renderer->projection, matrix);
+	wlr_matrix_multiply(gl_matrix, flip_180, gl_matrix);
 
 	// OpenGL ES 2 requires the glUniformMatrix3fv transpose parameter to be set
 	// to GL_FALSE
@@ -579,7 +586,7 @@ static bool gles2_blit_dmabuf(struct wlr_renderer *wlr_renderer,
 
 	// TODO: use ANGLE_framebuffer_blit if available
 	float mat[9];
-	wlr_matrix_projection(mat, 1, 1, WL_OUTPUT_TRANSFORM_NORMAL);
+	wlr_matrix_identity(mat);
 
 	wlr_renderer_begin(wlr_renderer, dst_attr->width, dst_attr->height);
 	wlr_renderer_clear(wlr_renderer, (float[]){ 0.0, 0.0, 0.0, 0.0 });
diff --git a/types/wlr_output.c b/types/wlr_output.c
index 1ce3a199..237ed161 100644
--- a/types/wlr_output.c
+++ b/types/wlr_output.c
@@ -141,8 +141,15 @@ void wlr_output_update_enabled(struct wlr_output *output, bool enabled) {
 }
 
 static void output_update_matrix(struct wlr_output *output) {
-	wlr_matrix_projection(output->transform_matrix, output->width,
-		output->height, output->transform);
+	float width = output->width / 2.f;
+	float height = output->height / 2.f;
+
+	wlr_matrix_identity(output->transform_matrix);
+	if (output->transform != WL_OUTPUT_TRANSFORM_NORMAL) {
+		wlr_matrix_translate(output->transform_matrix, width, height);
+		wlr_matrix_transform(output->transform_matrix, output->transform);
+		wlr_matrix_translate(output->transform_matrix, -width, -height);
+	}
 }
 
 void wlr_output_enable(struct wlr_output *output, bool enable) {