render/vulkan: Dynamically handle pipeline creation for textures

If we ever wanted to handle dynamic state that requires new pipelines
such as using different texture filters those can be added here with more
ease.

4 files changed, 107 insertions, 64 deletions

diff --git a/include/render/vulkan.h b/include/render/vulkan.h
index 3a51b0bf..a6b72292 100644
--- a/include/render/vulkan.h
+++ b/include/render/vulkan.h
@@ -142,6 +142,19 @@ enum wlr_vk_texture_transform {
 	WLR_VK_TEXTURE_TRANSFORM_SRGB = 1,
 };
 
+struct wlr_vk_pipeline_key {
+	enum wlr_vk_texture_transform texture_transform;
+	struct wlr_vk_pipeline_layout *layout;
+};
+
+struct wlr_vk_pipeline {
+	struct wlr_vk_pipeline_key key;
+
+	VkPipeline vk;
+	struct wlr_vk_render_format_setup *setup;
+	struct wl_list link; // struct wlr_vk_render_format_setup
+};
+
 // For each format we want to render, we need a separate renderpass
 // and therefore also separate pipelines.
 struct wlr_vk_render_format_setup {
@@ -149,12 +162,11 @@ struct wlr_vk_render_format_setup {
 	VkFormat render_format; // used in renderpass
 	VkRenderPass render_pass;
 
-	VkPipeline tex_identity_pipe;
-	VkPipeline tex_srgb_pipe;
 	VkPipeline quad_pipe;
 	VkPipeline output_pipe;
 
-	VkPipeline *tex_ycbcr_pipelines; // same length as wlr_vk_renderer.ycbcr_pipeline_layouts
+	struct wlr_vk_renderer *renderer;
+	struct wl_list pipelines; // struct wlr_vk_pipeline.link
 };
 
 // Renderer-internal represenation of an wlr_buffer imported for rendering.
@@ -268,6 +280,10 @@ struct wlr_vk_vert_pcr_data {
 	float uv_size[2];
 };
 
+struct wlr_vk_pipeline *setup_get_or_create_pipeline(
+	struct wlr_vk_render_format_setup *setup,
+	const struct wlr_vk_pipeline_key *key);
+
 // Creates a vulkan renderer for the given device.
 struct wlr_renderer *vulkan_renderer_create_for_device(struct wlr_vk_device *dev);
 
@@ -287,6 +303,7 @@ struct wlr_vk_render_pass {
 	struct wlr_vk_command_buffer *command_buffer;
 	VkPipeline bound_pipeline;
 	float projection[9];
+	bool failed;
 };
 
 struct wlr_vk_render_pass *vulkan_begin_render_pass(struct wlr_vk_renderer *renderer,
@@ -321,8 +338,6 @@ struct wlr_vk_renderer *vulkan_get_renderer(struct wlr_renderer *r);
 
 struct wlr_vk_pipeline_layout *vulkan_get_pipeline_layout(struct wlr_vk_renderer *renderer,
 	const struct wlr_vk_format *format);
-VkPipeline vulkan_get_texture_pipeline(struct wlr_vk_texture *texture,
-	struct wlr_vk_render_format_setup *render_setup);
 
 struct wlr_vk_command_buffer *vulkan_acquire_command_buffer(
 	struct wlr_vk_renderer *renderer);
@@ -346,6 +361,7 @@ struct wlr_vk_texture {
 	VkImageView image_view;
 	const struct wlr_vk_format *format;
 	struct wlr_vk_pipeline_layout *pipeline_layout;
+	enum wlr_vk_texture_transform transform;
 	VkDescriptorSet ds;
 	struct wlr_vk_descriptor_pool *ds_pool;
 	struct wlr_vk_command_buffer *last_used_cb; // to track when it can be destroyed
diff --git a/render/vulkan/pass.c b/render/vulkan/pass.c
index 55e18387..0a2d666d 100644
--- a/render/vulkan/pass.c
+++ b/render/vulkan/pass.c
@@ -70,6 +70,10 @@ static bool render_pass_submit(struct wlr_render_pass *wlr_pass) {
 	struct wlr_vk_command_buffer *stage_cb = NULL;
 	VkSemaphoreSubmitInfoKHR *render_wait = NULL;
 
+	if (pass->failed) {
+		goto error;
+	}
+
 	if (vulkan_record_stage_cb(renderer) == VK_NULL_HANDLE) {
 		goto error;
 	}
@@ -527,17 +531,24 @@ static void render_pass_add_texture(struct wlr_render_pass *wlr_pass,
 	};
 	mat3_to_mat4(matrix, vert_pcr_data.mat4);
 
-	VkPipelineLayout pipeline_layout = texture->pipeline_layout->vk;
-	VkPipeline pipeline = vulkan_get_texture_pipeline(texture, render_buffer->render_setup);
+	struct wlr_vk_pipeline *pipe = setup_get_or_create_pipeline(
+		render_buffer->render_setup,
+		&(struct wlr_vk_pipeline_key) {
+			.layout = texture->pipeline_layout,
+			.texture_transform = texture->transform,
+		});
+	if (!pipe) {
+		pass->failed = true;
+	}
 
-	bind_pipeline(pass, pipeline);
+	bind_pipeline(pass, pipe->vk);
 
 	vkCmdBindDescriptorSets(cb, VK_PIPELINE_BIND_POINT_GRAPHICS,
-		pipeline_layout, 0, 1, &texture->ds, 0, NULL);
+		pipe->key.layout->vk, 0, 1, &texture->ds, 0, NULL);
 
-	vkCmdPushConstants(cb, pipeline_layout,
+	vkCmdPushConstants(cb, pipe->key.layout->vk,
 		VK_SHADER_STAGE_VERTEX_BIT, 0, sizeof(vert_pcr_data), &vert_pcr_data);
-	vkCmdPushConstants(cb, pipeline_layout,
+	vkCmdPushConstants(cb, pipe->key.layout->vk,
 		VK_SHADER_STAGE_FRAGMENT_BIT, sizeof(vert_pcr_data), sizeof(float),
 		&alpha);
 
diff --git a/render/vulkan/renderer.c b/render/vulkan/renderer.c
index 70ec0ce0..799a5f3e 100644
--- a/render/vulkan/renderer.c
+++ b/render/vulkan/renderer.c
@@ -177,15 +177,14 @@ static void destroy_render_format_setup(struct wlr_vk_renderer *renderer,
 
 	VkDevice dev = renderer->dev->dev;
 	vkDestroyRenderPass(dev, setup->render_pass, NULL);
-	vkDestroyPipeline(dev, setup->tex_identity_pipe, NULL);
-	vkDestroyPipeline(dev, setup->tex_srgb_pipe, NULL);
 	vkDestroyPipeline(dev, setup->output_pipe, NULL);
 	vkDestroyPipeline(dev, setup->quad_pipe, NULL);
 
-	for (size_t i = 0; i < renderer->ycbcr_pipeline_layouts_len; i++) {
-		vkDestroyPipeline(dev, setup->tex_ycbcr_pipelines[i], NULL);
+	struct wlr_vk_pipeline *pipeline, *tmp_pipeline;
+	wl_list_for_each_safe(pipeline, tmp_pipeline, &setup->pipelines, link) {
+		vkDestroyPipeline(dev, pipeline->vk, NULL);
+		free(pipeline);
 	}
-	free(setup->tex_ycbcr_pipelines);
 }
 
 static void shared_buffer_destroy(struct wlr_vk_renderer *r,
@@ -1387,21 +1386,6 @@ static void vulkan_end(struct wlr_renderer *wlr_renderer) {
 	}
 }
 
-VkPipeline vulkan_get_texture_pipeline(struct wlr_vk_texture *texture,
-		struct wlr_vk_render_format_setup *render_setup) {
-	if (texture->format->is_ycbcr) {
-		size_t pipeline_layout_index = texture->pipeline_layout - texture->renderer->ycbcr_pipeline_layouts;
-		return render_setup->tex_ycbcr_pipelines[pipeline_layout_index];
-	} else {
-		if (texture->format->is_srgb) {
-			// sRGB formats already have the transfer function applied
-			return render_setup->tex_identity_pipe;
-		} else {
-			return render_setup->tex_srgb_pipe;
-		}
-	}
-}
-
 static bool vulkan_render_subtexture_with_matrix(struct wlr_renderer *wlr_renderer,
 		struct wlr_texture *wlr_texture, const struct wlr_fbox *box,
 		const float matrix[static 9], float alpha) {
@@ -1423,16 +1407,23 @@ static bool vulkan_render_subtexture_with_matrix(struct wlr_renderer *wlr_render
 		wl_list_insert(&renderer->foreign_textures, &texture->foreign_link);
 	}
 
-	VkPipelineLayout pipe_layout = texture->pipeline_layout->vk;
-	VkPipeline pipe = vulkan_get_texture_pipeline(texture, renderer->current_render_buffer->render_setup);
+	struct wlr_vk_pipeline *pipe = setup_get_or_create_pipeline(
+		renderer->current_render_buffer->render_setup,
+		&(struct wlr_vk_pipeline_key) {
+			.layout = texture->pipeline_layout,
+			.texture_transform = texture->transform,
+		});
+	if (!pipe) {
+		return false;
+	}
 
-	if (pipe != renderer->bound_pipe) {
-		vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_GRAPHICS, pipe);
-		renderer->bound_pipe = pipe;
+	if (pipe->vk != renderer->bound_pipe) {
+		vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_GRAPHICS, pipe->vk);
+		renderer->bound_pipe = pipe->vk;
 	}
 
 	vkCmdBindDescriptorSets(cb, VK_PIPELINE_BIND_POINT_GRAPHICS,
-		pipe_layout, 0, 1, &texture->ds, 0, NULL);
+		pipe->key.layout->vk, 0, 1, &texture->ds, 0, NULL);
 
 	float final_matrix[9];
 	wlr_matrix_multiply(final_matrix, renderer->projection, matrix);
@@ -1445,9 +1436,9 @@ static bool vulkan_render_subtexture_with_matrix(struct wlr_renderer *wlr_render
 	vert_pcr_data.uv_size[0] = box->width / wlr_texture->width;
 	vert_pcr_data.uv_size[1] = box->height / wlr_texture->height;
 
-	vkCmdPushConstants(cb, pipe_layout,
+	vkCmdPushConstants(cb, pipe->key.layout->vk,
 		VK_SHADER_STAGE_VERTEX_BIT, 0, sizeof(vert_pcr_data), &vert_pcr_data);
-	vkCmdPushConstants(cb, pipe_layout,
+	vkCmdPushConstants(cb, pipe->key.layout->vk,
 		VK_SHADER_STAGE_FRAGMENT_BIT, sizeof(vert_pcr_data), sizeof(float),
 		&alpha);
 	vkCmdDraw(cb, 4, 1, 0, 0);
@@ -2095,15 +2086,37 @@ static bool init_blend_to_output_layouts(struct wlr_vk_renderer *renderer,
 	return true;
 }
 
+static bool pipeline_key_equals(const struct wlr_vk_pipeline_key *a,
+		const struct wlr_vk_pipeline_key *b) {
+	return a->texture_transform == b->texture_transform && a->layout == b->layout;
+}
+
 // Initializes the pipeline for rendering textures and using the given
 // VkRenderPass and VkPipelineLayout.
-static bool init_tex_pipeline(struct wlr_vk_renderer *renderer,
-		VkRenderPass rp, VkPipelineLayout pipe_layout,
-		enum wlr_vk_texture_transform transform, VkPipeline *pipe) {
+struct wlr_vk_pipeline *setup_get_or_create_pipeline(
+		struct wlr_vk_render_format_setup *setup,
+		const struct wlr_vk_pipeline_key *key) {
+	struct wlr_vk_pipeline *pipeline;
+	wl_list_for_each(pipeline, &setup->pipelines, link) {
+		if (pipeline_key_equals(&pipeline->key, key)) {
+			return pipeline;
+		}
+	}
+
+	pipeline = calloc(1, sizeof(*pipeline));
+	if (!pipeline) {
+		return NULL;
+	}
+
+	struct wlr_vk_renderer *renderer = setup->renderer;
+
+	pipeline->setup = setup;
+	pipeline->key = *key;
+
 	VkResult res;
 	VkDevice dev = renderer->dev->dev;
 
-	uint32_t color_transform_type = transform;
+	uint32_t color_transform_type = key->texture_transform;
 
 	VkSpecializationMapEntry spec_entry = {
 		.constantID = 0,
@@ -2196,8 +2209,8 @@ static bool init_tex_pipeline(struct wlr_vk_renderer *renderer,
 
 	VkGraphicsPipelineCreateInfo pinfo = {
 		.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
-		.layout = pipe_layout,
-		.renderPass = rp,
+		.layout = key->layout->vk,
+		.renderPass = setup->render_pass,
 		.subpass = 0,
 		.stageCount = 2,
 		.pStages = tex_stages,
@@ -2212,13 +2225,15 @@ static bool init_tex_pipeline(struct wlr_vk_renderer *renderer,
 	};
 
 	VkPipelineCache cache = VK_NULL_HANDLE;
-	res = vkCreateGraphicsPipelines(dev, cache, 1, &pinfo, NULL, pipe);
+	res = vkCreateGraphicsPipelines(dev, cache, 1, &pinfo, NULL, &pipeline->vk);
 	if (res != VK_SUCCESS) {
 		wlr_vk_error("failed to create vulkan pipelines:", res);
-		return false;
+		free(pipeline);
+		return NULL;
 	}
 
-	return true;
+	wl_list_insert(&setup->pipelines, &pipeline->link);
+	return pipeline;
 }
 
 static bool init_quad_pipeline(struct wlr_vk_renderer *renderer,
@@ -2578,6 +2593,8 @@ static struct wlr_vk_render_format_setup *find_or_create_render_setup(
 	}
 
 	setup->render_format = format;
+	setup->renderer = renderer;
+	wl_list_init(&setup->pipelines);
 
 	VkDevice dev = renderer->dev->dev;
 	VkResult res;
@@ -2767,30 +2784,27 @@ static struct wlr_vk_render_format_setup *find_or_create_render_setup(
 		}
 	}
 
-	if (!init_tex_pipeline(renderer, setup->render_pass, renderer->default_pipeline_layout.vk,
-			WLR_VK_TEXTURE_TRANSFORM_IDENTITY, &setup->tex_identity_pipe)) {
+	if (!setup_get_or_create_pipeline(setup, &(struct wlr_vk_pipeline_key){
+		.texture_transform = WLR_VK_TEXTURE_TRANSFORM_IDENTITY,
+		.layout = &renderer->default_pipeline_layout,
+	})) {
 		goto error;
 	}
 
-	if (!init_tex_pipeline(renderer, setup->render_pass, renderer->default_pipeline_layout.vk,
-			WLR_VK_TEXTURE_TRANSFORM_SRGB, &setup->tex_srgb_pipe)) {
+	if (!setup_get_or_create_pipeline(setup, &(struct wlr_vk_pipeline_key){
+		.texture_transform = WLR_VK_TEXTURE_TRANSFORM_SRGB,
+		.layout = &renderer->default_pipeline_layout,
+	})) {
 		goto error;
 	}
 
-	if (renderer->ycbcr_pipeline_layouts_len > 0) {
-		setup->tex_ycbcr_pipelines =
-			calloc(renderer->ycbcr_pipeline_layouts_len, sizeof(*setup->tex_ycbcr_pipelines));
-		if (setup->tex_ycbcr_pipelines == NULL) {
+	for (size_t i = 0; i < renderer->ycbcr_pipeline_layouts_len; i++) {
+		if (!setup_get_or_create_pipeline(setup, &(struct wlr_vk_pipeline_key){
+			.texture_transform = WLR_VK_TEXTURE_TRANSFORM_SRGB,
+			.layout = &renderer->ycbcr_pipeline_layouts[i],
+		})) {
 			goto error;
 		}
-
-		for (size_t i = 0; i < renderer->ycbcr_pipeline_layouts_len; i++) {
-			struct wlr_vk_pipeline_layout *pipeline_layout = &renderer->ycbcr_pipeline_layouts[i];
-			if (!init_tex_pipeline(renderer, setup->render_pass, pipeline_layout->vk,
-					WLR_VK_TEXTURE_TRANSFORM_SRGB, &setup->tex_ycbcr_pipelines[i])) {
-				goto error;
-			}
-		}
 	}
 
 	if (!init_quad_pipeline(renderer, setup->render_pass, renderer->default_pipeline_layout.vk,
diff --git a/render/vulkan/texture.c b/render/vulkan/texture.c
index 71903d26..c1a4b7bb 100644
--- a/render/vulkan/texture.c
+++ b/render/vulkan/texture.c
@@ -689,6 +689,8 @@ static struct wlr_vk_texture *vulkan_texture_from_dmabuf(
 	}
 
 	texture->format = &fmt->format;
+	texture->transform = !texture->format->is_ycbcr && texture->format->is_srgb ?
+		WLR_VK_TEXTURE_TRANSFORM_IDENTITY : WLR_VK_TEXTURE_TRANSFORM_SRGB;
 
 	texture->pipeline_layout = vulkan_get_pipeline_layout(renderer, texture->format);
 	if (texture->pipeline_layout == NULL) {