render/vulkan: add support for RGB565 texture format

Since this does not have a matching _SRGB-type vulkan format, add a
new shader variant/pipeline to perform the sRGB->linear texture
conversion.

4 files changed, 78 insertions, 8 deletions

diff --git a/include/render/vulkan.h b/include/render/vulkan.h
index 5dfc3d37..8063291f 100644
--- a/include/render/vulkan.h
+++ b/include/render/vulkan.h
@@ -78,6 +78,7 @@ int vulkan_find_mem_type(struct wlr_vk_device *device,
 struct wlr_vk_format {
 	uint32_t drm_format;
 	VkFormat vk_format;
+	bool is_srgb;
 };
 
 // Returns all known format mappings.
@@ -110,6 +111,13 @@ struct wlr_vk_format_modifier_props *vulkan_format_props_find_modifier(
 	struct wlr_vk_format_props *props, uint64_t mod, bool render);
 void vulkan_format_props_finish(struct wlr_vk_format_props *props);
 
+// Constants used to pick the color transform for the texture drawing
+// fragment shader. Must match those in shaders/texture.frag
+enum wlr_vk_texture_transform {
+	WLR_VK_TEXTURE_TRANSFORM_IDENTITY = 0,
+	WLR_VK_TEXTURE_TRANSFORM_SRGB = 1,
+};
+
 // For each format we want to render, we need a separate renderpass
 // and therefore also separate pipelines.
 struct wlr_vk_render_format_setup {
@@ -117,7 +125,8 @@ struct wlr_vk_render_format_setup {
 	VkFormat render_format; // used in renderpass
 	VkRenderPass render_pass;
 
-	VkPipeline tex_pipe;
+	VkPipeline tex_identity_pipe;
+	VkPipeline tex_srgb_pipe;
 	VkPipeline quad_pipe;
 };
 
diff --git a/render/vulkan/pixel_format.c b/render/vulkan/pixel_format.c
index f55063d7..f315c8f3 100644
--- a/render/vulkan/pixel_format.c
+++ b/render/vulkan/pixel_format.c
@@ -10,19 +10,29 @@ static const struct wlr_vk_format formats[] = {
 	{
 		.drm_format = DRM_FORMAT_ARGB8888,
 		.vk_format = VK_FORMAT_B8G8R8A8_SRGB,
+		.is_srgb = true,
 	},
 	{
 		.drm_format = DRM_FORMAT_XRGB8888,
 		.vk_format = VK_FORMAT_B8G8R8A8_SRGB,
+		.is_srgb = true,
 	},
 	{
 		.drm_format = DRM_FORMAT_XBGR8888,
 		.vk_format = VK_FORMAT_R8G8B8A8_SRGB,
+		.is_srgb = true,
 	},
 	{
 		.drm_format = DRM_FORMAT_ABGR8888,
 		.vk_format = VK_FORMAT_R8G8B8A8_SRGB,
+		.is_srgb = true,
 	},
+#if WLR_LITTLE_ENDIAN
+	{
+		.drm_format = DRM_FORMAT_RGB565,
+		.vk_format = VK_FORMAT_R5G6B5_UNORM_PACK16,
+	},
+#endif
 };
 
 const struct wlr_vk_format *vulkan_get_format_list(size_t *len) {
@@ -139,7 +149,9 @@ static bool query_modifier_support(struct wlr_vk_device *dev,
 		const char *render_status, *texture_status;
 
 		// check that specific modifier for render usage
-		if ((m.drmFormatModifierTilingFeatures & render_features) == render_features) {
+		// also, only allow rendering to formats with SRGB encoding
+		if ((m.drmFormatModifierTilingFeatures & render_features) == render_features &&
+				props->format.is_srgb) {
 			fmti.usage = render_usage;
 
 			modi.drmFormatModifier = m.drmFormatModifier;
diff --git a/render/vulkan/renderer.c b/render/vulkan/renderer.c
index ef3184c3..0ef256c2 100644
--- a/render/vulkan/renderer.c
+++ b/render/vulkan/renderer.c
@@ -164,7 +164,8 @@ static void destroy_render_format_setup(struct wlr_vk_renderer *renderer,
 
 	VkDevice dev = renderer->dev->dev;
 	vkDestroyRenderPass(dev, setup->render_pass, NULL);
-	vkDestroyPipeline(dev, setup->tex_pipe, NULL);
+	vkDestroyPipeline(dev, setup->tex_identity_pipe, NULL);
+	vkDestroyPipeline(dev, setup->tex_srgb_pipe, NULL);
 	vkDestroyPipeline(dev, setup->quad_pipe, NULL);
 }
 
@@ -970,7 +971,13 @@ static bool vulkan_render_subtexture_with_matrix(struct wlr_renderer *wlr_render
 		wl_list_insert(&renderer->foreign_textures, &texture->foreign_link);
 	}
 
-	VkPipeline pipe = renderer->current_render_buffer->render_setup->tex_pipe;
+	VkPipeline pipe;
+	// SRGB formats already have the transfer function applied
+	if (texture->format->is_srgb) {
+		pipe = renderer->current_render_buffer->render_setup->tex_identity_pipe;
+	} else {
+		pipe = renderer->current_render_buffer->render_setup->tex_srgb_pipe;
+	}
 	if (pipe != renderer->bound_pipe) {
 		vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_GRAPHICS, pipe);
 		renderer->bound_pipe = pipe;
@@ -1501,10 +1508,26 @@ static bool init_tex_layouts(struct wlr_vk_renderer *renderer,
 // Initializes the pipeline for rendering textures and using the given
 // VkRenderPass and VkPipelineLayout.
 static bool init_tex_pipeline(struct wlr_vk_renderer *renderer,
-		VkRenderPass rp, VkPipelineLayout pipe_layout, VkPipeline *pipe) {
+		VkRenderPass rp, VkPipelineLayout pipe_layout,
+		enum wlr_vk_texture_transform transform, VkPipeline *pipe) {
 	VkResult res;
 	VkDevice dev = renderer->dev->dev;
 
+	uint32_t color_transform_type = transform;
+
+	VkSpecializationMapEntry spec_entry = {
+		.constantID = 0,
+		.offset = 0,
+		.size = sizeof(uint32_t),
+	};
+
+	VkSpecializationInfo specialization = {
+		.mapEntryCount = 1,
+		.pMapEntries = &spec_entry,
+		.dataSize = sizeof(uint32_t),
+		.pData = &color_transform_type,
+	};
+
 	// shaders
 	VkPipelineShaderStageCreateInfo tex_stages[2] = {
 		{
@@ -1518,6 +1541,7 @@ static bool init_tex_pipeline(struct wlr_vk_renderer *renderer,
 			.stage = VK_SHADER_STAGE_FRAGMENT_BIT,
 			.module = renderer->tex_frag_module,
 			.pName = "main",
+			.pSpecializationInfo = &specialization,
 		},
 	};
 
@@ -1657,7 +1681,7 @@ static bool init_static_render_data(struct wlr_vk_renderer *renderer) {
 		return false;
 	}
 
-	// tex frag
+	// tex frags
 	sinfo = (VkShaderModuleCreateInfo){
 		.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO,
 		.codeSize = sizeof(texture_frag_data),
@@ -1774,7 +1798,12 @@ static struct wlr_vk_render_format_setup *find_or_create_render_setup(
 	}
 
 	if (!init_tex_pipeline(renderer, setup->render_pass, renderer->pipe_layout,
-			&setup->tex_pipe)) {
+			WLR_VK_TEXTURE_TRANSFORM_IDENTITY, &setup->tex_identity_pipe)) {
+		goto error;
+	}
+
+	if (!init_tex_pipeline(renderer, setup->render_pass, renderer->pipe_layout,
+			WLR_VK_TEXTURE_TRANSFORM_SRGB, &setup->tex_srgb_pipe)) {
 		goto error;
 	}
 
diff --git a/render/vulkan/shaders/texture.frag b/render/vulkan/shaders/texture.frag
index 0f9c52f1..15ef52af 100644
--- a/render/vulkan/shaders/texture.frag
+++ b/render/vulkan/shaders/texture.frag
@@ -9,8 +9,28 @@ layout(push_constant) uniform UBO {
 	layout(offset = 80) float alpha;
 } data;
 
+layout (constant_id = 0) const int TEXTURE_TRANSFORM = 0;
+
+#define TEXTURE_TRANSFORM_IDENTITY 0
+#define TEXTURE_TRANSFORM_SRGB 1
+
+float srgb_to_linear(float x) {
+	return max(x / 12.92, pow((x + 0.055) / 1.055, 2.4));
+}
+
 void main() {
-	out_color = textureLod(tex, uv, 0);
+	vec4 val = textureLod(tex, uv, 0);
+	if (TEXTURE_TRANSFORM == TEXTURE_TRANSFORM_SRGB) {
+		out_color = vec4(
+			srgb_to_linear(val.r),
+			srgb_to_linear(val.g),
+			srgb_to_linear(val.b),
+			val.a
+		);
+	} else { // TEXTURE_TRANSFORM_IDENTITY
+		out_color = val;
+	}
+
 	out_color *= data.alpha;
 }