render/vulkan: add support for NV12

4 files changed, 124 insertions, 12 deletions

diff --git a/render/vulkan/pixel_format.c b/render/vulkan/pixel_format.c
index 156f6acf..c1931180 100644
--- a/render/vulkan/pixel_format.c
+++ b/render/vulkan/pixel_format.c
@@ -143,6 +143,15 @@ static const struct wlr_vk_format formats[] = {
 		.vk = VK_FORMAT_R16G16B16A16_SFLOAT,
 	},
 #endif
+
+	// YCbCr formats
+#if WLR_LITTLE_ENDIAN
+	{
+		.drm = DRM_FORMAT_NV12,
+		.vk = VK_FORMAT_G8_B8R8_2PLANE_420_UNORM,
+		.is_ycbcr = true,
+	},
+#endif
 };
 
 const struct wlr_vk_format *vulkan_get_format_list(size_t *len) {
@@ -183,6 +192,10 @@ static const VkFormatFeatureFlags dma_tex_features =
 	// NOTE: we don't strictly require this, we could create a NEAREST
 	// sampler for formats that need it, in case this ever makes problems.
 	VK_FORMAT_FEATURE_SAMPLED_IMAGE_FILTER_LINEAR_BIT;
+static const VkFormatFeatureFlags dma_tex_ycbcr_features =
+	VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT |
+	VK_FORMAT_FEATURE_SAMPLED_IMAGE_YCBCR_CONVERSION_LINEAR_FILTER_BIT |
+	VK_FORMAT_FEATURE_MIDPOINT_CHROMA_SAMPLES_BIT;
 
 static bool query_modifier_usage_support(struct wlr_vk_device *dev, VkFormat vk_format,
 		VkImageUsageFlags usage, const VkDrmFormatModifierPropertiesEXT *m,
@@ -285,7 +298,7 @@ static bool query_modifier_support(struct wlr_vk_device *dev,
 		// also, only allow rendering to formats with SRGB encoding
 		const char *errmsg = "unknown error";
 		if ((m.drmFormatModifierTilingFeatures & render_features) == render_features &&
-				props->format.is_srgb) {
+				props->format.is_srgb && !props->format.is_ycbcr) {
 			struct wlr_vk_format_modifier_props p = {0};
 			if (query_modifier_usage_support(dev, props->format.vk, render_usage, &m, &p, &errmsg)) {
 				props->dmabuf.render_mods[props->dmabuf.render_mod_count++] = p;
@@ -304,7 +317,8 @@ static bool query_modifier_support(struct wlr_vk_device *dev,
 
 		// check that specific modifier for texture usage
 		errmsg = "unknown error";
-		if ((m.drmFormatModifierTilingFeatures & dma_tex_features) == dma_tex_features) {
+		VkFormatFeatureFlags features = props->format.is_ycbcr ? dma_tex_ycbcr_features : dma_tex_features;
+		if ((m.drmFormatModifierTilingFeatures & features) == features) {
 			struct wlr_vk_format_modifier_props p = {0};
 			if (query_modifier_usage_support(dev, props->format.vk, dma_tex_usage, &m, &p, &errmsg)) {
 				props->dmabuf.texture_mods[props->dmabuf.texture_mod_count++] = p;
@@ -337,6 +351,10 @@ void vulkan_format_props_query(struct wlr_vk_device *dev,
 		const struct wlr_vk_format *format) {
 	VkResult res;
 
+	if (format->is_ycbcr && !dev->sampler_ycbcr_conversion) {
+		return;
+	}
+
 	char *format_name = drmGetFormatName(format->drm);
 	wlr_log(WLR_DEBUG, "  %s (0x%08"PRIX32")",
 		format_name ? format_name : "<unknown>", format->drm);
@@ -358,7 +376,8 @@ void vulkan_format_props_query(struct wlr_vk_device *dev,
 
 	// non-dmabuf texture properties
 	const char *shm_texture_status;
-	if ((fmtp.formatProperties.optimalTilingFeatures & tex_features) == tex_features) {
+	if ((fmtp.formatProperties.optimalTilingFeatures & tex_features) == tex_features &&
+			!format->is_ycbcr) {
 		VkPhysicalDeviceImageFormatInfo2 fmti = {
 			.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2,
 			.type = VK_IMAGE_TYPE_2D,
diff --git a/render/vulkan/renderer.c b/render/vulkan/renderer.c
index 2adb1822..82d81c00 100644
--- a/render/vulkan/renderer.c
+++ b/render/vulkan/renderer.c
@@ -87,7 +87,8 @@ static void mat3_to_mat4(const float mat3[9], float mat4[4][4]) {
 }
 
 struct wlr_vk_descriptor_pool *vulkan_alloc_texture_ds(
-		struct wlr_vk_renderer *renderer, VkDescriptorSet *ds) {
+		struct wlr_vk_renderer *renderer, VkDescriptorSetLayout ds_layout,
+		VkDescriptorSet *ds) {
 	VkResult res;
 
 	bool found = false;
@@ -139,7 +140,7 @@ struct wlr_vk_descriptor_pool *vulkan_alloc_texture_ds(
 	VkDescriptorSetAllocateInfo ds_info = {
 		.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO,
 		.descriptorSetCount = 1,
-		.pSetLayouts = &renderer->ds_layout,
+		.pSetLayouts = &ds_layout,
 		.descriptorPool = pool->pool,
 	};
 	res = vkAllocateDescriptorSets(renderer->dev->dev, &ds_info, ds);
@@ -168,6 +169,7 @@ static void destroy_render_format_setup(struct wlr_vk_renderer *renderer,
 	vkDestroyRenderPass(dev, setup->render_pass, NULL);
 	vkDestroyPipeline(dev, setup->tex_identity_pipe, NULL);
 	vkDestroyPipeline(dev, setup->tex_srgb_pipe, NULL);
+	vkDestroyPipeline(dev, setup->tex_nv12_pipe, NULL);
 	vkDestroyPipeline(dev, setup->quad_pipe, NULL);
 }
 
@@ -1182,11 +1184,14 @@ static bool vulkan_render_subtexture_with_matrix(struct wlr_renderer *wlr_render
 
 	VkPipeline pipe;
 	// SRGB formats already have the transfer function applied
-	if (texture->format->is_srgb) {
+	if (texture->format->drm == DRM_FORMAT_NV12) {
+		pipe = renderer->current_render_buffer->render_setup->tex_nv12_pipe;
+	} else if (texture->format->is_srgb) {
 		pipe = renderer->current_render_buffer->render_setup->tex_identity_pipe;
 	} else {
 		pipe = renderer->current_render_buffer->render_setup->tex_srgb_pipe;
 	}
+	assert(pipe != VK_NULL_HANDLE);
 	if (pipe != renderer->bound_pipe) {
 		vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_GRAPHICS, pipe);
 		renderer->bound_pipe = pipe;
@@ -1403,6 +1408,7 @@ static void vulkan_destroy(struct wlr_renderer *wlr_renderer) {
 	vkDestroyPipelineLayout(dev->dev, renderer->pipe_layout, NULL);
 	vkDestroyDescriptorSetLayout(dev->dev, renderer->ds_layout, NULL);
 	vkDestroySampler(dev->dev, renderer->sampler, NULL);
+	vkDestroySamplerYcbcrConversion(dev->dev, renderer->nv12_conversion, NULL);
 	vkDestroyCommandPool(dev->dev, renderer->command_pool, NULL);
 
 	if (renderer->read_pixels_cache.initialized) {
@@ -1671,6 +1677,47 @@ static const struct wlr_renderer_impl renderer_impl = {
 	.texture_from_buffer = vulkan_texture_from_buffer,
 };
 
+static bool init_nv12_sampler(struct wlr_vk_renderer *renderer, VkSampler *sampler) {
+	VkResult res;
+
+	VkSamplerYcbcrConversionCreateInfo conversion_create_info = {
+		.sType = VK_STRUCTURE_TYPE_SAMPLER_YCBCR_CONVERSION_CREATE_INFO,
+		.format = VK_FORMAT_G8_B8R8_2PLANE_420_UNORM,
+		.ycbcrModel = VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_601,
+		.ycbcrRange = VK_SAMPLER_YCBCR_RANGE_ITU_NARROW,
+		.xChromaOffset = VK_CHROMA_LOCATION_MIDPOINT,
+		.yChromaOffset = VK_CHROMA_LOCATION_MIDPOINT,
+		.chromaFilter = VK_FILTER_LINEAR,
+	};
+	res = vkCreateSamplerYcbcrConversion(renderer->dev->dev, &conversion_create_info, NULL, &renderer->nv12_conversion);
+	if (res != VK_SUCCESS) {
+		wlr_vk_error("vkCreateSamplerYcbcrConversion", res);
+		return false;
+	}
+
+	VkSamplerYcbcrConversionInfo conversion_info = {
+		.sType = VK_STRUCTURE_TYPE_SAMPLER_YCBCR_CONVERSION_INFO,
+		.conversion = renderer->nv12_conversion,
+	};
+	VkSamplerCreateInfo sampler_create_info = {
+		.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO,
+		.pNext = &conversion_info,
+		.magFilter = VK_FILTER_LINEAR,
+		.minFilter = VK_FILTER_LINEAR,
+		.addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
+		.addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
+		.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
+		.borderColor = VK_BORDER_COLOR_FLOAT_OPAQUE_BLACK,
+	};
+	res = vkCreateSampler(renderer->dev->dev, &sampler_create_info, NULL, sampler);
+	if (res != VK_SUCCESS) {
+		wlr_vk_error("vkCreateSampler", res);
+		return false;
+	}
+
+	return true;
+}
+
 // Initializes the VkDescriptorSetLayout and VkPipelineLayout needed
 // for the texture rendering pipeline using the given VkSampler.
 static bool init_tex_layouts(struct wlr_vk_renderer *renderer,
@@ -1889,10 +1936,18 @@ static bool init_static_render_data(struct wlr_vk_renderer *renderer) {
 		return false;
 	}
 
+	if (renderer->dev->sampler_ycbcr_conversion && !init_nv12_sampler(renderer, &renderer->nv12_sampler)) {
+		return false;
+	}
+
 	if (!init_tex_layouts(renderer, renderer->sampler,
 			&renderer->ds_layout, &renderer->pipe_layout)) {
 		return false;
 	}
+	if (renderer->dev->sampler_ycbcr_conversion && !init_tex_layouts(renderer, renderer->nv12_sampler,
+			&renderer->nv12_ds_layout, &renderer->nv12_pipe_layout)) {
+		return false;
+	}
 
 	// load vert module and tex frag module since they are needed to
 	// initialize the tex pipeline
@@ -2033,6 +2088,12 @@ static struct wlr_vk_render_format_setup *find_or_create_render_setup(
 		goto error;
 	}
 
+	if (renderer->dev->sampler_ycbcr_conversion && !init_tex_pipeline(renderer,
+			setup->render_pass, renderer->nv12_pipe_layout,
+			WLR_VK_TEXTURE_TRANSFORM_SRGB, &setup->tex_nv12_pipe)) {
+		goto error;
+	}
+
 	VkPipelineShaderStageCreateInfo quad_stages[2] = {
 		{
 			.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
diff --git a/render/vulkan/texture.c b/render/vulkan/texture.c
index 124b041f..04530135 100644
--- a/render/vulkan/texture.c
+++ b/render/vulkan/texture.c
@@ -1,5 +1,6 @@
 #define _POSIX_C_SOURCE 200809L
 #include <assert.h>
+#include <drm_fourcc.h>
 #include <fcntl.h>
 #include <stdbool.h>
 #include <stdlib.h>
@@ -271,7 +272,7 @@ static struct wlr_texture *vulkan_texture_from_pixels(
 
 	const struct wlr_vk_format_props *fmt =
 		vulkan_format_props_from_drm(renderer->dev, drm_fmt);
-	if (fmt == NULL) {
+	if (fmt == NULL && fmt->format.is_ycbcr) {
 		wlr_log(WLR_ERROR, "Unsupported pixel format %"PRIx32 " (%.4s)",
 			drm_fmt, (const char*) &drm_fmt);
 		return NULL;
@@ -369,7 +370,7 @@ static struct wlr_texture *vulkan_texture_from_pixels(
 	}
 
 	// descriptor
-	texture->ds_pool = vulkan_alloc_texture_ds(renderer, &texture->ds);
+	texture->ds_pool = vulkan_alloc_texture_ds(renderer, renderer->ds_layout, &texture->ds);
 	if (!texture->ds_pool) {
 		wlr_log(WLR_ERROR, "failed to allocate descriptor");
 		goto error;
@@ -671,9 +672,11 @@ static struct wlr_vk_texture *vulkan_texture_from_dmabuf(
 		goto error;
 	}
 
-	const struct wlr_pixel_format_info *format_info = drm_get_pixel_format_info(attribs->format);
-	assert(format_info);
-	texture->has_alpha = format_info->has_alpha;
+	if (!fmt->format.is_ycbcr) {
+		const struct wlr_pixel_format_info *format_info = drm_get_pixel_format_info(attribs->format);
+		assert(format_info);
+		texture->has_alpha = format_info->has_alpha;
+	}
 
 	// view
 	VkImageViewCreateInfo view_info = {
@@ -697,6 +700,17 @@ static struct wlr_vk_texture *vulkan_texture_from_dmabuf(
 		.image = texture->image,
 	};
 
+	VkDescriptorSetLayout ds_layout = renderer->ds_layout;
+	VkSamplerYcbcrConversionInfo ycbcr_conversion_info;
+	if (fmt->format.is_ycbcr) {
+		ds_layout = renderer->nv12_ds_layout;
+		ycbcr_conversion_info = (VkSamplerYcbcrConversionInfo){
+			.sType = VK_STRUCTURE_TYPE_SAMPLER_YCBCR_CONVERSION_INFO,
+			.conversion = renderer->nv12_conversion,
+		};
+		view_info.pNext = &ycbcr_conversion_info;
+	}
+
 	res = vkCreateImageView(dev, &view_info, NULL, &texture->image_view);
 	if (res != VK_SUCCESS) {
 		wlr_vk_error("vkCreateImageView failed", res);
@@ -704,7 +718,7 @@ static struct wlr_vk_texture *vulkan_texture_from_dmabuf(
 	}
 
 	// descriptor
-	texture->ds_pool = vulkan_alloc_texture_ds(renderer, &texture->ds);
+	texture->ds_pool = vulkan_alloc_texture_ds(renderer, ds_layout, &texture->ds);
 	if (!texture->ds_pool) {
 		wlr_log(WLR_ERROR, "failed to allocate descriptor");
 		goto error;
diff --git a/render/vulkan/vulkan.c b/render/vulkan/vulkan.c
index e2efee09..8ac9c7f5 100644
--- a/render/vulkan/vulkan.c
+++ b/render/vulkan/vulkan.c
@@ -540,6 +540,19 @@ struct wlr_vk_device *vulkan_device_create(struct wlr_vk_instance *ini,
 			"falling back to blocking");
 	}
 
+	VkPhysicalDeviceSamplerYcbcrConversionFeatures phdev_sampler_ycbcr_features = {
+		.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SAMPLER_YCBCR_CONVERSION_FEATURES,
+	};
+	VkPhysicalDeviceFeatures2 phdev_features = {
+		.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2,
+		.pNext = &phdev_sampler_ycbcr_features,
+	};
+	vkGetPhysicalDeviceFeatures2(phdev, &phdev_features);
+
+	dev->sampler_ycbcr_conversion = phdev_sampler_ycbcr_features.samplerYcbcrConversion;
+	wlr_log(WLR_DEBUG, "Sampler YCbCr conversion %s",
+		dev->sampler_ycbcr_conversion ? "supported" : "not supported");
+
 	const float prio = 1.f;
 	VkDeviceQueueCreateInfo qinfo = {
 		.sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO,
@@ -548,8 +561,13 @@ struct wlr_vk_device *vulkan_device_create(struct wlr_vk_instance *ini,
 		.pQueuePriorities = &prio,
 	};
 
+	VkPhysicalDeviceSamplerYcbcrConversionFeatures sampler_ycbcr_features = {
+		.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SAMPLER_YCBCR_CONVERSION_FEATURES,
+		.samplerYcbcrConversion = dev->sampler_ycbcr_conversion,
+	};
 	VkPhysicalDeviceSynchronization2FeaturesKHR sync2_features = {
 		.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SYNCHRONIZATION_2_FEATURES_KHR,
+		.pNext = &sampler_ycbcr_features,
 		.synchronization2 = VK_TRUE,
 	};
 	VkPhysicalDeviceTimelineSemaphoreFeaturesKHR timeline_features = {