layers: GH934 Bounds checking on SetViewport

Add viewport bounds checking.

Change-Id: I8ec8a663c5ce3d90447f8f6228ce3ed8fcbb4454

2 files changed, 77 insertions, 11 deletions

diff --git a/layers/parameter_validation.cpp b/layers/parameter_validation.cpp
index bf7db2e6..443d3bf5 100644
--- a/layers/parameter_validation.cpp
+++ b/layers/parameter_validation.cpp
@@ -75,6 +75,7 @@ struct layer_data {
 
     bool swapchain_enabled = false;
     bool display_swapchain_enabled = false;
+    bool amd_negative_viewport_height_enabled = false;
 };
 
 static std::unordered_map<void *, struct instance_extension_enables> instance_extension_map;
@@ -1614,6 +1615,7 @@ static void CheckDeviceRegisterExtensions(const VkDeviceCreateInfo *pCreateInfo,
     layer_data *device_data = get_my_data_ptr(get_dispatch_key(device), layer_data_map);
     device_data->swapchain_enabled = false;
     device_data->display_swapchain_enabled = false;
+    device_data->amd_negative_viewport_height_enabled = false;
 
     for (uint32_t i = 0; i < pCreateInfo->enabledExtensionCount; i++) {
         if (strcmp(pCreateInfo->ppEnabledExtensionNames[i], VK_KHR_SWAPCHAIN_EXTENSION_NAME) == 0) {
@@ -1622,6 +1624,9 @@ static void CheckDeviceRegisterExtensions(const VkDeviceCreateInfo *pCreateInfo,
         if (strcmp(pCreateInfo->ppEnabledExtensionNames[i], VK_KHR_DISPLAY_SWAPCHAIN_EXTENSION_NAME) == 0) {
             device_data->display_swapchain_enabled = true;
         }
+        if (strcmp(pCreateInfo->ppEnabledExtensionNames[i], VK_AMD_NEGATIVE_VIEWPORT_HEIGHT_EXTENSION_NAME) == 0) {
+            device_data->amd_negative_viewport_height_enabled = true;
+        }
     }
 }
 
@@ -3949,9 +3954,70 @@ VKAPI_ATTR void VKAPI_CALL CmdBindPipeline(VkCommandBuffer commandBuffer, VkPipe
     }
 }
 
-bool preCmdSetViewport(debug_report_data *report_data, uint32_t viewport_count, const VkViewport *viewports) {
+bool preCmdSetViewport(layer_data *my_data, uint32_t viewport_count, const VkViewport *viewports) {
+    debug_report_data *report_data = my_data->report_data;
+
     bool skip =
         validate_array(report_data, "vkCmdSetViewport", "viewportCount", "pViewports", viewport_count, viewports, true, true);
+
+    if (viewport_count > 0 && viewports != nullptr) {
+        const VkPhysicalDeviceLimits &limits = my_data->device_limits;
+        for (uint32_t viewportIndex = 0; viewportIndex < viewport_count; ++viewportIndex) {
+            const VkViewport &viewport = viewports[viewportIndex];
+
+            if (viewport.width <= 0 || viewport.width > limits.maxViewportDimensions[0]) {
+                skip |= log_msg(report_data, VK_DEBUG_REPORT_ERROR_BIT_EXT, VK_DEBUG_REPORT_OBJECT_TYPE_UNKNOWN_EXT, 0, __LINE__,
+                                VALIDATION_ERROR_01448, LayerName,
+                                "vkCmdSetViewport %d: width (%f) exceeds permitted bounds (0,%u). %s", viewportIndex,
+                                viewport.width, limits.maxViewportDimensions[0], validation_error_map[VALIDATION_ERROR_01448]);
+            }
+
+            bool invalid_height = (viewport.height <= 0 || viewport.height > limits.maxViewportDimensions[1]);
+            if (my_data->amd_negative_viewport_height_enabled && (viewport.height < 0)) {
+                // VALIDATION_ERROR_01790
+                invalid_height = false;
+            }
+            if (invalid_height) {
+                skip |= log_msg(report_data, VK_DEBUG_REPORT_ERROR_BIT_EXT, VK_DEBUG_REPORT_OBJECT_TYPE_UNKNOWN_EXT, 0, __LINE__,
+                                VALIDATION_ERROR_01449, LayerName,
+                                "vkCmdSetViewport %d: height (%f) exceeds permitted bounds (0,%u). %s", viewportIndex,
+                                viewport.height, limits.maxViewportDimensions[1], validation_error_map[VALIDATION_ERROR_01449]);
+            }
+
+            if (viewport.x < limits.viewportBoundsRange[0] || viewport.x > limits.viewportBoundsRange[1]) {
+                skip |=
+                    log_msg(report_data, VK_DEBUG_REPORT_ERROR_BIT_EXT, VK_DEBUG_REPORT_OBJECT_TYPE_UNKNOWN_EXT, 0, __LINE__,
+                            VALIDATION_ERROR_01450, LayerName, "vkCmdSetViewport %d: x (%f) exceeds permitted bounds (%f,%f). %s",
+                            viewportIndex, viewport.x, limits.viewportBoundsRange[0], limits.viewportBoundsRange[1],
+                            validation_error_map[VALIDATION_ERROR_01450]);
+            }
+
+            if (viewport.y < limits.viewportBoundsRange[0] || viewport.y > limits.viewportBoundsRange[1]) {
+                skip |=
+                    log_msg(report_data, VK_DEBUG_REPORT_ERROR_BIT_EXT, VK_DEBUG_REPORT_OBJECT_TYPE_UNKNOWN_EXT, 0, __LINE__,
+                            VALIDATION_ERROR_01450, LayerName, "vkCmdSetViewport %d: y (%f) exceeds permitted bounds (%f,%f). %s",
+                            viewportIndex, viewport.y, limits.viewportBoundsRange[0], limits.viewportBoundsRange[1],
+                            validation_error_map[VALIDATION_ERROR_01450]);
+            }
+
+            if (viewport.x + viewport.width > limits.viewportBoundsRange[1]) {
+                skip |=
+                    log_msg(report_data, VK_DEBUG_REPORT_ERROR_BIT_EXT, VK_DEBUG_REPORT_OBJECT_TYPE_UNKNOWN_EXT, 0, __LINE__,
+                            VALIDATION_ERROR_01451, LayerName,
+                            "vkCmdSetViewport %d: x (%f) + width (%f) exceeds permitted bound (%f). %s", viewportIndex, viewport.x,
+                            viewport.width, limits.viewportBoundsRange[1], validation_error_map[VALIDATION_ERROR_01451]);
+            }
+
+            if (viewport.y + viewport.height > limits.viewportBoundsRange[1]) {
+                skip |=
+                    log_msg(report_data, VK_DEBUG_REPORT_ERROR_BIT_EXT, VK_DEBUG_REPORT_OBJECT_TYPE_UNKNOWN_EXT, 0, __LINE__,
+                            VALIDATION_ERROR_01452, LayerName,
+                            "vkCmdSetViewport %d: y (%f) + height (%f) exceeds permitted bound (%f). %s", viewportIndex, viewport.y,
+                            viewport.height, limits.viewportBoundsRange[1], validation_error_map[VALIDATION_ERROR_01452]);
+            }
+        }
+    }
+
     return skip;
 }
 
@@ -3961,7 +4027,7 @@ VKAPI_ATTR void VKAPI_CALL CmdSetViewport(VkCommandBuffer commandBuffer, uint32_
     layer_data *my_data = get_my_data_ptr(get_dispatch_key(commandBuffer), layer_data_map);
     assert(my_data != NULL);
 
-    skip |= preCmdSetViewport(my_data->report_data, viewportCount, pViewports);
+    skip |= preCmdSetViewport(my_data, viewportCount, pViewports);
 
     if (!skip) {
         get_dispatch_table(pc_device_table_map, commandBuffer)
diff --git a/layers/vk_validation_error_database.txt b/layers/vk_validation_error_database.txt
index 404068b3..8139589f 100644
--- a/layers/vk_validation_error_database.txt
+++ b/layers/vk_validation_error_database.txt
@@ -1435,11 +1435,11 @@ VALIDATION_ERROR_01444~^~U~^~Unknown~^~vkCmdSetViewport~^~For more information r
 VALIDATION_ERROR_01445~^~U~^~Unknown~^~vkCmdSetViewport~^~For more information refer to Vulkan Spec Section '23.5. Controlling the Viewport' which states 'commandBuffer must be in the recording state' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#vkCmdSetViewport)~^~
 VALIDATION_ERROR_01446~^~U~^~Unknown~^~vkCmdSetViewport~^~For more information refer to Vulkan Spec Section '23.5. Controlling the Viewport' which states 'The VkCommandPool that commandBuffer was allocated from must support graphics operations' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#vkCmdSetViewport)~^~
 VALIDATION_ERROR_01447~^~U~^~Unknown~^~vkCmdSetViewport~^~For more information refer to Vulkan Spec Section '23.5. Controlling the Viewport' which states 'viewportCount must be greater than 0' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#vkCmdSetViewport)~^~
-VALIDATION_ERROR_01448~^~U~^~Unknown~^~vkCmdSetViewport~^~For more information refer to Vulkan Spec Section '23.5. Controlling the Viewport' which states 'width must be greater than 0.0 and less than or equal to VkPhysicalDeviceLimits::maxViewportDimensions[0]' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#VkViewport)~^~
-VALIDATION_ERROR_01449~^~U~^~Unknown~^~vkCmdSetViewport~^~For more information refer to Vulkan Spec Section '23.5. Controlling the Viewport' which states 'height must be greater than 0.0 and less than or equal to VkPhysicalDeviceLimits::maxViewportDimensions[1]' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#VkViewport)~^~
-VALIDATION_ERROR_01450~^~U~^~Unknown~^~vkCmdSetViewport~^~For more information refer to Vulkan Spec Section '23.5. Controlling the Viewport' which states 'x and y must each be between viewportBoundsRange[0] and viewportBoundsRange[1], inclusive' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#VkViewport)~^~
-VALIDATION_ERROR_01451~^~U~^~Unknown~^~vkCmdSetViewport~^~For more information refer to Vulkan Spec Section '23.5. Controlling the Viewport' which states 'x + width must be less than or equal to viewportBoundsRange[1]' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#VkViewport)~^~
-VALIDATION_ERROR_01452~^~U~^~Unknown~^~vkCmdSetViewport~^~For more information refer to Vulkan Spec Section '23.5. Controlling the Viewport' which states 'y + height must be less than or equal to viewportBoundsRange[1]' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#VkViewport)~^~
+VALIDATION_ERROR_01448~^~Y~^~ViewportAndScissorBoundsChecking~^~vkCmdSetViewport~^~For more information refer to Vulkan Spec Section '23.5. Controlling the Viewport' which states 'width must be greater than 0.0 and less than or equal to VkPhysicalDeviceLimits::maxViewportDimensions[0]' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#VkViewport)~^~
+VALIDATION_ERROR_01449~^~Y~^~ViewportAndScissorBoundsChecking~^~vkCmdSetViewport~^~For more information refer to Vulkan Spec Section '23.5. Controlling the Viewport' which states 'height must be greater than 0.0 and less than or equal to VkPhysicalDeviceLimits::maxViewportDimensions[1]' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#VkViewport)~^~
+VALIDATION_ERROR_01450~^~Y~^~ViewportAndScissorBoundsChecking~^~vkCmdSetViewport~^~For more information refer to Vulkan Spec Section '23.5. Controlling the Viewport' which states 'x and y must each be between viewportBoundsRange[0] and viewportBoundsRange[1], inclusive' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#VkViewport)~^~
+VALIDATION_ERROR_01451~^~Y~^~ViewportAndScissorBoundsChecking~^~vkCmdSetViewport~^~For more information refer to Vulkan Spec Section '23.5. Controlling the Viewport' which states 'x + width must be less than or equal to viewportBoundsRange[1]' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#VkViewport)~^~
+VALIDATION_ERROR_01452~^~Y~^~ViewportAndScissorBoundsChecking~^~vkCmdSetViewport~^~For more information refer to Vulkan Spec Section '23.5. Controlling the Viewport' which states 'y + height must be less than or equal to viewportBoundsRange[1]' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#VkViewport)~^~
 VALIDATION_ERROR_01453~^~U~^~Unknown~^~vkCmdSetViewport~^~For more information refer to Vulkan Spec Section '23.5. Controlling the Viewport' which states 'minDepth must be between 0.0 and 1.0, inclusive' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#VkViewport)~^~
 VALIDATION_ERROR_01454~^~U~^~Unknown~^~vkCmdSetViewport~^~For more information refer to Vulkan Spec Section '23.5. Controlling the Viewport' which states 'maxDepth must be between 0.0 and 1.0, inclusive' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#VkViewport)~^~
 VALIDATION_ERROR_01455~^~U~^~Unknown~^~vkCmdSetViewport~^~For more information refer to Vulkan Spec Section '23.5. Controlling the Viewport' which states 'If the depth clamping feature is not enabled, depthClampEnable must be VK_FALSE' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#VkPipelineRasterizationStateCreateInfo)~^~
@@ -1471,9 +1471,9 @@ VALIDATION_ERROR_01485~^~U~^~Unknown~^~vkCmdSetDepthBias~^~For more information
 VALIDATION_ERROR_01486~^~U~^~Unknown~^~vkCmdSetScissor~^~For more information refer to Vulkan Spec Section '25.2. Scissor Test' which states 'The currently bound graphics pipeline must have been created with the VK_DYNAMIC_STATE_SCISSOR dynamic state enabled' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#vkCmdSetScissor)~^~
 VALIDATION_ERROR_01487~^~U~^~Unknown~^~vkCmdSetScissor~^~For more information refer to Vulkan Spec Section '25.2. Scissor Test' which states 'firstScissor must be less than VkPhysicalDeviceLimits::maxViewports' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#vkCmdSetScissor)~^~
 VALIDATION_ERROR_01488~^~U~^~Unknown~^~vkCmdSetScissor~^~For more information refer to Vulkan Spec Section '25.2. Scissor Test' which states 'The sum of firstScissor and scissorCount must be between 1 and VkPhysicalDeviceLimits::maxViewports, inclusive' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#vkCmdSetScissor)~^~
-VALIDATION_ERROR_01489~^~Y~^~ViewportBoundsChecking~^~vkCmdSetScissor~^~For more information refer to Vulkan Spec Section '25.2. Scissor Test' which states 'The x and y members of offset must be greater than or equal to 0' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#vkCmdSetScissor)~^~
-VALIDATION_ERROR_01490~^~Y~^~ViewportBoundsChecking~^~vkCmdSetScissor~^~For more information refer to Vulkan Spec Section '25.2. Scissor Test' which states 'Evaluation of (offset.x + extent.width) must not cause a signed integer addition overflow' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#vkCmdSetScissor)~^~
-VALIDATION_ERROR_01491~^~Y~^~ViewportBoundsChecking~^~vkCmdSetScissor~^~For more information refer to Vulkan Spec Section '25.2. Scissor Test' which states 'Evaluation of (offset.y + extent.height) must not cause a signed integer addition overflow' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#vkCmdSetScissor)~^~
+VALIDATION_ERROR_01489~^~Y~^~ViewportAndScissorBoundsChecking~^~vkCmdSetScissor~^~For more information refer to Vulkan Spec Section '25.2. Scissor Test' which states 'The x and y members of offset must be greater than or equal to 0' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#vkCmdSetScissor)~^~
+VALIDATION_ERROR_01490~^~Y~^~ViewportAndScissorBoundsChecking~^~vkCmdSetScissor~^~For more information refer to Vulkan Spec Section '25.2. Scissor Test' which states 'Evaluation of (offset.x + extent.width) must not cause a signed integer addition overflow' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#vkCmdSetScissor)~^~
+VALIDATION_ERROR_01491~^~Y~^~ViewportAndScissorBoundsChecking~^~vkCmdSetScissor~^~For more information refer to Vulkan Spec Section '25.2. Scissor Test' which states 'Evaluation of (offset.y + extent.height) must not cause a signed integer addition overflow' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#vkCmdSetScissor)~^~
 VALIDATION_ERROR_01492~^~Y~^~Unknown~^~vkCmdSetScissor~^~For more information refer to Vulkan Spec Section '25.2. Scissor Test' which states 'commandBuffer must be a valid VkCommandBuffer handle' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#vkCmdSetScissor)~^~
 VALIDATION_ERROR_01493~^~U~^~Unknown~^~vkCmdSetScissor~^~For more information refer to Vulkan Spec Section '25.2. Scissor Test' which states 'pScissors must be a pointer to an array of scissorCount VkRect2D structures' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#vkCmdSetScissor)~^~
 VALIDATION_ERROR_01494~^~U~^~Unknown~^~vkCmdSetScissor~^~For more information refer to Vulkan Spec Section '25.2. Scissor Test' which states 'commandBuffer must be in the recording state' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#vkCmdSetScissor)~^~
@@ -1739,7 +1739,7 @@ VALIDATION_ERROR_01786~^~U~^~Unknown~^~vkCmdDrawIndexedIndirectCountAMD~^~For mo
 VALIDATION_ERROR_01787~^~U~^~Unknown~^~vkCmdDrawIndexedIndirectCountAMD~^~For more information refer to Vulkan Spec Section '19.2. Programmable Primitive Shading' which states 'The VkCommandPool that commandBuffer was allocated from must support graphics operations' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#vkCmdDrawIndexedIndirectCountAMD)~^~
 VALIDATION_ERROR_01788~^~U~^~Unknown~^~vkCmdDrawIndexedIndirectCountAMD~^~For more information refer to Vulkan Spec Section '19.2. Programmable Primitive Shading' which states 'This command must only be called inside of a render pass instance' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#vkCmdDrawIndexedIndirectCountAMD)~^~
 VALIDATION_ERROR_01789~^~U~^~Unknown~^~vkCmdDrawIndexedIndirectCountAMD~^~For more information refer to Vulkan Spec Section '19.2. Programmable Primitive Shading' which states 'Each of buffer, commandBuffer, and countBuffer must have been created, allocated, or retrieved from the same VkDevice' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#vkCmdDrawIndexedIndirectCountAMD)~^~
-VALIDATION_ERROR_01790~^~U~^~Unknown~^~vkCmdSetViewport~^~For more information refer to Vulkan Spec Section '23.5. Controlling the Viewport' which states 'If the VK_AMD_negative_viewport_height extension is enabled, height can also be negative.' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#VkViewport)~^~
+VALIDATION_ERROR_01790~^~Y~^~ViewportAndScissorBoundsChecking~^~vkCmdSetViewport~^~For more information refer to Vulkan Spec Section '23.5. Controlling the Viewport' which states 'If the VK_AMD_negative_viewport_height extension is enabled, height can also be negative.' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#VkViewport)~^~
 VALIDATION_ERROR_01791~^~U~^~Unknown~^~vkCmdSetViewport~^~For more information refer to Vulkan Spec Section '24.2. Rasterization Order' which states 'sType must be VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_RASTERIZATION_ORDER_AMD' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#VkPipelineRasterizationStateRasterizationOrderAMD)~^~
 VALIDATION_ERROR_01792~^~U~^~Unknown~^~vkCmdSetViewport~^~For more information refer to Vulkan Spec Section '24.2. Rasterization Order' which states 'pNext must be NULL' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#VkPipelineRasterizationStateRasterizationOrderAMD)~^~
 VALIDATION_ERROR_01793~^~U~^~Unknown~^~vkCmdSetViewport~^~For more information refer to Vulkan Spec Section '24.2. Rasterization Order' which states 'rasterizationOrder must be a valid VkRasterizationOrderAMD value' (https://www.khronos.org/registry/vulkan/specs/1.0-extensions/xhtml/vkspec.html#VkPipelineRasterizationStateRasterizationOrderAMD)~^~