bug-14746: add batched submit for semaphores and command buffers

11 files changed, 121 insertions, 62 deletions

diff --git a/demos/cube.c b/demos/cube.c
index aed2b931..7903ad8d 100644
--- a/demos/cube.c
+++ b/demos/cube.c
@@ -436,8 +436,16 @@ static void demo_flush_init_cmd(struct demo *demo)
 
     const VkCmdBuffer cmd_bufs[] = { demo->cmd };
     VkFence nullFence = { VK_NULL_HANDLE };
+    VkSubmitInfo submit_info = {
+        .waitSemCount = 0,
+        .pWaitSemaphores = NULL,
+        .cmdBufferCount = 1,
+        .pCommandBuffers = cmd_bufs,
+        .signalSemCount = 0,
+        .pSignalSemaphores = NULL
+    };
 
-    err = vkQueueSubmit(demo->queue, 1, cmd_bufs, nullFence);
+    err = vkQueueSubmit(demo->queue, 1, &submit_info, nullFence);
     assert(!err);
 
     err = vkQueueWaitIdle(demo->queue);
@@ -637,13 +645,6 @@ static void demo_draw(struct demo *demo)
     // return codes
     assert(!err);
 
-    // Wait for the present complete semaphore to be signaled to ensure
-    // that the image won't be rendered to until the presentation
-    // engine has fully released ownership to the application, and it is
-    // okay to render to the image.
-    err = vkQueueWaitSemaphore(demo->queue, presentCompleteSemaphore);
-    assert(!err);
-
     // Assume the command buffer has been run on current_buffer before so
     // we need to set the image layout back to COLOR_ATTACHMENT_OPTIMAL
     demo_set_image_layout(demo, demo->buffers[demo->current_buffer].image,
@@ -652,9 +653,22 @@ static void demo_draw(struct demo *demo)
                            VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
     demo_flush_init_cmd(demo);
 
+    // Wait for the present complete semaphore to be signaled to ensure
+    // that the image won't be rendered to until the presentation
+    // engine has fully released ownership to the application, and it is
+    // okay to render to the image.
+
 // FIXME/TODO: DEAL WITH VK_IMAGE_LAYOUT_PRESENT_SOURCE_KHR
-    err = vkQueueSubmit(demo->queue, 1, &demo->buffers[demo->current_buffer].cmd,
-            nullFence);
+    VkSubmitInfo submit_info = {
+        .waitSemCount = 1,
+        .pWaitSemaphores = &presentCompleteSemaphore,
+        .cmdBufferCount = 1,
+        .pCommandBuffers = &demo->buffers[demo->current_buffer].cmd,
+        .signalSemCount = 0,
+        .pSignalSemaphores = NULL
+    };
+
+    err = vkQueueSubmit(demo->queue, 1, &submit_info, nullFence);
     assert(!err);
 
     VkPresentInfoKHR present = {
diff --git a/demos/tri.c b/demos/tri.c
index 9b276e91..b4f1700d 100644
--- a/demos/tri.c
+++ b/demos/tri.c
@@ -276,8 +276,16 @@ static void demo_flush_init_cmd(struct demo *demo)
 
     const VkCmdBuffer cmd_bufs[] = { demo->setup_cmd };
     VkFence nullFence = {VK_NULL_HANDLE};
+    VkSubmitInfo submit_info = {
+        .waitSemCount = 0,
+        .pWaitSemaphores = NULL,
+        .cmdBufferCount = 1,
+        .pCommandBuffers = cmd_bufs,
+        .signalSemCount = 0,
+        .pSignalSemaphores = NULL
+    };
 
-    err = vkQueueSubmit(demo->queue, 1, cmd_bufs, nullFence);
+    err = vkQueueSubmit(demo->queue, 1, &submit_info, nullFence);
     assert(!err);
 
     err = vkQueueWaitIdle(demo->queue);
@@ -455,13 +463,6 @@ static void demo_draw(struct demo *demo)
     // return codes
     assert(!err);
 
-    // Wait for the present complete semaphore to be signaled to ensure
-    // that the image won't be rendered to until the presentation
-    // engine has fully released ownership to the application, and it is
-    // okay to render to the image.
-    err = vkQueueWaitSemaphore(demo->queue, presentCompleteSemaphore);
-    assert(!err);
-
     // Assume the command buffer has been run on current_buffer before so
     // we need to set the image layout back to COLOR_ATTACHMENT_OPTIMAL
     demo_set_image_layout(demo, demo->buffers[demo->current_buffer].image,
@@ -470,11 +471,25 @@ static void demo_draw(struct demo *demo)
                            VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
     demo_flush_init_cmd(demo);
 
+    // Wait for the present complete semaphore to be signaled to ensure
+    // that the image won't be rendered to until the presentation
+    // engine has fully released ownership to the application, and it is
+    // okay to render to the image.
+
 // FIXME/TODO: DEAL WITH VK_IMAGE_LAYOUT_PRESENT_SOURCE_KHR
     demo_draw_build_cmd(demo);
     VkFence nullFence = { VK_NULL_HANDLE };
 
-    err = vkQueueSubmit(demo->queue, 1, &demo->draw_cmd, nullFence);
+    VkSubmitInfo submit_info = {
+        .waitSemCount = 1,
+        .pWaitSemaphores = &presentCompleteSemaphore,
+        .cmdBufferCount = 1,
+        .pCommandBuffers = &demo->draw_cmd,
+        .signalSemCount = 0,
+        .pSignalSemaphores = NULL
+    };
+
+    err = vkQueueSubmit(demo->queue, 1, &submit_info, nullFence);
     assert(!err);
 
     VkPresentInfoKHR present = {
diff --git a/icd/nulldrv/nulldrv.c b/icd/nulldrv/nulldrv.c
index 2f082472..3814bb0b 100644
--- a/icd/nulldrv/nulldrv.c
+++ b/icd/nulldrv/nulldrv.c
@@ -1818,8 +1818,8 @@ ICD_EXPORT VkResult VKAPI vkQueueWaitIdle(
 
 ICD_EXPORT VkResult VKAPI vkQueueSubmit(
     VkQueue                                   queue_,
-    uint32_t                                    cmdBufferCount,
-    const VkCmdBuffer*                       pCmdBuffers,
+    uint32_t                                  submitCount,
+    const VkSubmitInfo*                       pSubmitInfo,
     VkFence                                   fence_)
 {
     NULLDRV_LOG_FUNC;
diff --git a/include/vulkan.h b/include/vulkan.h
index eb427f87..570e6be0 100644
--- a/include/vulkan.h
+++ b/include/vulkan.h
@@ -97,12 +97,12 @@ VK_DEFINE_HANDLE(VkInstance)
 VK_DEFINE_HANDLE(VkPhysicalDevice)
 VK_DEFINE_HANDLE(VkDevice)
 VK_DEFINE_HANDLE(VkQueue)
+VK_DEFINE_NONDISP_HANDLE(VkSemaphore)
 VK_DEFINE_HANDLE(VkCmdBuffer)
 VK_DEFINE_NONDISP_HANDLE(VkFence)
 VK_DEFINE_NONDISP_HANDLE(VkDeviceMemory)
 VK_DEFINE_NONDISP_HANDLE(VkBuffer)
 VK_DEFINE_NONDISP_HANDLE(VkImage)
-VK_DEFINE_NONDISP_HANDLE(VkSemaphore)
 VK_DEFINE_NONDISP_HANDLE(VkEvent)
 VK_DEFINE_NONDISP_HANDLE(VkQueryPool)
 VK_DEFINE_NONDISP_HANDLE(VkBufferView)
@@ -1384,6 +1384,15 @@ typedef struct {
 } VkLayerProperties;
 
 typedef struct {
+    uint32_t                                    waitSemCount;
+    const VkSemaphore*                          pWaitSemaphores;
+    uint32_t                                    cmdBufferCount;
+    const VkCmdBuffer*                          pCommandBuffers;
+    uint32_t                                    signalSemCount;
+    const VkSemaphore*                          pSignalSemaphores;
+} VkSubmitInfo;
+
+typedef struct {
     VkStructureType                             sType;
     const void*                                 pNext;
     VkDeviceSize                                allocationSize;
@@ -2115,7 +2124,7 @@ typedef VkResult (VKAPI *PFN_vkEnumerateDeviceExtensionProperties)(VkPhysicalDev
 typedef VkResult (VKAPI *PFN_vkEnumerateInstanceLayerProperties)(uint32_t* pCount, VkLayerProperties* pProperties);
 typedef VkResult (VKAPI *PFN_vkEnumerateDeviceLayerProperties)(VkPhysicalDevice physicalDevice, uint32_t* pCount, VkLayerProperties* pProperties);
 typedef void (VKAPI *PFN_vkGetDeviceQueue)(VkDevice device, uint32_t queueFamilyIndex, uint32_t queueIndex, VkQueue* pQueue);
-typedef VkResult (VKAPI *PFN_vkQueueSubmit)(VkQueue queue, uint32_t cmdBufferCount, const VkCmdBuffer* pCmdBuffers, VkFence fence);
+typedef VkResult (VKAPI *PFN_vkQueueSubmit)(VkQueue queue, uint32_t submitCount, const VkSubmitInfo* pSubmitInfo, VkFence fence);
 typedef VkResult (VKAPI *PFN_vkQueueWaitIdle)(VkQueue queue);
 typedef VkResult (VKAPI *PFN_vkDeviceWaitIdle)(VkDevice device);
 typedef VkResult (VKAPI *PFN_vkAllocMemory)(VkDevice device, const VkMemoryAllocInfo* pAllocInfo, VkDeviceMemory* pMem);
@@ -2330,8 +2339,8 @@ void VKAPI vkGetDeviceQueue(
 
 VkResult VKAPI vkQueueSubmit(
     VkQueue                                     queue,
-    uint32_t                                    cmdBufferCount,
-    const VkCmdBuffer*                          pCmdBuffers,
+    uint32_t                                    submitCount,
+    const VkSubmitInfo*                         pSubmitInfo,
     VkFence                                     fence);
 
 VkResult VKAPI vkQueueWaitIdle(
diff --git a/layers/draw_state.cpp b/layers/draw_state.cpp
index 7a33ffb9..d36b1af8 100755
--- a/layers/draw_state.cpp
+++ b/layers/draw_state.cpp
@@ -1543,32 +1543,35 @@ VK_LAYER_EXPORT VkResult VKAPI vkEnumerateDeviceLayerProperties(
                                    pCount, pProperties);
 }
 
-VK_LAYER_EXPORT VkResult VKAPI vkQueueSubmit(VkQueue queue, uint32_t cmdBufferCount, const VkCmdBuffer* pCmdBuffers, VkFence fence)
+VK_LAYER_EXPORT VkResult VKAPI vkQueueSubmit(VkQueue queue, uint32_t submitCount, const VkSubmitInfo* pSubmitInfo, VkFence fence)
 {
     VkBool32 skipCall = VK_FALSE;
     GLOBAL_CB_NODE* pCB = NULL;
     layer_data* dev_data = get_my_data_ptr(get_dispatch_key(queue), layer_data_map);
-    for (uint32_t i=0; i < cmdBufferCount; i++) {
-        // Validate that cmd buffers have been updated
-        pCB = getCBNode(dev_data, pCmdBuffers[i]);
-        loader_platform_thread_lock_mutex(&globalLock);
-        pCB->submitCount++; // increment submit count
-        if ((pCB->beginInfo.flags & VK_CMD_BUFFER_OPTIMIZE_ONE_TIME_SUBMIT_BIT) && (pCB->submitCount > 1)) {
-            skipCall |= log_msg(dev_data->report_data, VK_DBG_REPORT_ERROR_BIT, VK_OBJECT_TYPE_COMMAND_BUFFER, 0, 0, DRAWSTATE_CMD_BUFFER_SINGLE_SUBMIT_VIOLATION, "DS",
-                    "CB %#" PRIxLEAST64 " was begun w/ VK_CMD_BUFFER_OPTIMIZE_ONE_TIME_SUBMIT_BIT set, but has been submitted %#" PRIxLEAST64 " times.", reinterpret_cast<uint64_t>(pCB->cmdBuffer), pCB->submitCount);
-        }
-        if (CB_UPDATE_COMPLETE != pCB->state) {
-            // Flag error for using CB w/o vkEndCommandBuffer() called
-            // TODO : How to pass cb as srcObj?
-            skipCall |= log_msg(dev_data->report_data, VK_DBG_REPORT_ERROR_BIT, VK_OBJECT_TYPE_COMMAND_BUFFER, 0, 0, DRAWSTATE_NO_END_CMD_BUFFER, "DS",
-                    "You must call vkEndCommandBuffer() on CB %#" PRIxLEAST64 " before this call to vkQueueSubmit()!", reinterpret_cast<uint64_t>(pCB->cmdBuffer));
+    for (uint32_t submit_idx = 0; submit_idx < submitCount; submit_idx++) {
+        const VkSubmitInfo *submit = &pSubmitInfo[submit_idx];
+        for (uint32_t i=0; i < submit->cmdBufferCount; i++) {
+            // Validate that cmd buffers have been updated
+            pCB = getCBNode(dev_data, submit->pCommandBuffers[i]);
+            loader_platform_thread_lock_mutex(&globalLock);
+            pCB->submitCount++; // increment submit count
+            if ((pCB->beginInfo.flags & VK_CMD_BUFFER_OPTIMIZE_ONE_TIME_SUBMIT_BIT) && (pCB->submitCount > 1)) {
+                skipCall |= log_msg(dev_data->report_data, VK_DBG_REPORT_ERROR_BIT, VK_OBJECT_TYPE_COMMAND_BUFFER, 0, 0, DRAWSTATE_CMD_BUFFER_SINGLE_SUBMIT_VIOLATION, "DS",
+                        "CB %#" PRIxLEAST64 " was begun w/ VK_CMD_BUFFER_OPTIMIZE_ONE_TIME_SUBMIT_BIT set, but has been submitted %#" PRIxLEAST64 " times.", reinterpret_cast<uint64_t>(pCB->cmdBuffer), pCB->submitCount);
+            }
+            if (CB_UPDATE_COMPLETE != pCB->state) {
+                // Flag error for using CB w/o vkEndCommandBuffer() called
+                // TODO : How to pass cb as srcObj?
+                skipCall |= log_msg(dev_data->report_data, VK_DBG_REPORT_ERROR_BIT, VK_OBJECT_TYPE_COMMAND_BUFFER, 0, 0, DRAWSTATE_NO_END_CMD_BUFFER, "DS",
+                        "You must call vkEndCommandBuffer() on CB %#" PRIxLEAST64 " before this call to vkQueueSubmit()!", reinterpret_cast<uint64_t>(pCB->cmdBuffer));
+                loader_platform_thread_unlock_mutex(&globalLock);
+                return VK_ERROR_VALIDATION_FAILED;
+            }
             loader_platform_thread_unlock_mutex(&globalLock);
-            return VK_ERROR_VALIDATION_FAILED;
         }
-        loader_platform_thread_unlock_mutex(&globalLock);
     }
     if (VK_FALSE == skipCall)
-        return dev_data->device_dispatch_table->QueueSubmit(queue, cmdBufferCount, pCmdBuffers, fence);
+        return dev_data->device_dispatch_table->QueueSubmit(queue, submitCount, pSubmitInfo, fence);
     return VK_ERROR_VALIDATION_FAILED;
 }
 
diff --git a/layers/mem_tracker.cpp b/layers/mem_tracker.cpp
index 4a1e85f6..704e007e 100644
--- a/layers/mem_tracker.cpp
+++ b/layers/mem_tracker.cpp
@@ -1141,8 +1141,8 @@ VK_LAYER_EXPORT void VKAPI vkGetDeviceQueue(
 
 VK_LAYER_EXPORT VkResult VKAPI vkQueueSubmit(
     VkQueue             queue,
-    uint32_t            cmdBufferCount,
-    const VkCmdBuffer  *pCmdBuffers,
+    uint32_t            submitCount,
+    const VkSubmitInfo *pSubmitInfo,
     VkFence             fence)
 {
     VkResult result = VK_ERROR_VALIDATION_FAILED;
@@ -1155,17 +1155,20 @@ VK_LAYER_EXPORT VkResult VKAPI vkQueueSubmit(
 
     print_mem_list(queue);
     printCBList(queue);
-    for (uint32_t i = 0; i < cmdBufferCount; i++) {
-        pCBInfo = get_cmd_buf_info(pCmdBuffers[i]);
-        pCBInfo->fenceId = fenceId;
-        pCBInfo->lastSubmittedFence = fence;
-        pCBInfo->lastSubmittedQueue = queue;
+    for (uint32_t submit_idx = 0; submit_idx < submitCount; submit_idx++) {
+        const VkSubmitInfo *submit = &pSubmitInfo[submit_idx];
+        for (uint32_t i = 0; i < submit->cmdBufferCount; i++) {
+            pCBInfo = get_cmd_buf_info(submit->pCommandBuffers[i]);
+            pCBInfo->fenceId = fenceId;
+            pCBInfo->lastSubmittedFence = fence;
+            pCBInfo->lastSubmittedQueue = queue;
+        }
     }
 
     loader_platform_thread_unlock_mutex(&globalLock);
     if (VK_FALSE == skipCall) {
         result = get_dispatch_table(mem_tracker_device_table_map, queue)->QueueSubmit(
-            queue, cmdBufferCount, pCmdBuffers, fence);
+            queue, submitCount, pSubmitInfo, fence);
     }
     return result;
 }
diff --git a/layers/param_checker.cpp b/layers/param_checker.cpp
index de4890a0..97727793 100644
--- a/layers/param_checker.cpp
+++ b/layers/param_checker.cpp
@@ -2199,15 +2199,17 @@ bool PostQueueSubmit(
 
 VK_LAYER_EXPORT VkResult VKAPI vkQueueSubmit(
     VkQueue queue,
-    uint32_t cmdBufferCount,
-    const VkCmdBuffer* pCmdBuffers,
+    uint32_t submitCount,
+    const VkSubmitInfo* pSubmitInfo,
     VkFence fence)
 {
-    PreQueueSubmit(queue, pCmdBuffers);
+    for (uint32_t i = 0; i < submitCount; i++) {
+        PreQueueSubmit(queue, pSubmitInfo[i].pCommandBuffers);
+    }
 
-    VkResult result = get_dispatch_table(pc_device_table_map, queue)->QueueSubmit(queue, cmdBufferCount, pCmdBuffers, fence);
+    VkResult result = get_dispatch_table(pc_device_table_map, queue)->QueueSubmit(queue, submitCount, pSubmitInfo, fence);
 
-    PostQueueSubmit(queue, cmdBufferCount, fence, result);
+    PostQueueSubmit(queue, submitCount, fence, result);
 
     return result;
 }
diff --git a/layers/screenshot.cpp b/layers/screenshot.cpp
index c468257a..05e7414d 100644
--- a/layers/screenshot.cpp
+++ b/layers/screenshot.cpp
@@ -244,7 +244,17 @@ static void writePPM( const char *filename, VkImage image1)
     err = pTableCmdBuffer->EndCommandBuffer(cmdBuffer);
     assert(!err);
 
-    err = pTableQueue->QueueSubmit(queue, 1, &cmdBuffer, VK_NULL_HANDLE);
+    VkFence nullFence = { VK_NULL_HANDLE };
+    VkSubmitInfo submit_info = {
+        .waitSemCount = 0,
+        .pWaitSemaphores = NULL,
+        .cmdBufferCount = 1,
+        .pCommandBuffers = &cmdBuffer,
+        .signalSemCount = 0,
+        .pSignalSemaphores = NULL
+    };
+
+    err = pTableQueue->QueueSubmit(queue, 1, &submit_info, nullFence);
     assert(!err);
 
     err = pTableQueue->QueueWaitIdle(queue);
diff --git a/loader/trampoline.c b/loader/trampoline.c
index cbf8a861..2faf002d 100644
--- a/loader/trampoline.c
+++ b/loader/trampoline.c
@@ -342,13 +342,13 @@ LOADER_EXPORT void VKAPI vkGetDeviceQueue(VkDevice device, uint32_t queueNodeInd
     loader_set_dispatch(*pQueue, disp);
 }
 
-LOADER_EXPORT VkResult VKAPI vkQueueSubmit(VkQueue queue, uint32_t cmdBufferCount, const VkCmdBuffer* pCmdBuffers, VkFence fence)
+LOADER_EXPORT VkResult VKAPI vkQueueSubmit(VkQueue queue, uint32_t submitCount, const VkSubmitInfo* pSubmitInfo, VkFence fence)
 {
     const VkLayerDispatchTable *disp;
 
     disp = loader_get_dispatch(queue);
 
-    return disp->QueueSubmit(queue, cmdBufferCount, pCmdBuffers, fence);
+    return disp->QueueSubmit(queue, submitCount, pSubmitInfo, fence);
 }
 
 LOADER_EXPORT VkResult VKAPI vkQueueWaitIdle(VkQueue queue)
diff --git a/vk_helper.py b/vk_helper.py
index f44beb21..7122297c 100755
--- a/vk_helper.py
+++ b/vk_helper.py
@@ -868,7 +868,10 @@ class StructWrapperGen:
                         else:
                             sh_funcs.append('%s' % lineinfo.get())
                             addr_char = ''
-                            if stp_list[index]['type'] in vulkan.core.objects:
+                            if stp_list[index]['type'] in vulkan.object_dispatch_list:
+                                sh_funcs.append('%sss[%u] << %spStruct->%s[i];' % (indent, index, addr_char, stp_list[index]['name']))
+                                sh_funcs.append('%sstp_strs[%u] += " " + prefix + "%s[" + index_ss.str() + "] = " + ss[%u].str() + "\\n";' % (indent, index, stp_list[index]['name'], index))
+                            elif stp_list[index]['type'] in vulkan.object_non_dispatch_list:
                                 sh_funcs.append('%sss[%u] << %spStruct->%s[i].handle;' % (indent, index, addr_char, stp_list[index]['name']))
                                 sh_funcs.append('%sstp_strs[%u] += " " + prefix + "%s[" + index_ss.str() + "].handle = " + ss[%u].str() + "\\n";' % (indent, index, stp_list[index]['name'], index))
                             else:
diff --git a/vulkan.py b/vulkan.py
index 6bc70b28..00968576 100755
--- a/vulkan.py
+++ b/vulkan.py
@@ -298,8 +298,8 @@ core = Extension(
 
         Proto("VkResult", "QueueSubmit",
             [Param("VkQueue", "queue"),
-             Param("uint32_t", "cmdBufferCount"),
-             Param("const VkCmdBuffer*", "pCmdBuffers"),
+             Param("uint32_t", "submitCount"),
+             Param("const VkSubmitInfo*", "pSubmitInfo"),
              Param("VkFence", "fence")]),
 
         Proto("VkResult", "QueueWaitIdle",