From 5fa898decb2f22b30693d89355c0e67be93288ab Mon Sep 17 00:00:00 2001 From: Courtney Goeltzenleuchter Date: Tue, 20 Oct 2015 18:04:07 -0600 Subject: bug-14746: add batched submit for semaphores and command buffers --- demos/cube.c | 34 ++++++++++++++++++++++++---------- demos/tri.c | 33 ++++++++++++++++++++++++--------- icd/nulldrv/nulldrv.c | 4 ++-- include/vulkan.h | 17 +++++++++++++---- layers/draw_state.cpp | 39 +++++++++++++++++++++------------------ layers/mem_tracker.cpp | 19 +++++++++++-------- layers/param_checker.cpp | 12 +++++++----- layers/screenshot.cpp | 12 +++++++++++- loader/trampoline.c | 4 ++-- vk_helper.py | 5 ++++- vulkan.py | 4 ++-- 11 files changed, 121 insertions(+), 62 deletions(-) diff --git a/demos/cube.c b/demos/cube.c index aed2b931..7903ad8d 100644 --- a/demos/cube.c +++ b/demos/cube.c @@ -436,8 +436,16 @@ static void demo_flush_init_cmd(struct demo *demo) const VkCmdBuffer cmd_bufs[] = { demo->cmd }; VkFence nullFence = { VK_NULL_HANDLE }; + VkSubmitInfo submit_info = { + .waitSemCount = 0, + .pWaitSemaphores = NULL, + .cmdBufferCount = 1, + .pCommandBuffers = cmd_bufs, + .signalSemCount = 0, + .pSignalSemaphores = NULL + }; - err = vkQueueSubmit(demo->queue, 1, cmd_bufs, nullFence); + err = vkQueueSubmit(demo->queue, 1, &submit_info, nullFence); assert(!err); err = vkQueueWaitIdle(demo->queue); @@ -637,13 +645,6 @@ static void demo_draw(struct demo *demo) // return codes assert(!err); - // Wait for the present complete semaphore to be signaled to ensure - // that the image won't be rendered to until the presentation - // engine has fully released ownership to the application, and it is - // okay to render to the image. - err = vkQueueWaitSemaphore(demo->queue, presentCompleteSemaphore); - assert(!err); - // Assume the command buffer has been run on current_buffer before so // we need to set the image layout back to COLOR_ATTACHMENT_OPTIMAL demo_set_image_layout(demo, demo->buffers[demo->current_buffer].image, @@ -652,9 +653,22 @@ static void demo_draw(struct demo *demo) VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL); demo_flush_init_cmd(demo); + // Wait for the present complete semaphore to be signaled to ensure + // that the image won't be rendered to until the presentation + // engine has fully released ownership to the application, and it is + // okay to render to the image. + // FIXME/TODO: DEAL WITH VK_IMAGE_LAYOUT_PRESENT_SOURCE_KHR - err = vkQueueSubmit(demo->queue, 1, &demo->buffers[demo->current_buffer].cmd, - nullFence); + VkSubmitInfo submit_info = { + .waitSemCount = 1, + .pWaitSemaphores = &presentCompleteSemaphore, + .cmdBufferCount = 1, + .pCommandBuffers = &demo->buffers[demo->current_buffer].cmd, + .signalSemCount = 0, + .pSignalSemaphores = NULL + }; + + err = vkQueueSubmit(demo->queue, 1, &submit_info, nullFence); assert(!err); VkPresentInfoKHR present = { diff --git a/demos/tri.c b/demos/tri.c index 9b276e91..b4f1700d 100644 --- a/demos/tri.c +++ b/demos/tri.c @@ -276,8 +276,16 @@ static void demo_flush_init_cmd(struct demo *demo) const VkCmdBuffer cmd_bufs[] = { demo->setup_cmd }; VkFence nullFence = {VK_NULL_HANDLE}; + VkSubmitInfo submit_info = { + .waitSemCount = 0, + .pWaitSemaphores = NULL, + .cmdBufferCount = 1, + .pCommandBuffers = cmd_bufs, + .signalSemCount = 0, + .pSignalSemaphores = NULL + }; - err = vkQueueSubmit(demo->queue, 1, cmd_bufs, nullFence); + err = vkQueueSubmit(demo->queue, 1, &submit_info, nullFence); assert(!err); err = vkQueueWaitIdle(demo->queue); @@ -455,13 +463,6 @@ static void demo_draw(struct demo *demo) // return codes assert(!err); - // Wait for the present complete semaphore to be signaled to ensure - // that the image won't be rendered to until the presentation - // engine has fully released ownership to the application, and it is - // okay to render to the image. - err = vkQueueWaitSemaphore(demo->queue, presentCompleteSemaphore); - assert(!err); - // Assume the command buffer has been run on current_buffer before so // we need to set the image layout back to COLOR_ATTACHMENT_OPTIMAL demo_set_image_layout(demo, demo->buffers[demo->current_buffer].image, @@ -470,11 +471,25 @@ static void demo_draw(struct demo *demo) VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL); demo_flush_init_cmd(demo); + // Wait for the present complete semaphore to be signaled to ensure + // that the image won't be rendered to until the presentation + // engine has fully released ownership to the application, and it is + // okay to render to the image. + // FIXME/TODO: DEAL WITH VK_IMAGE_LAYOUT_PRESENT_SOURCE_KHR demo_draw_build_cmd(demo); VkFence nullFence = { VK_NULL_HANDLE }; - err = vkQueueSubmit(demo->queue, 1, &demo->draw_cmd, nullFence); + VkSubmitInfo submit_info = { + .waitSemCount = 1, + .pWaitSemaphores = &presentCompleteSemaphore, + .cmdBufferCount = 1, + .pCommandBuffers = &demo->draw_cmd, + .signalSemCount = 0, + .pSignalSemaphores = NULL + }; + + err = vkQueueSubmit(demo->queue, 1, &submit_info, nullFence); assert(!err); VkPresentInfoKHR present = { diff --git a/icd/nulldrv/nulldrv.c b/icd/nulldrv/nulldrv.c index 2f082472..3814bb0b 100644 --- a/icd/nulldrv/nulldrv.c +++ b/icd/nulldrv/nulldrv.c @@ -1818,8 +1818,8 @@ ICD_EXPORT VkResult VKAPI vkQueueWaitIdle( ICD_EXPORT VkResult VKAPI vkQueueSubmit( VkQueue queue_, - uint32_t cmdBufferCount, - const VkCmdBuffer* pCmdBuffers, + uint32_t submitCount, + const VkSubmitInfo* pSubmitInfo, VkFence fence_) { NULLDRV_LOG_FUNC; diff --git a/include/vulkan.h b/include/vulkan.h index eb427f87..570e6be0 100644 --- a/include/vulkan.h +++ b/include/vulkan.h @@ -97,12 +97,12 @@ VK_DEFINE_HANDLE(VkInstance) VK_DEFINE_HANDLE(VkPhysicalDevice) VK_DEFINE_HANDLE(VkDevice) VK_DEFINE_HANDLE(VkQueue) +VK_DEFINE_NONDISP_HANDLE(VkSemaphore) VK_DEFINE_HANDLE(VkCmdBuffer) VK_DEFINE_NONDISP_HANDLE(VkFence) VK_DEFINE_NONDISP_HANDLE(VkDeviceMemory) VK_DEFINE_NONDISP_HANDLE(VkBuffer) VK_DEFINE_NONDISP_HANDLE(VkImage) -VK_DEFINE_NONDISP_HANDLE(VkSemaphore) VK_DEFINE_NONDISP_HANDLE(VkEvent) VK_DEFINE_NONDISP_HANDLE(VkQueryPool) VK_DEFINE_NONDISP_HANDLE(VkBufferView) @@ -1383,6 +1383,15 @@ typedef struct { char description[VK_MAX_DESCRIPTION]; } VkLayerProperties; +typedef struct { + uint32_t waitSemCount; + const VkSemaphore* pWaitSemaphores; + uint32_t cmdBufferCount; + const VkCmdBuffer* pCommandBuffers; + uint32_t signalSemCount; + const VkSemaphore* pSignalSemaphores; +} VkSubmitInfo; + typedef struct { VkStructureType sType; const void* pNext; @@ -2115,7 +2124,7 @@ typedef VkResult (VKAPI *PFN_vkEnumerateDeviceExtensionProperties)(VkPhysicalDev typedef VkResult (VKAPI *PFN_vkEnumerateInstanceLayerProperties)(uint32_t* pCount, VkLayerProperties* pProperties); typedef VkResult (VKAPI *PFN_vkEnumerateDeviceLayerProperties)(VkPhysicalDevice physicalDevice, uint32_t* pCount, VkLayerProperties* pProperties); typedef void (VKAPI *PFN_vkGetDeviceQueue)(VkDevice device, uint32_t queueFamilyIndex, uint32_t queueIndex, VkQueue* pQueue); -typedef VkResult (VKAPI *PFN_vkQueueSubmit)(VkQueue queue, uint32_t cmdBufferCount, const VkCmdBuffer* pCmdBuffers, VkFence fence); +typedef VkResult (VKAPI *PFN_vkQueueSubmit)(VkQueue queue, uint32_t submitCount, const VkSubmitInfo* pSubmitInfo, VkFence fence); typedef VkResult (VKAPI *PFN_vkQueueWaitIdle)(VkQueue queue); typedef VkResult (VKAPI *PFN_vkDeviceWaitIdle)(VkDevice device); typedef VkResult (VKAPI *PFN_vkAllocMemory)(VkDevice device, const VkMemoryAllocInfo* pAllocInfo, VkDeviceMemory* pMem); @@ -2330,8 +2339,8 @@ void VKAPI vkGetDeviceQueue( VkResult VKAPI vkQueueSubmit( VkQueue queue, - uint32_t cmdBufferCount, - const VkCmdBuffer* pCmdBuffers, + uint32_t submitCount, + const VkSubmitInfo* pSubmitInfo, VkFence fence); VkResult VKAPI vkQueueWaitIdle( diff --git a/layers/draw_state.cpp b/layers/draw_state.cpp index 7a33ffb9..d36b1af8 100755 --- a/layers/draw_state.cpp +++ b/layers/draw_state.cpp @@ -1543,32 +1543,35 @@ VK_LAYER_EXPORT VkResult VKAPI vkEnumerateDeviceLayerProperties( pCount, pProperties); } -VK_LAYER_EXPORT VkResult VKAPI vkQueueSubmit(VkQueue queue, uint32_t cmdBufferCount, const VkCmdBuffer* pCmdBuffers, VkFence fence) +VK_LAYER_EXPORT VkResult VKAPI vkQueueSubmit(VkQueue queue, uint32_t submitCount, const VkSubmitInfo* pSubmitInfo, VkFence fence) { VkBool32 skipCall = VK_FALSE; GLOBAL_CB_NODE* pCB = NULL; layer_data* dev_data = get_my_data_ptr(get_dispatch_key(queue), layer_data_map); - for (uint32_t i=0; i < cmdBufferCount; i++) { - // Validate that cmd buffers have been updated - pCB = getCBNode(dev_data, pCmdBuffers[i]); - loader_platform_thread_lock_mutex(&globalLock); - pCB->submitCount++; // increment submit count - if ((pCB->beginInfo.flags & VK_CMD_BUFFER_OPTIMIZE_ONE_TIME_SUBMIT_BIT) && (pCB->submitCount > 1)) { - skipCall |= log_msg(dev_data->report_data, VK_DBG_REPORT_ERROR_BIT, VK_OBJECT_TYPE_COMMAND_BUFFER, 0, 0, DRAWSTATE_CMD_BUFFER_SINGLE_SUBMIT_VIOLATION, "DS", - "CB %#" PRIxLEAST64 " was begun w/ VK_CMD_BUFFER_OPTIMIZE_ONE_TIME_SUBMIT_BIT set, but has been submitted %#" PRIxLEAST64 " times.", reinterpret_cast(pCB->cmdBuffer), pCB->submitCount); - } - if (CB_UPDATE_COMPLETE != pCB->state) { - // Flag error for using CB w/o vkEndCommandBuffer() called - // TODO : How to pass cb as srcObj? - skipCall |= log_msg(dev_data->report_data, VK_DBG_REPORT_ERROR_BIT, VK_OBJECT_TYPE_COMMAND_BUFFER, 0, 0, DRAWSTATE_NO_END_CMD_BUFFER, "DS", - "You must call vkEndCommandBuffer() on CB %#" PRIxLEAST64 " before this call to vkQueueSubmit()!", reinterpret_cast(pCB->cmdBuffer)); + for (uint32_t submit_idx = 0; submit_idx < submitCount; submit_idx++) { + const VkSubmitInfo *submit = &pSubmitInfo[submit_idx]; + for (uint32_t i=0; i < submit->cmdBufferCount; i++) { + // Validate that cmd buffers have been updated + pCB = getCBNode(dev_data, submit->pCommandBuffers[i]); + loader_platform_thread_lock_mutex(&globalLock); + pCB->submitCount++; // increment submit count + if ((pCB->beginInfo.flags & VK_CMD_BUFFER_OPTIMIZE_ONE_TIME_SUBMIT_BIT) && (pCB->submitCount > 1)) { + skipCall |= log_msg(dev_data->report_data, VK_DBG_REPORT_ERROR_BIT, VK_OBJECT_TYPE_COMMAND_BUFFER, 0, 0, DRAWSTATE_CMD_BUFFER_SINGLE_SUBMIT_VIOLATION, "DS", + "CB %#" PRIxLEAST64 " was begun w/ VK_CMD_BUFFER_OPTIMIZE_ONE_TIME_SUBMIT_BIT set, but has been submitted %#" PRIxLEAST64 " times.", reinterpret_cast(pCB->cmdBuffer), pCB->submitCount); + } + if (CB_UPDATE_COMPLETE != pCB->state) { + // Flag error for using CB w/o vkEndCommandBuffer() called + // TODO : How to pass cb as srcObj? + skipCall |= log_msg(dev_data->report_data, VK_DBG_REPORT_ERROR_BIT, VK_OBJECT_TYPE_COMMAND_BUFFER, 0, 0, DRAWSTATE_NO_END_CMD_BUFFER, "DS", + "You must call vkEndCommandBuffer() on CB %#" PRIxLEAST64 " before this call to vkQueueSubmit()!", reinterpret_cast(pCB->cmdBuffer)); + loader_platform_thread_unlock_mutex(&globalLock); + return VK_ERROR_VALIDATION_FAILED; + } loader_platform_thread_unlock_mutex(&globalLock); - return VK_ERROR_VALIDATION_FAILED; } - loader_platform_thread_unlock_mutex(&globalLock); } if (VK_FALSE == skipCall) - return dev_data->device_dispatch_table->QueueSubmit(queue, cmdBufferCount, pCmdBuffers, fence); + return dev_data->device_dispatch_table->QueueSubmit(queue, submitCount, pSubmitInfo, fence); return VK_ERROR_VALIDATION_FAILED; } diff --git a/layers/mem_tracker.cpp b/layers/mem_tracker.cpp index 4a1e85f6..704e007e 100644 --- a/layers/mem_tracker.cpp +++ b/layers/mem_tracker.cpp @@ -1141,8 +1141,8 @@ VK_LAYER_EXPORT void VKAPI vkGetDeviceQueue( VK_LAYER_EXPORT VkResult VKAPI vkQueueSubmit( VkQueue queue, - uint32_t cmdBufferCount, - const VkCmdBuffer *pCmdBuffers, + uint32_t submitCount, + const VkSubmitInfo *pSubmitInfo, VkFence fence) { VkResult result = VK_ERROR_VALIDATION_FAILED; @@ -1155,17 +1155,20 @@ VK_LAYER_EXPORT VkResult VKAPI vkQueueSubmit( print_mem_list(queue); printCBList(queue); - for (uint32_t i = 0; i < cmdBufferCount; i++) { - pCBInfo = get_cmd_buf_info(pCmdBuffers[i]); - pCBInfo->fenceId = fenceId; - pCBInfo->lastSubmittedFence = fence; - pCBInfo->lastSubmittedQueue = queue; + for (uint32_t submit_idx = 0; submit_idx < submitCount; submit_idx++) { + const VkSubmitInfo *submit = &pSubmitInfo[submit_idx]; + for (uint32_t i = 0; i < submit->cmdBufferCount; i++) { + pCBInfo = get_cmd_buf_info(submit->pCommandBuffers[i]); + pCBInfo->fenceId = fenceId; + pCBInfo->lastSubmittedFence = fence; + pCBInfo->lastSubmittedQueue = queue; + } } loader_platform_thread_unlock_mutex(&globalLock); if (VK_FALSE == skipCall) { result = get_dispatch_table(mem_tracker_device_table_map, queue)->QueueSubmit( - queue, cmdBufferCount, pCmdBuffers, fence); + queue, submitCount, pSubmitInfo, fence); } return result; } diff --git a/layers/param_checker.cpp b/layers/param_checker.cpp index de4890a0..97727793 100644 --- a/layers/param_checker.cpp +++ b/layers/param_checker.cpp @@ -2199,15 +2199,17 @@ bool PostQueueSubmit( VK_LAYER_EXPORT VkResult VKAPI vkQueueSubmit( VkQueue queue, - uint32_t cmdBufferCount, - const VkCmdBuffer* pCmdBuffers, + uint32_t submitCount, + const VkSubmitInfo* pSubmitInfo, VkFence fence) { - PreQueueSubmit(queue, pCmdBuffers); + for (uint32_t i = 0; i < submitCount; i++) { + PreQueueSubmit(queue, pSubmitInfo[i].pCommandBuffers); + } - VkResult result = get_dispatch_table(pc_device_table_map, queue)->QueueSubmit(queue, cmdBufferCount, pCmdBuffers, fence); + VkResult result = get_dispatch_table(pc_device_table_map, queue)->QueueSubmit(queue, submitCount, pSubmitInfo, fence); - PostQueueSubmit(queue, cmdBufferCount, fence, result); + PostQueueSubmit(queue, submitCount, fence, result); return result; } diff --git a/layers/screenshot.cpp b/layers/screenshot.cpp index c468257a..05e7414d 100644 --- a/layers/screenshot.cpp +++ b/layers/screenshot.cpp @@ -244,7 +244,17 @@ static void writePPM( const char *filename, VkImage image1) err = pTableCmdBuffer->EndCommandBuffer(cmdBuffer); assert(!err); - err = pTableQueue->QueueSubmit(queue, 1, &cmdBuffer, VK_NULL_HANDLE); + VkFence nullFence = { VK_NULL_HANDLE }; + VkSubmitInfo submit_info = { + .waitSemCount = 0, + .pWaitSemaphores = NULL, + .cmdBufferCount = 1, + .pCommandBuffers = &cmdBuffer, + .signalSemCount = 0, + .pSignalSemaphores = NULL + }; + + err = pTableQueue->QueueSubmit(queue, 1, &submit_info, nullFence); assert(!err); err = pTableQueue->QueueWaitIdle(queue); diff --git a/loader/trampoline.c b/loader/trampoline.c index cbf8a861..2faf002d 100644 --- a/loader/trampoline.c +++ b/loader/trampoline.c @@ -342,13 +342,13 @@ LOADER_EXPORT void VKAPI vkGetDeviceQueue(VkDevice device, uint32_t queueNodeInd loader_set_dispatch(*pQueue, disp); } -LOADER_EXPORT VkResult VKAPI vkQueueSubmit(VkQueue queue, uint32_t cmdBufferCount, const VkCmdBuffer* pCmdBuffers, VkFence fence) +LOADER_EXPORT VkResult VKAPI vkQueueSubmit(VkQueue queue, uint32_t submitCount, const VkSubmitInfo* pSubmitInfo, VkFence fence) { const VkLayerDispatchTable *disp; disp = loader_get_dispatch(queue); - return disp->QueueSubmit(queue, cmdBufferCount, pCmdBuffers, fence); + return disp->QueueSubmit(queue, submitCount, pSubmitInfo, fence); } LOADER_EXPORT VkResult VKAPI vkQueueWaitIdle(VkQueue queue) diff --git a/vk_helper.py b/vk_helper.py index f44beb21..7122297c 100755 --- a/vk_helper.py +++ b/vk_helper.py @@ -868,7 +868,10 @@ class StructWrapperGen: else: sh_funcs.append('%s' % lineinfo.get()) addr_char = '' - if stp_list[index]['type'] in vulkan.core.objects: + if stp_list[index]['type'] in vulkan.object_dispatch_list: + sh_funcs.append('%sss[%u] << %spStruct->%s[i];' % (indent, index, addr_char, stp_list[index]['name'])) + sh_funcs.append('%sstp_strs[%u] += " " + prefix + "%s[" + index_ss.str() + "] = " + ss[%u].str() + "\\n";' % (indent, index, stp_list[index]['name'], index)) + elif stp_list[index]['type'] in vulkan.object_non_dispatch_list: sh_funcs.append('%sss[%u] << %spStruct->%s[i].handle;' % (indent, index, addr_char, stp_list[index]['name'])) sh_funcs.append('%sstp_strs[%u] += " " + prefix + "%s[" + index_ss.str() + "].handle = " + ss[%u].str() + "\\n";' % (indent, index, stp_list[index]['name'], index)) else: diff --git a/vulkan.py b/vulkan.py index 6bc70b28..00968576 100755 --- a/vulkan.py +++ b/vulkan.py @@ -298,8 +298,8 @@ core = Extension( Proto("VkResult", "QueueSubmit", [Param("VkQueue", "queue"), - Param("uint32_t", "cmdBufferCount"), - Param("const VkCmdBuffer*", "pCmdBuffers"), + Param("uint32_t", "submitCount"), + Param("const VkSubmitInfo*", "pSubmitInfo"), Param("VkFence", "fence")]), Proto("VkResult", "QueueWaitIdle", -- cgit v1.2.3