loader: gh851 wrap debug report callbacks

Allow layers to wrap the debug report callbacks so they can
enable more messaging than the application, but also filter
the items returned.

Change-Id: I3fe8feecfa1838869de8a7338ff610e5ebca2e61

2 files changed, 44 insertions, 14 deletions

diff --git a/loader/debug_report.c b/loader/debug_report.c
index 4da1413a..83c5a6a1 100644
--- a/loader/debug_report.c
+++ b/loader/debug_report.c
@@ -65,22 +65,27 @@ util_CreateDebugReportCallback(struct loader_instance *inst,
                                VkDebugReportCallbackCreateInfoEXT *pCreateInfo,
                                const VkAllocationCallbacks *pAllocator,
                                VkDebugReportCallbackEXT callback) {
-    VkLayerDbgFunctionNode *pNewDbgFuncNode;
+    VkLayerDbgFunctionNode *pNewDbgFuncNode = NULL;
+
 #if (DEBUG_DISABLE_APP_ALLOCATORS == 1)
     {
 #else
     if (pAllocator != NULL) {
-        pNewDbgFuncNode = (VkLayerDbgFunctionNode *)pAllocator->pfnAllocation(
-            pAllocator->pUserData, sizeof(VkLayerDbgFunctionNode),
-            sizeof(int *), VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
+        pNewDbgFuncNode =
+            (VkLayerDbgFunctionNode *)pAllocator->pfnAllocation(
+                pAllocator->pUserData, sizeof(VkLayerDbgFunctionNode),
+                sizeof(int *), VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
     } else {
 #endif
-        pNewDbgFuncNode = (VkLayerDbgFunctionNode *)loader_instance_heap_alloc(
-            inst, sizeof(VkLayerDbgFunctionNode),
-            VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
+        pNewDbgFuncNode =
+            (VkLayerDbgFunctionNode *)loader_instance_heap_alloc(
+                inst, sizeof(VkLayerDbgFunctionNode),
+                VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
     }
-    if (!pNewDbgFuncNode)
+    if (!pNewDbgFuncNode) {
         return VK_ERROR_OUT_OF_HOST_MEMORY;
+    }
+    memset(pNewDbgFuncNode, 0, sizeof(VkLayerDbgFunctionNode));
 
     pNewDbgFuncNode->msgCallback = callback;
     pNewDbgFuncNode->pfnMsgCallback = pCreateInfo->pfnCallback;
@@ -99,10 +104,6 @@ static VKAPI_ATTR VkResult VKAPI_CALL debug_report_CreateDebugReportCallback(
     loader_platform_thread_lock_mutex(&loader_lock);
     VkResult result = inst->disp->CreateDebugReportCallbackEXT(
         instance, pCreateInfo, pAllocator, pCallback);
-    if (result == VK_SUCCESS) {
-        result = util_CreateDebugReportCallback(inst, pCreateInfo, pAllocator,
-                                                *pCallback);
-    }
     loader_platform_thread_unlock_mutex(&loader_lock);
     return result;
 }
@@ -311,6 +312,7 @@ VKAPI_ATTR VkResult VKAPI_CALL terminator_CreateDebugReportCallback(
     struct loader_instance *inst = (struct loader_instance *)instance;
     VkResult res = VK_SUCCESS;
     uint32_t storage_idx;
+    VkLayerDbgFunctionNode *pNewDbgFuncNode = NULL;
 
 #if (DEBUG_DISABLE_APP_ALLOCATORS == 1)
     {
@@ -346,6 +348,36 @@ VKAPI_ATTR VkResult VKAPI_CALL terminator_CreateDebugReportCallback(
         storage_idx++;
     }
 
+    // Setup the debug report callback in the terminator since a layer may want
+    // to grab the information itself (RenderDoc) and then return back to the
+    // user callback a sub-set of the messages.
+#if (DEBUG_DISABLE_APP_ALLOCATORS == 0)
+    if (pAllocator != NULL) {
+        pNewDbgFuncNode =
+            (VkLayerDbgFunctionNode *)pAllocator->pfnAllocation(
+                pAllocator->pUserData, sizeof(VkLayerDbgFunctionNode),
+                sizeof(int *), VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
+    } else {
+#else
+    {
+#endif
+        pNewDbgFuncNode =
+            (VkLayerDbgFunctionNode *)loader_instance_heap_alloc(
+                inst, sizeof(VkLayerDbgFunctionNode),
+                VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
+    }
+    if (!pNewDbgFuncNode) {
+        return VK_ERROR_OUT_OF_HOST_MEMORY;
+    }
+    memset(pNewDbgFuncNode, 0, sizeof(VkLayerDbgFunctionNode));
+
+    pNewDbgFuncNode->msgCallback = *pCallback;
+    pNewDbgFuncNode->pfnMsgCallback = pCreateInfo->pfnCallback;
+    pNewDbgFuncNode->msgFlags = pCreateInfo->flags;
+    pNewDbgFuncNode->pUserData = pCreateInfo->pUserData;
+    pNewDbgFuncNode->pNext = inst->DbgFunctionHead;
+    inst->DbgFunctionHead = pNewDbgFuncNode;
+
     /* roll back on errors */
     if (icd) {
         storage_idx = 0;
diff --git a/loader/trampoline.c b/loader/trampoline.c
index 54a71057..7ed168f5 100644
--- a/loader/trampoline.c
+++ b/loader/trampoline.c
@@ -418,8 +418,6 @@ LOADER_EXPORT VKAPI_ATTR VkResult VKAPI_CALL vkCreateInstance(
          * if enabled.
          */
         loader_activate_instance_layer_extensions(ptr_instance, *pInstance);
-    } else {
-        // TODO: cleanup here.
     }
 
 out: