layers: Use tmp callback for msgs during vk{Create|Destroy}Instance().

This is implements some relatively-new functionality of the VK_EXT_debug_report
extension.  An application can pass VkDebugReportCallbackCreateInfoEXT structs
on the pNext chain given to vkCreateInstance(), in order to setup one or more
callbacks that can be used during vk{Create|Destroy}Instance().  These special,
"temporary callbacks" allow messages (e.g. errors) to be logged during the time
when the debug_report extension is normally not setup.

A set of utilities copy VkDebugReportCallbackCreateInfoEXT structs from the
pNext chain given to vkCreateInstance().  These utilities are used by the
validation layers that may have messages (e.g. errors) during
vk{Create|Destroy}Instance().

8 files changed, 232 insertions, 6 deletions

diff --git a/layers/object_tracker.h b/layers/object_tracker.h
index aab1ce11..591484d2 100644
--- a/layers/object_tracker.h
+++ b/layers/object_tracker.h
@@ -72,12 +72,19 @@ typedef uint64_t (*OBJ_TRACK_GET_OBJECTS_OF_TYPE_COUNT)(VkDevice, VkDebugReportO
 
 struct layer_data {
     debug_report_data *report_data;
+    // The following are for keeping track of the temporary callbacks that can
+    // be used in vkCreateInstance and vkDestroyInstance:
+    uint32_t num_tmp_callbacks;
+    VkDebugReportCallbackCreateInfoEXT *tmp_dbg_create_infos;
+    VkDebugReportCallbackEXT *tmp_callbacks;
     // TODO: put instance data here
     std::vector<VkDebugReportCallbackEXT> logging_callback;
     bool wsi_enabled;
     bool objtrack_extensions_enabled;
 
-    layer_data() : report_data(nullptr), wsi_enabled(false), objtrack_extensions_enabled(false){};
+    layer_data()
+        : report_data(nullptr), wsi_enabled(false), objtrack_extensions_enabled(false), num_tmp_callbacks(0),
+          tmp_dbg_create_infos(nullptr), tmp_callbacks(nullptr){};
 };
 
 struct instExts {
@@ -616,6 +623,11 @@ VkResult explicit_CreateInstance(const VkInstanceCreateInfo *pCreateInfo, const
     initInstanceTable(*pInstance, fpGetInstanceProcAddr, object_tracker_instance_table_map);
     VkLayerInstanceDispatchTable *pInstanceTable = get_dispatch_table(object_tracker_instance_table_map, *pInstance);
 
+    // Look for one or more debug report create info structures, and copy the
+    // callback(s) for each one found (for use by vkDestroyInstance)
+    layer_copy_tmp_callbacks(pCreateInfo->pNext, &my_data->num_tmp_callbacks, &my_data->tmp_dbg_create_infos,
+                             &my_data->tmp_callbacks);
+
     my_data->report_data = debug_report_create_instance(pInstanceTable, *pInstance, pCreateInfo->enabledExtensionCount,
                                                         pCreateInfo->ppEnabledExtensionNames);
 
diff --git a/layers/parameter_validation.cpp b/layers/parameter_validation.cpp
index 9a995d98..9546e4c3 100644
--- a/layers/parameter_validation.cpp
+++ b/layers/parameter_validation.cpp
@@ -50,12 +50,18 @@ struct layer_data {
     debug_report_data *report_data;
     std::vector<VkDebugReportCallbackEXT> logging_callback;
 
+    // The following are for keeping track of the temporary callbacks that can
+    // be used in vkCreateInstance and vkDestroyInstance:
+    uint32_t num_tmp_callbacks;
+    VkDebugReportCallbackCreateInfoEXT *tmp_dbg_create_infos;
+    VkDebugReportCallbackEXT *tmp_callbacks;
+
     // TODO: Split instance/device structs
     // Device Data
     // Map for queue family index to queue count
     std::unordered_map<uint32_t, uint32_t> queueFamilyIndexMap;
 
-    layer_data() : report_data(nullptr){};
+    layer_data() : report_data(nullptr), num_tmp_callbacks(0), tmp_dbg_create_infos(nullptr), tmp_callbacks(nullptr){};
 };
 
 static std::unordered_map<void *, layer_data *> layer_data_map;
@@ -1306,6 +1312,22 @@ vkCreateInstance(const VkInstanceCreateInfo *pCreateInfo, const VkAllocationCall
         my_instance_data->report_data = debug_report_create_instance(pTable, *pInstance, pCreateInfo->enabledExtensionCount,
                                                                      pCreateInfo->ppEnabledExtensionNames);
 
+        // Look for one or more debug report create info structures
+        // and setup a callback(s) for each one found.
+        if (!layer_copy_tmp_callbacks(pCreateInfo->pNext, &my_instance_data->num_tmp_callbacks,
+                                      &my_instance_data->tmp_dbg_create_infos, &my_instance_data->tmp_callbacks)) {
+            if (my_instance_data->num_tmp_callbacks > 0) {
+                // Setup the temporary callback(s) here to catch early issues:
+                if (layer_enable_tmp_callbacks(my_instance_data->report_data, my_instance_data->num_tmp_callbacks,
+                                               my_instance_data->tmp_dbg_create_infos, my_instance_data->tmp_callbacks)) {
+                    // Failure of setting up one or more of the callback.
+                    // Therefore, clean up and don't use those callbacks:
+                    layer_free_tmp_callbacks(my_instance_data->tmp_dbg_create_infos, my_instance_data->tmp_callbacks);
+                    my_instance_data->num_tmp_callbacks = 0;
+                }
+            }
+        }
+
         init_parameter_validation(my_instance_data, pAllocator);
 
         // Ordinarily we'd check these before calling down the chain, but none of the layer
@@ -1324,6 +1346,12 @@ vkCreateInstance(const VkInstanceCreateInfo *pCreateInfo, const VkAllocationCall
                                 pCreateInfo->pApplicationInfo->pEngineName);
             }
         }
+
+        // Disable the tmp callbacks:
+        if (my_instance_data->num_tmp_callbacks > 0) {
+            layer_disable_tmp_callbacks(my_instance_data->report_data, my_instance_data->num_tmp_callbacks,
+                                        my_instance_data->tmp_callbacks);
+        }
     }
 
     return result;
@@ -1336,8 +1364,26 @@ VK_LAYER_EXPORT VKAPI_ATTR void VKAPI_CALL vkDestroyInstance(VkInstance instance
     layer_data *my_data = get_my_data_ptr(key, layer_data_map);
     assert(my_data != NULL);
 
+    // Enable the temporary callback(s) here to catch vkDestroyInstance issues:
+    bool callback_setup = false;
+    if (my_data->num_tmp_callbacks > 0) {
+        if (!layer_enable_tmp_callbacks(my_data->report_data, my_data->num_tmp_callbacks, my_data->tmp_dbg_create_infos,
+                                        my_data->tmp_callbacks)) {
+            callback_setup = true;
+        }
+    }
+
     skipCall |= parameter_validation_vkDestroyInstance(my_data->report_data, pAllocator);
 
+    // Disable and cleanup the temporary callback(s):
+    if (callback_setup) {
+        layer_disable_tmp_callbacks(my_data->report_data, my_data->num_tmp_callbacks, my_data->tmp_callbacks);
+    }
+    if (my_data->num_tmp_callbacks > 0) {
+        layer_free_tmp_callbacks(my_data->tmp_dbg_create_infos, my_data->tmp_callbacks);
+        my_data->num_tmp_callbacks = 0;
+    }
+
     if (!skipCall) {
         VkLayerInstanceDispatchTable *pTable = get_dispatch_table(pc_instance_table_map, instance);
         pTable->DestroyInstance(instance, pAllocator);
diff --git a/layers/swapchain.cpp b/layers/swapchain.cpp
index 8e035c1e..b999d666 100644
--- a/layers/swapchain.cpp
+++ b/layers/swapchain.cpp
@@ -176,6 +176,11 @@ static void createInstanceRegisterExtensions(const VkInstanceCreateInfo *pCreate
     my_data->instanceMap[instance].xlibSurfaceExtensionEnabled = false;
 #endif // VK_USE_PLATFORM_XLIB_KHR
 
+    // Look for one or more debug report create info structures, and copy the
+    // callback(s) for each one found (for use by vkDestroyInstance)
+    layer_copy_tmp_callbacks(pCreateInfo->pNext, &my_data->num_tmp_callbacks, &my_data->tmp_dbg_create_infos,
+                             &my_data->tmp_callbacks);
+
     // Record whether the WSI instance extension was enabled for this
     // VkInstance.  No need to check if the extension was advertised by
     // vkEnumerateInstanceExtensionProperties(), since the loader handles that.
@@ -292,6 +297,15 @@ VK_LAYER_EXPORT VKAPI_ATTR void VKAPI_CALL vkDestroyInstance(VkInstance instance
 
     std::lock_guard<std::mutex> lock(global_lock);
 
+    // Enable the temporary callback(s) here to catch cleanup issues:
+    bool callback_setup = false;
+    if (my_data->num_tmp_callbacks > 0) {
+        if (!layer_enable_tmp_callbacks(my_data->report_data, my_data->num_tmp_callbacks, my_data->tmp_dbg_create_infos,
+                                        my_data->tmp_callbacks)) {
+            callback_setup = true;
+        }
+    }
+
     // Do additional internal cleanup:
     if (pInstance) {
         // Delete all of the SwpPhysicalDevice's, SwpSurface's, and the
@@ -329,6 +343,15 @@ VK_LAYER_EXPORT VKAPI_ATTR void VKAPI_CALL vkDestroyInstance(VkInstance instance
         my_data->instanceMap.erase(instance);
     }
 
+    // Disable and cleanup the temporary callback(s):
+    if (callback_setup) {
+        layer_disable_tmp_callbacks(my_data->report_data, my_data->num_tmp_callbacks, my_data->tmp_callbacks);
+    }
+    if (my_data->num_tmp_callbacks > 0) {
+        layer_free_tmp_callbacks(my_data->tmp_dbg_create_infos, my_data->tmp_callbacks);
+        my_data->num_tmp_callbacks = 0;
+    }
+
     // Clean up logging callback, if any
     while (my_data->logging_callback.size() > 0) {
         VkDebugReportCallbackEXT callback = my_data->logging_callback.back();
diff --git a/layers/swapchain.h b/layers/swapchain.h
index 0bfb8029..21a0dc6f 100644
--- a/layers/swapchain.h
+++ b/layers/swapchain.h
@@ -324,6 +324,13 @@ struct layer_data {
     std::vector<VkDebugReportCallbackEXT> logging_callback;
     VkLayerDispatchTable *device_dispatch_table;
     VkLayerInstanceDispatchTable *instance_dispatch_table;
+
+    // The following are for keeping track of the temporary callbacks that can
+    // be used in vkCreateInstance and vkDestroyInstance:
+    uint32_t num_tmp_callbacks;
+    VkDebugReportCallbackCreateInfoEXT *tmp_dbg_create_infos;
+    VkDebugReportCallbackEXT *tmp_callbacks;
+
     // NOTE: The following are for keeping track of info that is used for
     // validating the WSI extensions.
     std::unordered_map<void *, SwpInstance> instanceMap;
@@ -333,7 +340,9 @@ struct layer_data {
     std::unordered_map<VkSwapchainKHR, SwpSwapchain> swapchainMap;
     std::unordered_map<void *, SwpQueue> queueMap;
 
-    layer_data() : report_data(nullptr), device_dispatch_table(nullptr), instance_dispatch_table(nullptr){};
+    layer_data()
+        : report_data(nullptr), device_dispatch_table(nullptr), instance_dispatch_table(nullptr), num_tmp_callbacks(0),
+          tmp_dbg_create_infos(nullptr), tmp_callbacks(nullptr){};
 };
 
 #endif // SWAPCHAIN_H
diff --git a/layers/threading.cpp b/layers/threading.cpp
index 79f047c5..4d9fca0d 100644
--- a/layers/threading.cpp
+++ b/layers/threading.cpp
@@ -69,6 +69,11 @@ vkCreateInstance(const VkInstanceCreateInfo *pCreateInfo, const VkAllocationCall
     my_data->report_data = debug_report_create_instance(my_data->instance_dispatch_table, *pInstance,
                                                         pCreateInfo->enabledExtensionCount, pCreateInfo->ppEnabledExtensionNames);
     initThreading(my_data, pAllocator);
+
+    // Look for one or more debug report create info structures, and copy the
+    // callback(s) for each one found (for use by vkDestroyInstance)
+    layer_copy_tmp_callbacks(pCreateInfo->pNext, &my_data->num_tmp_callbacks, &my_data->tmp_dbg_create_infos,
+                             &my_data->tmp_callbacks);
     return result;
 }
 
@@ -76,10 +81,29 @@ VK_LAYER_EXPORT VKAPI_ATTR void VKAPI_CALL vkDestroyInstance(VkInstance instance
     dispatch_key key = get_dispatch_key(instance);
     layer_data *my_data = get_my_data_ptr(key, layer_data_map);
     VkLayerInstanceDispatchTable *pTable = my_data->instance_dispatch_table;
+
+    // Enable the temporary callback(s) here to catch cleanup issues:
+    bool callback_setup = false;
+    if (my_data->num_tmp_callbacks > 0) {
+        if (!layer_enable_tmp_callbacks(my_data->report_data, my_data->num_tmp_callbacks, my_data->tmp_dbg_create_infos,
+                                        my_data->tmp_callbacks)) {
+            callback_setup = true;
+        }
+    }
+
     startWriteObject(my_data, instance);
     pTable->DestroyInstance(instance, pAllocator);
     finishWriteObject(my_data, instance);
 
+    // Disable and cleanup the temporary callback(s):
+    if (callback_setup) {
+        layer_disable_tmp_callbacks(my_data->report_data, my_data->num_tmp_callbacks, my_data->tmp_callbacks);
+    }
+    if (my_data->num_tmp_callbacks > 0) {
+        layer_free_tmp_callbacks(my_data->tmp_dbg_create_infos, my_data->tmp_callbacks);
+        my_data->num_tmp_callbacks = 0;
+    }
+
     // Clean up logging callback, if any
     while (my_data->logging_callback.size() > 0) {
         VkDebugReportCallbackEXT callback = my_data->logging_callback.back();
diff --git a/layers/threading.h b/layers/threading.h
index 31881618..96175394 100644
--- a/layers/threading.h
+++ b/layers/threading.h
@@ -193,6 +193,11 @@ struct layer_data {
     std::vector<VkDebugReportCallbackEXT> logging_callback;
     VkLayerDispatchTable *device_dispatch_table;
     VkLayerInstanceDispatchTable *instance_dispatch_table;
+    // The following are for keeping track of the temporary callbacks that can
+    // be used in vkCreateInstance and vkDestroyInstance:
+    uint32_t num_tmp_callbacks;
+    VkDebugReportCallbackCreateInfoEXT *tmp_dbg_create_infos;
+    VkDebugReportCallbackEXT *tmp_callbacks;
     counter<VkCommandBuffer> c_VkCommandBuffer;
     counter<VkDevice> c_VkDevice;
     counter<VkInstance> c_VkInstance;
@@ -223,7 +228,8 @@ struct layer_data {
     counter<uint64_t> c_uint64_t;
 #endif // DISTINCT_NONDISPATCHABLE_HANDLES
     layer_data()
-        : report_data(nullptr), c_VkCommandBuffer("VkCommandBuffer", VK_DEBUG_REPORT_OBJECT_TYPE_COMMAND_BUFFER_EXT),
+        : report_data(nullptr), num_tmp_callbacks(0), tmp_dbg_create_infos(nullptr), tmp_callbacks(nullptr),
+          c_VkCommandBuffer("VkCommandBuffer", VK_DEBUG_REPORT_OBJECT_TYPE_COMMAND_BUFFER_EXT),
           c_VkDevice("VkDevice", VK_DEBUG_REPORT_OBJECT_TYPE_DEVICE_EXT),
           c_VkInstance("VkInstance", VK_DEBUG_REPORT_OBJECT_TYPE_INSTANCE_EXT),
           c_VkQueue("VkQueue", VK_DEBUG_REPORT_OBJECT_TYPE_QUEUE_EXT),
diff --git a/layers/vk_layer_logging.h b/layers/vk_layer_logging.h
index 424031a4..bbd24d0f 100644
--- a/layers/vk_layer_logging.h
+++ b/layers/vk_layer_logging.h
@@ -192,6 +192,87 @@ static inline PFN_vkVoidFunction debug_report_get_instance_proc_addr(debug_repor
     return NULL;
 }
 
+// This utility (called at vkCreateInstance() time), looks at a pNext chain.
+// It counts any VkDebugReportCallbackCreateInfoEXT structs that it finds.  It
+// then allocates an array that can hold that many structs, as well as that
+// many VkDebugReportCallbackEXT handles.  It then copies each
+// VkDebugReportCallbackCreateInfoEXT, and initializes each handle.
+static VkResult layer_copy_tmp_callbacks(const void *pChain, uint32_t *num_callbacks, VkDebugReportCallbackCreateInfoEXT **infos,
+                                         VkDebugReportCallbackEXT **callbacks) {
+    uint32_t n = *num_callbacks = 0;
+
+    const void *pNext = pChain;
+    while (pNext) {
+        // 1st, count the number VkDebugReportCallbackCreateInfoEXT:
+        if (((VkDebugReportCallbackCreateInfoEXT *)pNext)->sType == VK_STRUCTURE_TYPE_DEBUG_REPORT_CREATE_INFO_EXT) {
+            n++;
+        }
+        pNext = (void *)((VkDebugReportCallbackCreateInfoEXT *)pNext)->pNext;
+    }
+    if (n == 0) {
+        return VK_SUCCESS;
+    }
+
+    // 2nd, allocate memory for each VkDebugReportCallbackCreateInfoEXT:
+    VkDebugReportCallbackCreateInfoEXT *pInfos = *infos =
+        ((VkDebugReportCallbackCreateInfoEXT *)malloc(n * sizeof(VkDebugReportCallbackCreateInfoEXT)));
+    if (!pInfos) {
+        return VK_ERROR_OUT_OF_HOST_MEMORY;
+    }
+    // 3rd, allocate memory for a unique handle for each callback:
+    VkDebugReportCallbackEXT *pCallbacks = *callbacks = ((VkDebugReportCallbackEXT *)malloc(n * sizeof(VkDebugReportCallbackEXT)));
+    if (!pCallbacks) {
+        free(pInfos);
+        return VK_ERROR_OUT_OF_HOST_MEMORY;
+    }
+    // 4th, copy each VkDebugReportCallbackCreateInfoEXT for use by
+    // vkDestroyInstance, and assign a unique handle to each callback (just
+    // use the address of the copied VkDebugReportCallbackCreateInfoEXT):
+    pNext = pChain;
+    while (pNext) {
+        if (((VkInstanceCreateInfo *)pNext)->sType == VK_STRUCTURE_TYPE_DEBUG_REPORT_CREATE_INFO_EXT) {
+            memcpy(pInfos, pNext, sizeof(VkDebugReportCallbackCreateInfoEXT));
+            *pCallbacks++ = (VkDebugReportCallbackEXT)pInfos++;
+        }
+        pNext = (void *)((VkInstanceCreateInfo *)pNext)->pNext;
+    }
+
+    *num_callbacks = n;
+    return VK_SUCCESS;
+}
+
+// This utility frees the arrays allocated by layer_copy_tmp_callbacks()
+static void layer_free_tmp_callbacks(VkDebugReportCallbackCreateInfoEXT *infos, VkDebugReportCallbackEXT *callbacks) {
+    free(infos);
+    free(callbacks);
+}
+
+// This utility enables all of the VkDebugReportCallbackCreateInfoEXT structs
+// that were copied by layer_copy_tmp_callbacks()
+static VkResult layer_enable_tmp_callbacks(debug_report_data *debug_data, uint32_t num_callbacks,
+                                           VkDebugReportCallbackCreateInfoEXT *infos, VkDebugReportCallbackEXT *callbacks) {
+    VkResult rtn = VK_SUCCESS;
+    for (uint32_t i = 0; i < num_callbacks; i++) {
+        rtn = layer_create_msg_callback(debug_data, &infos[i], NULL, &callbacks[i]);
+        if (rtn != VK_SUCCESS) {
+            for (uint32_t j = 0; j < i; j++) {
+                layer_destroy_msg_callback(debug_data, callbacks[j], NULL);
+            }
+            return rtn;
+        }
+    }
+    return rtn;
+}
+
+// This utility disables all of the VkDebugReportCallbackCreateInfoEXT structs
+// that were copied by layer_copy_tmp_callbacks()
+static void layer_disable_tmp_callbacks(debug_report_data *debug_data, uint32_t num_callbacks,
+                                        VkDebugReportCallbackEXT *callbacks) {
+    for (uint32_t i = 0; i < num_callbacks; i++) {
+        layer_destroy_msg_callback(debug_data, callbacks[i], NULL);
+    }
+}
+
 /*
  * Checks if the message will get logged.
  * Allows layer to defer collecting & formating data if the
diff --git a/vk-layer-generate.py b/vk-layer-generate.py
index ba9084c5..f3194c27 100755
--- a/vk-layer-generate.py
+++ b/vk-layer-generate.py
@@ -949,6 +949,21 @@ class ObjectTrackerSubcommand(Subcommand):
         gedi_txt.append('const VkAllocationCallbacks* pAllocator)')
         gedi_txt.append('{')
         gedi_txt.append('    std::unique_lock<std::mutex> lock(global_lock);')
+        gedi_txt.append('')
+        gedi_txt.append('    dispatch_key key = get_dispatch_key(instance);')
+        gedi_txt.append('    layer_data *my_data = get_my_data_ptr(key, layer_data_map);')
+        gedi_txt.append('')
+        gedi_txt.append('    // Enable the temporary callback(s) here to catch cleanup issues:')
+        gedi_txt.append('    bool callback_setup = false;')
+        gedi_txt.append('    if (my_data->num_tmp_callbacks > 0) {')
+        gedi_txt.append('        if (!layer_enable_tmp_callbacks(my_data->report_data,')
+        gedi_txt.append('                                        my_data->num_tmp_callbacks,')
+        gedi_txt.append('                                        my_data->tmp_dbg_create_infos,')
+        gedi_txt.append('                                        my_data->tmp_callbacks)) {')
+        gedi_txt.append('            callback_setup = true;')
+        gedi_txt.append('        }')
+        gedi_txt.append('    }')
+        gedi_txt.append('')
         gedi_txt.append('    validate_instance(instance, instance, VK_DEBUG_REPORT_OBJECT_TYPE_INSTANCE_EXT, false);')
         gedi_txt.append('')
         gedi_txt.append('    destroy_instance(instance, instance);')
@@ -980,11 +995,21 @@ class ObjectTrackerSubcommand(Subcommand):
         gedi_txt.append('        }')
         gedi_txt.append('    }')
         gedi_txt.append('')
-        gedi_txt.append('    dispatch_key key = get_dispatch_key(instance);')
         gedi_txt.append('    VkLayerInstanceDispatchTable *pInstanceTable = get_dispatch_table(object_tracker_instance_table_map, instance);')
         gedi_txt.append('    pInstanceTable->DestroyInstance(instance, pAllocator);')
         gedi_txt.append('')
-        gedi_txt.append('    layer_data *my_data = get_my_data_ptr(key, layer_data_map);')
+        gedi_txt.append('    // Disable and cleanup the temporary callback(s):')
+        gedi_txt.append('    if (callback_setup) {')
+        gedi_txt.append('        layer_disable_tmp_callbacks(my_data->report_data,')
+        gedi_txt.append('                                    my_data->num_tmp_callbacks,')
+        gedi_txt.append('                                    my_data->tmp_callbacks);')
+        gedi_txt.append('    }')
+        gedi_txt.append('    if (my_data->num_tmp_callbacks > 0) {')
+        gedi_txt.append('        layer_free_tmp_callbacks(my_data->tmp_dbg_create_infos,')
+        gedi_txt.append('                                 my_data->tmp_callbacks);')
+        gedi_txt.append('        my_data->num_tmp_callbacks = 0;')
+        gedi_txt.append('    }')
+        gedi_txt.append('')
         gedi_txt.append('    // Clean up logging callback, if any')
         gedi_txt.append('    while (my_data->logging_callback.size() > 0) {')
         gedi_txt.append('        VkDebugReportCallbackEXT callback = my_data->logging_callback.back();')