loader: Only allow layer in chain once

If an application were to specify a "Validatio" layer
and the environment specified a ParamChecker layer
it was possible to load the same layer twice in
one chain. Layers don't like that can only be referenced
once and that's all they need to support multiple
extensions from the one layer.
This patch adds an alias pointer so that when putting
together the lists of available extensions the loader
can keep track of multiple extensions that are servied
by the same library. Also now create specific
activated_layers_list that contains only the list
of layers that need to be activated.

2 files changed, 142 insertions, 95 deletions

diff --git a/loader/loader.c b/loader/loader.c
index d4ddc847..83010a2f 100644
--- a/loader/loader.c
+++ b/loader/loader.c
@@ -249,6 +249,7 @@ static struct loader_extension_property *get_extension_property_from_vkext(
 
 static void get_global_extensions(
         const PFN_vkGetGlobalExtensionInfo fp_get,
+        const char *get_extension_info_name,
         const char *lib_name,
         const enum extension_origin origin,
         struct loader_extension_list *ext_list)
@@ -265,12 +266,13 @@ static void get_global_extensions(
     }
     siz = sizeof(VkExtensionProperties);
     for (i = 0; i < count; i++) {
-        memset(&ext_props, 1, sizeof(ext_props));
+        memset(&ext_props, 0, sizeof(ext_props));
         res = fp_get(VK_EXTENSION_INFO_TYPE_PROPERTIES, i, &siz, &ext_props.info);
         if (res == VK_SUCCESS) {
             ext_props.hosted = false;
             ext_props.origin = origin;
             ext_props.lib_name = lib_name;
+            strncpy(ext_props.get_extension_info_name, get_extension_info_name, MAX_EXTENSION_NAME_SIZE);
             loader_add_to_ext_list(ext_list, 1, &ext_props);
         }
     }
@@ -300,7 +302,12 @@ static void get_physical_device_layer_extensions(
     char funcStr[256];
     snprintf(funcStr, 256, "%sGetPhysicalDeviceExtensionInfo", ext_props.info.name);
     lib_handle = loader_add_layer_lib("device", &ext_props);
-    fp_get = (PFN_vkGetPhysicalDeviceExtensionInfo) loader_platform_get_proc_address(lib_handle, "vkGetPhysicalDeviceExtensionInfo");
+
+    /* first try extension specific function, then generic */
+    fp_get = (PFN_vkGetPhysicalDeviceExtensionInfo) loader_platform_get_proc_address(lib_handle, funcStr);
+    if (!fp_get) {
+        fp_get = (PFN_vkGetPhysicalDeviceExtensionInfo) loader_platform_get_proc_address(lib_handle, "vkGetPhysicalDeviceExtensionInfo");
+    }
     if (fp_get) {
         res = fp_get(physical_device, VK_EXTENSION_INFO_TYPE_COUNT, 0, &siz, &count);
         if (res == VK_SUCCESS) {
@@ -402,6 +409,27 @@ void loader_add_to_ext_list(
             ext_list->capacity *= 2;
             ext_list->list = realloc(ext_list->list, ext_list->capacity);
         }
+
+        /*
+         * Check if any extensions already on the list come from the same
+         * library and use the same Get*ExtensionInfo. If so, link this
+         * extension to the previous as an alias. That way when we activate
+         * extensions we only activiate the associated layer once no
+         * matter how many extensions are used.
+         */
+        for (uint32_t j = 0; j < ext_list->count; j++) {
+            struct loader_extension_property *active_property = &ext_list->list[j];
+            if (cur_ext->lib_name &&
+                cur_ext->origin == VK_EXTENSION_ORIGIN_LAYER &&
+                active_property->origin == VK_EXTENSION_ORIGIN_LAYER &&
+                strcmp(cur_ext->lib_name, active_property->lib_name) == 0 &&
+                strcmp(cur_ext->get_extension_info_name, active_property->get_extension_info_name) == 0 &&
+                    active_property->alias == NULL) {
+                cur_ext->alias = active_property;
+                break;
+            }
+        }
+
         memcpy(&ext_list->list[ext_list->count], cur_ext, sizeof(struct loader_extension_property));
         ext_list->count++;
     }
@@ -559,6 +587,7 @@ static void loader_scanned_icd_add(const char *filename)
 
     get_global_extensions(
                 (PFN_vkGetGlobalExtensionInfo) fp_get_global_ext_info,
+                "vkGetGlobalExtensionInfo",
                 new_node->lib_name,
                 VK_EXTENSION_ORIGIN_ICD,
                 &new_node->global_extension_list);
@@ -751,95 +780,96 @@ void layer_lib_scan(void)
     for (p = libPaths; *p; p = next) {
         next = strchr(p, PATH_SEPERATOR);
         if (next == NULL) {
-           len = (uint32_t) strlen(p);
-           next = p + len;
+            len = (uint32_t) strlen(p);
+            next = p + len;
         }
         else {
-           len = (uint32_t) (next - p);
-           *(char *) next = '\0';
-           next++;
+            len = (uint32_t) (next - p);
+            *(char *) next = '\0';
+            next++;
         }
 
-       curdir = opendir(p);
-       if (curdir) {
-          dent = readdir(curdir);
-          while (dent) {
-             /* Look for layers starting with VK_LAYER_LIBRARY_PREFIX and
-              * ending with VK_LIBRARY_SUFFIX
-              */
-              if (!strncmp(dent->d_name,
-                          VK_LAYER_LIBRARY_PREFIX,
-                          VK_LAYER_LIBRARY_PREFIX_LEN)) {
-                 uint32_t nlen = (uint32_t) strlen(dent->d_name);
-                 const char *suf = dent->d_name + nlen - VK_LIBRARY_SUFFIX_LEN;
-                 if ((nlen > VK_LIBRARY_SUFFIX_LEN) &&
-                     !strncmp(suf,
-                              VK_LIBRARY_SUFFIX,
-                              VK_LIBRARY_SUFFIX_LEN)) {
-                     loader_platform_dl_handle handle;
-                     snprintf(temp_str, sizeof(temp_str),
-                              "%s" DIRECTORY_SYMBOL "%s",p,dent->d_name);
-                     // Used to call: dlopen(temp_str, RTLD_LAZY)
-                     fprintf(stderr, "Attempt to open library: %s\n", temp_str);
-                     if ((handle = loader_platform_open_library(temp_str)) == NULL) {
-                         dent = readdir(curdir);
-                         continue;
-                     }
-                     fprintf(stderr, "Opened library: %s\n", temp_str);
-
-                     /* TODO: Remove fixed count */
-                     if (count == MAX_LAYER_LIBRARIES) {
-                         loader_log(VK_DBG_REPORT_ERROR_BIT, 0,
-                                    "%s ignored: max layer libraries exceed",
-                                    temp_str);
-                         break;
-                     }
-                     fp_get_ext = loader_platform_get_proc_address(handle,
-                                               "vkGetGlobalExtensionInfo");
-
-                     if (!fp_get_ext) {
-                         fprintf(stderr, "Unable to find vkGetGlobalExtensionInfo\n");
-                         loader_log(VK_DBG_REPORT_WARN_BIT, 0,
-                              "Couldn't dlsym vkGetGlobalExtensionInfo from library %s",
-                               temp_str);
-                         dent = readdir(curdir);
-                         loader_platform_close_library(handle);
-                         continue;
-                     }
-
-                     loader.scanned_layers[count].lib_name =
-                                                   malloc(strlen(temp_str) + 1);
-                     if (loader.scanned_layers[count].lib_name == NULL) {
-                         loader_log(VK_DBG_REPORT_ERROR_BIT, 0, "%s ignored: out of memory", temp_str);
-                         break;
-                     }
-
-                     strcpy(loader.scanned_layers[count].lib_name, temp_str);
-
-                     fprintf(stderr, "Collecting global extensions for %s\n", temp_str);
-                     get_global_extensions(
-                                 fp_get_ext,
-                                 loader.scanned_layers[count].lib_name,
-                                 VK_EXTENSION_ORIGIN_LAYER,
-                                 &loader.scanned_layers[count].global_extension_list);
-
-                     fp_get_ext = loader_platform_get_proc_address(handle,
-                                               "vkGetPhysicalDeviceExtensionInfo");
-                     if (fp_get_ext) {
-                         loader.scanned_layers[count].physical_device_extensions_supported = true;
-                     }
-
-                     count++;
-                     loader_platform_close_library(handle);
-                 }
-             }
-
-             dent = readdir(curdir);
-          } // while (dir_entry)
-          if (count == MAX_LAYER_LIBRARIES)
-             break;
-          closedir(curdir);
-       } // if (curdir))
+        curdir = opendir(p);
+        if (curdir) {
+            dent = readdir(curdir);
+            while (dent) {
+                /* Look for layers starting with VK_LAYER_LIBRARY_PREFIX and
+                 * ending with VK_LIBRARY_SUFFIX
+                 */
+                if (!strncmp(dent->d_name,
+                             VK_LAYER_LIBRARY_PREFIX,
+                             VK_LAYER_LIBRARY_PREFIX_LEN)) {
+                    uint32_t nlen = (uint32_t) strlen(dent->d_name);
+                    const char *suf = dent->d_name + nlen - VK_LIBRARY_SUFFIX_LEN;
+                    if ((nlen > VK_LIBRARY_SUFFIX_LEN) &&
+                            !strncmp(suf,
+                                     VK_LIBRARY_SUFFIX,
+                                     VK_LIBRARY_SUFFIX_LEN)) {
+                        loader_platform_dl_handle handle;
+                        snprintf(temp_str, sizeof(temp_str),
+                                 "%s" DIRECTORY_SYMBOL "%s",p,dent->d_name);
+                        // Used to call: dlopen(temp_str, RTLD_LAZY)
+                        loader_log(VK_DBG_REPORT_DEBUG_BIT, 0,
+                                   "Attempt to open library: %s\n", temp_str);
+                        if ((handle = loader_platform_open_library(temp_str)) == NULL) {
+                            dent = readdir(curdir);
+                            continue;
+                        }
+                        loader_log(VK_DBG_REPORT_DEBUG_BIT, 0,
+                                   "Opened library: %s\n", temp_str);
+
+                        /* TODO: Remove fixed count */
+                        if (count == MAX_LAYER_LIBRARIES) {
+                            loader_log(VK_DBG_REPORT_ERROR_BIT, 0,
+                                       "%s ignored: max layer libraries exceed",
+                                       temp_str);
+                            break;
+                        }
+
+                        fp_get_ext = loader_platform_get_proc_address(handle, "vkGetGlobalExtensionInfo");
+                        if (!fp_get_ext) {
+                            loader_log(VK_DBG_REPORT_WARN_BIT, 0,
+                                       "Couldn't dlsym vkGetGlobalExtensionInfo from library %s",
+                                       temp_str);
+                            dent = readdir(curdir);
+                            loader_platform_close_library(handle);
+                            continue;
+                        }
+
+                        loader.scanned_layers[count].lib_name =
+                                malloc(strlen(temp_str) + 1);
+                        if (loader.scanned_layers[count].lib_name == NULL) {
+                            loader_log(VK_DBG_REPORT_ERROR_BIT, 0, "%s ignored: out of memory", temp_str);
+                            break;
+                        }
+
+                        strcpy(loader.scanned_layers[count].lib_name, temp_str);
+
+                        fprintf(stderr, "Collecting global extensions for %s\n", temp_str);
+                        get_global_extensions(
+                                    fp_get_ext,
+                                    "vkGetGlobalExtensionInfo",
+                                    loader.scanned_layers[count].lib_name,
+                                    VK_EXTENSION_ORIGIN_LAYER,
+                                    &loader.scanned_layers[count].global_extension_list);
+
+                        fp_get_ext = loader_platform_get_proc_address(handle,
+                                                                      "vkGetPhysicalDeviceExtensionInfo");
+                        if (fp_get_ext) {
+                            loader.scanned_layers[count].physical_device_extensions_supported = true;
+                        }
+
+                        count++;
+                        loader_platform_close_library(handle);
+                    }
+                }
+
+                dent = readdir(curdir);
+            } // while (dir_entry)
+            if (count == MAX_LAYER_LIBRARIES)
+                break;
+            closedir(curdir);
+        } // if (curdir))
     } // for (libpaths)
 
     loader.scanned_layer_count = count;
@@ -1113,14 +1143,19 @@ uint32_t loader_activate_instance_layers(struct loader_instance *inst)
     /*
      * Figure out how many actual layers will need to be wrapped.
      */
-    inst->layer_count = 0;
     for (uint32_t i = 0; i < inst->enabled_instance_extensions.count; i++) {
         struct loader_extension_property *ext_prop = &inst->enabled_instance_extensions.list[i];
-        if (ext_prop->origin == VK_EXTENSION_ORIGIN_LAYER) {
-            inst->layer_count++;
+        if (ext_prop->alias) {
+            ext_prop = ext_prop->alias;
+        }
+        if (ext_prop->origin != VK_EXTENSION_ORIGIN_LAYER) {
+            continue;
         }
+        loader_add_to_ext_list(&inst->activated_layer_list, 1, ext_prop);
     }
 
+    inst->layer_count = inst->activated_layer_list.count;
+
     if (!inst->layer_count) {
         return 0;
     }
@@ -1134,8 +1169,8 @@ uint32_t loader_activate_instance_layers(struct loader_instance *inst)
 
     /* Create instance chain of enabled layers */
     layer_idx = inst->layer_count - 1;
-    for (int32_t i = inst->enabled_instance_extensions.count - 1; i >= 0; i--) {
-        struct loader_extension_property *ext_prop = &inst->enabled_instance_extensions.list[i];
+    for (int32_t i = inst->activated_layer_list.count - 1; i >= 0; i--) {
+        struct loader_extension_property *ext_prop = &inst->activated_layer_list.list[i];
         loader_platform_dl_handle lib_handle;
 
         /*
@@ -1167,9 +1202,7 @@ uint32_t loader_activate_instance_layers(struct loader_instance *inst)
          * will not use a wrapped object and must look up their local dispatch table from
          * the given baseObject.
          */
-        if (ext_prop->origin != VK_EXTENSION_ORIGIN_LAYER) {
-            continue;
-        }
+        assert(ext_prop->origin == VK_EXTENSION_ORIGIN_LAYER);
 
         nextInstObj = (inst->wrappedInstance + layer_idx);
         nextInstObj->pGPA = nextGPA;
diff --git a/loader/loader.h b/loader/loader.h
index e2befdc4..0e62ec6e 100644
--- a/loader/loader.h
+++ b/loader/loader.h
@@ -43,6 +43,7 @@
 #  define LOADER_EXPORT
 #endif
 
+#define MAX_EXTENSION_NAME_SIZE 63
 #define MAX_LAYER_LIBRARIES 64
 #define MAX_GPUS_FOR_LAYER 16
 
@@ -56,6 +57,18 @@ struct loader_extension_property {
     VkExtensionProperties info;
     const char *lib_name;
     enum extension_origin origin;
+    // An extension library can export the same extension
+    // under different names. Handy to provide a "grouping"
+    // such as Validation. However, the loader requires
+    // that a layer be included only once in a chain.
+    // During layer scanning the loader will check if
+    // the vkGet*ProcAddr is the same as an existing extension
+    // If so, it will link them together via the alias pointer.
+    // At initialization time we'll follow the alias pointer
+    // to the "base" extension and then use that extension
+    // internally to ensure we reject duplicates
+    char get_extension_info_name[MAX_EXTENSION_NAME_SIZE+1];
+    struct loader_extension_property *alias;
     bool hosted;        // does the extension reside in one driver/layer
 };
 
@@ -189,6 +202,7 @@ struct loader_instance {
     struct loader_msg_callback_map_entry *icd_msg_callback_map;
 
     struct loader_extension_list enabled_instance_extensions;
+    struct loader_extension_list activated_layer_list;
 
     uint32_t  app_extension_count;
     VkExtensionProperties *app_extension_props;