tests: Update document validator to check for valid tests

This adds a set of checks to the document validator that will iterate through
all of the checks documented in vk_validation_layer_details.md and compare the
names of the test(s) for each check against the names of the actual tests in
layer_validation_tests.cpp.

Currently, all checks not ending in "_NONE" that don't have a valid test
recorded in the table will report a warning with a final count of the warnings
given. There are initially 131 such warnings, but I suspect at least a handful
of them are implemented but just need to be correctly documented.

Once all the test holes are filled in we can promote the warnings to errors
to make sure that any new checks are both documented and have a valid test
implemented.

Rough support of wild-carded testnames such as "CreatePipeline*NotConsumed" is
included.

1 files changed, 72 insertions, 6 deletions

diff --git a/vk_layer_documentation_generate.py b/vk_layer_documentation_generate.py
index a14e16bc..5e793696 100755
--- a/vk_layer_documentation_generate.py
+++ b/vk_layer_documentation_generate.py
@@ -85,6 +85,7 @@ layer_inputs = { 'draw_state' : {'header' : 'layers/core_validation_error_enums.
 
 builtin_headers = [layer_inputs[ln]['header'] for ln in layer_inputs]
 builtin_source = [layer_inputs[ln]['source'] for ln in layer_inputs]
+builtin_tests = ['tests/layer_validation_tests.cpp', ]
 
 # List of extensions in layers that are included in documentation, but not in vulkan.py API set
 layer_extension_functions = ['objTrackGetObjects', 'objTrackGetObjectsOfType']
@@ -93,6 +94,7 @@ def handle_args():
     parser = argparse.ArgumentParser(description='Generate layer documenation from source.')
     parser.add_argument('--in_headers', required=False, default=builtin_headers, help='The input layer header files from which code will be generated.')
     parser.add_argument('--in_source', required=False, default=builtin_source, help='The input layer source files from which code will be generated.')
+    parser.add_argument('--test_source', required=False, default=builtin_tests, help='The input test source files from which code will be generated.')
     parser.add_argument('--layer_doc', required=False, default='layers/vk_validation_layer_details.md', help='Existing layer document to be validated against actual layers.')
     parser.add_argument('--validate', action='store_true', default=False, help='Validate that there are no mismatches between layer documentation and source. This includes cross-checking the validation checks, and making sure documented Vulkan API calls exist.')
     parser.add_argument('--print_structs', action='store_true', default=False, help='Primarily a debug option that prints out internal data structs used to generate layer docs.')
@@ -105,10 +107,12 @@ class bcolors:
     def __init__(self):
         self.GREEN = '\033[0;32m'
         self.RED = '\033[0;31m'
+        self.YELLOW = '\033[1;33m'
         self.ENDC = '\033[0m'
         if 'Linux' != platform.system():
             self.GREEN = ''
             self.RED = ''
+            self.YELLOW = ''
             self.ENDC = ''
 
     def green(self):
@@ -117,9 +121,43 @@ class bcolors:
     def red(self):
         return self.RED
 
+    def yellow(self):
+        return self.YELLOW
+
     def endc(self):
         return self.ENDC
 
+# Class to parse the validation layer test source and store testnames
+class TestParser:
+    def __init__(self, test_file_list, test_group_name='VkLayerTest'):
+        self.test_files = test_file_list
+        self.tests_set = set()
+        self.test_trigger_txt = 'TEST_F(%s' % test_group_name
+
+    # Parse test files into internal data struct
+    def parse(self):
+        # For each test file, parse test names into set
+        grab_next_line = False # handle testname on separate line than wildcard
+        for test_file in self.test_files:
+            with open(test_file) as tf:
+                for line in tf:
+                    if True in [line.strip().startswith(comment) for comment in ['//', '/*']]:
+                        continue
+
+                    if self.test_trigger_txt in line:
+                        #print('Test wildcard in line: %s' % (line))
+                        testname = line.split(',')[-1]
+                        testname = testname.strip().strip(' {)')
+                        #print('Inserting test: "%s"' % (testname))
+                        if ('' == testname):
+                            grab_next_line = True
+                            continue
+                        self.tests_set.add(testname)
+                    if grab_next_line: # test name on its own line
+                        grab_next_line = False
+                        testname = testname.strip().strip(' {)')
+                        self.tests_set.add(testname)
+
 # Class to parse the layer source code and store details in internal data structs
 class LayerParser:
     def __init__(self, header_file_list, source_file_list):
@@ -145,7 +183,6 @@ class LayerParser:
                         # For now skipping lines starting w/ comment, may use these to capture
                         #  documentation in the future
                         continue
-
                     # Find enums
                     if store_enum:
                         if '}' in line: # we're done with enum definition
@@ -251,13 +288,16 @@ class LayerDoc:
                     self.layer_doc_dict[layer_name] = {}
                     self.layer_doc_dict[layer_name]['overview'] = ''
 
-    # Verify that checks and api references in layer doc match reality
+    # Verify that checks, tests and api references in layer doc match reality
     #  Report API calls from doc that are not found in API
     #  Report checks from doc that are not in actual layers
     #  Report checks from layers that are not captured in doc
-    def validate(self, layer_dict):
+    #  Report checks from doc that do not have a valid test
+    def validate(self, layer_dict, tests_set):
+        #print("tests_set: %s" % (tests_set))
         # Count number of errors found and return it
         errors_found = 0
+        warnings_found = 0
         # First we'll go through the doc datastructures and flag any issues
         for chk in self.enum_list:
             doc_layer_found = False
@@ -269,6 +309,7 @@ class LayerDoc:
             if not doc_layer_found:
                 print(self.txt_color.red() + 'Actual layers do not contain documented check: %s' % (chk) + self.txt_color.endc())
                 errors_found += 1
+
         # Now go through API names in doc and verify they're real
         # First we're going to transform proto names from vulkan.py into single list
         core_api_names = [p.name for p in vulkan.core.protos]
@@ -284,6 +325,24 @@ class LayerDoc:
                     if api[2:] not in api_names and api not in layer_extension_functions:
                         print(self.txt_color.red() + 'Doc references invalid function: %s' % (api) + self.txt_color.endc())
                         errors_found += 1
+                # For now warn on missing or invalid tests
+                for test in self.layer_doc_dict[ln][chk]['tests']:
+                    if '*' in test:
+                        # naive way to handle wildcards, just make sure we have matches on parts
+                        test_parts = test.split('*')
+                        for part in test_parts:
+                            part_found = False
+                            for t in tests_set:
+                                if part in t:
+                                    part_found = True
+                                    break
+                            if not part_found:
+                                print(self.txt_color.yellow() + 'Validation check %s has missing or invalid test : %s' % (chk, test))
+                                warnings_found += 1
+                                break
+                    elif test not in tests_set and not chk.endswith('_NONE'):
+                        print(self.txt_color.yellow() + 'Validation check %s has missing or invalid test : %s' % (chk, test))
+                        warnings_found += 1
         # Now go through all of the actual checks in the layers and make sure they're covered in the doc
         for ln in layer_dict:
             for chk in layer_dict[ln]['CHECKS']:
@@ -291,7 +350,7 @@ class LayerDoc:
                     print(self.txt_color.red() + 'Doc is missing check: %s' % (chk) + self.txt_color.endc())
                     errors_found += 1
 
-        return errors_found
+        return (errors_found, warnings_found)
 
     # Print all of the checks captured in the doc
     def print_checks(self):
@@ -304,6 +363,9 @@ def main(argv=None):
     layer_parser = LayerParser(opts.in_headers, opts.in_source)
     # Parse files into internal data structs
     layer_parser.parse()
+    # Parse test files
+    test_parser = TestParser(opts.test_source)
+    test_parser.parse()
 
     # Generate requested types of output
     if opts.print_structs: # Print details of internal data structs
@@ -315,9 +377,13 @@ def main(argv=None):
         layer_doc.print_checks()
 
     if opts.validate:
-        num_errors = layer_doc.validate(layer_parser.layer_dict)
+        (num_errors, num_warnings) = layer_doc.validate(layer_parser.layer_dict, test_parser.tests_set)
+        txt_color = bcolors()
+        if (0 == num_warnings):
+            print(txt_color.green() + 'No warning cases found between %s and implementation' % (os.path.basename(opts.layer_doc)) + txt_color.endc())
+        else:
+            print(txt_color.yellow() + 'Found %s warnings. See above for details' % num_warnings)
         if (0 == num_errors):
-            txt_color = bcolors()
             print(txt_color.green() + 'No mismatches found between %s and implementation' % (os.path.basename(opts.layer_doc)) + txt_color.endc())
         else:
             return num_errors