diff options
Diffstat (limited to 'scripts/generate_source.py')
| -rwxr-xr-x | scripts/generate_source.py | 295 |
1 files changed, 203 insertions, 92 deletions
diff --git a/scripts/generate_source.py b/scripts/generate_source.py index 336e5ffb..82bc213e 100755 --- a/scripts/generate_source.py +++ b/scripts/generate_source.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 -# Copyright (c) 2019-2025 The Khronos Group Inc. -# Copyright (c) 2019-2025 Valve Corporation -# Copyright (c) 2019-2025 LunarG, Inc. -# Copyright (c) 2019-2025 Google Inc. -# Copyright (c) 2023-2025 RasterGrid Kft. +# Copyright (c) 2019 The Khronos Group Inc. +# Copyright (c) 2019 Valve Corporation +# Copyright (c) 2019 LunarG, Inc. +# Copyright (c) 2019 Google Inc. +# Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2023-2023 RasterGrid Kft. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,118 +23,218 @@ import argparse import filecmp import os -import re -import json import shutil -import subprocess import sys +import re import tempfile - +import json import common_codegen -# files to exclude from --verify check -verify_exclude = ['.clang-format'] +from xml.etree import ElementTree + +# Because we have special logic to import the Registry from input arguments and the BaseGenerator comes from the registry, we have to delay defining it until *after* +# the Registry has been imported. Yes this is awkward, but it was the least awkward way to make --verify work. +generators = {} + +def RunGenerators(api: str, registry: str, video_registry: str, directory: str, styleFile: str, targetFilter: str, flatOutput: bool): + + try: + common_codegen.RunShellCmd(f'clang-format --version') + has_clang_format = True + except: + has_clang_format = False + if not has_clang_format: + print("WARNING: Unable to find clang-format!") + + # These live in the Vulkan-Docs repo, but are pulled in via the + # Vulkan-Headers/registry folder + # At runtime we inject python path to find these helper scripts + scripts = os.path.dirname(registry) + scripts_directory_path = os.path.dirname(os.path.abspath(__file__)) + registry_headers_path = os.path.join(scripts_directory_path, scripts) + sys.path.insert(0, registry_headers_path) + try: + from reg import Registry + except: + print("ModuleNotFoundError: No module named 'reg'") # normal python error message + print(f'{registry_headers_path} is not pointing to the Vulkan-Headers registry directory.') + print("Inside Vulkan-Headers there is a registry/reg.py file that is used.") + sys.exit(1) # Return without call stack so easy to spot error + + from base_generator import BaseGeneratorOptions + from generators.mock_icd_generator import MockICDOutputGenerator + from generators.vulkan_tools_helper_file_generator import HelperFileOutputGenerator + from generators.vulkaninfo_generator import VulkanInfoGenerator + + # These set fields that are needed by both OutputGenerator and BaseGenerator, + # but are uniform and don't need to be set at a per-generated file level + from base_generator import (SetTargetApiName, SetMergedApiNames) + SetTargetApiName(api) + + # Generated directory and dispatch table helper file name may be API specific (e.g. Vulkan SC) + mock_icd_generated_directory = 'icd/generated' + vulkaninfo_generated_directory = 'vulkaninfo/generated' + + generators.update({ + 'vk_typemap_helper.h': { + 'generator' : HelperFileOutputGenerator, + 'genCombined': False, + 'directory' : mock_icd_generated_directory, + }, + 'function_declarations.h': { + 'generator' : MockICDOutputGenerator, + 'genCombined': False, + 'directory' : mock_icd_generated_directory, + }, + 'function_definitions.h': { + 'generator' : MockICDOutputGenerator, + 'genCombined': False, + 'directory' : mock_icd_generated_directory, + }, + 'vulkaninfo.hpp': { + 'generator' : VulkanInfoGenerator, + 'genCombined': False, + 'directory' : vulkaninfo_generated_directory, + }, + }) + + unknownTargets = [x for x in (targetFilter if targetFilter else []) if x not in generators.keys()] + if unknownTargets: + print(f'ERROR: No generator options for unknown target(s): {", ".join(unknownTargets)}', file=sys.stderr) + return 1 + + # Filter if --target is passed in + targets = [x for x in generators.keys() if not targetFilter or x in targetFilter] + + for index, target in enumerate(targets, start=1): + print(f'[{index}|{len(targets)}] Generating {target}') + + # First grab a class constructor object and create an instance + generator = generators[target]['generator'] + gen = generator() + + # This code and the 'genCombined' generator metadata is used by downstream + # users to generate code with all Vulkan APIs merged into the target API variant + # (e.g. Vulkan SC) when needed. The constructed apiList is also used to filter + # out non-applicable extensions later below. + apiList = [api] + if api != 'vulkan' and generators[target]['genCombined']: + SetMergedApiNames('vulkan') + apiList.append('vulkan') + else: + SetMergedApiNames(None) + + # For people who want to generate all the files in a single director + if flatOutput: + outDirectory = os.path.abspath(os.path.join(directory)) + else: + outDirectory = os.path.abspath(os.path.join(directory, generators[target]['directory'])) + + options = BaseGeneratorOptions( + customFileName = target, + customDirectory = outDirectory) + + if not os.path.exists(outDirectory): + os.makedirs(outDirectory) + + # Create the registry object with the specified generator and generator + # options. The options are set before XML loading as they may affect it. + reg = Registry(gen, options) + + # Parse the specified registry XML into an ElementTree object + tree = ElementTree.parse(registry) + + # Load the XML tree into the registry object + reg.loadElementTree(tree) + + # Set the path to the video registry so that videoStd is available + reg.genOpts.videoXmlPath = video_registry + + # Finally, use the output generator to create the requested target + reg.apiGen() + + # Run clang-format on the file + if has_clang_format and styleFile: + common_codegen.RunShellCmd(f'clang-format -i --style=file:{styleFile} {os.path.join(outDirectory, target)}') + def main(argv): + + # files to exclude from --verify check + verify_exclude = ['.clang-format'] # None currently + parser = argparse.ArgumentParser(description='Generate source code for this repository') + parser.add_argument('registry', metavar='REGISTRY_PATH', help='path to the Vulkan-Headers registry directory') parser.add_argument('--api', default='vulkan', choices=['vulkan', 'vulkansc'], help='Specify API name to generate') parser.add_argument('--generated-version', help='sets the header version used to generate the repo') - parser.add_argument('registry', metavar='REGISTRY_PATH', help='path to the Vulkan-Headers registry directory') group = parser.add_mutually_exclusive_group() + group.add_argument('--target', nargs='+', help='only generate file names passed in') group.add_argument('-i', '--incremental', action='store_true', help='only update repo files that change') group.add_argument('-v', '--verify', action='store_true', help='verify repo files match generator output') + group.add_argument('-o', action='store', dest='directory', help='Create target and related files in specified directory') args = parser.parse_args(argv) - # output paths and the list of files in the path - files_to_gen = {str(os.path.join('icd','generated')) : ['vk_typemap_helper.h', - 'function_definitions.h', - 'function_declarations.h'], - str(os.path.join('vulkaninfo','generated')): ['vulkaninfo.hpp']} - - #base directory for the source repository - repo_dir = common_codegen.repo_relative('') - - # Update the api_version in the respective json files - if args.generated_version: - json_files = [] - json_files.append(common_codegen.repo_relative('icd/VkICD_mock_icd.json.in')) - for json_file in json_files: - with open(json_file) as f: - data = json.load(f) - - data["ICD"]["api_version"] = args.generated_version - - with open(json_file, mode='w', encoding='utf-8', newline='\n') as f: - f.write(json.dumps(data, indent=4)) - - # get directory where generators will run if needed - if args.verify or args.incremental: - # generate in temp directory so we can compare or copy later - temp_obj = tempfile.TemporaryDirectory(prefix='VulkanLoader_generated_source_') - temp_dir = temp_obj.name - for path in files_to_gen.keys(): - os.makedirs(os.path.join(temp_dir, path)) + repo_dir = common_codegen.repo_relative('.') registry = os.path.abspath(os.path.join(args.registry, 'vk.xml')) - if not os.path.isfile(registry): + video_registry = os.path.abspath(os.path.join(args.registry, 'video.xml')) + if not os.path.isfile(registry) and not os.path.isfile(registry): registry = os.path.abspath(os.path.join(args.registry, 'Vulkan-Headers/registry/vk.xml')) if not os.path.isfile(registry): print(f'cannot find vk.xml in {args.registry}') return -1 + video_registry = os.path.abspath(os.path.join(args.registry, 'Vulkan-Headers/registry/video.xml')) + if not os.path.isfile(video_registry): + print(f'{video_registry} does not exist') + return -1 + + # Need pass style file incase running with --verify and it can't find the file automatically in the temp directory + style_file = os.path.join(repo_dir, '.clang-format') - # run each code generator - for path, filenames in files_to_gen.items(): - for filename in filenames: - if args.verify or args.incremental: - output_path = os.path.join(temp_dir, path) - else: - output_path = common_codegen.repo_relative(path) - - cmd = [common_codegen.repo_relative(os.path.join('scripts','kvt_genvk.py')), - '-api', args.api, - '-registry', registry, - '-quiet', '-directory', output_path, filename] - print(' '.join(cmd)) - try: - if args.verify or args.incremental: - subprocess.check_call([sys.executable] + cmd, cwd=temp_dir) - else: - subprocess.check_call([sys.executable] + cmd, cwd=repo_dir) - - except Exception as e: - print('ERROR:', str(e)) - return 1 + # get directory where generators will run + if args.verify or args.incremental: + # generate in temp directory so we can compare or copy later + temp_obj = tempfile.TemporaryDirectory(prefix='vulkan_tools_codegen_') + temp_dir = temp_obj.name + gen_dir = temp_dir + elif args.directory: + gen_dir = args.directory + else: + # generate directly in the repo + gen_dir = repo_dir + + RunGenerators(api=args.api,registry=registry, video_registry=video_registry, directory=gen_dir, styleFile=style_file, targetFilter=args.target, flatOutput=False) # optional post-generation steps if args.verify: # compare contents of temp dir and repo temp_files = {} - for path in files_to_gen.keys(): - temp_files[path] = set() - temp_files[path].update(set(os.listdir(os.path.join(temp_dir, path)))) - repo_files = {} - for path in files_to_gen.keys(): - repo_files[path] = set() - repo_files[path].update(set(os.listdir(os.path.join(repo_dir, path))) - set(verify_exclude)) + for details in generators.values(): + if details['directory'] not in temp_files: + temp_files[details['directory']] = set() + temp_files[details['directory']].update(set(os.listdir(os.path.join(temp_dir, details['directory'])))) + if details['directory'] not in repo_files: + repo_files[details['directory']] = set() + repo_files[details['directory']].update(set(os.listdir(os.path.join(repo_dir, details['directory']))) - set(verify_exclude)) + # compare contents of temp dir and repo files_match = True - for path in files_to_gen.keys(): - for filename in sorted((temp_files[path] | repo_files[path])): - if filename not in repo_files[path]: - print('ERROR: Missing repo file', filename) - files_match = False - elif filename not in temp_files[path]: - print('ERROR: Missing generator for', filename) - files_match = False - elif not filecmp.cmp(os.path.join(temp_dir, path, filename), - os.path.join(repo_dir, path, filename), - shallow=False): - print('ERROR: Repo files do not match generator output for', filename) - files_match = False + for filename, details in generators.items(): + if filename not in repo_files[details['directory']]: + print('ERROR: Missing repo file', filename) + files_match = False + elif filename not in temp_files[details['directory']]: + print('ERROR: Missing generator for', filename) + files_match = False + elif not filecmp.cmp(os.path.join(temp_dir, details['directory'], filename), + os.path.join(repo_dir, details['directory'], filename), + shallow=False): + print('ERROR: Repo files do not match generator output for', filename) + files_match = False # return code for test scripts if files_match: @@ -143,17 +244,27 @@ def main(argv): elif args.incremental: # copy missing or differing files from temp directory to repo - for path in files_to_gen.keys(): - for filename in os.listdir(os.path.join(temp_dir,path)): - temp_filename = os.path.join(temp_dir, path, filename) - repo_filename = os.path.join(repo_dir, path, filename) - if not os.path.exists(repo_filename) or \ - not filecmp.cmp(temp_filename, repo_filename, shallow=False): - print('update', repo_filename) - shutil.copyfile(temp_filename, repo_filename) + for filename, details in generators.items(): + temp_filename = os.path.join(temp_dir, details['directory'], filename) + repo_filename = os.path.join(repo_dir, details['directory'], filename) + if not os.path.exists(repo_filename) or \ + not filecmp.cmp(temp_filename, repo_filename, shallow=False): + print('update', repo_filename) + shutil.copyfile(temp_filename, repo_filename) # write out the header version used to generate the code to a checked in CMake file if args.generated_version: + json_files = [] + json_files.append(common_codegen.repo_relative('icd/VkICD_mock_icd.json.in')) + for json_file in json_files: + with open(json_file) as f: + data = json.load(f) + + data["ICD"]["api_version"] = args.generated_version + + with open(json_file, mode='w', encoding='utf-8', newline='\n') as f: + f.write(json.dumps(data, indent=4)) + # Update the CMake project version with open(common_codegen.repo_relative('CMakeLists.txt'), "r+") as f: data = f.read() |