From 0a314bca16f6de199c319ffb0d84a5d5c3a61387 Mon Sep 17 00:00:00 2001 From: mat Date: Tue, 24 May 2022 20:28:08 -0500 Subject: rename code-generator to codegen --- codegen/.gitignore | 3 + codegen/README.md | 13 ++++ codegen/burger.json | 1 + codegen/lib/download.py | 89 ++++++++++++++++++++++ codegen/lib/mappings.py | 60 +++++++++++++++ codegen/lib/packetcodegen.py | 170 +++++++++++++++++++++++++++++++++++++++++++ codegen/lib/utils.py | 15 ++++ codegen/newpacket.py | 17 +++++ 8 files changed, 368 insertions(+) create mode 100644 codegen/.gitignore create mode 100644 codegen/README.md create mode 100644 codegen/burger.json create mode 100644 codegen/lib/download.py create mode 100644 codegen/lib/mappings.py create mode 100644 codegen/lib/packetcodegen.py create mode 100644 codegen/lib/utils.py create mode 100644 codegen/newpacket.py (limited to 'codegen') diff --git a/codegen/.gitignore b/codegen/.gitignore new file mode 100644 index 00000000..2ef6e1be --- /dev/null +++ b/codegen/.gitignore @@ -0,0 +1,3 @@ +downloads +__pycache__ +*.tmp diff --git a/codegen/README.md b/codegen/README.md new file mode 100644 index 00000000..fa00b63f --- /dev/null +++ b/codegen/README.md @@ -0,0 +1,13 @@ +Tools for automatically generating code to help with updating Minecraft versions. + +The directory name doesn't start with `azalea-` because it's not a Rust crate. + +## Usage + +Generate packet:\ +`python newpacket.py [packet id] [clientbound or serverbound] \[game/handshake/login/status\]`\ +This will create a new file in the `azalea-protocol/src/packets/\[state\] directory`. You will probably have to manually fix up the auto generated code. + +Migrate to a new Minecraft version:\ +`python migrate.py [new version]`\ +This updates all the packet ids in `azalea-protocol/src/packets/mod.rs` and creates all the new packets. diff --git a/codegen/burger.json b/codegen/burger.json new file mode 100644 index 00000000..4fa39aa3 --- /dev/null +++ b/codegen/burger.json @@ -0,0 +1 @@ +{"message": "Not Found", "documentation_url": "https://docs.github.com/rest/reference/repos#get-the-latest-release"} \ No newline at end of file diff --git a/codegen/lib/download.py b/codegen/lib/download.py new file mode 100644 index 00000000..284591ab --- /dev/null +++ b/codegen/lib/download.py @@ -0,0 +1,89 @@ +from .mappings import Mappings +import requests +import json +import os + +# make sure the downloads directory exists +if not os.path.exists('downloads'): + os.mkdir('downloads') + + +def get_burger(): + if not os.path.exists('downloads/Burger'): + with open('burger.json', 'w') as f: + json.dump(requests.get( + 'https://api.github.com/repos/Burger/Burger/releases/latest').json(), f) + print('\033[92mDownloading Burger...\033[m') + os.system( + 'cd downloads && git clone https://github.com/pokechu22/Burger && cd Burger && git pull') + + print('\033[92mInstalling dependencies...\033[m') + os.system('cd downloads/Burger && pip install six jawa') + + +def get_version_manifest(): + if not os.path.exists(f'downloads/version_manifest.json'): + print( + f'\033[92mDownloading version manifest...\033[m') + version_manifest_data = requests.get( + 'https://launchermeta.mojang.com/mc/game/version_manifest.json').json() + with open(f'downloads/version_manifest.json', 'w') as f: + json.dump(version_manifest_data, f) + else: + with open(f'downloads/version_manifest.json', 'r') as f: + version_manifest_data = json.load(f) + return version_manifest_data + + +def get_version_data(version_id: str): + if not os.path.exists(f'downloads/{version_id}.json'): + version_manifest_data = get_version_manifest() + + print( + f'\033[92mGetting data for \033[1m{version_id}..\033[m') + package_url = next( + filter(lambda v: v['id'] == version_id, version_manifest_data['versions']))['url'] + package_data = requests.get(package_url).json() + with open(f'downloads/{version_id}.json', 'w') as f: + json.dump(package_data, f) + else: + with open(f'downloads/{version_id}.json', 'r') as f: + package_data = json.load(f) + return package_data + + +def get_client_jar(version_id: str): + if not os.path.exists(f'downloads/client-{version_id}.jar'): + package_data = get_version_data(version_id) + print('\033[92mDownloading client jar...\033[m') + client_jar_url = package_data['downloads']['client']['url'] + with open(f'downloads/client-{version_id}.jar', 'wb') as f: + f.write(requests.get(client_jar_url).content) + + +def get_burger_data_for_version(version_id: str): + if not os.path.exists(f'downloads/burger-{version_id}.json'): + get_burger() + get_client_jar(version_id) + + os.system( + f'cd downloads/Burger && python munch.py ../client-{version_id}.jar --output ../burger-{version_id}.json' + ) + with open(f'downloads/burger-{version_id}.json', 'r') as f: + return json.load(f) + + +def get_mappings_for_version(version_id: str): + if not os.path.exists(f'downloads/mappings-{version_id}.txt'): + package_data = get_version_data(version_id) + + client_mappings_url = package_data['downloads']['client_mappings']['url'] + + mappings_text = requests.get(client_mappings_url).text + + with open(f'downloads/mappings-{version_id}.txt', 'w') as f: + f.write(mappings_text) + else: + with open(f'downloads/mappings-{version_id}.txt', 'r') as f: + mappings_text = f.read() + return Mappings.parse(mappings_text) diff --git a/codegen/lib/mappings.py b/codegen/lib/mappings.py new file mode 100644 index 00000000..fb3e8bda --- /dev/null +++ b/codegen/lib/mappings.py @@ -0,0 +1,60 @@ +class Mappings: + __slots__ = ('classes', 'fields', 'methods') + + def __init__(self, classes, fields, methods): + self.classes = classes + self.fields = fields + self.methods = methods + + @staticmethod + def parse(mappings_txt): + classes = {} + fields = {} + methods = {} + + current_obfuscated_class_name = None + + for line in mappings_txt.splitlines(): + if line.startswith('#') or line == '': + continue + + if line.startswith(' '): + # if a line starts with 4 spaces, that means it's a method or a field + if '(' in line: + # if it has an opening parenthesis, it's a method + real_name_with_parameters_and_line, obfuscated_name = line.strip().split(' -> ') + real_name_with_parameters = real_name_with_parameters_and_line.split( + ':')[-1] + + real_name = real_name_with_parameters.split('(')[0] + parameters = real_name_with_parameters.split('(')[1] + + if current_obfuscated_class_name not in methods: + methods[current_obfuscated_class_name] = {} + methods[current_obfuscated_class_name][ + f'{obfuscated_name}({parameters})'] = real_name + else: + # otherwise, it's a field + real_name_with_type, obfuscated_name = line.strip().split(' -> ') + real_name = real_name_with_type.split(' ')[1] + + if current_obfuscated_class_name not in fields: + fields[current_obfuscated_class_name] = {} + fields[current_obfuscated_class_name][obfuscated_name] = real_name + else: + # otherwise it's a class + real_name, obfuscated_name = line.strip(':').split(' -> ') + current_obfuscated_class_name = obfuscated_name + + classes[obfuscated_name] = real_name + + return Mappings(classes, fields, methods) + + def get_field(self, obfuscated_class_name, obfuscated_field_name): + return self.fields.get(obfuscated_class_name, {}).get(obfuscated_field_name) + + def get_class(self, obfuscated_class_name): + return self.classes[obfuscated_class_name] + + def get_method(self, obfuscated_class_name, obfuscated_method_name, obfuscated_signature): + return self.methods[obfuscated_class_name][f'{obfuscated_method_name}({obfuscated_signature})'] diff --git a/codegen/lib/packetcodegen.py b/codegen/lib/packetcodegen.py new file mode 100644 index 00000000..b4c6a83b --- /dev/null +++ b/codegen/lib/packetcodegen.py @@ -0,0 +1,170 @@ +from .utils import to_snake_case, to_camel_case +from .mappings import Mappings + + +def burger_type_to_rust_type(burger_type): + is_var = False + uses = set() + + if burger_type == 'byte': + field_type_rs = 'i8' + elif burger_type == 'short': + field_type_rs = 'i16' + elif burger_type == 'int': + field_type_rs = 'i32' + elif burger_type == 'long': + field_type_rs = 'i64' + elif burger_type == 'float': + field_type_rs = 'f32' + elif burger_type == 'double': + field_type_rs = 'f64' + + elif burger_type == 'varint': + is_var = True + field_type_rs = 'i32' + elif burger_type == 'varlong': + is_var = True + field_type_rs = 'i64' + + elif burger_type == 'boolean': + field_type_rs = 'bool' + elif burger_type == 'string': + field_type_rs = 'String' + + elif burger_type == 'chatcomponent': + field_type_rs = 'Component' + uses.add('azalea_chat::component::Component') + elif burger_type == 'identifier': + field_type_rs = 'ResourceLocation' + uses.add('azalea_core::resource_location::ResourceLocation') + elif burger_type == 'uuid': + field_type_rs = 'Uuid' + uses.add('uuid::Uuid') + elif burger_type == 'position': + field_type_rs = 'BlockPos' + uses.add('azalea_core::BlockPos') + elif burger_type == 'nbtcompound': + field_type_rs = 'azalea_nbt::Tag' + elif burger_type == 'itemstack': + field_type_rs = 'Slot' + uses.add('azalea_core::Slot') + elif burger_type == 'metadata': + field_type_rs = 'EntityMetadata' + uses.add('crate::mc_buf::EntityMetadata') + elif burger_type == 'enum': + # enums are too complicated, leave those to the user + field_type_rs = 'todo!()' + elif burger_type.endswith('[]'): + field_type_rs, is_var, uses = burger_type_to_rust_type( + burger_type[:-2]) + field_type_rs = f'Vec<{field_type_rs}>' + else: + print('Unknown field type:', burger_type) + exit() + return field_type_rs, is_var, uses + + +def write_packet_file(state, packet_name_snake_case, code): + with open(f'../azalea-protocol/src/packets/{state}/{packet_name_snake_case}.rs', 'w') as f: + f.write(code) + + +def generate(burger_packets, mappings: Mappings, target_packet_id, target_packet_direction, target_packet_state): + for packet in burger_packets.values(): + if packet['id'] != target_packet_id: + continue + + direction = packet['direction'].lower() # serverbound or clientbound + state = {'PLAY': 'game'}.get(packet['state'], packet['state'].lower()) + + if state != target_packet_state or direction != target_packet_direction: + continue + + generated_packet_code = [] + uses = set() + generated_packet_code.append( + f'#[derive(Clone, Debug, McBuf, {to_camel_case(state)}Packet)]') + uses.add(f'packet_macros::{{{to_camel_case(state)}Packet, McBuf}}') + + obfuscated_class_name = packet['class'].split('.')[0].split('$')[0] + class_name = mappings.get_class( + obfuscated_class_name).split('.')[-1].split('$')[0] + + generated_packet_code.append( + f'pub struct {to_camel_case(class_name)} {{') + + for instruction in packet.get('instructions', []): + if instruction['operation'] == 'write': + obfuscated_field_name = instruction['field'] + if '.' in obfuscated_field_name or ' ' in obfuscated_field_name or '(' in obfuscated_field_name: + generated_packet_code.append(f'// TODO: {instruction}') + continue + field_name = mappings.get_field( + obfuscated_class_name, obfuscated_field_name) + if not field_name: + generated_packet_code.append( + f'// TODO: unknown field {instruction}') + continue + + field_type = instruction['type'] + field_type_rs, is_var, instruction_uses = burger_type_to_rust_type( + field_type) + if is_var: + generated_packet_code.append('#[var]') + generated_packet_code.append( + f'pub {to_snake_case(field_name)}: {field_type_rs},') + uses.update(instruction_uses) + else: + generated_packet_code.append(f'// TODO: {instruction}') + continue + + generated_packet_code.append('}') + + if uses: + # empty line before the `use` statements + generated_packet_code.insert(0, '') + for use in uses: + generated_packet_code.insert(0, f'use {use};') + + print(generated_packet_code) + write_packet_file(state, to_snake_case(class_name), + '\n'.join(generated_packet_code)) + print() + + mod_rs_dir = f'../azalea-protocol/src/packets/{state}/mod.rs' + with open(mod_rs_dir, 'r') as f: + mod_rs = f.read().splitlines() + + pub_mod_line = f'pub mod {to_snake_case(class_name)};' + if pub_mod_line not in mod_rs: + mod_rs.insert(0, pub_mod_line) + packet_mod_rs_line = f' {hex(packet["id"])}: {to_snake_case(class_name)}::{to_camel_case(class_name)},' + + in_serverbound = False + in_clientbound = False + for i, line in enumerate(mod_rs): + if line.strip() == 'Serverbound => {': + in_serverbound = True + continue + elif line.strip() == 'Clientbound => {': + in_clientbound = True + continue + elif line.strip() in ('}', '},'): + if (in_serverbound and direction == 'serverbound') or (in_clientbound and direction == 'clientbound'): + mod_rs.insert(i, packet_mod_rs_line) + break + in_serverbound = in_clientbound = False + continue + + if line.strip() == '' or line.strip().startswith('//') or (not in_serverbound and direction == 'serverbound') or (not in_clientbound and direction == 'clientbound'): + continue + + line_packet_id_hex = line.strip().split(':')[0] + assert line_packet_id_hex.startswith('0x') + line_packet_id = int(line_packet_id_hex[2:], 16) + if line_packet_id > packet['id']: + mod_rs.insert(i, packet_mod_rs_line) + break + + with open(mod_rs_dir, 'w') as f: + f.write('\n'.join(mod_rs)) diff --git a/codegen/lib/utils.py b/codegen/lib/utils.py new file mode 100644 index 00000000..5336d574 --- /dev/null +++ b/codegen/lib/utils.py @@ -0,0 +1,15 @@ +import urllib.request +import gzip +import json +import re +import io + + +def to_snake_case(name): + s = re.sub('([A-Z])', r'_\1', name) + return s.lower().strip('_') + + +def to_camel_case(name): + s = re.sub('_([a-z])', lambda m: m.group(1).upper(), name) + return s[0].upper() + s[1:] diff --git a/codegen/newpacket.py b/codegen/newpacket.py new file mode 100644 index 00000000..f4dc172e --- /dev/null +++ b/codegen/newpacket.py @@ -0,0 +1,17 @@ +from lib import download, packetcodegen # type: ignore +import sys +import os + +mappings = download.get_mappings_for_version('1.18.2') +burger_data = download.get_burger_data_for_version('1.18.2') + +burger_packets_data = burger_data[0]['packets']['packet'] +packet_id, direction, state = int(sys.argv[1]), sys.argv[2], sys.argv[3] +print( + f'Generating code for packet id: {packet_id} with direction {direction} and state {state}') +packetcodegen.generate(burger_packets_data, mappings, + packet_id, direction, state) + +os.system('cd .. && cargo fmt') + +print('Done!') -- cgit v1.2.3