path: root/codegen/lib/extract.py

# Extracting data from the Minecraft jars

from typing import TYPE_CHECKING
from lib.download import get_mappings_for_version, get_server_jar, get_burger, get_client_jar, get_pixlyzer, get_yarn_data, get_fabric_api_versions, get_fabric_loader_versions
from lib.utils import get_dir_location, to_camel_case, upper_first_letter
from zipfile import ZipFile
import subprocess
import requests
import json
import sys
import re
import os


def generate_data_from_server_jar(version_id: str):
    if os.path.exists(get_dir_location(f'__cache__/generated-{version_id}')):
        return

    get_server_jar(version_id)
    os.system(
        f'cd {get_dir_location(f"__cache__")} && java -DbundlerMainClass=net.minecraft.data.Main -jar {get_dir_location(f"__cache__/server-{version_id}.jar")} --all --output \"{get_dir_location(f"__cache__/generated-{version_id}")}\"'
    )


def get_block_states_report(version_id: str):
    return get_report(version_id, 'blocks')
def get_registries_report(version_id: str):
    return get_report(version_id, 'registries')
def get_packets_report(version_id: str):
    return get_report(version_id, 'packets')
def get_report(version_id: str, name: str):
    generate_data_from_server_jar(version_id)
    with open(get_dir_location(f'__cache__/generated-{version_id}/reports/{name}.json'), 'r') as f:
        return json.load(f)

def get_registry_tags(version_id: str, name: str):
    generate_data_from_server_jar(version_id)
    tags_directory = get_dir_location(f'__cache__/generated-{version_id}/data/minecraft/tags/{name}')
    if not os.path.exists(tags_directory):
        return {}
    tags = {}
    for root, dirs, files in os.walk(tags_directory, topdown=False):
        for name in files:
            file = os.path.join(root, name)
            relative_path = file.replace(tags_directory, '')[1:]
            if not file.endswith('.json'):
                continue
            with open(file, 'r') as f:
                tags[relative_path[:-5]] = json.load(f)
    return tags

python_command = None


def determine_python_command():
    return 'venv/bin/python'
    


def run_python_command_and_download_deps(command):
    print('>', command)
    for _ in range(10):
        p = subprocess.Popen(
            command,
            stderr=subprocess.PIPE,
            shell=True
        )

        stderr = b''
        while True:
            data = p.stderr.read()
            if data == b'':
                break
            print(data.decode(), end='', flush=True)
            stderr += data

        regex_match = re.search(
            r'ModuleNotFoundError: No module named \'(\w+?)\'', stderr.decode())
        if not regex_match:
            out, err = p.communicate()
            if out:
                print(out)
            if err:
                print(err)
            break
        missing_lib = regex_match.group(1)
        print('Missing required lib:', missing_lib)
        subprocess.run(f'venv/bin/pip install {missing_lib}', cwd=os.path.dirname(os.path.dirname(__file__)))
    print('ok')


def get_burger_data_for_version(version_id: str):
    if not os.path.exists(get_dir_location(f'__cache__/burger-{version_id}.json')):
        get_burger()
        get_client_jar(version_id)
        get_mappings_for_version(version_id)

        print('\033[92mRunning Burger...\033[m')
        run_python_command_and_download_deps(
            f'cd {get_dir_location("__cache__/Burger")} && '\
            f'venv/bin/python munch.py {get_dir_location("__cache__")}/client-{version_id}.jar '\
            f'--output {get_dir_location("__cache__")}/burger-{version_id}.json '\
            f'--mappings {get_dir_location("__cache__")}/mappings-{version_id}.txt'
        )
    with open(get_dir_location(f'__cache__/burger-{version_id}.json'), 'r') as f:
        return json.load(f)


def get_pixlyzer_data(version_id: str, category: str):
    '''
    Gets data from Pixlyzer. Note that this requires Yarn to release updates first.
    '''

    target_dir = get_dir_location(f'__cache__/pixlyzer-{version_id}')

    # TODO: right now this False is hard-coded, it should retry with this
    # enabled if # initially getting the data fails
    if True or (os.path.exists(target_dir) and not os.path.exists(f'{target_dir}/{category}.min.json')):
        print('Downloading', category, 'from pixlyzer-data.')
        data = requests.get(f'https://gitlab.com/Bixilon/pixlyzer-data/-/raw/master/version/{version_id}/{category}.min.json?inline=false').text
        try:
            os.mkdir(target_dir)
        except:
            pass
        with open(f'{target_dir}/{category}.min.json', 'w') as f:
            f.write(data)
        return json.loads(data)

    if not os.path.exists(target_dir):
        pixlyzer_dir = get_pixlyzer()

        # for some reason pixlyzer doesn't work right unless the mvn clean
        # instruction looks like that
        # and pixlyzer.py doesn't do it right

        # map jar + download dependencies
        run_python_command_and_download_deps(
            f'cd {pixlyzer_dir}/wrapper && {determine_python_command()} PixLyzer.py --only-version={version_id} --dont-compile --only-map'
        )
        # update the pom.xml <dependencies>
        # list directories in pixlyzer/wrapper/data/data/dependencies/libraries
        pom_xml_dependencies = '''<dependency>
            <groupId>org.jetbrains.kotlin</groupId>
            <artifactId>kotlin-test-junit</artifactId>
            <version>1.7.21</version>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>org.jetbrains.kotlin</groupId>
            <artifactId>kotlin-stdlib-jdk8</artifactId>
            <version>1.7.21</version>
        </dependency>

        <dependency>
            <groupId>net.minecraft</groupId>
            <artifactId>client</artifactId>
            <version>${minecraft.version}</version>
            <scope>system</scope>
            <systemPath>${project.basedir}/wrapper/data/data/${minecraft.version}_yarn/${minecraft.version}-exhibitionism.jar</systemPath>
        </dependency>
        <dependency>
            <groupId>de.bixilon</groupId>
            <artifactId>mbf-kotlin</artifactId>
            <version>0.2.1</version>
        </dependency>
        <dependency>
            <groupId>org.objenesis</groupId>
            <artifactId>objenesis</artifactId>
            <version>3.3</version>
        </dependency>
        <dependency>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-lang3</artifactId>
            <version>3.12.0</version>
        </dependency>
        <dependency>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-databind</artifactId>
            <version>2.14.0</version>
        </dependency>
        <dependency>
            <groupId>de.bixilon</groupId>
            <artifactId>kutil</artifactId>
            <version>1.17.1</version>
        </dependency>'''
        # walk dir f'{pixlyzer_dir}/wrapper/data/data/dependencies/libraries'
        for root, dirs, files in os.walk(f'{pixlyzer_dir}/wrapper/data/data/dependencies/libraries'):
            for file in files:
                full_path = os.path.join(
                    root.replace('\\', '/').replace(
                        f'{pixlyzer_dir}/wrapper/data/data/dependencies/libraries/'.replace('\\', '/'), ''),
                    file
                ).replace('\\', '/')
                print(full_path)
                if not full_path.endswith('.jar'):
                    continue
                split_path = full_path.split('/')
                group = ''
                for group_index in range(0, len(split_path) - 3):
                    group += split_path[group_index] + '.'
                if group.endswith('.'):
                    group = group[:-1]
                artifact = split_path[-3]
                version = split_path[-2]
                path = '${project.basedir}/wrapper/data/data/dependencies/libraries/' + full_path
                pom_xml_dependencies += """
                    <dependency>
                        <groupId>""" + group + """</groupId>
                        <artifactId>""" + artifact + """</artifactId>
                        <version>""" + version + """</version>
                        <scope>system</scope>
                        <systemPath>""" + path + """</systemPath>
                    </dependency>
                    """
        print('pom_xml_dependencies', pom_xml_dependencies)
        assert pom_xml_dependencies != ''
        pom_xml = open(f'{pixlyzer_dir}/pom.xml', 'r').read()
        pom_xml = re.sub(
            r'<dependencies>.*?</dependencies>', f'<dependencies>{pom_xml_dependencies}</dependencies>', pom_xml, flags=re.DOTALL)
        pom_xml = re.sub(
            r'<minecraft\.version>.*?</minecraft\.version>', f'<minecraft.version>{version_id}</minecraft.version>', pom_xml, flags=re.DOTALL)
        open(f'{pixlyzer_dir}/pom.xml', 'w').write(pom_xml)

        # compile
        os.system(
            f'cd {pixlyzer_dir} && mvn clean -Dmaven.repo.local=. verify')
        # run pixlyzer.py again lol
        run_python_command_and_download_deps(
            f'cd {pixlyzer_dir}/wrapper && {determine_python_command()} PixLyzer.py --only-version={version_id} --no-compile'
        )

        source_dir = get_dir_location(
            f'{pixlyzer_dir}/wrapper/data/version/{version_id}')

        if not os.path.exists(source_dir):
            print('PixLyzer failed, no output!')
            exit()
        if os.path.exists(target_dir):
            os.unlink(target_dir)
        os.rename(
            source_dir,
            target_dir
        )

    with open(f'{target_dir}/{category}.min.json', 'r') as f:
        return json.load(f)

def get_file_from_jar(version_id: str, file_dir: str):
    get_client_jar(version_id)
    with ZipFile(get_dir_location(f'__cache__/client-{version_id}.jar')) as z:
        with z.open(file_dir) as f:
            return f.read()


def get_en_us_lang(version_id: str):
    return json.loads(
        get_file_from_jar(version_id, 'assets/minecraft/lang/en_us.json')
    )

# burger packet id extraction is broken since 1.20.5 (always returns -1, so we have to determine packet id ourselves from the mappings).
# this is very much not ideal.

if TYPE_CHECKING: from codegen.lib.mappings import Mappings
def get_packet_list(version_id: str):
    if version_id != '1.21':
        return []

    generate_data_from_server_jar(version_id)
    with open(get_dir_location(f'__cache__/generated-{version_id}/reports/packets.json'), 'r') as f:
        packets_report = json.load(f)
    packet_list = []
    for state, state_value in packets_report.items():
        for direction, direction_value in state_value.items():
            for packet_resourcelocation, packet_value in direction_value.items():
                assert packet_resourcelocation.startswith('minecraft:')
                packet_resourcelocation = upper_first_letter(to_camel_case(packet_resourcelocation[len('minecraft:'):]))
                packet_list.append({
                    'state': state,
                    'direction': direction,
                    'name': packet_resourcelocation,
                    'id': packet_value['protocol_id']
                })