drop dependency on pixlyzer and start using pumpkin extractor

7 files changed, 144 insertions, 211 deletions

diff --git a/codegen/README.md b/codegen/README.md
index 465aec47..6ff95348 100644
--- a/codegen/README.md
+++ b/codegen/README.md
@@ -42,6 +42,6 @@ At the time of writing, the following data generators are used:
 
 -   [Vanilla data generator](https://minecraft.wiki/w/Minecraft_Wiki:Projects/wiki.vg_merge/Data_Generators)
 -   [Burger](https://github.com/mat-1/Burger)
--   [PixLyzer](https://gitlab.bixilon.de/bixilon/pixlyzer)
+-   [Pumpkin Extractor](https://github.com/Pumpkin-MC/Extractor)
 
 Some things can be obtained from multiple generators. You should prefer them by the order above (the vanilla generator is the most reliable).
diff --git a/codegen/genblocks.py b/codegen/genblocks.py
index d0bea909..d4d4b9ae 100644
--- a/codegen/genblocks.py
+++ b/codegen/genblocks.py
@@ -8,22 +8,15 @@ import lib.extract
 import lib.utils
 
 def generate(version_id):
-    # TODO: pixlyzer is broken so we use old data
-    shape_datas = lib.extract.get_pixlyzer_data(
-        '1.20.3-pre4', 'shapes')
-    pixlyzer_block_datas = lib.extract.get_pixlyzer_data(
-        '1.20.3-pre4', 'blocks')
+    pumpkin_block_datas = lib.extract.get_pumpkin_data(version_id, 'blocks')
     burger_data = lib.extract.get_burger_data_for_version(version_id)
 
     block_states_report = lib.extract.get_block_states_report(version_id)
     registries = lib.extract.get_registries_report(version_id)
     ordered_blocks = lib.code.blocks.get_ordered_blocks(registries)
 
-    lib.code.blocks.generate_blocks(
-        block_states_report, pixlyzer_block_datas, ordered_blocks, burger_data)
-
-    lib.code.shapes.generate_block_shapes(
-        pixlyzer_block_datas, shape_datas['shapes'], shape_datas['aabbs'], block_states_report)
+    lib.code.blocks.generate_blocks(block_states_report, pumpkin_block_datas, ordered_blocks, burger_data)
+    lib.code.shapes.generate_block_shapes(pumpkin_block_datas, block_states_report)
 
     lib.code.utils.fmt()
 
diff --git a/codegen/lib/code/blocks.py b/codegen/lib/code/blocks.py
index aaa95ffa..212d080c 100644
--- a/codegen/lib/code/blocks.py
+++ b/codegen/lib/code/blocks.py
@@ -12,7 +12,7 @@ BLOCKS_RS_DIR = get_dir_location('../azalea-block/src/generated.rs')
 # - Block: Has properties and states.
 
 
-def generate_blocks(blocks_report: dict, pixlyzer_block_datas: dict, ordered_blocks: list[str], burger_data: dict):
+def generate_blocks(blocks_report: dict, pumpkin_block_datas: dict, ordered_blocks: list[str], burger_data: dict):
     with open(BLOCKS_RS_DIR, 'r') as f:
         existing_code = f.read().splitlines()
 
@@ -21,6 +21,11 @@ def generate_blocks(blocks_report: dict, pixlyzer_block_datas: dict, ordered_blo
 
     burger_block_datas = burger_data[0]['blocks']['block']
 
+    pumpkin_block_map = {}
+    for block_data in pumpkin_block_datas['blocks']:
+        block_id = block_data['name']
+        pumpkin_block_map[block_id] = block_data
+
     # Find properties
     properties = {}
 
@@ -77,8 +82,8 @@ def generate_blocks(blocks_report: dict, pixlyzer_block_datas: dict, ordered_blo
     new_make_block_states_macro_code.append('    Blocks => {')
     for block_id in ordered_blocks:
         block_data_report = blocks_report['minecraft:' + block_id]
-        block_data_pixlyzer = pixlyzer_block_datas.get(f'minecraft:{block_id}', {})
         block_data_burger = burger_block_datas.get(block_id, {})
+        block_data_pumpkin = pumpkin_block_map[block_id]
 
         default_property_variants: dict[str, str] = {}
         for state in block_data_report['states']:
@@ -116,11 +121,11 @@ def generate_blocks(blocks_report: dict, pixlyzer_block_datas: dict, ordered_blo
         # make the block behavior
         behavior_constructor = 'BlockBehavior::new()'
         # requires tool
-        if block_data_pixlyzer.get('requires_tool'):
+        if block_data_burger.get('requires_correct_tool_for_drops'):
             behavior_constructor += '.requires_correct_tool_for_drops()'
         # strength
-        destroy_time = block_data_pixlyzer.get('hardness')
-        explosion_resistance = block_data_pixlyzer.get('explosion_resistance')
+        destroy_time = block_data_pumpkin.get('hardness')
+        explosion_resistance = block_data_pumpkin.get('blast_resistance')
         if destroy_time and explosion_resistance:
             behavior_constructor += f'.strength({destroy_time}, {explosion_resistance})'
         elif destroy_time:
@@ -128,7 +133,7 @@ def generate_blocks(blocks_report: dict, pixlyzer_block_datas: dict, ordered_blo
         elif explosion_resistance:
             behavior_constructor += f'.explosion_resistance({explosion_resistance})'
         # friction
-        friction = block_data_pixlyzer.get('friction')
+        friction = block_data_burger.get('friction')
         if friction != None:
             behavior_constructor += f'.friction({friction})'
         
diff --git a/codegen/lib/code/shapes.py b/codegen/lib/code/shapes.py
index 0896bd82..3a3e6d83 100644
--- a/codegen/lib/code/shapes.py
+++ b/codegen/lib/code/shapes.py
@@ -5,64 +5,69 @@ COLLISION_BLOCKS_RS_DIR = get_dir_location(
     '../azalea-physics/src/collision/blocks.rs')
 
 
-def generate_block_shapes(blocks_pixlyzer: dict, shapes: dict, aabbs: dict, block_states_report):
-    blocks, shapes = simplify_shapes(blocks_pixlyzer, shapes, aabbs)
+def generate_block_shapes(pumpkin_block_datas: dict, block_states_report):
+    blocks, shapes = simplify_shapes(pumpkin_block_datas)
 
     code = generate_block_shapes_code(blocks, shapes, block_states_report)
     with open(COLLISION_BLOCKS_RS_DIR, 'w') as f:
         f.write(code)
 
 
-def simplify_shapes(blocks: dict, shapes: dict, aabbs: dict):
-    new_id_increment = 0
-
-    new_shapes = {}
-    old_id_to_new_id = {}
-
-    old_id_to_new_id[None] = 0
-    new_shapes[0] = ()
-    new_id_increment += 1
-
-    used_shape_ids = set()
-    # determine the used shape ids
-    for _block_id, block_data in blocks.items():
-        block_shapes = {state.get('collision_shape') for state in block_data['states'].values()}
-        block_shapes.update({state.get('outline_shape') for state in block_data['states'].values()})
-        for s in block_shapes:
-            used_shape_ids.add(s)
-
-    for shape_id, shape in enumerate(shapes):
-        if shape_id not in used_shape_ids: continue
-        # pixlyzer gives us shapes as an index or list of indexes into the
-        # aabbs list
-        # and aabbs look like { "from": number or [x, y, z], "to": (number or vec3) }
-        # convert them to [x1, y1, z1, x2, y2, z2]
-        shape = [shape] if isinstance(shape, int) else shape
-        shape = [aabbs[shape_aabb] for shape_aabb in shape]
-        shape = tuple([(
-            (tuple(part['from']) if isinstance(
-                part['from'], list) else ((part['from'],)*3))
-            + (tuple(part['to']) if isinstance(part['to'], list)
-               else ((part['to'],)*3))
-        ) for part in shape])
-
-        old_id_to_new_id[shape_id] = new_id_increment
-        new_shapes[new_id_increment] = shape
-        new_id_increment += 1
-
-    # now map the blocks to the new shape ids
+def simplify_shapes(blocks: dict) -> tuple[dict, dict]:
+    '''
+    Returns new_blocks and new_shapes,
+    where new_blocks is like { grass_block: { collision: [1, 1], outline: [1, 1] } }
+    and new_shapes is like { 1: [ [0, 0, 0, 1, 1, 1] ] }
+    '''
     new_blocks = {}
-    for block_id, block_data in blocks.items():
-        block_id = block_id.split(':')[-1]
+    new_shapes = {}
 
-        block_collision_shapes = [state.get('collision_shape') for state in block_data['states'].values()]
-        block_outline_shapes = [state.get('outline_shape') for state in block_data['states'].values()]
+    all_shapes_ids = {}
+
+    for block_data in blocks['blocks']:
+        new_block_collision_shapes = []
+        new_block_outline_shapes = []
+
+        for state in block_data['states']:
+            collision_shape = []
+            for box_id in state['collision_shapes']:
+                box = blocks['shapes'][box_id]
+                collision_shape.append(
+                    tuple(box['min'] + box['max'])
+                )
+            outline_shape = []
+            for box_id in state['outline_shapes']:
+                box = blocks['shapes'][box_id]
+                outline_shape.append(
+                    tuple(box['min'] + box['max'])
+                )
+
+            collision_shape = tuple(collision_shape)
+            outline_shape = tuple(outline_shape)
+
+            if collision_shape in all_shapes_ids:
+                collision_shape_id = all_shapes_ids[collision_shape]
+            else:
+                collision_shape_id = len(all_shapes_ids)
+                all_shapes_ids[collision_shape] = collision_shape_id
+                new_shapes[collision_shape_id] = collision_shape
+            if outline_shape in all_shapes_ids:
+                outline_shape_id = all_shapes_ids[outline_shape]
+            else:
+                outline_shape_id = len(all_shapes_ids)
+                all_shapes_ids[outline_shape] = outline_shape_id
+                new_shapes[outline_shape_id] = outline_shape
+
+            block_id = block_data['name']
+            new_block_collision_shapes.append(collision_shape_id)
+            new_block_outline_shapes.append(outline_shape_id)
 
         new_blocks[block_id] = {
-            'collision': [old_id_to_new_id[shape_id] for shape_id in block_collision_shapes],
-            'outline': [old_id_to_new_id[shape_id] for shape_id in block_outline_shapes]
+            'collision': new_block_collision_shapes,
+            'outline': new_block_outline_shapes
         }
 
+
     return new_blocks, new_shapes
 
 
diff --git a/codegen/lib/download.py b/codegen/lib/download.py
index 599e0b72..43ca8d2a 100644
--- a/codegen/lib/download.py
+++ b/codegen/lib/download.py
@@ -22,12 +22,41 @@ def get_burger():
         os.system(f'cd {get_dir_location("__cache__")}/Burger && python -m venv venv && venv/bin/pip install six jawa')
 
 
-def get_pixlyzer():
-    if not os.path.exists(get_dir_location('__cache__/pixlyzer')):
-        print('\033[92mDownloading bixilon/pixlyzer...\033[m')
+def get_pumpkin_extractor():
+    if not os.path.exists(get_dir_location('__cache__/pumpkin-extractor')):
+        print('\033[92mDownloading Pumpkin-MC/Extractor...\033[m')
         os.system(
-            f'cd {get_dir_location("__cache__")} && git clone https://gitlab.bixilon.de/bixilon/pixlyzer.git && cd pixlyzer && git pull')
-    return get_dir_location('__cache__/pixlyzer')
+            f'cd {get_dir_location("__cache__")} && git clone https://github.com/Pumpkin-MC/Extractor pumpkin-extractor && cd pumpkin-extractor && git pull')
+        
+        GIT_PATCH = '''diff --git a/src/main/kotlin/de/snowii/extractor/extractors/Blocks.kt b/src/main/kotlin/de/snowii/extractor/extractors/Blocks.kt
+index 936cd7b..9876a4b 100644
+--- a/src/main/kotlin/de/snowii/extractor/extractors/Blocks.kt
++++ b/src/main/kotlin/de/snowii/extractor/extractors/Blocks.kt
+@@ -106,12 +106,18 @@ class Blocks : Extractor.Extractor {
+                 }
+
+                 val collisionShapeIdxsJson = JsonArray()
++                val outlineShapeIdxsJson = JsonArray()
+                 for (box in state.getCollisionShape(EmptyBlockView.INSTANCE, BlockPos.ORIGIN).boundingBoxes) {
+                     val idx = shapes.putIfAbsent(box, shapes.size)
+                     collisionShapeIdxsJson.add(Objects.requireNonNullElseGet(idx) { shapes.size - 1 })
+                 }
++                for (box in state.getOutlineShape(EmptyBlockView.INSTANCE, BlockPos.ORIGIN).boundingBoxes) {
++                    val idx = shapes.putIfAbsent(box, shapes.size)
++                    outlineShapeIdxsJson.add(Objects.requireNonNullElseGet(idx) { shapes.size - 1 })
++                }
+
+                 stateJson.add("collision_shapes", collisionShapeIdxsJson)
++                stateJson.add("outline_shapes", outlineShapeIdxsJson)
+
+                 for (blockEntity in Registries.BLOCK_ENTITY_TYPE) {
+                     if (blockEntity.supports(state)) {
+'''
+        os.system(
+            f'cd {get_dir_location("__cache__")}/pumpkin-extractor && git apply - <<EOF\n{GIT_PATCH}\nEOF'
+        )
+        
+    return get_dir_location('__cache__/pumpkin-extractor')
 
 
 def get_version_manifest():
@@ -181,7 +210,7 @@ def clear_version_cache():
     if os.path.exists(burger_path):
         os.system(
             f'cd {burger_path} && git pull')
-    pixlyzer_path = get_dir_location('__cache__/pixlyzer')
-    if os.path.exists(pixlyzer_path):
+    pumpkin_path = get_dir_location('__cache__/pumpkin-extractor')
+    if os.path.exists(pumpkin_path):
         os.system(
-            f'cd {pixlyzer_path} && git pull')
\ No newline at end of file
+            f'cd {pumpkin_path} && git add . && git stash && git pull && git stash pop')
\ No newline at end of file
diff --git a/codegen/lib/extract.py b/codegen/lib/extract.py
index 564133e2..e8fc5b4d 100644
--- a/codegen/lib/extract.py
+++ b/codegen/lib/extract.py
@@ -1,7 +1,7 @@
 # Extracting data from the Minecraft jars
 
 from typing import TYPE_CHECKING
-from lib.download import get_mappings_for_version, get_server_jar, get_burger, get_client_jar, get_pixlyzer, get_yarn_data, get_fabric_api_versions, get_fabric_loader_versions
+from lib.download import get_mappings_for_version, get_pumpkin_extractor, get_server_jar, get_burger, get_client_jar
 from lib.utils import get_dir_location, to_camel_case, upper_first_letter
 from zipfile import ZipFile
 import subprocess
@@ -106,145 +106,50 @@ def get_burger_data_for_version(version_id: str):
         return json.load(f)
 
 
-def get_pixlyzer_data(version_id: str, category: str):
-    '''
-    Gets data from Pixlyzer. Note that this requires Yarn to release updates first.
-    '''
+def get_pumpkin_data(version_id: str, category: str):
+    assert '/' not in version_id
+    assert '\\' not in version_id
+    target_parent_dir = get_dir_location(f'__cache__/pumpkin-{version_id}')
+    category_dir = f'{target_parent_dir}/{category}.json'
 
-    target_dir = get_dir_location(f'__cache__/pixlyzer-{version_id}')
+    if os.path.exists(category_dir):
+        with open(category_dir, 'r') as f:
+            return json.load(f)
 
-    # TODO: right now this False is hard-coded, it should retry with this
-    # enabled if # initially getting the data fails
-    if True or (os.path.exists(target_dir) and not os.path.exists(f'{target_dir}/{category}.min.json')):
-        print('Downloading', category, 'from pixlyzer-data.')
-        data = requests.get(f'https://gitlab.com/Bixilon/pixlyzer-data/-/raw/master/version/{version_id}/{category}.min.json?inline=false').text
-        try:
-            os.mkdir(target_dir)
-        except:
-            pass
-        with open(f'{target_dir}/{category}.min.json', 'w') as f:
-            f.write(data)
-        return json.loads(data)
+    pumpkin_dir = get_pumpkin_extractor()
+    os.makedirs(f'{pumpkin_dir}/run', exist_ok=True)
+    with open(f'{pumpkin_dir}/run/eula.txt', 'w') as f:
+        f.write('eula=true')
 
-    if not os.path.exists(target_dir):
-        pixlyzer_dir = get_pixlyzer()
+    # run ./gradlew runServer until it logs "(pumpkin_extractor) Done"
+    p = subprocess.Popen(
+        f'cd {pumpkin_dir} && ./gradlew runServer',
+        stderr=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        shell=True
+    )
 
-        # for some reason pixlyzer doesn't work right unless the mvn clean
-        # instruction looks like that
-        # and pixlyzer.py doesn't do it right
+    while True:
+        data = p.stdout.readline().decode()
+        print('>' + data, end='', flush=True)
+        if '[Server thread/INFO] (pumpkin_extractor) Done' in data:
+            print('Pumpkin extractor done')
+            break
+        if data == b'':
+            break
 
-        # map jar + download dependencies
-        run_python_command_and_download_deps(
-            f'cd {pixlyzer_dir}/wrapper && {determine_python_command()} PixLyzer.py --only-version={version_id} --dont-compile --only-map'
-        )
-        # update the pom.xml <dependencies>
-        # list directories in pixlyzer/wrapper/data/data/dependencies/libraries
-        pom_xml_dependencies = '''<dependency>
-            <groupId>org.jetbrains.kotlin</groupId>
-            <artifactId>kotlin-test-junit</artifactId>
-            <version>1.7.21</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.jetbrains.kotlin</groupId>
-            <artifactId>kotlin-stdlib-jdk8</artifactId>
-            <version>1.7.21</version>
-        </dependency>
-
-        <dependency>
-            <groupId>net.minecraft</groupId>
-            <artifactId>client</artifactId>
-            <version>${minecraft.version}</version>
-            <scope>system</scope>
-            <systemPath>${project.basedir}/wrapper/data/data/${minecraft.version}_yarn/${minecraft.version}-exhibitionism.jar</systemPath>
-        </dependency>
-        <dependency>
-            <groupId>de.bixilon</groupId>
-            <artifactId>mbf-kotlin</artifactId>
-            <version>0.2.1</version>
-        </dependency>
-        <dependency>
-            <groupId>org.objenesis</groupId>
-            <artifactId>objenesis</artifactId>
-            <version>3.3</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.commons</groupId>
-            <artifactId>commons-lang3</artifactId>
-            <version>3.12.0</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-databind</artifactId>
-            <version>2.14.0</version>
-        </dependency>
-        <dependency>
-            <groupId>de.bixilon</groupId>
-            <artifactId>kutil</artifactId>
-            <version>1.17.1</version>
-        </dependency>'''
-        # walk dir f'{pixlyzer_dir}/wrapper/data/data/dependencies/libraries'
-        for root, dirs, files in os.walk(f'{pixlyzer_dir}/wrapper/data/data/dependencies/libraries'):
-            for file in files:
-                full_path = os.path.join(
-                    root.replace('\\', '/').replace(
-                        f'{pixlyzer_dir}/wrapper/data/data/dependencies/libraries/'.replace('\\', '/'), ''),
-                    file
-                ).replace('\\', '/')
-                print(full_path)
-                if not full_path.endswith('.jar'):
-                    continue
-                split_path = full_path.split('/')
-                group = ''
-                for group_index in range(0, len(split_path) - 3):
-                    group += split_path[group_index] + '.'
-                if group.endswith('.'):
-                    group = group[:-1]
-                artifact = split_path[-3]
-                version = split_path[-2]
-                path = '${project.basedir}/wrapper/data/data/dependencies/libraries/' + full_path
-                pom_xml_dependencies += """
-                    <dependency>
-                        <groupId>""" + group + """</groupId>
-                        <artifactId>""" + artifact + """</artifactId>
-                        <version>""" + version + """</version>
-                        <scope>system</scope>
-                        <systemPath>""" + path + """</systemPath>
-                    </dependency>
-                    """
-        print('pom_xml_dependencies', pom_xml_dependencies)
-        assert pom_xml_dependencies != ''
-        pom_xml = open(f'{pixlyzer_dir}/pom.xml', 'r').read()
-        pom_xml = re.sub(
-            r'<dependencies>.*?</dependencies>', f'<dependencies>{pom_xml_dependencies}</dependencies>', pom_xml, flags=re.DOTALL)
-        pom_xml = re.sub(
-            r'<minecraft\.version>.*?</minecraft\.version>', f'<minecraft.version>{version_id}</minecraft.version>', pom_xml, flags=re.DOTALL)
-        open(f'{pixlyzer_dir}/pom.xml', 'w').write(pom_xml)
-
-        # compile
-        os.system(
-            f'cd {pixlyzer_dir} && mvn clean -Dmaven.repo.local=. verify')
-        # run pixlyzer.py again lol
-        run_python_command_and_download_deps(
-            f'cd {pixlyzer_dir}/wrapper && {determine_python_command()} PixLyzer.py --only-version={version_id} --no-compile'
-        )
+    p.terminate()
 
-        source_dir = get_dir_location(
-            f'{pixlyzer_dir}/wrapper/data/version/{version_id}')
-
-        if not os.path.exists(source_dir):
-            print('PixLyzer failed, no output!')
-            exit()
-        if os.path.exists(target_dir):
-            os.unlink(target_dir)
-        os.rename(
-            source_dir,
-            target_dir
-        )
+    # move the run/pumpkin_extractor_output directory to target_parent_dir
+    # delete target_parent_dir if it's empty
+    if os.path.exists(target_parent_dir):
+        os.rmdir(target_parent_dir)
+    os.rename(f'{pumpkin_dir}/run/pumpkin_extractor_output', target_parent_dir)
 
-    with open(f'{target_dir}/{category}.min.json', 'r') as f:
+    with open(category_dir, 'r') as f:
         return json.load(f)
 
+
 def get_file_from_jar(version_id: str, file_dir: str):
     get_client_jar(version_id)
     with ZipFile(get_dir_location(f'__cache__/client-{version_id}.jar')) as z:
diff --git a/codegen/migrate.py b/codegen/migrate.py
index 30e33812..bfdf68ec 100644
--- a/codegen/migrate.py
+++ b/codegen/migrate.py
@@ -40,18 +40,14 @@ lib.code.version.set_version_name(new_version_id)
 print('Updated protocol!')
 
 print('Generating blocks and shapes...')
-# TODO: pixlyzer is broken so we use old data
-new_shape_datas = lib.extract.get_pixlyzer_data(
-    '1.20.3-pre4', 'shapes')
-new_pixlyzer_block_datas = lib.extract.get_pixlyzer_data(
-    '1.20.3-pre4', 'blocks')
+new_pumpkin_block_datas = lib.extract.get_pumpkin_data(new_version_id, 'blocks')
+
 new_block_states_report = lib.extract.get_block_states_report(new_version_id)
 new_registries = lib.extract.get_registries_report(new_version_id)
 new_ordered_blocks = lib.code.blocks.get_ordered_blocks(new_registries)
-lib.code.blocks.generate_blocks(
-    new_block_states_report, new_pixlyzer_block_datas, new_ordered_blocks, new_burger_data)
-lib.code.shapes.generate_block_shapes(
-    new_pixlyzer_block_datas, new_shape_datas['shapes'], new_shape_datas['aabbs'], new_block_states_report)
+
+lib.code.blocks.generate_blocks(new_block_states_report, new_pumpkin_block_datas, new_ordered_blocks, new_burger_data)
+lib.code.shapes.generate_block_shapes(new_pumpkin_block_datas, new_block_states_report)
 
 print('Getting en_us.json...')
 language = lib.extract.get_en_us_lang(new_version_id)