From 54c19fd662812a37758569be7d18c85d8435d7dc Mon Sep 17 00:00:00 2001 From: "Dylan K. Taylor" Date: Wed, 8 Mar 2023 23:04:09 +0000 Subject: [PATCH] Open-source script that generates recipes and other goodies for BedrockData this script has been lurking in my workspace for years, waiting to be cleaned up and open-sourced. --- tools/generate-bedrock-data-from-packets.php | 558 +++++++++++++++++++ 1 file changed, 558 insertions(+) create mode 100644 tools/generate-bedrock-data-from-packets.php diff --git a/tools/generate-bedrock-data-from-packets.php b/tools/generate-bedrock-data-from-packets.php new file mode 100644 index 000000000..3dc8ad172 --- /dev/null +++ b/tools/generate-bedrock-data-from-packets.php @@ -0,0 +1,558 @@ +blockMapping = new RuntimeBlockMapping( + BlockStateDictionary::loadFromString( + Filesystem::fileGetContents(Path::join($this->bedrockDataPath, "canonical_block_states.nbt")), + Filesystem::fileGetContents(Path::join($this->bedrockDataPath, "block_state_meta_map.json")), + ), + GlobalBlockStateHandlers::getSerializer() + ); + } + + private static function blockStatePropertiesToString(BlockStateData $blockStateData) : string{ + $statePropertiesTag = CompoundTag::create(); + foreach(Utils::stringifyKeys($blockStateData->getStates()) as $name => $value){ + $statePropertiesTag->setTag($name, $value); + } + return base64_encode((new LittleEndianNbtSerializer())->write(new TreeRoot($statePropertiesTag))); + } + + private function itemStackToJson(ItemStack $itemStack) : ItemStackData{ + if($itemStack->getId() === 0){ + throw new InvalidArgumentException("Cannot serialize a null itemstack"); + } + if($this->itemTypeDictionary === null){ + throw new PacketHandlingException("Can't process item yet; haven't received item type dictionary"); + } + $data = new ItemStackData($this->itemTypeDictionary->fromIntId($itemStack->getId())); + + if($itemStack->getCount() !== 1){ + $data->count = $itemStack->getCount(); + } + + $meta = $itemStack->getMeta(); + if($meta === 32767){ + $meta = 0; //kick wildcard magic bullshit + } + if($itemStack->getBlockRuntimeId() !== 0){ + if($meta !== 0){ + throw new PacketHandlingException("Unexpected non-zero blockitem meta"); + } + $blockState = $this->blockMapping->getBlockStateDictionary()->getDataFromStateId($itemStack->getBlockRuntimeId()) ?? null; + if($blockState === null){ + throw new PacketHandlingException("Unmapped blockstate ID " . $itemStack->getBlockRuntimeId()); + } + + $stateProperties = $blockState->getStates(); + if(count($stateProperties) > 0){ + $data->block_states = self::blockStatePropertiesToString($blockState); + } + }elseif($meta !== 0){ + $data->meta = $meta; + } + + $nbt = $itemStack->getNbt(); + if($nbt !== null && count($nbt) > 0){ + $data->nbt = base64_encode((new LittleEndianNbtSerializer())->write(new TreeRoot($nbt))); + } + + if(count($itemStack->getCanPlaceOn()) > 0){ + $data->can_place_on = $itemStack->getCanPlaceOn(); + } + if(count($itemStack->getCanDestroy()) > 0){ + $data->can_destroy = $itemStack->getCanDestroy(); + } + + return $data; + } + + /** + * @return mixed[] + */ + private static function objectToOrderedArray(object $object) : array{ + $result = (array) $object; + ksort($result, SORT_STRING); + + foreach($result as $property => $value){ + if(is_object($value)){ + $result[$property] = self::objectToOrderedArray($value); + }elseif(is_array($value)){ + $array = []; + foreach($value as $k => $v){ + if(is_object($v)){ + $array[$k] = self::objectToOrderedArray($v); + }else{ + $array[$k] = $v; + } + } + + $result[$property] = $array; + } + } + + return $result; + } + + private static function sort(mixed $object) : mixed{ + if(is_object($object)){ + return self::objectToOrderedArray($object); + } + if(is_array($object)){ + $result = []; + foreach($object as $k => $v){ + $result[$k] = self::sort($v); + } + return $result; + } + + return $object; + } + + public function handleStartGame(StartGamePacket $packet) : bool{ + $this->itemTypeDictionary = new ItemTypeDictionary($packet->itemTable); + + echo "updating legacy item ID mapping table\n"; + $table = []; + foreach($packet->itemTable as $entry){ + $table[$entry->getStringId()] = [ + "runtime_id" => $entry->getNumericId(), + "component_based" => $entry->isComponentBased() + ]; + } + ksort($table, SORT_STRING); + file_put_contents($this->bedrockDataPath . '/required_item_list.json', json_encode($table, JSON_PRETTY_PRINT) . "\n"); + + foreach($packet->levelSettings->experiments->getExperiments() as $name => $experiment){ + echo "Experiment \"$name\" is " . ($experiment ? "" : "not ") . "active\n"; + } + return true; + } + + public function handleCreativeContent(CreativeContentPacket $packet) : bool{ + echo "updating creative inventory data\n"; + $items = array_map(function(CreativeContentEntry $entry) : array{ + return self::objectToOrderedArray($this->itemStackToJson($entry->getItem())); + }, $packet->getEntries()); + file_put_contents($this->bedrockDataPath . '/creativeitems.json', json_encode($items, JSON_PRETTY_PRINT) . "\n"); + return true; + } + + private function recipeIngredientToJson(RecipeIngredient $itemStack) : RecipeIngredientData{ + if($this->itemTypeDictionary === null){ + throw new PacketHandlingException("Can't process item yet; haven't received item type dictionary"); + } + + $descriptor = $itemStack->getDescriptor(); + if($descriptor === null){ + throw new PacketHandlingException("Can't json-serialize a null recipe ingredient"); + } + $data = new RecipeIngredientData(); + + if($descriptor instanceof IntIdMetaItemDescriptor || $descriptor instanceof StringIdMetaItemDescriptor){ + if($descriptor instanceof IntIdMetaItemDescriptor){ + $data->name = $this->itemTypeDictionary->fromIntId($descriptor->getId()); + }else{ + $data->name = $descriptor->getId(); + } + $meta = $descriptor->getMeta(); + if($meta !== 32767){ + $blockStateId = $this->blockMapping->getBlockStateDictionary()->lookupStateIdFromIdMeta($data->name, $meta); + if($blockStateId !== null){ + $blockState = $this->blockMapping->getBlockStateDictionary()->getDataFromStateId($blockStateId); + if($blockState !== null && count($blockState->getStates()) > 0){ + $data->block_states = self::blockStatePropertiesToString($blockState); + } + }elseif($meta !== 0){ + $data->meta = $meta; + } + }else{ + $data->meta = $meta; + } + }elseif($descriptor instanceof TagItemDescriptor){ + $data->tag = $descriptor->getTag(); + }elseif($descriptor instanceof MolangItemDescriptor){ + $data->molang_expression = $descriptor->getMolangExpression(); + $data->molang_version = $descriptor->getMolangVersion(); + }elseif($descriptor instanceof ComplexAliasItemDescriptor){ + $data->name = $descriptor->getAlias(); + }else{ + throw new \UnexpectedValueException("Unknown item descriptor type " . get_class($descriptor)); + } + if($itemStack->getCount() !== 1){ + $data->count = $itemStack->getCount(); + } + + return $data; + } + + private function shapedRecipeToJson(ShapedRecipe $entry) : ShapedRecipeData{ + $keys = []; + + $shape = []; + $char = ord("A"); + + $outputsByKey = []; + foreach($entry->getInput() as $x => $row){ + foreach($row as $y => $ingredient){ + if($ingredient->getDescriptor() === null){ + $shape[$x][$y] = " "; + }else{ + $jsonIngredient = $this->recipeIngredientToJson($ingredient); + $hash = json_encode($jsonIngredient, JSON_THROW_ON_ERROR); + if(isset($keys[$hash])){ + $shape[$x][$y] = $keys[$hash]; + }else{ + $key = chr($char); + $keys[$hash] = $shape[$x][$y] = $key; + $outputsByKey[$key] = $jsonIngredient; + $char++; + } + } + } + } + return new ShapedRecipeData( + array_map(fn(array $array) => implode('', $array), $shape), + $outputsByKey, + array_map(fn(ItemStack $output) => $this->itemStackToJson($output), $entry->getOutput()), + $entry->getBlockName(), + $entry->getPriority() + ); + } + + private function shapelessRecipeToJson(ShapelessRecipe $recipe) : ShapelessRecipeData{ + return new ShapelessRecipeData( + array_map(fn(RecipeIngredient $input) => $this->recipeIngredientToJson($input), $recipe->getInputs()), + array_map(fn(ItemStack $output) => $this->itemStackToJson($output), $recipe->getOutputs()), + $recipe->getBlockName(), + $recipe->getPriority() + ); + } + + private function furnaceRecipeToJson(FurnaceRecipe $recipe) : FurnaceRecipeData{ + return new FurnaceRecipeData( + $this->recipeIngredientToJson(new RecipeIngredient(new IntIdMetaItemDescriptor($recipe->getInputId(), $recipe->getInputMeta() ?? 32767), 1)), + $this->itemStackToJson($recipe->getResult()), + $recipe->getBlockName() + ); + } + + private function smithingRecipeToJson(SmithingTransformRecipe $recipe) : SmithingTransformRecipeData{ + return new SmithingTransformRecipeData( + $this->recipeIngredientToJson($recipe->getInput()), + $this->recipeIngredientToJson($recipe->getAddition()), + $this->itemStackToJson($recipe->getOutput()), + $recipe->getBlockName() + ); + } + + public function handleCraftingData(CraftingDataPacket $packet) : bool{ + echo "updating crafting data\n"; + + $recipesPath = Path::join($this->bedrockDataPath, "recipes"); + Filesystem::recursiveUnlink($recipesPath); + @mkdir($recipesPath); + + $recipes = []; + foreach($packet->recipesWithTypeIds as $entry){ + static $typeMap = [ + CraftingDataPacket::ENTRY_SHAPELESS => "shapeless_crafting", + CraftingDataPacket::ENTRY_SHAPED => "shaped_crafting", + CraftingDataPacket::ENTRY_FURNACE => "smelting", + CraftingDataPacket::ENTRY_FURNACE_DATA => "smelting", + CraftingDataPacket::ENTRY_MULTI => "special_hardcoded", + CraftingDataPacket::ENTRY_SHULKER_BOX => "shapeless_shulker_box", + CraftingDataPacket::ENTRY_SHAPELESS_CHEMISTRY => "shapeless_chemistry", + CraftingDataPacket::ENTRY_SHAPED_CHEMISTRY => "shaped_chemistry", + CraftingDataPacket::ENTRY_SMITHING_TRANSFORM => "smithing", + ]; + if(!isset($typeMap[$entry->getTypeId()])){ + throw new \UnexpectedValueException("Unknown recipe type ID " . $entry->getTypeId()); + } + $mappedType = $typeMap[$entry->getTypeId()]; + + if($entry instanceof ShapedRecipe){ + $recipes[$mappedType][] = $this->shapedRecipeToJson($entry); + }elseif($entry instanceof ShapelessRecipe){ + $recipes[$mappedType][] = $this->shapelessRecipeToJson($entry); + }elseif($entry instanceof MultiRecipe){ + $recipes[$mappedType][] = $entry->getRecipeId()->toString(); + }elseif($entry instanceof FurnaceRecipe){ + $recipes[$mappedType][] = $this->furnaceRecipeToJson($entry); + }elseif($entry instanceof SmithingTransformRecipe){ + $recipes[$mappedType][] = $this->smithingRecipeToJson($entry); + }else{ + throw new AssumptionFailedError("Unknown recipe type " . get_class($entry)); + } + } + + foreach($packet->potionTypeRecipes as $recipe){ + $recipes["potion_type"][] = new PotionTypeRecipeData( + $this->recipeIngredientToJson(new RecipeIngredient(new IntIdMetaItemDescriptor($recipe->getInputItemId(), $recipe->getInputItemMeta()), 1)), + $this->recipeIngredientToJson(new RecipeIngredient(new IntIdMetaItemDescriptor($recipe->getIngredientItemId(), $recipe->getIngredientItemMeta()), 1)), + $this->itemStackToJson(new ItemStack($recipe->getOutputItemId(), $recipe->getOutputItemMeta(), 1, 0, null, [], [], null)), + ); + } + + if($this->itemTypeDictionary === null){ + throw new AssumptionFailedError("We should have already crashed if this was null"); + } + foreach($packet->potionContainerRecipes as $recipe){ + $recipes["potion_container_change"][] = new PotionContainerChangeRecipeData( + $this->itemTypeDictionary->fromIntId($recipe->getInputItemId()), + $this->recipeIngredientToJson(new RecipeIngredient(new IntIdMetaItemDescriptor($recipe->getIngredientItemId(), 0), 1)), + $this->itemTypeDictionary->fromIntId($recipe->getOutputItemId()), + ); + } + + //this sorts the data into a canonical order to make diffs between versions reliable + //how the data is ordered doesn't matter as long as it's reproducible + foreach($recipes as $_type => $entries){ + $_sortedRecipes = []; + foreach($entries as $entry){ + $entry = self::sort($entry); + $_key = json_encode($entry); + while(isset($_sortedRecipes[$_key])){ + echo "warning: duplicated $_type recipe: $_key\n"; + $_key .= "a"; + } + $_sortedRecipes[$_key] = $entry; + } + ksort($_sortedRecipes, SORT_STRING); + $recipes[$_type] = array_values($_sortedRecipes); + } + + ksort($recipes, SORT_STRING); + foreach($recipes as $type => $entries){ + echo "$type: " . count($entries) . "\n"; + } + foreach($recipes as $type => $entries){ + file_put_contents(Path::join($recipesPath, $type . '.json'), json_encode($entries, JSON_PRETTY_PRINT) . "\n"); + } + + return true; + } + + public function handleAvailableActorIdentifiers(AvailableActorIdentifiersPacket $packet) : bool{ + echo "storing actor identifiers" . PHP_EOL; + + $tag = $packet->identifiers->getRoot(); + if(!($tag instanceof CompoundTag)){ + throw new AssumptionFailedError(); + } + $idList = $tag->getTag("idlist"); + if(!($idList instanceof ListTag) || $idList->getTagType() !== NBT::TAG_Compound){ + echo $tag . "\n"; + throw new \RuntimeException("expected TAG_List(\"idlist\") tag inside root TAG_Compound"); + } + if($tag->count() > 1){ + echo $tag . "\n"; + echo "!!! unexpected extra data found in available actor identifiers\n"; + } + echo "updating legacy => string entity ID mapping table\n"; + $map = []; + /** + * @var CompoundTag $thing + */ + foreach($idList as $thing){ + $map[$thing->getString("id")] = $thing->getInt("rid"); + } + asort($map, SORT_NUMERIC); + file_put_contents($this->bedrockDataPath . '/entity_id_map.json', json_encode($map, JSON_PRETTY_PRINT) . "\n"); + echo "storing entity identifiers\n"; + file_put_contents($this->bedrockDataPath . '/entity_identifiers.nbt', $packet->identifiers->getEncodedNbt()); + return true; + } + + public function handleBiomeDefinitionList(BiomeDefinitionListPacket $packet) : bool{ + echo "storing biome definitions" . PHP_EOL; + + file_put_contents($this->bedrockDataPath . '/biome_definitions_full.nbt', $packet->definitions->getEncodedNbt()); + + $nbt = $packet->definitions->getRoot(); + if(!$nbt instanceof CompoundTag){ + throw new AssumptionFailedError(); + } + $strippedNbt = clone $nbt; + foreach($strippedNbt as $compound){ + if($compound instanceof CompoundTag){ + foreach([ + "minecraft:capped_surface", + "minecraft:consolidated_features", + "minecraft:frozen_ocean_surface", + "minecraft:legacy_world_generation_rules", + "minecraft:mesa_surface", + "minecraft:mountain_parameters", + "minecraft:multinoise_generation_rules", + "minecraft:overworld_generation_rules", + "minecraft:surface_material_adjustments", + "minecraft:surface_parameters", + "minecraft:swamp_surface", + ] as $remove){ + $compound->removeTag($remove); + } + } + } + + file_put_contents($this->bedrockDataPath . '/biome_definitions.nbt', (new CacheableNbt($strippedNbt))->getEncodedNbt()); + + return true; + } +} + +/** + * @param string[] $argv + */ +function main(array $argv) : int{ + if(count($argv) !== 3){ + fwrite(STDERR, 'Usage: ' . PHP_BINARY . ' ' . __FILE__ . ' '); + return 1; + } + [, $inputFile, $bedrockDataPath] = $argv; + + $handler = new ParserPacketHandler($bedrockDataPath); + + $packets = file($inputFile, FILE_IGNORE_NEW_LINES); + if($packets === false){ + fwrite(STDERR, 'File ' . $inputFile . ' not found or permission denied'); + return 1; + } + + foreach($packets as $lineNum => $line){ + $parts = explode(':', $line); + if(count($parts) !== 2){ + fwrite(STDERR, 'Wrong packet format at line ' . ($lineNum + 1) . ', expected read:base64 or write:base64'); + return 1; + } + $raw = base64_decode($parts[1], true); + if($raw === false){ + fwrite(STDERR, 'Invalid base64\'d packet on line ' . ($lineNum + 1) . ' could not be parsed'); + return 1; + } + + $pk = PacketPool::getInstance()->getPacket($raw); + if($pk === null){ + fwrite(STDERR, "Unknown packet on line " . ($lineNum + 1) . ": " . $parts[1]); + continue; + } + $serializer = PacketSerializer::decoder($raw, 0, new PacketSerializerContext( + $handler->itemTypeDictionary ?? + new ItemTypeDictionary([new ItemTypeEntry("minecraft:shield", 0, false)])) + ); + + $pk->decode($serializer); + $pk->handle($handler); + if(!$serializer->feof()){ + echo "Packet on line " . ($lineNum + 1) . ": didn't read all data from " . get_class($pk) . " (stopped at offset " . $serializer->getOffset() . " of " . strlen($serializer->getBuffer()) . " bytes): " . bin2hex($serializer->getRemaining()) . "\n"; + } + } + return 0; +} + +exit(main($argv));