BlockTranslator: cut memory usage in half

this was achieved by storing binary representations of the blockstates, rather than the original BlockStateData.

Due to the insane object:data ratio of Tag objects (40:1 for ByteTag for example), modestly sized NBT can explode in memory footprint. This has been previously seen with the absurd 25 MB footprint on file load.
Previously, I attempted to mitigate this by deduplicating tag objects, but this was mitigating a symptom rather than addressing the cause.

We don't actually need to keep the NBT around in memory, since we don't actually use it for anything other than matching blockstates. In this case, we can allow the code to be possibly a little slower, since the lookup is anyway slow and the result will be cached.
In fact, using encoded ordered states as hash keys significantly improves the speed of lookups for stuff like walls, which have many thousands of states.

We keep around generateStateData(), since it's still possible we may need the BlockStateData associated, and it can be easily reconstructed from the binary-encoded representation in BlockStateDictionaryEntry.

src/network/mcpe/convert/BlockStateDictionary.php

@@ -25,10 +25,6 @@ namespace pocketmine\network\mcpe\convert;
 
 use pocketmine\data\bedrock\block\BlockStateData;
 use pocketmine\nbt\NbtDataException;
-use pocketmine\nbt\tag\ByteTag;
-use pocketmine\nbt\tag\CompoundTag;
-use pocketmine\nbt\tag\IntTag;
-use pocketmine\nbt\tag\StringTag;
 use pocketmine\nbt\TreeRoot;
 use pocketmine\network\mcpe\protocol\serializer\NetworkNbtSerializer;
 use function array_map;
@@ -58,7 +54,7 @@ final class BlockStateDictionary{
 	public function __construct(
 		private array $states
 	){
-		$this->stateDataToStateIdLookupCache = new BlockStateLookupCache(array_map(fn(BlockStateDictionaryEntry $entry) => $entry->getStateData(), $this->states));
+		$this->stateDataToStateIdLookupCache = new BlockStateLookupCache($this->states);
 	}
 
 	/**
@@ -71,15 +67,15 @@ final class BlockStateDictionary{
 			$this->idMetaToStateIdLookupCache = [];
 
 			foreach($this->states as $i => $state){
-				$this->idMetaToStateIdLookupCache[$state->getMeta()][$state->getStateData()->getName()] = $i;
+				$this->idMetaToStateIdLookupCache[$state->getMeta()][$state->getStateName()] = $i;
 			}
 		}
 
 		return $this->idMetaToStateIdLookupCache;
 	}
 
-	public function getDataFromStateId(int $networkRuntimeId) : ?BlockStateData{
-		return ($this->states[$networkRuntimeId] ?? null)?->getStateData();
+	public function generateDataFromStateId(int $networkRuntimeId) : ?BlockStateData{
+		return ($this->states[$networkRuntimeId] ?? null)?->generateStateData();
 	}
 
 	/**
@@ -113,33 +109,6 @@ final class BlockStateDictionary{
 	 */
 	public function getStates() : array{ return $this->states; }
 
-	/**
-	 * @param string[] $keyIndex
-	 * @param (ByteTag|StringTag|IntTag)[][] $valueIndex
-	 * @phpstan-param array<string, string> $keyIndex
-	 * @phpstan-param array<int, array<int|string, ByteTag|IntTag|StringTag>> $valueIndex
-	 */
-	private static function deduplicateCompound(CompoundTag $tag, array &$keyIndex, array &$valueIndex) : CompoundTag{
-		if($tag->count() === 0){
-			return $tag;
-		}
-
-		$newTag = CompoundTag::create();
-		foreach($tag as $key => $value){
-			$key = $keyIndex[$key] ??= $key;
-
-			if($value instanceof CompoundTag){
-				$value = self::deduplicateCompound($value, $keyIndex, $valueIndex);
-			}elseif($value instanceof ByteTag || $value instanceof IntTag || $value instanceof StringTag){
-				$value = $valueIndex[$value->getType()][$value->getValue()] ??= $value;
-			}
-
-			$newTag->setTag($key, $value);
-		}
-
-		return $newTag;
-	}
-
 	/**
 	 * @return BlockStateData[]
 	 * @phpstan-return list<BlockStateData>
@@ -147,13 +116,8 @@ final class BlockStateDictionary{
 	 * @throws NbtDataException
 	 */
 	public static function loadPaletteFromString(string $blockPaletteContents) : array{
-		$keyIndex = [];
-		$valueIndex = [];
-
 		return array_map(
-			function(TreeRoot $root) use (&$keyIndex, &$valueIndex) : BlockStateData{
-				return BlockStateData::fromNbt(self::deduplicateCompound($root->mustGetCompoundTag(), $keyIndex, $valueIndex));
-			},
+			fn(TreeRoot $root) => BlockStateData::fromNbt($root->mustGetCompoundTag()),
 			(new NetworkNbtSerializer())->readMultiple($blockPaletteContents)
 		);
 	}

src/network/mcpe/convert/BlockStateDictionaryEntry.php

@@ -24,15 +24,60 @@ declare(strict_types=1);
 namespace pocketmine\network\mcpe\convert;
 
 use pocketmine\data\bedrock\block\BlockStateData;
+use pocketmine\nbt\LittleEndianNbtSerializer;
+use pocketmine\nbt\tag\Tag;
+use pocketmine\nbt\TreeRoot;
+use function array_map;
+use function count;
+use function ksort;
+use const SORT_STRING;
 
 final class BlockStateDictionaryEntry{
 
-	public function __construct(
-		private BlockStateData $stateData,
-		private int $meta
-	){}
+	private string $stateName;
+	private string $rawStateProperties;
 
-	public function getStateData() : BlockStateData{ return $this->stateData; }
+	public function __construct(
+		BlockStateData $stateData,
+		private int $meta
+	){
+		$this->stateName = $stateData->getName();
+		$this->rawStateProperties = self::encodeStateProperties($stateData->getStates());
+	}
+
+	public function getStateName() : string{ return $this->stateName; }
+
+	public function getRawStateProperties() : string{ return $this->rawStateProperties; }
+
+	public function generateStateData() : BlockStateData{
+		return new BlockStateData(
+			$this->stateName,
+			self::decodeStateProperties($this->rawStateProperties),
+			BlockStateData::CURRENT_VERSION
+		);
+	}
 
 	public function getMeta() : int{ return $this->meta; }
+
+	/**
+	 * @return Tag[]
+	 */
+	public static function decodeStateProperties(string $rawProperties) : array{
+		if($rawProperties === ""){
+			return [];
+		}
+		return array_map(fn(TreeRoot $root) => $root->getTag(), (new LittleEndianNbtSerializer())->readMultiple($rawProperties));
+	}
+
+	/**
+	 * @param Tag[] $properties
+	 */
+	public static function encodeStateProperties(array $properties) : string{
+		if(count($properties) === 0){
+			return "";
+		}
+		//TODO: make a more efficient encoding - NBT will do for now, but it's not very compact
+		ksort($properties, SORT_STRING);
+		return (new LittleEndianNbtSerializer())->writeMultiple(array_map(fn(Tag $tag) => new TreeRoot($tag), $properties));
+	}
 }

src/network/mcpe/convert/BlockStateLookupCache.php

@@ -35,7 +35,7 @@ final class BlockStateLookupCache{
 
 	/**
 	 * @var int[][]
-	 * @phpstan-var array<string, array<int, BlockStateData>>
+	 * @phpstan-var array<string, array<string, int>>
 	 */
 	private array $nameToNetworkIdsLookup = [];
 
@@ -46,18 +46,18 @@ final class BlockStateLookupCache{
 	private array $nameToSingleNetworkIdLookup = [];
 
 	/**
-	 * @param BlockStateData[] $blockStates
-	 * @phpstan-param list<BlockStateData> $blockStates
+	 * @param BlockStateDictionaryEntry[] $blockStates
+	 * @phpstan-param list<BlockStateDictionaryEntry> $blockStates
 	 */
 	public function __construct(array $blockStates){
 		foreach($blockStates as $stateId => $stateNbt){
-			$this->nameToNetworkIdsLookup[$stateNbt->getName()][$stateId] = $stateNbt;
+			$this->nameToNetworkIdsLookup[$stateNbt->getStateName()][$stateNbt->getRawStateProperties()] = $stateId;
 		}
 
 		//setup fast path for stateless blocks
 		foreach(Utils::stringifyKeys($this->nameToNetworkIdsLookup) as $name => $stateIds){
 			if(count($stateIds) === 1){
-				$this->nameToSingleNetworkIdLookup[$name] = array_key_first($stateIds);
+				$this->nameToSingleNetworkIdLookup[$name] = $stateIds[array_key_first($stateIds)];
 			}
 		}
 	}
@@ -73,14 +73,6 @@ final class BlockStateLookupCache{
 			return $this->nameToSingleNetworkIdLookup[$name];
 		}
 
-		if(isset($this->nameToNetworkIdsLookup[$name])){
-			foreach($this->nameToNetworkIdsLookup[$name] as $stateId => $stateNbt){
-				if($stateNbt->equals($data)){
-					return $stateId;
-				}
-			}
-		}
-
-		return null;
+		return $this->nameToNetworkIdsLookup[$name][BlockStateDictionaryEntry::encodeStateProperties($data->getStates())] ?? null;
 	}
 }

src/network/mcpe/convert/BlockTranslator.php

@@ -51,7 +51,7 @@ final class BlockTranslator{
 				BlockStateData::current(BlockTypeNames::INFO_UPDATE, [])
 			) ?? throw new AssumptionFailedError(BlockTypeNames::INFO_UPDATE . " should always exist");
 		//lookup the state data from the dictionary to avoid keeping two copies of the same data around
-		$this->fallbackStateData = $this->blockStateDictionary->getDataFromStateId($this->fallbackStateId) ?? throw new AssumptionFailedError("We just looked up this state data, so it must exist");
+		$this->fallbackStateData = $this->blockStateDictionary->generateDataFromStateId($this->fallbackStateId) ?? throw new AssumptionFailedError("We just looked up this state data, so it must exist");
 	}
 
 	public function toRuntimeId(int $internalStateId) : int{
@@ -84,7 +84,7 @@ final class BlockTranslator{
 		//case someone wants to implement multi version).
 		$networkRuntimeId = $this->toRuntimeId($internalStateId);
 
-		return $this->blockStateDictionary->getDataFromStateId($networkRuntimeId) ?? throw new AssumptionFailedError("We just looked up this state ID, so it must exist");
+		return $this->blockStateDictionary->generateDataFromStateId($networkRuntimeId) ?? throw new AssumptionFailedError("We just looked up this state ID, so it must exist");
 	}
 
 	public function getBlockStateDictionary() : BlockStateDictionary{ return $this->blockStateDictionary; }

src/network/mcpe/convert/ItemTranslator.php

@@ -107,7 +107,7 @@ final class ItemTranslator{
 
 		$blockStateData = null;
 		if($networkBlockRuntimeId !== self::NO_BLOCK_RUNTIME_ID){
-			$blockStateData = $this->blockStateDictionary->getDataFromStateId($networkBlockRuntimeId);
+			$blockStateData = $this->blockStateDictionary->generateDataFromStateId($networkBlockRuntimeId);
 			if($blockStateData === null){
 				throw new TypeConversionException("Blockstate runtimeID $networkBlockRuntimeId does not correspond to any known blockstate");
 			}

src/network/mcpe/serializer/ChunkSerializer.php

@@ -109,7 +109,7 @@ final class ChunkSerializer{
 				$nbtSerializer = new NetworkNbtSerializer();
 				foreach($palette as $p){
 					//TODO: introduce a binary cache for this
-					$state = $blockStateDictionary->getDataFromStateId($blockTranslator->toRuntimeId($p));
+					$state = $blockStateDictionary->generateDataFromStateId($blockTranslator->toRuntimeId($p));
 					if($state === null){
 						$state = $blockTranslator->getFallbackStateData();
 					}

tools/generate-bedrock-data-from-packets.php

@@ -150,7 +150,7 @@ class ParserPacketHandler extends PacketHandler{
 			if($meta !== 0){
 				throw new PacketHandlingException("Unexpected non-zero blockitem meta");
 			}
-			$blockState = $this->blockTranslator->getBlockStateDictionary()->getDataFromStateId($itemStack->getBlockRuntimeId()) ?? null;
+			$blockState = $this->blockTranslator->getBlockStateDictionary()->generateDataFromStateId($itemStack->getBlockRuntimeId()) ?? null;
 			if($blockState === null){
 				throw new PacketHandlingException("Unmapped blockstate ID " . $itemStack->getBlockRuntimeId());
 			}
@@ -270,7 +270,7 @@ class ParserPacketHandler extends PacketHandler{
 			if($meta !== 32767){
 				$blockStateId = $this->blockTranslator->getBlockStateDictionary()->lookupStateIdFromIdMeta($data->name, $meta);
 				if($blockStateId !== null){
-					$blockState = $this->blockTranslator->getBlockStateDictionary()->getDataFromStateId($blockStateId);
+					$blockState = $this->blockTranslator->getBlockStateDictionary()->generateDataFromStateId($blockStateId);
 					if($blockState !== null && count($blockState->getStates()) > 0){
 						$data->block_states = self::blockStatePropertiesToString($blockState);
 					}