From ed021d193d47dadb6bdf00a991a0967f76c1fe5d Mon Sep 17 00:00:00 2001 From: "Dylan K. Taylor" Date: Wed, 3 May 2023 23:11:00 +0100 Subject: [PATCH] BlockTranslator: cut memory usage in half this was achieved by storing binary representations of the blockstates, rather than the original BlockStateData. Due to the insane object:data ratio of Tag objects (40:1 for ByteTag for example), modestly sized NBT can explode in memory footprint. This has been previously seen with the absurd 25 MB footprint on file load. Previously, I attempted to mitigate this by deduplicating tag objects, but this was mitigating a symptom rather than addressing the cause. We don't actually need to keep the NBT around in memory, since we don't actually use it for anything other than matching blockstates. In this case, we can allow the code to be possibly a little slower, since the lookup is anyway slow and the result will be cached. In fact, using encoded ordered states as hash keys significantly improves the speed of lookups for stuff like walls, which have many thousands of states. We keep around generateStateData(), since it's still possible we may need the BlockStateData associated, and it can be easily reconstructed from the binary-encoded representation in BlockStateDictionaryEntry. --- .../mcpe/convert/BlockStateDictionary.php | 46 ++-------------- .../convert/BlockStateDictionaryEntry.php | 55 +++++++++++++++++-- .../mcpe/convert/BlockStateLookupCache.php | 20 ++----- src/network/mcpe/convert/BlockTranslator.php | 4 +- src/network/mcpe/convert/ItemTranslator.php | 2 +- .../mcpe/serializer/ChunkSerializer.php | 2 +- tools/generate-bedrock-data-from-packets.php | 4 +- 7 files changed, 67 insertions(+), 66 deletions(-) diff --git a/src/network/mcpe/convert/BlockStateDictionary.php b/src/network/mcpe/convert/BlockStateDictionary.php index 4c6e8aec0..b70038220 100644 --- a/src/network/mcpe/convert/BlockStateDictionary.php +++ b/src/network/mcpe/convert/BlockStateDictionary.php @@ -25,10 +25,6 @@ namespace pocketmine\network\mcpe\convert; use pocketmine\data\bedrock\block\BlockStateData; use pocketmine\nbt\NbtDataException; -use pocketmine\nbt\tag\ByteTag; -use pocketmine\nbt\tag\CompoundTag; -use pocketmine\nbt\tag\IntTag; -use pocketmine\nbt\tag\StringTag; use pocketmine\nbt\TreeRoot; use pocketmine\network\mcpe\protocol\serializer\NetworkNbtSerializer; use function array_map; @@ -58,7 +54,7 @@ final class BlockStateDictionary{ public function __construct( private array $states ){ - $this->stateDataToStateIdLookupCache = new BlockStateLookupCache(array_map(fn(BlockStateDictionaryEntry $entry) => $entry->getStateData(), $this->states)); + $this->stateDataToStateIdLookupCache = new BlockStateLookupCache($this->states); } /** @@ -71,15 +67,15 @@ final class BlockStateDictionary{ $this->idMetaToStateIdLookupCache = []; foreach($this->states as $i => $state){ - $this->idMetaToStateIdLookupCache[$state->getMeta()][$state->getStateData()->getName()] = $i; + $this->idMetaToStateIdLookupCache[$state->getMeta()][$state->getStateName()] = $i; } } return $this->idMetaToStateIdLookupCache; } - public function getDataFromStateId(int $networkRuntimeId) : ?BlockStateData{ - return ($this->states[$networkRuntimeId] ?? null)?->getStateData(); + public function generateDataFromStateId(int $networkRuntimeId) : ?BlockStateData{ + return ($this->states[$networkRuntimeId] ?? null)?->generateStateData(); } /** @@ -113,33 +109,6 @@ final class BlockStateDictionary{ */ public function getStates() : array{ return $this->states; } - /** - * @param string[] $keyIndex - * @param (ByteTag|StringTag|IntTag)[][] $valueIndex - * @phpstan-param array $keyIndex - * @phpstan-param array> $valueIndex - */ - private static function deduplicateCompound(CompoundTag $tag, array &$keyIndex, array &$valueIndex) : CompoundTag{ - if($tag->count() === 0){ - return $tag; - } - - $newTag = CompoundTag::create(); - foreach($tag as $key => $value){ - $key = $keyIndex[$key] ??= $key; - - if($value instanceof CompoundTag){ - $value = self::deduplicateCompound($value, $keyIndex, $valueIndex); - }elseif($value instanceof ByteTag || $value instanceof IntTag || $value instanceof StringTag){ - $value = $valueIndex[$value->getType()][$value->getValue()] ??= $value; - } - - $newTag->setTag($key, $value); - } - - return $newTag; - } - /** * @return BlockStateData[] * @phpstan-return list @@ -147,13 +116,8 @@ final class BlockStateDictionary{ * @throws NbtDataException */ public static function loadPaletteFromString(string $blockPaletteContents) : array{ - $keyIndex = []; - $valueIndex = []; - return array_map( - function(TreeRoot $root) use (&$keyIndex, &$valueIndex) : BlockStateData{ - return BlockStateData::fromNbt(self::deduplicateCompound($root->mustGetCompoundTag(), $keyIndex, $valueIndex)); - }, + fn(TreeRoot $root) => BlockStateData::fromNbt($root->mustGetCompoundTag()), (new NetworkNbtSerializer())->readMultiple($blockPaletteContents) ); } diff --git a/src/network/mcpe/convert/BlockStateDictionaryEntry.php b/src/network/mcpe/convert/BlockStateDictionaryEntry.php index 53229963b..746e93977 100644 --- a/src/network/mcpe/convert/BlockStateDictionaryEntry.php +++ b/src/network/mcpe/convert/BlockStateDictionaryEntry.php @@ -24,15 +24,60 @@ declare(strict_types=1); namespace pocketmine\network\mcpe\convert; use pocketmine\data\bedrock\block\BlockStateData; +use pocketmine\nbt\LittleEndianNbtSerializer; +use pocketmine\nbt\tag\Tag; +use pocketmine\nbt\TreeRoot; +use function array_map; +use function count; +use function ksort; +use const SORT_STRING; final class BlockStateDictionaryEntry{ - public function __construct( - private BlockStateData $stateData, - private int $meta - ){} + private string $stateName; + private string $rawStateProperties; - public function getStateData() : BlockStateData{ return $this->stateData; } + public function __construct( + BlockStateData $stateData, + private int $meta + ){ + $this->stateName = $stateData->getName(); + $this->rawStateProperties = self::encodeStateProperties($stateData->getStates()); + } + + public function getStateName() : string{ return $this->stateName; } + + public function getRawStateProperties() : string{ return $this->rawStateProperties; } + + public function generateStateData() : BlockStateData{ + return new BlockStateData( + $this->stateName, + self::decodeStateProperties($this->rawStateProperties), + BlockStateData::CURRENT_VERSION + ); + } public function getMeta() : int{ return $this->meta; } + + /** + * @return Tag[] + */ + public static function decodeStateProperties(string $rawProperties) : array{ + if($rawProperties === ""){ + return []; + } + return array_map(fn(TreeRoot $root) => $root->getTag(), (new LittleEndianNbtSerializer())->readMultiple($rawProperties)); + } + + /** + * @param Tag[] $properties + */ + public static function encodeStateProperties(array $properties) : string{ + if(count($properties) === 0){ + return ""; + } + //TODO: make a more efficient encoding - NBT will do for now, but it's not very compact + ksort($properties, SORT_STRING); + return (new LittleEndianNbtSerializer())->writeMultiple(array_map(fn(Tag $tag) => new TreeRoot($tag), $properties)); + } } diff --git a/src/network/mcpe/convert/BlockStateLookupCache.php b/src/network/mcpe/convert/BlockStateLookupCache.php index 40b08e37d..cea2c9c57 100644 --- a/src/network/mcpe/convert/BlockStateLookupCache.php +++ b/src/network/mcpe/convert/BlockStateLookupCache.php @@ -35,7 +35,7 @@ final class BlockStateLookupCache{ /** * @var int[][] - * @phpstan-var array> + * @phpstan-var array> */ private array $nameToNetworkIdsLookup = []; @@ -46,18 +46,18 @@ final class BlockStateLookupCache{ private array $nameToSingleNetworkIdLookup = []; /** - * @param BlockStateData[] $blockStates - * @phpstan-param list $blockStates + * @param BlockStateDictionaryEntry[] $blockStates + * @phpstan-param list $blockStates */ public function __construct(array $blockStates){ foreach($blockStates as $stateId => $stateNbt){ - $this->nameToNetworkIdsLookup[$stateNbt->getName()][$stateId] = $stateNbt; + $this->nameToNetworkIdsLookup[$stateNbt->getStateName()][$stateNbt->getRawStateProperties()] = $stateId; } //setup fast path for stateless blocks foreach(Utils::stringifyKeys($this->nameToNetworkIdsLookup) as $name => $stateIds){ if(count($stateIds) === 1){ - $this->nameToSingleNetworkIdLookup[$name] = array_key_first($stateIds); + $this->nameToSingleNetworkIdLookup[$name] = $stateIds[array_key_first($stateIds)]; } } } @@ -73,14 +73,6 @@ final class BlockStateLookupCache{ return $this->nameToSingleNetworkIdLookup[$name]; } - if(isset($this->nameToNetworkIdsLookup[$name])){ - foreach($this->nameToNetworkIdsLookup[$name] as $stateId => $stateNbt){ - if($stateNbt->equals($data)){ - return $stateId; - } - } - } - - return null; + return $this->nameToNetworkIdsLookup[$name][BlockStateDictionaryEntry::encodeStateProperties($data->getStates())] ?? null; } } diff --git a/src/network/mcpe/convert/BlockTranslator.php b/src/network/mcpe/convert/BlockTranslator.php index a475109f4..d3f07fb30 100644 --- a/src/network/mcpe/convert/BlockTranslator.php +++ b/src/network/mcpe/convert/BlockTranslator.php @@ -51,7 +51,7 @@ final class BlockTranslator{ BlockStateData::current(BlockTypeNames::INFO_UPDATE, []) ) ?? throw new AssumptionFailedError(BlockTypeNames::INFO_UPDATE . " should always exist"); //lookup the state data from the dictionary to avoid keeping two copies of the same data around - $this->fallbackStateData = $this->blockStateDictionary->getDataFromStateId($this->fallbackStateId) ?? throw new AssumptionFailedError("We just looked up this state data, so it must exist"); + $this->fallbackStateData = $this->blockStateDictionary->generateDataFromStateId($this->fallbackStateId) ?? throw new AssumptionFailedError("We just looked up this state data, so it must exist"); } public function toRuntimeId(int $internalStateId) : int{ @@ -84,7 +84,7 @@ final class BlockTranslator{ //case someone wants to implement multi version). $networkRuntimeId = $this->toRuntimeId($internalStateId); - return $this->blockStateDictionary->getDataFromStateId($networkRuntimeId) ?? throw new AssumptionFailedError("We just looked up this state ID, so it must exist"); + return $this->blockStateDictionary->generateDataFromStateId($networkRuntimeId) ?? throw new AssumptionFailedError("We just looked up this state ID, so it must exist"); } public function getBlockStateDictionary() : BlockStateDictionary{ return $this->blockStateDictionary; } diff --git a/src/network/mcpe/convert/ItemTranslator.php b/src/network/mcpe/convert/ItemTranslator.php index 6309d774c..ea7a64d40 100644 --- a/src/network/mcpe/convert/ItemTranslator.php +++ b/src/network/mcpe/convert/ItemTranslator.php @@ -107,7 +107,7 @@ final class ItemTranslator{ $blockStateData = null; if($networkBlockRuntimeId !== self::NO_BLOCK_RUNTIME_ID){ - $blockStateData = $this->blockStateDictionary->getDataFromStateId($networkBlockRuntimeId); + $blockStateData = $this->blockStateDictionary->generateDataFromStateId($networkBlockRuntimeId); if($blockStateData === null){ throw new TypeConversionException("Blockstate runtimeID $networkBlockRuntimeId does not correspond to any known blockstate"); } diff --git a/src/network/mcpe/serializer/ChunkSerializer.php b/src/network/mcpe/serializer/ChunkSerializer.php index 88c03ae38..84682e114 100644 --- a/src/network/mcpe/serializer/ChunkSerializer.php +++ b/src/network/mcpe/serializer/ChunkSerializer.php @@ -109,7 +109,7 @@ final class ChunkSerializer{ $nbtSerializer = new NetworkNbtSerializer(); foreach($palette as $p){ //TODO: introduce a binary cache for this - $state = $blockStateDictionary->getDataFromStateId($blockTranslator->toRuntimeId($p)); + $state = $blockStateDictionary->generateDataFromStateId($blockTranslator->toRuntimeId($p)); if($state === null){ $state = $blockTranslator->getFallbackStateData(); } diff --git a/tools/generate-bedrock-data-from-packets.php b/tools/generate-bedrock-data-from-packets.php index 83f684884..989310e25 100644 --- a/tools/generate-bedrock-data-from-packets.php +++ b/tools/generate-bedrock-data-from-packets.php @@ -150,7 +150,7 @@ class ParserPacketHandler extends PacketHandler{ if($meta !== 0){ throw new PacketHandlingException("Unexpected non-zero blockitem meta"); } - $blockState = $this->blockTranslator->getBlockStateDictionary()->getDataFromStateId($itemStack->getBlockRuntimeId()) ?? null; + $blockState = $this->blockTranslator->getBlockStateDictionary()->generateDataFromStateId($itemStack->getBlockRuntimeId()) ?? null; if($blockState === null){ throw new PacketHandlingException("Unmapped blockstate ID " . $itemStack->getBlockRuntimeId()); } @@ -270,7 +270,7 @@ class ParserPacketHandler extends PacketHandler{ if($meta !== 32767){ $blockStateId = $this->blockTranslator->getBlockStateDictionary()->lookupStateIdFromIdMeta($data->name, $meta); if($blockStateId !== null){ - $blockState = $this->blockTranslator->getBlockStateDictionary()->getDataFromStateId($blockStateId); + $blockState = $this->blockTranslator->getBlockStateDictionary()->generateDataFromStateId($blockStateId); if($blockState !== null && count($blockState->getStates()) > 0){ $data->block_states = self::blockStatePropertiesToString($blockState); }