mirrors/PocketMine-MP
1
0
Fork 0
mirror of https://github.com/pmmp/PocketMine-MP.git synced 2025-06-11 05:55:33 +00:00
Code Issues Projects Releases Wiki Activity

Introduced a custom LevelDB impl with regions and better key format

Browse Source 
This new impl (which is not loadable by vanilla) is targeted at very large worlds, which experience significant I/O performance issues due to a variety of issues described in #6580.

Two main changes are made in RegionizedLevelDB:
- First, multiple LevelDBs are used, which cover a fixed NxN segment of terrain, similar to Anvil in Java. However, there's no constraint on these region sizes. Several experimental sizes are supported by default in WorldProviderManager.
- Second, bigEndianLong(morton2d(chunkX, chunkZ)) is used for chunk keys instead of littleEndianInt(chunkX).littleEndianInt(chunkZ). This new scheme has much better cache locality than Mojang's version, which reduces overlap and costly DB compactions.

The following new provider options are available as a result of this change:
- custom-leveldb-regions-32
- custom-leveldb-regions-64
- custom-leveldb-regions-128
- custom-leveldb-regions-256

Smaller sizes will likely be less space-efficient, but will also probably have better performance.
Once a sweet spot is found, a default will be introduced.

Note that the different variations of custom-leveldb-regions-* are not cross-compatible.
Conversion between the different formats is necessary if you want to change formats.
This commit is contained in:
Dylan K. Taylor 2024-12-21 17:56:09 +00:00
parent 306623e890
commit 1224055f66
No known key found for this signature in database
GPG Key ID: 8927471A91CAFD3D
4 changed files with 1084 additions and 755 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/world/format/io/WorldProviderManager.php
View File

@ -25,9 +25,11 @@ namespace pocketmine\world\format\io;
use pocketmine\utils\Utils;
use pocketmine\world\format\io\leveldb\LevelDB;
use pocketmine\world\format\io\leveldb\RegionizedLevelDB;
use pocketmine\world\format\io\region\Anvil;
use pocketmine\world\format\io\region\McRegion;
use pocketmine\world\format\io\region\PMAnvil;
use pocketmine\world\WorldCreationOptions;
use function strtolower;
use function trim;
@ -41,10 +43,24 @@ final class WorldProviderManager{
private WritableWorldProviderManagerEntry $default;
public function __construct(){
$leveldb = new WritableWorldProviderManagerEntry(LevelDB::isValid(...), fn(string $path, \Logger $logger) => new LevelDB($path, $logger), LevelDB::generate(...));
$leveldb = new WritableWorldProviderManagerEntry(
LevelDB::isValid(...),
fn(string $path, \Logger $logger) => new LevelDB($path, $logger),
LevelDB::generate(...)
);
$this->default = $leveldb;
$this->addProvider($leveldb, "leveldb");
//any arbitrary size is supported, but powers of 2 are best
//these are the most likely to be useful
foreach([32, 64, 128, 256] as $regionLength){
$this->addProvider(new WritableWorldProviderManagerEntry(
fn(string $path) => RegionizedLevelDB::isValid($path, $regionLength),
fn(string $path, \Logger $logger) => new RegionizedLevelDB($path, $logger, $regionLength),
fn(string $path, string $name, WorldCreationOptions $options) => RegionizedLevelDB::generate($path, $name, $options, $regionLength)
), "custom-leveldb-regions-$regionLength");
}
$this->addProvider(new ReadOnlyWorldProviderManagerEntry(Anvil::isValid(...), fn(string $path, \Logger $logger) => new Anvil($path, $logger)), "anvil");
$this->addProvider(new ReadOnlyWorldProviderManagerEntry(McRegion::isValid(...), fn(string $path, \Logger $logger) => new McRegion($path, $logger)), "mcregion");
$this->addProvider(new ReadOnlyWorldProviderManagerEntry(PMAnvil::isValid(...), fn(string $path, \Logger $logger) => new PMAnvil($path, $logger)), "pmanvil");

824
src/world/format/io/leveldb/BaseLevelDB.php Normal file
View File

@ -0,0 +1,824 @@
<?php
/*
*
* ____ _ _ __ __ _ __ __ ____
* | _ \ ___ ___| | _____| |_| \/ (_)_ __ ___ | \/ | _ \
* | |_) / _ \ / __| |/ / _ \ __| |\/| | | '_ \ / _ \_____| |\/| | |_) |
* | __/ (_) | (__| < __/ |_| | | | | | | | __/_____| | | | __/
* |_| \___/ \___|_|\_\___|\__|_| |_|_|_| |_|\___| |_| |_|_|
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* @author PocketMine Team
* @link http://www.pocketmine.net/
*
*
*/
declare(strict_types=1);
namespace pocketmine\world\format\io\leveldb;
use pocketmine\block\Block;
use pocketmine\data\bedrock\BiomeIds;
use pocketmine\data\bedrock\block\BlockStateDeserializeException;
use pocketmine\data\bedrock\block\convert\UnsupportedBlockStateException;
use pocketmine\nbt\LittleEndianNbtSerializer;
use pocketmine\nbt\NBT;
use pocketmine\nbt\NbtDataException;
use pocketmine\nbt\tag\CompoundTag;
use pocketmine\nbt\TreeRoot;
use pocketmine\utils\Binary;
use pocketmine\utils\BinaryDataException;
use pocketmine\utils\BinaryStream;
use pocketmine\VersionInfo;
use pocketmine\world\format\Chunk;
use pocketmine\world\format\io\BaseWorldProvider;
use pocketmine\world\format\io\ChunkData;
use pocketmine\world\format\io\ChunkUtils;
use pocketmine\world\format\io\data\BedrockWorldData;
use pocketmine\world\format\io\exception\CorruptedChunkException;
use pocketmine\world\format\io\exception\CorruptedWorldException;
use pocketmine\world\format\io\exception\UnsupportedWorldFormatException;
use pocketmine\world\format\io\GlobalBlockStateHandlers;
use pocketmine\world\format\io\LoadedChunkData;
use pocketmine\world\format\io\WorldData;
use pocketmine\world\format\io\WritableWorldProvider;
use pocketmine\world\format\PalettedBlockArray;
use pocketmine\world\format\SubChunk;
use pocketmine\world\WorldCreationOptions;
use Symfony\Component\Filesystem\Path;
use function array_map;
use function array_values;
use function chr;
use function count;
use function defined;
use function extension_loaded;
use function implode;
use function mkdir;
use function ord;
use function str_repeat;
use function strlen;
use function substr;
use function unpack;
abstract class BaseLevelDB extends BaseWorldProvider implements WritableWorldProvider{
protected const FINALISATION_NEEDS_INSTATICKING = 0;
protected const FINALISATION_NEEDS_POPULATION = 1;
protected const FINALISATION_DONE = 2;
protected const ENTRY_FLAT_WORLD_LAYERS = "game_flatworldlayers";
protected const CURRENT_LEVEL_CHUNK_VERSION = ChunkVersion::v1_21_40;
protected const CURRENT_LEVEL_SUBCHUNK_VERSION = SubChunkVersion::PALETTED_MULTI;
private const CAVES_CLIFFS_EXPERIMENTAL_SUBCHUNK_KEY_OFFSET = 4;
private static function checkForLevelDBExtension() : void{
if(!extension_loaded('leveldb')){
throw new UnsupportedWorldFormatException("The leveldb PHP extension is required to use this world format");
}
if(!defined('LEVELDB_ZLIB_RAW_COMPRESSION')){
throw new UnsupportedWorldFormatException("Given version of php-leveldb doesn't support zlib raw compression");
}
}
public function __construct(string $path, \Logger $logger){
self::checkForLevelDBExtension();
parent::__construct($path, $logger);
}
protected function loadLevelData() : WorldData{
return new BedrockWorldData(Path::join($this->getPath(), "level.dat"));
}
public function getWorldMinY() : int{
return -64;
}
public function getWorldMaxY() : int{
return 320;
}
protected static function baseGenerate(string $path, string $name, WorldCreationOptions $options) : void{
self::checkForLevelDBExtension();
@mkdir($path, 0777, true);
BedrockWorldData::generate($path, $name, $options);
}
/**
* @throws CorruptedChunkException
*/
protected function deserializeBlockPalette(BinaryStream $stream, \Logger $logger) : PalettedBlockArray{
$bitsPerBlock = $stream->getByte() >> 1;
try{
$words = $stream->get(PalettedBlockArray::getExpectedWordArraySize($bitsPerBlock));
}catch(\InvalidArgumentException $e){
throw new CorruptedChunkException("Failed to deserialize paletted storage: " . $e->getMessage(), 0, $e);
}
$nbt = new LittleEndianNbtSerializer();
$palette = [];
if($bitsPerBlock === 0){
$paletteSize = 1;
/*
* Due to code copy-paste in a public plugin, some PM4 worlds have 0 bpb palettes with a length prefix.
* This is invalid and does not happen in vanilla.
* These palettes were accepted by PM4 despite being invalid, but PM5 considered them corrupt, causing loss
* of data. Since many users were affected by this, a workaround is therefore necessary to allow PM5 to read
* these worlds without data loss.
*
* References:
* - https://github.com/Refaltor77/CustomItemAPI/issues/68
* - https://github.com/pmmp/PocketMine-MP/issues/5911
*/
$offset = $stream->getOffset();
$byte1 = $stream->getByte();
$stream->setOffset($offset); //reset offset
if($byte1 !== NBT::TAG_Compound){ //normally the first byte would be the NBT of the blockstate
$susLength = $stream->getLInt();
if($susLength !== 1){ //make sure the data isn't complete garbage
throw new CorruptedChunkException("CustomItemAPI borked 0 bpb palette should always have a length of 1");
}
$logger->error("Unexpected palette size for 0 bpb palette");
}
}else{
$paletteSize = $stream->getLInt();
}
$blockDecodeErrors = [];
for($i = 0; $i < $paletteSize; ++$i){
try{
$offset = $stream->getOffset();
$blockStateNbt = $nbt->read($stream->getBuffer(), $offset)->mustGetCompoundTag();
$stream->setOffset($offset);
}catch(NbtDataException $e){
//NBT borked, unrecoverable
throw new CorruptedChunkException("Invalid blockstate NBT at offset $i in paletted storage: " . $e->getMessage(), 0, $e);
}
//TODO: remember data for unknown states so we can implement them later
try{
$blockStateData = $this->blockDataUpgrader->upgradeBlockStateNbt($blockStateNbt);
}catch(BlockStateDeserializeException $e){
//while not ideal, this is not a fatal error
$blockDecodeErrors[] = "Palette offset $i / Upgrade error: " . $e->getMessage() . ", NBT: " . $blockStateNbt->toString();
$palette[] = $this->blockStateDeserializer->deserialize(GlobalBlockStateHandlers::getUnknownBlockStateData());
continue;
}
try{
$palette[] = $this->blockStateDeserializer->deserialize($blockStateData);
}catch(UnsupportedBlockStateException $e){
$blockDecodeErrors[] = "Palette offset $i / " . $e->getMessage();
$palette[] = $this->blockStateDeserializer->deserialize(GlobalBlockStateHandlers::getUnknownBlockStateData());
}catch(BlockStateDeserializeException $e){
$blockDecodeErrors[] = "Palette offset $i / Deserialize error: " . $e->getMessage() . ", NBT: " . $blockStateNbt->toString();
$palette[] = $this->blockStateDeserializer->deserialize(GlobalBlockStateHandlers::getUnknownBlockStateData());
}
}
if(count($blockDecodeErrors) > 0){
//$logger->error("Errors decoding blocks:\n - " . implode("\n - ", $blockDecodeErrors));
}
//TODO: exceptions
return PalettedBlockArray::fromData($bitsPerBlock, $words, $palette);
}
private function serializeBlockPalette(BinaryStream $stream, PalettedBlockArray $blocks) : void{
$stream->putByte($blocks->getBitsPerBlock() << 1);
$stream->put($blocks->getWordArray());
$palette = $blocks->getPalette();
if($blocks->getBitsPerBlock() !== 0){
$stream->putLInt(count($palette));
}
$tags = [];
foreach($palette as $p){
$tags[] = new TreeRoot($this->blockStateSerializer->serialize($p)->toNbt());
}
$stream->put((new LittleEndianNbtSerializer())->writeMultiple($tags));
}
/**
* @throws CorruptedChunkException
*/
private static function getExpected3dBiomesCount(int $chunkVersion) : int{
return match(true){
$chunkVersion >= ChunkVersion::v1_18_30 => 24,
$chunkVersion >= ChunkVersion::v1_18_0_25_beta => 25,
$chunkVersion >= ChunkVersion::v1_18_0_24_beta => 32,
$chunkVersion >= ChunkVersion::v1_18_0_22_beta => 65,
$chunkVersion >= ChunkVersion::v1_17_40_20_beta_experimental_caves_cliffs => 32,
default => throw new CorruptedChunkException("Chunk version $chunkVersion should not have 3D biomes")
};
}
/**
* @throws CorruptedChunkException
*/
private static function deserializeBiomePalette(BinaryStream $stream, int $bitsPerBlock) : PalettedBlockArray{
try{
$words = $stream->get(PalettedBlockArray::getExpectedWordArraySize($bitsPerBlock));
}catch(\InvalidArgumentException $e){
throw new CorruptedChunkException("Failed to deserialize paletted biomes: " . $e->getMessage(), 0, $e);
}
$palette = [];
$paletteSize = $bitsPerBlock === 0 ? 1 : $stream->getLInt();
for($i = 0; $i < $paletteSize; ++$i){
$palette[] = $stream->getLInt();
}
//TODO: exceptions
return PalettedBlockArray::fromData($bitsPerBlock, $words, $palette);
}
private static function serializeBiomePalette(BinaryStream $stream, PalettedBlockArray $biomes) : void{
$stream->putByte($biomes->getBitsPerBlock() << 1);
$stream->put($biomes->getWordArray());
$palette = $biomes->getPalette();
if($biomes->getBitsPerBlock() !== 0){
$stream->putLInt(count($palette));
}
foreach($palette as $p){
$stream->putLInt($p);
}
}
/**
* @throws CorruptedChunkException
* @return PalettedBlockArray[]
* @phpstan-return array<int, PalettedBlockArray>
*/
private static function deserialize3dBiomes(BinaryStream $stream, int $chunkVersion, \Logger $logger) : array{
$previous = null;
$result = [];
$nextIndex = Chunk::MIN_SUBCHUNK_INDEX;
$expectedCount = self::getExpected3dBiomesCount($chunkVersion);
for($i = 0; $i < $expectedCount; ++$i){
try{
$bitsPerBlock = $stream->getByte() >> 1;
if($bitsPerBlock === 127){
if($previous === null){
throw new CorruptedChunkException("Serialized biome palette $i has no previous palette to copy from");
}
$decoded = clone $previous;
}else{
$decoded = self::deserializeBiomePalette($stream, $bitsPerBlock);
}
$previous = $decoded;
if($nextIndex <= Chunk::MAX_SUBCHUNK_INDEX){ //older versions wrote additional superfluous biome palettes
$result[$nextIndex++] = $decoded;
}elseif($stream->feof()){
//not enough padding biome arrays for the given version - this is non-critical since we discard the excess anyway, but this should be logged
$logger->error("Wrong number of 3D biome palettes for this chunk version: expected $expectedCount, but got " . ($i + 1) . " - this is not a problem, but may indicate a corrupted chunk");
break;
}
}catch(BinaryDataException $e){
throw new CorruptedChunkException("Failed to deserialize biome palette $i: " . $e->getMessage(), 0, $e);
}
}
if(!$stream->feof()){
//maybe bad output produced by a third-party conversion tool like Chunker
$logger->error("Unexpected trailing data after 3D biomes data");
}
return $result;
}
/**
* @param SubChunk[] $subChunks
*/
private static function serialize3dBiomes(BinaryStream $stream, array $subChunks) : void{
//TODO: the server-side min/max may not coincide with the world storage min/max - we may need additional logic to handle this
for($y = Chunk::MIN_SUBCHUNK_INDEX; $y <= Chunk::MAX_SUBCHUNK_INDEX; $y++){
//TODO: is it worth trying to use the previous palette if it's the same as the current one? vanilla supports
//this, but it's not clear if it's worth the effort to implement.
self::serializeBiomePalette($stream, $subChunks[$y]->getBiomeArray());
}
}
/**
* @phpstan-param-out int $x
* @phpstan-param-out int $y
* @phpstan-param-out int $z
*/
protected static function deserializeExtraDataKey(int $chunkVersion, int $key, ?int &$x, ?int &$y, ?int &$z) : void{
if($chunkVersion >= ChunkVersion::v1_0_0){
$x = ($key >> 12) & 0xf;
$z = ($key >> 8) & 0xf;
$y = $key & 0xff;
}else{ //pre-1.0, 7 bits were used because the build height limit was lower
$x = ($key >> 11) & 0xf;
$z = ($key >> 7) & 0xf;
$y = $key & 0x7f;
}
}
/**
* @return PalettedBlockArray[]
*/
protected function deserializeLegacyExtraData(\LevelDB $db, string $index, int $chunkVersion, \Logger $logger) : array{
if(($extraRawData = $db->get($index . ChunkDataKey::LEGACY_BLOCK_EXTRA_DATA)) === false || $extraRawData === ""){
return [];
}
/** @var PalettedBlockArray[] $extraDataLayers */
$extraDataLayers = [];
$binaryStream = new BinaryStream($extraRawData);
$count = $binaryStream->getLInt();
for($i = 0; $i < $count; ++$i){
$key = $binaryStream->getLInt();
$value = $binaryStream->getLShort();
self::deserializeExtraDataKey($chunkVersion, $key, $x, $fullY, $z);
$ySub = ($fullY >> SubChunk::COORD_BIT_SIZE);
$y = $key & SubChunk::COORD_MASK;
$blockId = $value & 0xff;
$blockData = ($value >> 8) & 0xf;
try{
$blockStateData = $this->blockDataUpgrader->upgradeIntIdMeta($blockId, $blockData);
}catch(BlockStateDeserializeException $e){
//TODO: we could preserve this in case it's supported in the future, but this was historically only
//used for grass anyway, so we probably don't need to care
$logger->error("Failed to upgrade legacy extra block: " . $e->getMessage() . " ($blockId:$blockData)");
continue;
}
//assume this won't throw
$blockStateId = $this->blockStateDeserializer->deserialize($blockStateData);
if(!isset($extraDataLayers[$ySub])){
$extraDataLayers[$ySub] = new PalettedBlockArray(Block::EMPTY_STATE_ID);
}
$extraDataLayers[$ySub]->set($x, $y, $z, $blockStateId);
}
return $extraDataLayers;
}
private function readVersion(\LevelDB $db, int $chunkX, int $chunkZ) : ?int{
$index = $this->coordsToChunkIndex($chunkX, $chunkZ);
$chunkVersionRaw = $db->get($index . ChunkDataKey::NEW_VERSION);
if($chunkVersionRaw === false){
$chunkVersionRaw = $db->get($index . ChunkDataKey::OLD_VERSION);
if($chunkVersionRaw === false){
return null;
}
}
return ord($chunkVersionRaw);
}
/**
* Deserializes terrain data stored in the 0.9 full-chunk format into subchunks.
*
* @return SubChunk[]
* @phpstan-return array<int, SubChunk>
* @throws CorruptedWorldException
*/
private function deserializeLegacyTerrainData(\LevelDB $db, string $index, int $chunkVersion, \Logger $logger) : array{
$convertedLegacyExtraData = $this->deserializeLegacyExtraData($db, $index, $chunkVersion, $logger);
$legacyTerrain = $db->get($index . ChunkDataKey::LEGACY_TERRAIN);
if($legacyTerrain === false){
throw new CorruptedChunkException("Missing expected LEGACY_TERRAIN tag for format version $chunkVersion");
}
$binaryStream = new BinaryStream($legacyTerrain);
try{
$fullIds = $binaryStream->get(32768);
$fullData = $binaryStream->get(16384);
$binaryStream->get(32768); //legacy light info, discard it
}catch(BinaryDataException $e){
throw new CorruptedChunkException($e->getMessage(), 0, $e);
}
try{
$binaryStream->get(256); //heightmap, discard it
/** @var int[] $unpackedBiomeArray */
$unpackedBiomeArray = unpack("N*", $binaryStream->get(1024)); //unpack() will never fail here
$biomes3d = ChunkUtils::extrapolate3DBiomes(ChunkUtils::convertBiomeColors(array_values($unpackedBiomeArray))); //never throws
}catch(BinaryDataException $e){
throw new CorruptedChunkException($e->getMessage(), 0, $e);
}
if(!$binaryStream->feof()){
$logger->error("Unexpected trailing data in legacy terrain data");
}
$subChunks = [];
for($yy = 0; $yy < 8; ++$yy){
$storages = [$this->palettizeLegacySubChunkFromColumn($fullIds, $fullData, $yy, new \PrefixedLogger($logger, "Subchunk y=$yy"))];
if(isset($convertedLegacyExtraData[$yy])){
$storages[] = $convertedLegacyExtraData[$yy];
}
$subChunks[$yy] = new SubChunk(Block::EMPTY_STATE_ID, $storages, clone $biomes3d);
}
//make sure extrapolated biomes get filled in correctly
for($yy = Chunk::MIN_SUBCHUNK_INDEX; $yy <= Chunk::MAX_SUBCHUNK_INDEX; ++$yy){
if(!isset($subChunks[$yy])){
$subChunks[$yy] = new SubChunk(Block::EMPTY_STATE_ID, [], clone $biomes3d);
}
}
return $subChunks;
}
/**
* Deserializes a subchunk stored in the legacy non-paletted format used from 1.0 until 1.2.13.
*/
private function deserializeNonPalettedSubChunkData(BinaryStream $binaryStream, int $chunkVersion, ?PalettedBlockArray $convertedLegacyExtraData, PalettedBlockArray $biomePalette, \Logger $logger) : SubChunk{
try{
$blocks = $binaryStream->get(4096);
$blockData = $binaryStream->get(2048);
}catch(BinaryDataException $e){
throw new CorruptedChunkException($e->getMessage(), 0, $e);
}
if($chunkVersion < ChunkVersion::v1_1_0){
try{
$binaryStream->get(4096); //legacy light info, discard it
if(!$binaryStream->feof()){
$logger->error("Unexpected trailing data in legacy subchunk data");
}
}catch(BinaryDataException $e){
$logger->error("Failed to read legacy subchunk light info: " . $e->getMessage());
}
}
$storages = [$this->palettizeLegacySubChunkXZY($blocks, $blockData, $logger)];
if($convertedLegacyExtraData !== null){
$storages[] = $convertedLegacyExtraData;
}
return new SubChunk(Block::EMPTY_STATE_ID, $storages, $biomePalette);
}
/**
* Deserializes subchunk data stored under a subchunk LevelDB key.
*
* @see ChunkDataKey::SUBCHUNK
* @throws CorruptedChunkException
*/
private function deserializeSubChunkData(BinaryStream $binaryStream, int $chunkVersion, int $subChunkVersion, ?PalettedBlockArray $convertedLegacyExtraData, PalettedBlockArray $biomePalette, \Logger $logger) : SubChunk{
switch($subChunkVersion){
case SubChunkVersion::CLASSIC:
case SubChunkVersion::CLASSIC_BUG_2: //these are all identical to version 0, but vanilla respects these so we should also
case SubChunkVersion::CLASSIC_BUG_3:
case SubChunkVersion::CLASSIC_BUG_4:
case SubChunkVersion::CLASSIC_BUG_5:
case SubChunkVersion::CLASSIC_BUG_6:
case SubChunkVersion::CLASSIC_BUG_7:
return $this->deserializeNonPalettedSubChunkData($binaryStream, $chunkVersion, $convertedLegacyExtraData, $biomePalette, $logger);
case SubChunkVersion::PALETTED_SINGLE:
$storages = [$this->deserializeBlockPalette($binaryStream, $logger)];
if($convertedLegacyExtraData !== null){
$storages[] = $convertedLegacyExtraData;
}
return new SubChunk(Block::EMPTY_STATE_ID, $storages, $biomePalette);
case SubChunkVersion::PALETTED_MULTI:
case SubChunkVersion::PALETTED_MULTI_WITH_OFFSET:
//legacy extradata layers intentionally ignored because they aren't supposed to exist in v8
$storageCount = $binaryStream->getByte();
if($subChunkVersion >= SubChunkVersion::PALETTED_MULTI_WITH_OFFSET){
//height ignored; this seems pointless since this is already in the key anyway
$binaryStream->getByte();
}
$storages = [];
for($k = 0; $k < $storageCount; ++$k){
$storages[] = $this->deserializeBlockPalette($binaryStream, $logger);
}
return new SubChunk(Block::EMPTY_STATE_ID, $storages, $biomePalette);
default:
//this should never happen - an unsupported chunk appearing in a supported world is a sign of corruption
throw new CorruptedChunkException("don't know how to decode LevelDB subchunk format version $subChunkVersion");
}
}
private static function hasOffsetCavesAndCliffsSubChunks(int $chunkVersion) : bool{
return $chunkVersion >= ChunkVersion::v1_16_220_50_unused && $chunkVersion <= ChunkVersion::v1_16_230_50_unused;
}
/**
* Deserializes any subchunks stored under subchunk LevelDB keys, upgrading them to the current format if necessary.
*
* @param PalettedBlockArray[] $convertedLegacyExtraData
* @param PalettedBlockArray[] $biomeArrays
*
* @phpstan-param array<int, PalettedBlockArray> $convertedLegacyExtraData
* @phpstan-param array<int, PalettedBlockArray> $biomeArrays
* @phpstan-param-out bool $hasBeenUpgraded
*
* @return SubChunk[]
* @phpstan-return array<int, SubChunk>
*/
private function deserializeAllSubChunkData(\LevelDB $db, string $index, int $chunkVersion, bool &$hasBeenUpgraded, array $convertedLegacyExtraData, array $biomeArrays, \Logger $logger) : array{
$subChunks = [];
$subChunkKeyOffset = self::hasOffsetCavesAndCliffsSubChunks($chunkVersion) ? self::CAVES_CLIFFS_EXPERIMENTAL_SUBCHUNK_KEY_OFFSET : 0;
for($y = Chunk::MIN_SUBCHUNK_INDEX; $y <= Chunk::MAX_SUBCHUNK_INDEX; ++$y){
if(($data = $db->get($index . ChunkDataKey::SUBCHUNK . chr($y + $subChunkKeyOffset))) === false){
$subChunks[$y] = new SubChunk(Block::EMPTY_STATE_ID, [], $biomeArrays[$y]);
continue;
}
$binaryStream = new BinaryStream($data);
if($binaryStream->feof()){
throw new CorruptedChunkException("Unexpected empty data for subchunk $y");
}
$subChunkVersion = $binaryStream->getByte();
if($subChunkVersion < self::CURRENT_LEVEL_SUBCHUNK_VERSION){
$hasBeenUpgraded = true;
}
$subChunks[$y] = $this->deserializeSubChunkData(
$binaryStream,
$chunkVersion,
$subChunkVersion,
$convertedLegacyExtraData[$y] ?? null,
$biomeArrays[$y],
new \PrefixedLogger($logger, "Subchunk y=$y v$subChunkVersion")
);
}
return $subChunks;
}
/**
* Deserializes any available biome data into an array of paletted biomes. Old 2D biomes are extrapolated to 3D.
*
* @return PalettedBlockArray[]
* @phpstan-return array<int, PalettedBlockArray>
*/
private function deserializeBiomeData(\LevelDB $db, string $index, int $chunkVersion, \Logger $logger) : array{
$biomeArrays = [];
if(($maps2d = $db->get($index . ChunkDataKey::HEIGHTMAP_AND_2D_BIOMES)) !== false){
$binaryStream = new BinaryStream($maps2d);
try{
$binaryStream->get(512); //heightmap, discard it
$biomes3d = ChunkUtils::extrapolate3DBiomes($binaryStream->get(256)); //never throws
if(!$binaryStream->feof()){
$logger->error("Unexpected trailing data after 2D biome data");
}
}catch(BinaryDataException $e){
throw new CorruptedChunkException($e->getMessage(), 0, $e);
}
for($i = Chunk::MIN_SUBCHUNK_INDEX; $i <= Chunk::MAX_SUBCHUNK_INDEX; ++$i){
$biomeArrays[$i] = clone $biomes3d;
}
}elseif(($maps3d = $db->get($index . ChunkDataKey::HEIGHTMAP_AND_3D_BIOMES)) !== false){
$binaryStream = new BinaryStream($maps3d);
try{
$binaryStream->get(512);
$biomeArrays = self::deserialize3dBiomes($binaryStream, $chunkVersion, $logger);
}catch(BinaryDataException $e){
throw new CorruptedChunkException($e->getMessage(), 0, $e);
}
}else{
$logger->error("Missing biome data, using default ocean biome");
for($i = Chunk::MIN_SUBCHUNK_INDEX; $i <= Chunk::MAX_SUBCHUNK_INDEX; ++$i){
$biomeArrays[$i] = new PalettedBlockArray(BiomeIds::OCEAN); //polyfill
}
}
return $biomeArrays;
}
protected function loadChunkFromDB(\LevelDB $db, int $chunkX, int $chunkZ) : ?LoadedChunkData{
$index = $this->coordsToChunkIndex($chunkX, $chunkZ);
$chunkVersion = $this->readVersion($db, $chunkX, $chunkZ);
if($chunkVersion === null){
//TODO: this might be a slightly-corrupted chunk with a missing version field
return null;
}
//TODO: read PM_DATA_VERSION - we'll need it to fix up old chunks
$logger = new \PrefixedLogger($this->logger, "Loading chunk x=$chunkX z=$chunkZ v$chunkVersion");
$hasBeenUpgraded = $chunkVersion < self::CURRENT_LEVEL_CHUNK_VERSION;
switch($chunkVersion){
case ChunkVersion::v1_21_40:
//TODO: BiomeStates became shorts instead of bytes
case ChunkVersion::v1_18_30:
case ChunkVersion::v1_18_0_25_beta:
case ChunkVersion::v1_18_0_24_unused:
case ChunkVersion::v1_18_0_24_beta:
case ChunkVersion::v1_18_0_22_unused:
case ChunkVersion::v1_18_0_22_beta:
case ChunkVersion::v1_18_0_20_unused:
case ChunkVersion::v1_18_0_20_beta:
case ChunkVersion::v1_17_40_unused:
case ChunkVersion::v1_17_40_20_beta_experimental_caves_cliffs:
case ChunkVersion::v1_17_30_25_unused:
case ChunkVersion::v1_17_30_25_beta_experimental_caves_cliffs:
case ChunkVersion::v1_17_30_23_unused:
case ChunkVersion::v1_17_30_23_beta_experimental_caves_cliffs:
case ChunkVersion::v1_16_230_50_unused:
case ChunkVersion::v1_16_230_50_beta_experimental_caves_cliffs:
case ChunkVersion::v1_16_220_50_unused:
case ChunkVersion::v1_16_220_50_beta_experimental_caves_cliffs:
case ChunkVersion::v1_16_210:
case ChunkVersion::v1_16_100_57_beta:
case ChunkVersion::v1_16_100_52_beta:
case ChunkVersion::v1_16_0:
case ChunkVersion::v1_16_0_51_beta:
//TODO: check walls
case ChunkVersion::v1_12_0_unused2:
case ChunkVersion::v1_12_0_unused1:
case ChunkVersion::v1_12_0_4_beta:
case ChunkVersion::v1_11_1:
case ChunkVersion::v1_11_0_4_beta:
case ChunkVersion::v1_11_0_3_beta:
case ChunkVersion::v1_11_0_1_beta:
case ChunkVersion::v1_9_0:
case ChunkVersion::v1_8_0:
case ChunkVersion::v1_2_13:
case ChunkVersion::v1_2_0:
case ChunkVersion::v1_2_0_2_beta:
case ChunkVersion::v1_1_0_converted_from_console:
case ChunkVersion::v1_1_0:
//TODO: check beds
case ChunkVersion::v1_0_0:
$convertedLegacyExtraData = $this->deserializeLegacyExtraData($db, $index, $chunkVersion, $logger);
$biomeArrays = $this->deserializeBiomeData($db, $index, $chunkVersion, $logger);
$subChunks = $this->deserializeAllSubChunkData($db, $index, $chunkVersion, $hasBeenUpgraded, $convertedLegacyExtraData, $biomeArrays, $logger);
break;
case ChunkVersion::v0_9_5:
case ChunkVersion::v0_9_2:
case ChunkVersion::v0_9_0:
$subChunks = $this->deserializeLegacyTerrainData($db, $index, $chunkVersion, $logger);
break;
default:
throw new CorruptedChunkException("don't know how to decode chunk format version $chunkVersion");
}
$nbt = new LittleEndianNbtSerializer();
/** @var CompoundTag[] $entities */
$entities = [];
if(($entityData = $db->get($index . ChunkDataKey::ENTITIES)) !== false && $entityData !== ""){
try{
$entities = array_map(fn(TreeRoot $root) => $root->mustGetCompoundTag(), $nbt->readMultiple($entityData));
}catch(NbtDataException $e){
throw new CorruptedChunkException($e->getMessage(), 0, $e);
}
}
/** @var CompoundTag[] $tiles */
$tiles = [];
if(($tileData = $db->get($index . ChunkDataKey::BLOCK_ENTITIES)) !== false && $tileData !== ""){
try{
$tiles = array_map(fn(TreeRoot $root) => $root->mustGetCompoundTag(), $nbt->readMultiple($tileData));
}catch(NbtDataException $e){
throw new CorruptedChunkException($e->getMessage(), 0, $e);
}
}
$finalisationChr = $db->get($index . ChunkDataKey::FINALIZATION);
if($finalisationChr !== false){
$finalisation = ord($finalisationChr);
$terrainPopulated = $finalisation === self::FINALISATION_DONE;
}else{ //older versions didn't have this tag
$terrainPopulated = true;
}
//TODO: tile ticks, biome states (?)
return new LoadedChunkData(
data: new ChunkData($subChunks, $terrainPopulated, $entities, $tiles),
upgraded: $hasBeenUpgraded,
fixerFlags: LoadedChunkData::FIXER_FLAG_ALL //TODO: fill this by version rather than just setting all flags
);
}
protected function saveChunkToDB(\LevelDB $db, int $chunkX, int $chunkZ, ChunkData $chunkData, int $dirtyFlags) : void{
$index = $this->coordsToChunkIndex($chunkX, $chunkZ);
$write = new \LevelDBWriteBatch();
$write->put($index . ChunkDataKey::NEW_VERSION, chr(self::CURRENT_LEVEL_CHUNK_VERSION));
$write->put($index . ChunkDataKey::PM_DATA_VERSION, Binary::writeLLong(VersionInfo::WORLD_DATA_VERSION));
$subChunks = $chunkData->getSubChunks();
if(($dirtyFlags & Chunk::DIRTY_FLAG_BLOCKS) !== 0){
foreach($subChunks as $y => $subChunk){
$key = $index . ChunkDataKey::SUBCHUNK . chr($y);
if($subChunk->isEmptyAuthoritative()){
$write->delete($key);
}else{
$subStream = new BinaryStream();
$subStream->putByte(self::CURRENT_LEVEL_SUBCHUNK_VERSION);
$layers = $subChunk->getBlockLayers();
$subStream->putByte(count($layers));
foreach($layers as $blocks){
$this->serializeBlockPalette($subStream, $blocks);
}
$write->put($key, $subStream->getBuffer());
}
}
}
if(($dirtyFlags & Chunk::DIRTY_FLAG_BIOMES) !== 0){
$write->delete($index . ChunkDataKey::HEIGHTMAP_AND_2D_BIOMES);
$stream = new BinaryStream();
$stream->put(str_repeat("\x00", 512)); //fake heightmap
self::serialize3dBiomes($stream, $subChunks);
$write->put($index . ChunkDataKey::HEIGHTMAP_AND_3D_BIOMES, $stream->getBuffer());
}
//TODO: use this properly
$write->put($index . ChunkDataKey::FINALIZATION, chr($chunkData->isPopulated() ? self::FINALISATION_DONE : self::FINALISATION_NEEDS_POPULATION));
$this->writeTags($chunkData->getTileNBT(), $index . ChunkDataKey::BLOCK_ENTITIES, $write);
$this->writeTags($chunkData->getEntityNBT(), $index . ChunkDataKey::ENTITIES, $write);
$write->delete($index . ChunkDataKey::HEIGHTMAP_AND_2D_BIOME_COLORS);
$write->delete($index . ChunkDataKey::LEGACY_TERRAIN);
$db->write($write);
}
/**
* @param CompoundTag[] $targets
*/
private function writeTags(array $targets, string $index, \LevelDBWriteBatch $write) : void{
if(count($targets) > 0){
$nbt = new LittleEndianNbtSerializer();
$write->put($index, $nbt->writeMultiple(array_map(fn(CompoundTag $tag) => new TreeRoot($tag), $targets)));
}else{
$write->delete($index);
}
}
abstract protected function coordsToChunkIndex(int $chunkX, int $chunkZ) : string;
/**
* @return int[]
* @phpstan-return array{int, int}
*/
abstract protected function coordsFromChunkIndex(string $chunkIndex) : array;
/**
* Returns a generator which yields all the chunks in this database.
*
* @return \Generator|LoadedChunkData[]
* @phpstan-return \Generator<array{int, int}, LoadedChunkData, void, void>
* @throws CorruptedChunkException
*/
protected function getAllChunksFromDB(\LevelDB $db, bool $skipCorrupted, ?\Logger $logger) : \Generator{
foreach($db->getIterator() as $key => $_){
if(strlen($key) === 9 && ($key[8] === ChunkDataKey::NEW_VERSION || $key[8] === ChunkDataKey::OLD_VERSION)){
[$chunkX, $chunkZ] = $this->coordsFromChunkIndex(substr($key, 0, 8));
try{
if(($chunk = $this->loadChunk($chunkX, $chunkZ)) !== null){
yield [$chunkX, $chunkZ] => $chunk;
}
}catch(CorruptedChunkException $e){
if(!$skipCorrupted){
throw $e;
}
if($logger !== null){
$logger->error("Skipped corrupted chunk $chunkX $chunkZ (" . $e->getMessage() . ")");
}
}
}
}
}
protected function calculateChunkCountInDB(\LevelDB $db) : int{
$count = 0;
foreach($db->getIterator() as $key => $_){
if(strlen($key) === 9 && ($key[8] === ChunkDataKey::NEW_VERSION || $key[8] === ChunkDataKey::OLD_VERSION)){
$count++;
}
}
return $count;
}
}

775
src/world/format/io/leveldb/LevelDB.php
View File

@ -23,78 +23,24 @@ declare(strict_types=1);
namespace pocketmine\world\format\io\leveldb;
use pocketmine\block\Block;
use pocketmine\data\bedrock\BiomeIds;
use pocketmine\data\bedrock\block\BlockStateDeserializeException;
use pocketmine\data\bedrock\block\convert\UnsupportedBlockStateException;
use pocketmine\nbt\LittleEndianNbtSerializer;
use pocketmine\nbt\NBT;
use pocketmine\nbt\NbtDataException;
use pocketmine\nbt\tag\CompoundTag;
use pocketmine\nbt\TreeRoot;
use pocketmine\utils\Binary;
use pocketmine\utils\BinaryDataException;
use pocketmine\utils\BinaryStream;
use pocketmine\VersionInfo;
use pocketmine\world\format\Chunk;
use pocketmine\world\format\io\BaseWorldProvider;
use pocketmine\world\format\io\ChunkData;
use pocketmine\world\format\io\ChunkUtils;
use pocketmine\world\format\io\data\BedrockWorldData;
use pocketmine\world\format\io\exception\CorruptedChunkException;
use pocketmine\world\format\io\exception\CorruptedWorldException;
use pocketmine\world\format\io\exception\UnsupportedWorldFormatException;
use pocketmine\world\format\io\GlobalBlockStateHandlers;
use pocketmine\world\format\io\LoadedChunkData;
use pocketmine\world\format\io\WorldData;
use pocketmine\world\format\io\WritableWorldProvider;
use pocketmine\world\format\PalettedBlockArray;
use pocketmine\world\format\SubChunk;
use pocketmine\world\WorldCreationOptions;
use Symfony\Component\Filesystem\Path;
use function array_map;
use function array_values;
use function chr;
use function count;
use function defined;
use function extension_loaded;
use function file_exists;
use function implode;
use function is_dir;
use function mkdir;
use function ord;
use function str_repeat;
use function strlen;
use function substr;
use function trim;
use function unpack;
use const LEVELDB_ZLIB_RAW_COMPRESSION;
class LevelDB extends BaseWorldProvider implements WritableWorldProvider{
protected const FINALISATION_NEEDS_INSTATICKING = 0;
protected const FINALISATION_NEEDS_POPULATION = 1;
protected const FINALISATION_DONE = 2;
protected const ENTRY_FLAT_WORLD_LAYERS = "game_flatworldlayers";
protected const CURRENT_LEVEL_CHUNK_VERSION = ChunkVersion::v1_21_40;
protected const CURRENT_LEVEL_SUBCHUNK_VERSION = SubChunkVersion::PALETTED_MULTI;
private const CAVES_CLIFFS_EXPERIMENTAL_SUBCHUNK_KEY_OFFSET = 4;
class LevelDB extends BaseLevelDB{
protected \LevelDB $db;
private static function checkForLevelDBExtension() : void{
if(!extension_loaded('leveldb')){
throw new UnsupportedWorldFormatException("The leveldb PHP extension is required to use this world format");
}
if(!defined('LEVELDB_ZLIB_RAW_COMPRESSION')){
throw new UnsupportedWorldFormatException("Given version of php-leveldb doesn't support zlib raw compression");
}
}
/**
* @throws \LevelDBException
*/
@ -106,7 +52,6 @@ class LevelDB extends BaseWorldProvider implements WritableWorldProvider{
}
public function __construct(string $path, \Logger $logger){
self::checkForLevelDBExtension();
parent::__construct($path, $logger);
try{
@ -117,708 +62,53 @@ class LevelDB extends BaseWorldProvider implements WritableWorldProvider{
}
}
protected function loadLevelData() : WorldData{
return new BedrockWorldData(Path::join($this->getPath(), "level.dat"));
}
public function getWorldMinY() : int{
return -64;
}
public function getWorldMaxY() : int{
return 320;
}
public static function isValid(string $path) : bool{
return file_exists(Path::join($path, "level.dat")) && is_dir(Path::join($path, "db"));
}
public static function generate(string $path, string $name, WorldCreationOptions $options) : void{
self::checkForLevelDBExtension();
self::baseGenerate($path, $name, $options);
$dbPath = Path::join($path, "db");
if(!file_exists($dbPath)){
mkdir($dbPath, 0777, true);
}
BedrockWorldData::generate($path, $name, $options);
}
/**
* @throws CorruptedChunkException
*/
protected function deserializeBlockPalette(BinaryStream $stream, \Logger $logger) : PalettedBlockArray{
$bitsPerBlock = $stream->getByte() >> 1;
try{
$words = $stream->get(PalettedBlockArray::getExpectedWordArraySize($bitsPerBlock));
}catch(\InvalidArgumentException $e){
throw new CorruptedChunkException("Failed to deserialize paletted storage: " . $e->getMessage(), 0, $e);
}
$nbt = new LittleEndianNbtSerializer();
$palette = [];
if($bitsPerBlock === 0){
$paletteSize = 1;
/*
* Due to code copy-paste in a public plugin, some PM4 worlds have 0 bpb palettes with a length prefix.
* This is invalid and does not happen in vanilla.
* These palettes were accepted by PM4 despite being invalid, but PM5 considered them corrupt, causing loss
* of data. Since many users were affected by this, a workaround is therefore necessary to allow PM5 to read
* these worlds without data loss.
*
* References:
* - https://github.com/Refaltor77/CustomItemAPI/issues/68
* - https://github.com/pmmp/PocketMine-MP/issues/5911
*/
$offset = $stream->getOffset();
$byte1 = $stream->getByte();
$stream->setOffset($offset); //reset offset
if($byte1 !== NBT::TAG_Compound){ //normally the first byte would be the NBT of the blockstate
$susLength = $stream->getLInt();
if($susLength !== 1){ //make sure the data isn't complete garbage
throw new CorruptedChunkException("CustomItemAPI borked 0 bpb palette should always have a length of 1");
}
$logger->error("Unexpected palette size for 0 bpb palette");
}
}else{
$paletteSize = $stream->getLInt();
}
$blockDecodeErrors = [];
for($i = 0; $i < $paletteSize; ++$i){
try{
$offset = $stream->getOffset();
$blockStateNbt = $nbt->read($stream->getBuffer(), $offset)->mustGetCompoundTag();
$stream->setOffset($offset);
}catch(NbtDataException $e){
//NBT borked, unrecoverable
throw new CorruptedChunkException("Invalid blockstate NBT at offset $i in paletted storage: " . $e->getMessage(), 0, $e);
}
//TODO: remember data for unknown states so we can implement them later
try{
$blockStateData = $this->blockDataUpgrader->upgradeBlockStateNbt($blockStateNbt);
}catch(BlockStateDeserializeException $e){
//while not ideal, this is not a fatal error
$blockDecodeErrors[] = "Palette offset $i / Upgrade error: " . $e->getMessage() . ", NBT: " . $blockStateNbt->toString();
$palette[] = $this->blockStateDeserializer->deserialize(GlobalBlockStateHandlers::getUnknownBlockStateData());
continue;
}
try{
$palette[] = $this->blockStateDeserializer->deserialize($blockStateData);
}catch(UnsupportedBlockStateException $e){
$blockDecodeErrors[] = "Palette offset $i / " . $e->getMessage();
$palette[] = $this->blockStateDeserializer->deserialize(GlobalBlockStateHandlers::getUnknownBlockStateData());
}catch(BlockStateDeserializeException $e){
$blockDecodeErrors[] = "Palette offset $i / Deserialize error: " . $e->getMessage() . ", NBT: " . $blockStateNbt->toString();
$palette[] = $this->blockStateDeserializer->deserialize(GlobalBlockStateHandlers::getUnknownBlockStateData());
}
}
if(count($blockDecodeErrors) > 0){
$logger->error("Errors decoding blocks:\n - " . implode("\n - ", $blockDecodeErrors));
}
//TODO: exceptions
return PalettedBlockArray::fromData($bitsPerBlock, $words, $palette);
}
private function serializeBlockPalette(BinaryStream $stream, PalettedBlockArray $blocks) : void{
$stream->putByte($blocks->getBitsPerBlock() << 1);
$stream->put($blocks->getWordArray());
$palette = $blocks->getPalette();
if($blocks->getBitsPerBlock() !== 0){
$stream->putLInt(count($palette));
}
$tags = [];
foreach($palette as $p){
$tags[] = new TreeRoot($this->blockStateSerializer->serialize($p)->toNbt());
}
$stream->put((new LittleEndianNbtSerializer())->writeMultiple($tags));
}
/**
* @throws CorruptedChunkException
*/
private static function getExpected3dBiomesCount(int $chunkVersion) : int{
return match(true){
$chunkVersion >= ChunkVersion::v1_18_30 => 24,
$chunkVersion >= ChunkVersion::v1_18_0_25_beta => 25,
$chunkVersion >= ChunkVersion::v1_18_0_24_beta => 32,
$chunkVersion >= ChunkVersion::v1_18_0_22_beta => 65,
$chunkVersion >= ChunkVersion::v1_17_40_20_beta_experimental_caves_cliffs => 32,
default => throw new CorruptedChunkException("Chunk version $chunkVersion should not have 3D biomes")
};
}
/**
* @throws CorruptedChunkException
*/
private static function deserializeBiomePalette(BinaryStream $stream, int $bitsPerBlock) : PalettedBlockArray{
try{
$words = $stream->get(PalettedBlockArray::getExpectedWordArraySize($bitsPerBlock));
}catch(\InvalidArgumentException $e){
throw new CorruptedChunkException("Failed to deserialize paletted biomes: " . $e->getMessage(), 0, $e);
}
$palette = [];
$paletteSize = $bitsPerBlock === 0 ? 1 : $stream->getLInt();
for($i = 0; $i < $paletteSize; ++$i){
$palette[] = $stream->getLInt();
}
//TODO: exceptions
return PalettedBlockArray::fromData($bitsPerBlock, $words, $palette);
}
private static function serializeBiomePalette(BinaryStream $stream, PalettedBlockArray $biomes) : void{
$stream->putByte($biomes->getBitsPerBlock() << 1);
$stream->put($biomes->getWordArray());
$palette = $biomes->getPalette();
if($biomes->getBitsPerBlock() !== 0){
$stream->putLInt(count($palette));
}
foreach($palette as $p){
$stream->putLInt($p);
}
}
/**
* @throws CorruptedChunkException
* @return PalettedBlockArray[]
* @phpstan-return array<int, PalettedBlockArray>
*/
private static function deserialize3dBiomes(BinaryStream $stream, int $chunkVersion, \Logger $logger) : array{
$previous = null;
$result = [];
$nextIndex = Chunk::MIN_SUBCHUNK_INDEX;
$expectedCount = self::getExpected3dBiomesCount($chunkVersion);
for($i = 0; $i < $expectedCount; ++$i){
try{
$bitsPerBlock = $stream->getByte() >> 1;
if($bitsPerBlock === 127){
if($previous === null){
throw new CorruptedChunkException("Serialized biome palette $i has no previous palette to copy from");
}
$decoded = clone $previous;
}else{
$decoded = self::deserializeBiomePalette($stream, $bitsPerBlock);
}
$previous = $decoded;
if($nextIndex <= Chunk::MAX_SUBCHUNK_INDEX){ //older versions wrote additional superfluous biome palettes
$result[$nextIndex++] = $decoded;
}elseif($stream->feof()){
//not enough padding biome arrays for the given version - this is non-critical since we discard the excess anyway, but this should be logged
$logger->error("Wrong number of 3D biome palettes for this chunk version: expected $expectedCount, but got " . ($i + 1) . " - this is not a problem, but may indicate a corrupted chunk");
break;
}
}catch(BinaryDataException $e){
throw new CorruptedChunkException("Failed to deserialize biome palette $i: " . $e->getMessage(), 0, $e);
}
}
if(!$stream->feof()){
//maybe bad output produced by a third-party conversion tool like Chunker
$logger->error("Unexpected trailing data after 3D biomes data");
}
return $result;
}
/**
* @param SubChunk[] $subChunks
*/
private static function serialize3dBiomes(BinaryStream $stream, array $subChunks) : void{
//TODO: the server-side min/max may not coincide with the world storage min/max - we may need additional logic to handle this
for($y = Chunk::MIN_SUBCHUNK_INDEX; $y <= Chunk::MAX_SUBCHUNK_INDEX; $y++){
//TODO: is it worth trying to use the previous palette if it's the same as the current one? vanilla supports
//this, but it's not clear if it's worth the effort to implement.
self::serializeBiomePalette($stream, $subChunks[$y]->getBiomeArray());
}
}
/**
* @phpstan-param-out int $x
* @phpstan-param-out int $y
* @phpstan-param-out int $z
*/
protected static function deserializeExtraDataKey(int $chunkVersion, int $key, ?int &$x, ?int &$y, ?int &$z) : void{
if($chunkVersion >= ChunkVersion::v1_0_0){
$x = ($key >> 12) & 0xf;
$z = ($key >> 8) & 0xf;
$y = $key & 0xff;
}else{ //pre-1.0, 7 bits were used because the build height limit was lower
$x = ($key >> 11) & 0xf;
$z = ($key >> 7) & 0xf;
$y = $key & 0x7f;
}
}
/**
* @return PalettedBlockArray[]
*/
protected function deserializeLegacyExtraData(string $index, int $chunkVersion, \Logger $logger) : array{
if(($extraRawData = $this->db->get($index . ChunkDataKey::LEGACY_BLOCK_EXTRA_DATA)) === false || $extraRawData === ""){
return [];
}
/** @var PalettedBlockArray[] $extraDataLayers */
$extraDataLayers = [];
$binaryStream = new BinaryStream($extraRawData);
$count = $binaryStream->getLInt();
for($i = 0; $i < $count; ++$i){
$key = $binaryStream->getLInt();
$value = $binaryStream->getLShort();
self::deserializeExtraDataKey($chunkVersion, $key, $x, $fullY, $z);
$ySub = ($fullY >> SubChunk::COORD_BIT_SIZE);
$y = $key & SubChunk::COORD_MASK;
$blockId = $value & 0xff;
$blockData = ($value >> 8) & 0xf;
try{
$blockStateData = $this->blockDataUpgrader->upgradeIntIdMeta($blockId, $blockData);
}catch(BlockStateDeserializeException $e){
//TODO: we could preserve this in case it's supported in the future, but this was historically only
//used for grass anyway, so we probably don't need to care
$logger->error("Failed to upgrade legacy extra block: " . $e->getMessage() . " ($blockId:$blockData)");
continue;
}
//assume this won't throw
$blockStateId = $this->blockStateDeserializer->deserialize($blockStateData);
if(!isset($extraDataLayers[$ySub])){
$extraDataLayers[$ySub] = new PalettedBlockArray(Block::EMPTY_STATE_ID);
}
$extraDataLayers[$ySub]->set($x, $y, $z, $blockStateId);
}
return $extraDataLayers;
}
private function readVersion(int $chunkX, int $chunkZ) : ?int{
$index = self::chunkIndex($chunkX, $chunkZ);
$chunkVersionRaw = $this->db->get($index . ChunkDataKey::NEW_VERSION);
if($chunkVersionRaw === false){
$chunkVersionRaw = $this->db->get($index . ChunkDataKey::OLD_VERSION);
if($chunkVersionRaw === false){
return null;
}
}
return ord($chunkVersionRaw);
}
/**
* Deserializes terrain data stored in the 0.9 full-chunk format into subchunks.
*
* @return SubChunk[]
* @phpstan-return array<int, SubChunk>
* @throws CorruptedWorldException
*/
private function deserializeLegacyTerrainData(string $index, int $chunkVersion, \Logger $logger) : array{
$convertedLegacyExtraData = $this->deserializeLegacyExtraData($index, $chunkVersion, $logger);
$legacyTerrain = $this->db->get($index . ChunkDataKey::LEGACY_TERRAIN);
if($legacyTerrain === false){
throw new CorruptedChunkException("Missing expected LEGACY_TERRAIN tag for format version $chunkVersion");
}
$binaryStream = new BinaryStream($legacyTerrain);
try{
$fullIds = $binaryStream->get(32768);
$fullData = $binaryStream->get(16384);
$binaryStream->get(32768); //legacy light info, discard it
}catch(BinaryDataException $e){
throw new CorruptedChunkException($e->getMessage(), 0, $e);
}
try{
$binaryStream->get(256); //heightmap, discard it
/** @var int[] $unpackedBiomeArray */
$unpackedBiomeArray = unpack("N*", $binaryStream->get(1024)); //unpack() will never fail here
$biomes3d = ChunkUtils::extrapolate3DBiomes(ChunkUtils::convertBiomeColors(array_values($unpackedBiomeArray))); //never throws
}catch(BinaryDataException $e){
throw new CorruptedChunkException($e->getMessage(), 0, $e);
}
if(!$binaryStream->feof()){
$logger->error("Unexpected trailing data in legacy terrain data");
}
$subChunks = [];
for($yy = 0; $yy < 8; ++$yy){
$storages = [$this->palettizeLegacySubChunkFromColumn($fullIds, $fullData, $yy, new \PrefixedLogger($logger, "Subchunk y=$yy"))];
if(isset($convertedLegacyExtraData[$yy])){
$storages[] = $convertedLegacyExtraData[$yy];
}
$subChunks[$yy] = new SubChunk(Block::EMPTY_STATE_ID, $storages, clone $biomes3d);
}
//make sure extrapolated biomes get filled in correctly
for($yy = Chunk::MIN_SUBCHUNK_INDEX; $yy <= Chunk::MAX_SUBCHUNK_INDEX; ++$yy){
if(!isset($subChunks[$yy])){
$subChunks[$yy] = new SubChunk(Block::EMPTY_STATE_ID, [], clone $biomes3d);
}
}
return $subChunks;
}
/**
* Deserializes a subchunk stored in the legacy non-paletted format used from 1.0 until 1.2.13.
*/
private function deserializeNonPalettedSubChunkData(BinaryStream $binaryStream, int $chunkVersion, ?PalettedBlockArray $convertedLegacyExtraData, PalettedBlockArray $biomePalette, \Logger $logger) : SubChunk{
try{
$blocks = $binaryStream->get(4096);
$blockData = $binaryStream->get(2048);
}catch(BinaryDataException $e){
throw new CorruptedChunkException($e->getMessage(), 0, $e);
}
if($chunkVersion < ChunkVersion::v1_1_0){
try{
$binaryStream->get(4096); //legacy light info, discard it
if(!$binaryStream->feof()){
$logger->error("Unexpected trailing data in legacy subchunk data");
}
}catch(BinaryDataException $e){
$logger->error("Failed to read legacy subchunk light info: " . $e->getMessage());
}
}
$storages = [$this->palettizeLegacySubChunkXZY($blocks, $blockData, $logger)];
if($convertedLegacyExtraData !== null){
$storages[] = $convertedLegacyExtraData;
}
return new SubChunk(Block::EMPTY_STATE_ID, $storages, $biomePalette);
}
/**
* Deserializes subchunk data stored under a subchunk LevelDB key.
*
* @see ChunkDataKey::SUBCHUNK
* @throws CorruptedChunkException
*/
private function deserializeSubChunkData(BinaryStream $binaryStream, int $chunkVersion, int $subChunkVersion, ?PalettedBlockArray $convertedLegacyExtraData, PalettedBlockArray $biomePalette, \Logger $logger) : SubChunk{
switch($subChunkVersion){
case SubChunkVersion::CLASSIC:
case SubChunkVersion::CLASSIC_BUG_2: //these are all identical to version 0, but vanilla respects these so we should also
case SubChunkVersion::CLASSIC_BUG_3:
case SubChunkVersion::CLASSIC_BUG_4:
case SubChunkVersion::CLASSIC_BUG_5:
case SubChunkVersion::CLASSIC_BUG_6:
case SubChunkVersion::CLASSIC_BUG_7:
return $this->deserializeNonPalettedSubChunkData($binaryStream, $chunkVersion, $convertedLegacyExtraData, $biomePalette, $logger);
case SubChunkVersion::PALETTED_SINGLE:
$storages = [$this->deserializeBlockPalette($binaryStream, $logger)];
if($convertedLegacyExtraData !== null){
$storages[] = $convertedLegacyExtraData;
}
return new SubChunk(Block::EMPTY_STATE_ID, $storages, $biomePalette);
case SubChunkVersion::PALETTED_MULTI:
case SubChunkVersion::PALETTED_MULTI_WITH_OFFSET:
//legacy extradata layers intentionally ignored because they aren't supposed to exist in v8
$storageCount = $binaryStream->getByte();
if($subChunkVersion >= SubChunkVersion::PALETTED_MULTI_WITH_OFFSET){
//height ignored; this seems pointless since this is already in the key anyway
$binaryStream->getByte();
}
$storages = [];
for($k = 0; $k < $storageCount; ++$k){
$storages[] = $this->deserializeBlockPalette($binaryStream, $logger);
}
return new SubChunk(Block::EMPTY_STATE_ID, $storages, $biomePalette);
default:
//this should never happen - an unsupported chunk appearing in a supported world is a sign of corruption
throw new CorruptedChunkException("don't know how to decode LevelDB subchunk format version $subChunkVersion");
}
}
private static function hasOffsetCavesAndCliffsSubChunks(int $chunkVersion) : bool{
return $chunkVersion >= ChunkVersion::v1_16_220_50_unused && $chunkVersion <= ChunkVersion::v1_16_230_50_unused;
}
/**
* Deserializes any subchunks stored under subchunk LevelDB keys, upgrading them to the current format if necessary.
*
* @param PalettedBlockArray[] $convertedLegacyExtraData
* @param PalettedBlockArray[] $biomeArrays
*
* @phpstan-param array<int, PalettedBlockArray> $convertedLegacyExtraData
* @phpstan-param array<int, PalettedBlockArray> $biomeArrays
* @phpstan-param-out bool $hasBeenUpgraded
*
* @return SubChunk[]
* @phpstan-return array<int, SubChunk>
*/
private function deserializeAllSubChunkData(string $index, int $chunkVersion, bool &$hasBeenUpgraded, array $convertedLegacyExtraData, array $biomeArrays, \Logger $logger) : array{
$subChunks = [];
$subChunkKeyOffset = self::hasOffsetCavesAndCliffsSubChunks($chunkVersion) ? self::CAVES_CLIFFS_EXPERIMENTAL_SUBCHUNK_KEY_OFFSET : 0;
for($y = Chunk::MIN_SUBCHUNK_INDEX; $y <= Chunk::MAX_SUBCHUNK_INDEX; ++$y){
if(($data = $this->db->get($index . ChunkDataKey::SUBCHUNK . chr($y + $subChunkKeyOffset))) === false){
$subChunks[$y] = new SubChunk(Block::EMPTY_STATE_ID, [], $biomeArrays[$y]);
continue;
}
$binaryStream = new BinaryStream($data);
if($binaryStream->feof()){
throw new CorruptedChunkException("Unexpected empty data for subchunk $y");
}
$subChunkVersion = $binaryStream->getByte();
if($subChunkVersion < self::CURRENT_LEVEL_SUBCHUNK_VERSION){
$hasBeenUpgraded = true;
}
$subChunks[$y] = $this->deserializeSubChunkData(
$binaryStream,
$chunkVersion,
$subChunkVersion,
$convertedLegacyExtraData[$y] ?? null,
$biomeArrays[$y],
new \PrefixedLogger($logger, "Subchunk y=$y v$subChunkVersion")
);
}
return $subChunks;
}
/**
* Deserializes any available biome data into an array of paletted biomes. Old 2D biomes are extrapolated to 3D.
*
* @return PalettedBlockArray[]
* @phpstan-return array<int, PalettedBlockArray>
*/
private function deserializeBiomeData(string $index, int $chunkVersion, \Logger $logger) : array{
$biomeArrays = [];
if(($maps2d = $this->db->get($index . ChunkDataKey::HEIGHTMAP_AND_2D_BIOMES)) !== false){
$binaryStream = new BinaryStream($maps2d);
try{
$binaryStream->get(512); //heightmap, discard it
$biomes3d = ChunkUtils::extrapolate3DBiomes($binaryStream->get(256)); //never throws
if(!$binaryStream->feof()){
$logger->error("Unexpected trailing data after 2D biome data");
}
}catch(BinaryDataException $e){
throw new CorruptedChunkException($e->getMessage(), 0, $e);
}
for($i = Chunk::MIN_SUBCHUNK_INDEX; $i <= Chunk::MAX_SUBCHUNK_INDEX; ++$i){
$biomeArrays[$i] = clone $biomes3d;
}
}elseif(($maps3d = $this->db->get($index . ChunkDataKey::HEIGHTMAP_AND_3D_BIOMES)) !== false){
$binaryStream = new BinaryStream($maps3d);
try{
$binaryStream->get(512);
$biomeArrays = self::deserialize3dBiomes($binaryStream, $chunkVersion, $logger);
}catch(BinaryDataException $e){
throw new CorruptedChunkException($e->getMessage(), 0, $e);
}
}else{
$logger->error("Missing biome data, using default ocean biome");
for($i = Chunk::MIN_SUBCHUNK_INDEX; $i <= Chunk::MAX_SUBCHUNK_INDEX; ++$i){
$biomeArrays[$i] = new PalettedBlockArray(BiomeIds::OCEAN); //polyfill
}
}
return $biomeArrays;
}
/**
* @throws CorruptedChunkException
*/
public function loadChunk(int $chunkX, int $chunkZ) : ?LoadedChunkData{
$index = LevelDB::chunkIndex($chunkX, $chunkZ);
$chunkVersion = $this->readVersion($chunkX, $chunkZ);
if($chunkVersion === null){
//TODO: this might be a slightly-corrupted chunk with a missing version field
return null;
}
//TODO: read PM_DATA_VERSION - we'll need it to fix up old chunks
$logger = new \PrefixedLogger($this->logger, "Loading chunk x=$chunkX z=$chunkZ v$chunkVersion");
$hasBeenUpgraded = $chunkVersion < self::CURRENT_LEVEL_CHUNK_VERSION;
switch($chunkVersion){
case ChunkVersion::v1_21_40:
//TODO: BiomeStates became shorts instead of bytes
case ChunkVersion::v1_18_30:
case ChunkVersion::v1_18_0_25_beta:
case ChunkVersion::v1_18_0_24_unused:
case ChunkVersion::v1_18_0_24_beta:
case ChunkVersion::v1_18_0_22_unused:
case ChunkVersion::v1_18_0_22_beta:
case ChunkVersion::v1_18_0_20_unused:
case ChunkVersion::v1_18_0_20_beta:
case ChunkVersion::v1_17_40_unused:
case ChunkVersion::v1_17_40_20_beta_experimental_caves_cliffs:
case ChunkVersion::v1_17_30_25_unused:
case ChunkVersion::v1_17_30_25_beta_experimental_caves_cliffs:
case ChunkVersion::v1_17_30_23_unused:
case ChunkVersion::v1_17_30_23_beta_experimental_caves_cliffs:
case ChunkVersion::v1_16_230_50_unused:
case ChunkVersion::v1_16_230_50_beta_experimental_caves_cliffs:
case ChunkVersion::v1_16_220_50_unused:
case ChunkVersion::v1_16_220_50_beta_experimental_caves_cliffs:
case ChunkVersion::v1_16_210:
case ChunkVersion::v1_16_100_57_beta:
case ChunkVersion::v1_16_100_52_beta:
case ChunkVersion::v1_16_0:
case ChunkVersion::v1_16_0_51_beta:
//TODO: check walls
case ChunkVersion::v1_12_0_unused2:
case ChunkVersion::v1_12_0_unused1:
case ChunkVersion::v1_12_0_4_beta:
case ChunkVersion::v1_11_1:
case ChunkVersion::v1_11_0_4_beta:
case ChunkVersion::v1_11_0_3_beta:
case ChunkVersion::v1_11_0_1_beta:
case ChunkVersion::v1_9_0:
case ChunkVersion::v1_8_0:
case ChunkVersion::v1_2_13:
case ChunkVersion::v1_2_0:
case ChunkVersion::v1_2_0_2_beta:
case ChunkVersion::v1_1_0_converted_from_console:
case ChunkVersion::v1_1_0:
//TODO: check beds
case ChunkVersion::v1_0_0:
$convertedLegacyExtraData = $this->deserializeLegacyExtraData($index, $chunkVersion, $logger);
$biomeArrays = $this->deserializeBiomeData($index, $chunkVersion, $logger);
$subChunks = $this->deserializeAllSubChunkData($index, $chunkVersion, $hasBeenUpgraded, $convertedLegacyExtraData, $biomeArrays, $logger);
break;
case ChunkVersion::v0_9_5:
case ChunkVersion::v0_9_2:
case ChunkVersion::v0_9_0:
$subChunks = $this->deserializeLegacyTerrainData($index, $chunkVersion, $logger);
break;
default:
throw new CorruptedChunkException("don't know how to decode chunk format version $chunkVersion");
}
$nbt = new LittleEndianNbtSerializer();
/** @var CompoundTag[] $entities */
$entities = [];
if(($entityData = $this->db->get($index . ChunkDataKey::ENTITIES)) !== false && $entityData !== ""){
try{
$entities = array_map(fn(TreeRoot $root) => $root->mustGetCompoundTag(), $nbt->readMultiple($entityData));
}catch(NbtDataException $e){
throw new CorruptedChunkException($e->getMessage(), 0, $e);
}
}
/** @var CompoundTag[] $tiles */
$tiles = [];
if(($tileData = $this->db->get($index . ChunkDataKey::BLOCK_ENTITIES)) !== false && $tileData !== ""){
try{
$tiles = array_map(fn(TreeRoot $root) => $root->mustGetCompoundTag(), $nbt->readMultiple($tileData));
}catch(NbtDataException $e){
throw new CorruptedChunkException($e->getMessage(), 0, $e);
}
}
$finalisationChr = $this->db->get($index . ChunkDataKey::FINALIZATION);
if($finalisationChr !== false){
$finalisation = ord($finalisationChr);
$terrainPopulated = $finalisation === self::FINALISATION_DONE;
}else{ //older versions didn't have this tag
$terrainPopulated = true;
}
//TODO: tile ticks, biome states (?)
return new LoadedChunkData(
data: new ChunkData($subChunks, $terrainPopulated, $entities, $tiles),
upgraded: $hasBeenUpgraded,
fixerFlags: LoadedChunkData::FIXER_FLAG_ALL //TODO: fill this by version rather than just setting all flags
);
return $this->loadChunkFromDB($this->db, $chunkX, $chunkZ);
}
public function saveChunk(int $chunkX, int $chunkZ, ChunkData $chunkData, int $dirtyFlags) : void{
$index = LevelDB::chunkIndex($chunkX, $chunkZ);
$write = new \LevelDBWriteBatch();
$write->put($index . ChunkDataKey::NEW_VERSION, chr(self::CURRENT_LEVEL_CHUNK_VERSION));
$write->put($index . ChunkDataKey::PM_DATA_VERSION, Binary::writeLLong(VersionInfo::WORLD_DATA_VERSION));
$subChunks = $chunkData->getSubChunks();
if(($dirtyFlags & Chunk::DIRTY_FLAG_BLOCKS) !== 0){
foreach($subChunks as $y => $subChunk){
$key = $index . ChunkDataKey::SUBCHUNK . chr($y);
if($subChunk->isEmptyAuthoritative()){
$write->delete($key);
}else{
$subStream = new BinaryStream();
$subStream->putByte(self::CURRENT_LEVEL_SUBCHUNK_VERSION);
$layers = $subChunk->getBlockLayers();
$subStream->putByte(count($layers));
foreach($layers as $blocks){
$this->serializeBlockPalette($subStream, $blocks);
}
$write->put($key, $subStream->getBuffer());
}
}
}
if(($dirtyFlags & Chunk::DIRTY_FLAG_BIOMES) !== 0){
$write->delete($index . ChunkDataKey::HEIGHTMAP_AND_2D_BIOMES);
$stream = new BinaryStream();
$stream->put(str_repeat("\x00", 512)); //fake heightmap
self::serialize3dBiomes($stream, $subChunks);
$write->put($index . ChunkDataKey::HEIGHTMAP_AND_3D_BIOMES, $stream->getBuffer());
}
//TODO: use this properly
$write->put($index . ChunkDataKey::FINALIZATION, chr($chunkData->isPopulated() ? self::FINALISATION_DONE : self::FINALISATION_NEEDS_POPULATION));
$this->writeTags($chunkData->getTileNBT(), $index . ChunkDataKey::BLOCK_ENTITIES, $write);
$this->writeTags($chunkData->getEntityNBT(), $index . ChunkDataKey::ENTITIES, $write);
$write->delete($index . ChunkDataKey::HEIGHTMAP_AND_2D_BIOME_COLORS);
$write->delete($index . ChunkDataKey::LEGACY_TERRAIN);
$this->db->write($write);
}
/**
* @param CompoundTag[] $targets
*/
private function writeTags(array $targets, string $index, \LevelDBWriteBatch $write) : void{
if(count($targets) > 0){
$nbt = new LittleEndianNbtSerializer();
$write->put($index, $nbt->writeMultiple(array_map(fn(CompoundTag $tag) => new TreeRoot($tag), $targets)));
}else{
$write->delete($index);
}
$this->saveChunkToDB($this->db, $chunkX, $chunkZ, $chunkData, $dirtyFlags);
}
public function getDatabase() : \LevelDB{
return $this->db;
}
/**
* @deprecated
*/
public static function chunkIndex(int $chunkX, int $chunkZ) : string{
return Binary::writeLInt($chunkX) . Binary::writeLInt($chunkZ);
}
protected function coordsToChunkIndex(int $chunkX, int $chunkZ) : string{
return Binary::writeLInt($chunkX) . Binary::writeLInt($chunkZ);
}
/**
* @return int[]
* @phpstan-return array{int, int}
*/
protected function coordsFromChunkIndex(string $chunkIndex) : array{
return [Binary::readLInt(substr($chunkIndex, 0, 4)), Binary::readLInt(substr($chunkIndex, 4, 4))];
}
public function doGarbageCollection() : void{
}
@ -828,33 +118,10 @@ class LevelDB extends BaseWorldProvider implements WritableWorldProvider{
}
public function getAllChunks(bool $skipCorrupted = false, ?\Logger $logger = null) : \Generator{
foreach($this->db->getIterator() as $key => $_){
if(strlen($key) === 9 && ($key[8] === ChunkDataKey::NEW_VERSION || $key[8] === ChunkDataKey::OLD_VERSION)){
$chunkX = Binary::readLInt(substr($key, 0, 4));
$chunkZ = Binary::readLInt(substr($key, 4, 4));
try{
if(($chunk = $this->loadChunk($chunkX, $chunkZ)) !== null){
yield [$chunkX, $chunkZ] => $chunk;
}
}catch(CorruptedChunkException $e){
if(!$skipCorrupted){
throw $e;
}
if($logger !== null){
$logger->error("Skipped corrupted chunk $chunkX $chunkZ (" . $e->getMessage() . ")");
}
}
}
}
yield from $this->getAllChunksFromDB($this->db, $skipCorrupted, $logger);
}
public function calculateChunkCount() : int{
$count = 0;
foreach($this->db->getIterator() as $key => $_){
if(strlen($key) === 9 && ($key[8] === ChunkDataKey::NEW_VERSION || $key[8] === ChunkDataKey::OLD_VERSION)){
$count++;
}
}
return $count;
return $this->calculateChunkCountInDB($this->db);
}
}

222
src/world/format/io/leveldb/RegionizedLevelDB.php Normal file
View File

@ -0,0 +1,222 @@
<?php
/*
*
* ____ _ _ __ __ _ __ __ ____
* | _ \ ___ ___| | _____| |_| \/ (_)_ __ ___ | \/ | _ \
* | |_) / _ \ / __| |/ / _ \ __| |\/| | | '_ \ / _ \_____| |\/| | |_) |
* | __/ (_) | (__| < __/ |_| | | | | | | | __/_____| | | | __/
* |_| \___/ \___|_|\_\___|\__|_| |_|_|_| |_|\___| |_| |_|_|
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* @author PocketMine Team
* @link http://www.pocketmine.net/
*
*
*/
declare(strict_types=1);
namespace pocketmine\world\format\io\leveldb;
use pocketmine\utils\AssumptionFailedError;
use pocketmine\utils\Binary;
use pocketmine\world\format\io\ChunkData;
use pocketmine\world\format\io\exception\CorruptedChunkException;
use pocketmine\world\format\io\LoadedChunkData;
use pocketmine\world\WorldCreationOptions;
use Symfony\Component\Filesystem\Path;
use function array_key_exists;
use function file_exists;
use function intdiv;
use function is_dir;
use function mkdir;
use function morton2d_decode;
use function morton2d_encode;
use function sprintf;
use function str_contains;
use function time;
use function touch;
use const LEVELDB_ZLIB_RAW_COMPRESSION;
final class RegionizedLevelDB extends BaseLevelDB{
/**
* Any DBs that haven't been accessed for the last 5 minutes will be removed from memory
*/
private const MAX_DB_CACHE_AGE = 5 * 60;
private const DB_DEFAULT_OPTIONS = [
"compression" => LEVELDB_ZLIB_RAW_COMPRESSION,
"block_size" => 64 * 1024 //64KB, big enough for most chunks
];
private static function dbRegionPath(string $base, int $regionLength) : string{
return Path::join($base, "leveldb-regions-$regionLength");
}
public static function isValid(string $path, int $regionLength) : bool{
return file_exists(Path::join($path, "level.dat")) && is_dir(self::dbRegionPath($path, $regionLength));
}
public static function generate(string $path, string $name, WorldCreationOptions $options, int $regionLength) : void{
self::baseGenerate($path, $name, $options);
touch(Path::join($path, 'NOT_BEDROCK_COMPATIBLE.txt'));
@mkdir(self::dbRegionPath($path, $regionLength), 0777, true);
}
/**
* @var \LevelDB[]|null[]
* @phpstan-var array<int, \LevelDB|null>
*/
private array $databases = [];
/**
* @var int[]
* @phpstan-var array<int, int>
*/
private array $databasesLastUsed = [];
public function __construct(
string $path,
\Logger $logger,
private readonly int $regionLength
){
parent::__construct($path, $logger);
}
protected function coordsFromChunkIndex(string $chunkIndex) : array{
//TODO: these indexes don't need to use long in separated DBs, we could make them smaller and save space
return morton2d_decode(Binary::readLong($chunkIndex));
}
protected function coordsToChunkIndex(int $chunkX, int $chunkZ) : string{
return Binary::writeLong(morton2d_encode($chunkX, $chunkZ));
}
protected function getDBPathForCoords(int $chunkX, int $chunkZ) : string{
return Path::join(self::dbRegionPath($this->path, $this->regionLength), sprintf(
"db.%d.%d",
intdiv($chunkX, $this->regionLength),
intdiv($chunkZ, $this->regionLength)
));
}
protected function getDBIndexForCoords(int $chunkX, int $chunkZ) : int{
return morton2d_encode(intdiv($chunkX, $this->regionLength), intdiv($chunkZ, $this->regionLength));
}
protected function fetchDBForCoords(int $chunkX, int $chunkZ, bool $createIfMissing) : ?\LevelDB{
$index = $this->getDBIndexForCoords($chunkX, $chunkZ);
$db = $this->databases[$index] ?? null;
if(
!array_key_exists($index, $this->databases) || //we haven't tried to fetch this DB yet
($db === null && $createIfMissing) //or we know it doesn't exist and want to create it (for writing)
){
$options = self::DB_DEFAULT_OPTIONS;
$options["create_if_missing"] = $createIfMissing;
$dbPath = $this->getDBPathForCoords($chunkX, $chunkZ);
try{
$this->databases[$index] = new \LevelDB($dbPath, $options);
}catch(\LevelDBException $e){
//no other way to detect error type :(
if(!str_contains($e->getMessage(), "(create_if_missing is false)")){
throw new CorruptedChunkException("Couldn't open LevelDB region $dbPath for $chunkX.$chunkZ: " . $e->getMessage(), 0, $e);
}
//remember that this DB doesn't exist, so we don't have to hit the disk hundreds of times looking for it
$this->databases[$index] = null;
}
}
$this->databasesLastUsed[$index] = time();
return $this->databases[$index];
}
public function loadChunk(int $chunkX, int $chunkZ) : ?LoadedChunkData{
$db = $this->fetchDBForCoords($chunkX, $chunkZ, createIfMissing: false);
return $db !== null ? $this->loadChunkFromDB($db, $chunkX, $chunkZ) : null;
}
public function saveChunk(int $chunkX, int $chunkZ, ChunkData $chunkData, int $dirtyFlags) : void{
$db = $this->fetchDBForCoords($chunkX, $chunkZ, createIfMissing: true) ??
throw new AssumptionFailedError("We asked fetch to create a DB, it shouldn't return null");
$this->saveChunkToDB($db, $chunkX, $chunkZ, $chunkData, $dirtyFlags);
}
public function doGarbageCollection() : void{
$minLastUsed = time() - self::MAX_DB_CACHE_AGE;
foreach($this->databasesLastUsed as $index => $time){
if($time < $minLastUsed){
//unset will close the DB
unset(
$this->databases[$index],
$this->databasesLastUsed[$index]
);
}
}
}
public function close() : void{
//no explicit actions needed to close DBs
$this->databases = [];
$this->databasesLastUsed = [];
}
private function createRegionIterator() : \RegexIterator{
return new \RegexIterator(
new \FilesystemIterator(
self::dbRegionPath($this->path, $this->regionLength),
\FilesystemIterator::CURRENT_AS_PATHNAME | \FilesystemIterator::SKIP_DOTS | \FilesystemIterator::UNIX_PATHS
),
'/\/db\.(-?\d+)\.(-?\d+)\$/',
\RegexIterator::GET_MATCH
);
}
public function getAllChunks(bool $skipCorrupted = false, ?\Logger $logger = null) : \Generator{
$iterator = $this->createRegionIterator();
/** @var string[] $region */
foreach($iterator as $region){
try{
$db = new \LevelDB($region[0], self::DB_DEFAULT_OPTIONS);
//TODO: we don't need the DB name coords for now, but we might in the future if the key format is
//changed to be relative
yield from $this->getAllChunksFromDB($db, $skipCorrupted, $logger);
}catch(\LevelDBException $e){
//TODO: detect permission errors - although I'm not sure what we could do differently
if(!$skipCorrupted){
throw new CorruptedChunkException($e->getMessage(), 0, $e);
}
if($logger !== null){
$logger->error($e->getMessage());
}
}
}
}
public function calculateChunkCount() : int{
$iterator = $this->createRegionIterator();
$total = 0;
/** @var string[] $region */
foreach($iterator as $region){
//TODO: calculateChunkCount has no accounting for corruption errors
$db = new \LevelDB($region[0], self::DB_DEFAULT_OPTIONS);
//TODO: we'd need a specialized calculate impl if we change the key length
$total += $this->calculateChunkCountInDB($db);
}
return $total;
}
}