Updated leveldb implementation to support MCPE 1.1 worlds

src/pocketmine/level/format/Chunk.php

@@ -28,7 +28,7 @@ namespace pocketmine\level\format;
 
 use pocketmine\block\Block;
 use pocketmine\entity\Entity;
-use pocketmine\level\format\io\ChunkException;
+use pocketmine\level\format\ChunkException;
 use pocketmine\level\Level;
 use pocketmine\nbt\NBT;
 use pocketmine\nbt\tag\CompoundTag;

src/pocketmine/level/format/ChunkException.php

@@ -21,7 +21,7 @@
 
 declare(strict_types=1);
 
-namespace pocketmine\level\format\io;
+namespace pocketmine\level\format;
 
 class ChunkException extends \RuntimeException{
 

src/pocketmine/level/format/EmptySubChunk.php

@@ -25,7 +25,7 @@ namespace pocketmine\level\format;
 
 class EmptySubChunk implements SubChunkInterface{
 
-	public function isEmpty() : bool{
+	public function isEmpty(bool $checkLight = true) : bool{
 		return true;
 	}
 

src/pocketmine/level/format/SubChunk.php

@@ -46,11 +46,13 @@ class SubChunk implements SubChunkInterface{
 		self::assignData($this->blockLight, $blockLight, 2048);
 	}
 
-	public function isEmpty() : bool{
+	public function isEmpty(bool $checkLight = true) : bool{
 		return (
 			substr_count($this->ids, "\x00") === 4096 and
-			substr_count($this->skyLight, "\xff") === 2048 and
+			(!$checkLight or (
-			substr_count($this->blockLight, "\x00") === 2048
+				substr_count($this->skyLight, "\xff") === 2048 and
+				substr_count($this->blockLight, "\x00") === 2048
+			))
 		);
 	}
 

src/pocketmine/level/format/SubChunkInterface.php

@@ -26,9 +26,10 @@ namespace pocketmine\level\format;
 interface SubChunkInterface{
 
 	/**
+	 * @param bool $checkLight
 	 * @return bool
 	 */
-	public function isEmpty() : bool;
+	public function isEmpty(bool $checkLight = true) : bool;
 
 	/**
 	 * @param int $x

src/pocketmine/level/format/io/BaseLevelProvider.php

@@ -24,6 +24,7 @@ declare(strict_types=1);
 namespace pocketmine\level\format\io;
 
 use pocketmine\level\format\Chunk;
+use pocketmine\level\format\ChunkException;
 use pocketmine\level\generator\Generator;
 use pocketmine\level\Level;
 use pocketmine\level\LevelException;

src/pocketmine/level/format/io/exception/UnsupportedChunkFormatException.php

@@ -0,0 +1,30 @@
+<?php
+
+/*
+ *
+ *  ____            _        _   __  __ _                  __  __ ____
+ * |  _ \ ___   ___| | _____| |_|  \/  (_)_ __   ___      |  \/  |  _ \
+ * | |_) / _ \ / __| |/ / _ \ __| |\/| | | '_ \ / _ \_____| |\/| | |_) |
+ * |  __/ (_) | (__|   <  __/ |_| |  | | | | | |  __/_____| |  | |  __/
+ * |_|   \___/ \___|_|\_\___|\__|_|  |_|_|_| |_|\___|     |_|  |_|_|
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * @author PocketMine Team
+ * @link http://www.pocketmine.net/
+ *
+ *
+*/
+
+declare(strict_types=1);
+
+namespace pocketmine\level\format\io\exception;
+
+use pocketmine\level\format\ChunkException;
+
+class UnsupportedChunkFormatException extends ChunkException{
+
+}

src/pocketmine/level/format/io/leveldb/LevelDB.php

@@ -23,9 +23,12 @@ declare(strict_types=1);
 
 namespace pocketmine\level\format\io\leveldb;
 
+use pocketmine\entity\Entity;
 use pocketmine\level\format\Chunk;
+use pocketmine\level\format\ChunkException;
 use pocketmine\level\format\io\BaseLevelProvider;
 use pocketmine\level\format\io\ChunkUtils;
+use pocketmine\level\format\io\exception\UnsupportedChunkFormatException;
 use pocketmine\level\format\SubChunk;
 use pocketmine\level\generator\Flat;
 use pocketmine\level\generator\Generator;
@@ -36,7 +39,9 @@ use pocketmine\nbt\tag\{
 	ByteTag, CompoundTag, FloatTag, IntTag, LongTag, StringTag
 };
 use pocketmine\network\mcpe\protocol\ProtocolInfo;
+use pocketmine\tile\Tile;
 use pocketmine\utils\Binary;
+use pocketmine\utils\BinaryStream;
 use pocketmine\utils\MainLogger;
 
 class LevelDB extends BaseLevelProvider{
@@ -51,6 +56,11 @@ class LevelDB extends BaseLevelProvider{
 	const TAG_PENDING_TICK = "3";
 	const TAG_BLOCK_EXTRA_DATA = "4";
 	const TAG_BIOME_STATE = "5";
+	const TAG_STATE_FINALISATION = "6";
+
+	const FINALISATION_NEEDS_INSTATICKING = 0;
+	const FINALISATION_NEEDS_POPULATION = 1;
+	const FINALISATION_DONE = 2;
 
 	const TAG_VERSION = "v";
 
@@ -61,6 +71,8 @@ class LevelDB extends BaseLevelProvider{
 	const GENERATOR_FLAT = 2;
 
 	const CURRENT_STORAGE_VERSION = 5; //Current MCPE level format version
+	const CURRENT_LEVEL_CHUNK_VERSION = 4;
+	const CURRENT_LEVEL_SUBCHUNK_VERSION = 0;
 
 	/** @var Chunk[] */
 	protected $chunks = [];
@@ -217,6 +229,9 @@ class LevelDB extends BaseLevelProvider{
 	}
 
 	public function saveLevelData(){
+		$this->levelData->NetworkVersion = new IntTag("NetworkVersion", ProtocolInfo::CURRENT_PROTOCOL);
+		$this->levelData->StorageVersion = new IntTag("StorageVersion", self::CURRENT_STORAGE_VERSION);
+
 		$nbt = new NBT(NBT::LITTLE_ENDIAN);
 		$nbt->setData($this->levelData);
 		$buffer = $nbt->write();
@@ -287,78 +302,94 @@ class LevelDB extends BaseLevelProvider{
 		}
 
 		try{
+			/** @var SubChunk[] $subChunks */
 			$subChunks = [];
-			$heightMap = [];
-			$biomeIds = "";
 
-			for($y = Chunk::MAX_SUBCHUNKS - 1; $y >= 0; --$y){
+			/** @var bool $lightPopulated */
-				if($this->db->get($index . self::TAG_SUBCHUNK_PREFIX . chr($y)) !== false){ //Found subchunk data!
+			$lightPopulated = true;
+
+			$chunkVersion = ord($this->db->get($index . self::TAG_VERSION));
+
+			$binaryStream = new BinaryStream();
+
+			switch($chunkVersion){
+				case 4: //MCPE 1.1
+					//TODO: check beds
+				case 3: //MCPE 1.0
+					for($y = 0; $y < Chunk::MAX_SUBCHUNKS; ++$y){
+						if(($data = $this->db->get($index . self::TAG_SUBCHUNK_PREFIX . chr($y))) === false){
+							continue;
+						}
+
+						$binaryStream->setBuffer($data, 0);
+						$subChunkVersion = $binaryStream->getByte();
+
+						switch($subChunkVersion){
+							case 0:
+								$blocks = $binaryStream->get(4096);
+								$blockData = $binaryStream->get(2048);
+								if($chunkVersion < 4){
+									$blockSkyLight = $binaryStream->get(2048);
+									$blockLight = $binaryStream->get(2048);
+								}else{
+									//Mojang didn't bother changing the subchunk version when they stopped saving sky light -_-
+									$blockSkyLight = "";
+									$blockLight = "";
+									$lightPopulated = false;
+								}
+
+								$subChunks[$y] = new SubChunk($blocks, $blockData, $blockSkyLight, $blockLight);
+								break;
+							default:
+								throw new UnsupportedChunkFormatException("don't know how to decode LevelDB subchunk format version $subChunkVersion");
+						}
+					}
+
+					$binaryStream->setBuffer($this->db->get($index . self::TAG_DATA_2D), 0);
+
+					$heightMap = array_values(unpack("v*", $binaryStream->get(512)));
+					$biomeIds = $binaryStream->get(256);
 					break;
-				}
+				case 2: // < MCPE 1.0
-			}
+					$binaryStream->setBuffer($this->db->get($index . self::TAG_LEGACY_TERRAIN));
+					$fullIds = $binaryStream->get(32768);
+					$fullData = $binaryStream->get(16384);
+					$fullSkyLight = $binaryStream->get(16384);
+					$fullBlockLight = $binaryStream->get(16384);
 
-			if($y <= 0 and ($legacyTerrain = $this->db->get($index . self::TAG_LEGACY_TERRAIN)) !== false){ //didn't find any subchunk data but found old (pre-1.0) data
+					for($yy = 0; $yy < 8; ++$yy){
-				$offset = 0;
+						$subOffset = ($yy << 4);
-				$fullIds = substr($legacyTerrain, $offset, 32768);
+						$ids = "";
-				$offset += 32768;
+						for($i = 0; $i < 256; ++$i){
-				$fullData = substr($legacyTerrain, $offset, 16384);
+							$ids .= substr($fullIds, $subOffset, 16);
-				$offset += 16384;
+							$subOffset += 128;
-				$fullSkyLight = substr($legacyTerrain, $offset, 16384);
+						}
-				$offset += 16384;
+						$data = "";
-				$fullBlockLight = substr($legacyTerrain, $offset, 16384);
+						$subOffset = ($yy << 3);
-				$offset += 16384;
+						for($i = 0; $i < 256; ++$i){
+							$data .= substr($fullData, $subOffset, 8);
+							$subOffset += 64;
+						}
+						$skyLight = "";
+						$subOffset = ($yy << 3);
+						for($i = 0; $i < 256; ++$i){
+							$skyLight .= substr($fullSkyLight, $subOffset, 8);
+							$subOffset += 64;
+						}
+						$blockLight = "";
+						$subOffset = ($yy << 3);
+						for($i = 0; $i < 256; ++$i){
+							$blockLight .= substr($fullBlockLight, $subOffset, 8);
+							$subOffset += 64;
+						}
+						$subChunks[$yy] = new SubChunk($ids, $data, $skyLight, $blockLight);
+					}
 
-				for($yy = 0; $yy < 8; ++$yy){
+					$heightMap = array_values(unpack("C*", $binaryStream->get(256)));
-					$subOffset = ($yy << 4);
+					$biomeIds = ChunkUtils::convertBiomeColors(array_values(unpack("N*", $binaryStream->get(1024))));
-					$ids = "";
+					break;
-					for($i = 0; $i < 256; ++$i){
+				default:
-						$ids .= substr($fullIds, $subOffset, 16);
+					throw new UnsupportedChunkFormatException("don't know how to decode chunk format version $chunkVersion");
-						$subOffset += 128;
-					}
-					$data = "";
-					$subOffset = ($yy << 3);
-					for($i = 0; $i < 256; ++$i){
-						$data .= substr($fullData, $subOffset, 8);
-						$subOffset += 64;
-					}
-					$skyLight = "";
-					$subOffset = ($yy << 3);
-					for($i = 0; $i < 256; ++$i){
-						$skyLight .= substr($fullSkyLight, $subOffset, 8);
-						$subOffset += 64;
-					}
-					$blockLight = "";
-					$subOffset = ($yy << 3);
-					for($i = 0; $i < 256; ++$i){
-						$blockLight .= substr($fullBlockLight, $subOffset, 8);
-						$subOffset += 64;
-					}
-					$subChunks[$yy] = new SubChunk($ids, $data, $skyLight, $blockLight);
-				}
-
-				$heightMap = array_values(unpack("C*", substr($legacyTerrain, $offset, 256)));
-				$offset += 256;
-				$biomeIds = ChunkUtils::convertBiomeColors(array_values(unpack("N*", substr($legacyTerrain, $offset, 1024))));
-				$offset += 1024;
-			}else{
-				for(; $y >= 0; --$y){ //If one subchunk exists, all subchunks below it are also guaranteed to exist.
-					$offset = 1; //Skip subchunk version byte
-					$subChunkData = $this->db->get($index . self::TAG_SUBCHUNK_PREFIX . chr($y));
-					$subChunks[$y] = new SubChunk(
-						substr($subChunkData, $offset,         4096), //block ids
-						substr($subChunkData, $offset += 4096, 2048), //block meta
-						substr($subChunkData, $offset += 2048, 2048), //sky light
-						substr($subChunkData, $offset += 2048, 2048) //block light
-					);
-				}
-
-				if(($data2dLegacy = $this->db->get($index . self::TAG_DATA_2D_LEGACY)) !== false){ //Found old data, convert it to new format
-					$heightMap = array_values(unpack("C*", substr($data2dLegacy, 0, 256)));
-					$biomeIds = ChunkUtils::convertBiomeColors(array_values(unpack("N*", substr($data2dLegacy, 256, 1024))));
-				}elseif(($data2d = $this->db->get($index . self::TAG_DATA_2D)) !== false){
-					$heightMap = array_values(unpack("v*", substr($data2d, 0, 512)));
-					$biomeIds = substr($data2d, 512, 256);
-				}
 			}
 
 			$nbt = new NBT(NBT::LITTLE_ENDIAN);
@@ -381,17 +412,16 @@ class LevelDB extends BaseLevelProvider{
 				}
 			}
 
-			/*
 			$extraData = [];
-			if(($extraRawData = $this->db->get($index . self::TAG_EXTRA_DATA)) !== false){
+			if(($extraRawData = $this->db->get($index . self::TAG_BLOCK_EXTRA_DATA)) !== false and strlen($extraRawData) > 0){
-				$stream = new BinaryStream($extraRawData);
+				$binaryStream->setBuffer($extraRawData, 0);
-				$count = $stream->getLInt(); //TODO: check if the extra data is BE or LE
+				$count = $binaryStream->getLInt();
 				for($i = 0; $i < $count; ++$i){
-					$key = $stream->getInt();
+					$key = $binaryStream->getLInt();
-					$value = $stream->getShort(false);
+					$value = $binaryStream->getLShort();
 					$extraData[$key] = $value;
 				}
-			}*/ //TODO
+			}
 
 			$chunk = new Chunk(
 				$chunkX,
@@ -405,17 +435,18 @@ class LevelDB extends BaseLevelProvider{
 
 			//TODO: tile ticks, biome states (?)
 
-			/*
-			$flags = $this->db->get($index . self::ENTRY_FLAGS);
-			if($flags === false){
-				$flags = "\x03";
-			}*/
-			// TODO: check this, add flags (?)
 			$chunk->setGenerated(true);
 			$chunk->setPopulated(true);
-			$chunk->setLightPopulated(true);
+			$chunk->setLightPopulated($lightPopulated);
 
 			return $chunk;
+		}catch(UnsupportedChunkFormatException $e){
+			//TODO: set chunks read-only so the version on disk doesn't get overwritten
+
+			$logger = MainLogger::getLogger();
+			$logger->error("Failed to decode LevelDB chunk: " . $e->getMessage());
+
+			return null;
 		}catch(\Throwable $t){
 			$logger = MainLogger::getLogger();
 			$logger->error("LevelDB chunk decode error");
@@ -428,28 +459,52 @@ class LevelDB extends BaseLevelProvider{
 
 	private function writeChunk(Chunk $chunk){
 		$index = LevelDB::chunkIndex($chunk->getX(), $chunk->getZ());
-		$this->db->put($index . self::TAG_VERSION, chr(self::CURRENT_STORAGE_VERSION));
+		$this->db->put($index . self::TAG_VERSION, chr(self::CURRENT_LEVEL_CHUNK_VERSION));
-		$highestIndex = $chunk->getHighestSubChunkIndex();
-		$subChunks = $chunk->getSubChunks();
-		for($y = $highestIndex; $y >= 0; --$y){ //Subchunks behave like a stack
 
-			$this->db->put($index . self::TAG_SUBCHUNK_PREFIX . chr($y),
+		$subChunks = $chunk->getSubChunks();
-						   "\x00" . //Subchunk version byte
+		foreach($subChunks as $y => $subChunk){
-						   $subChunks[$y]->getBlockIdArray() .
+			$key = $index . self::TAG_SUBCHUNK_PREFIX . chr($y);
-						   $subChunks[$y]->getBlockDataArray() .
+			if($subChunk->isEmpty(false)){ //MCPE doesn't save light anymore as of 1.1
-						   $subChunks[$y]->getBlockSkyLightArray() .
+				$this->db->delete($key);
-						   $subChunks[$y]->getBlockLightArray()
+			}else{
-			);
+				$this->db->put($key,
+					chr(self::CURRENT_LEVEL_SUBCHUNK_VERSION) .
+					$subChunks[$y]->getBlockIdArray() .
+					$subChunks[$y]->getBlockDataArray()
+				);
+			}
 		}
 
 		$this->db->put($index . self::TAG_DATA_2D, pack("v*", ...$chunk->getHeightMapArray()) . $chunk->getBiomeIdArray());
 
+		$extraData = $chunk->getBlockExtraDataArray();
+		if(count($extraData) > 0){
+			$stream = new BinaryStream();
+			$stream->putLInt(count($extraData));
+			foreach($extraData as $key => $value){
+				$stream->putLInt($key);
+				$stream->putLShort($value);
+			}
+
+			$this->db->put($index . self::TAG_BLOCK_EXTRA_DATA, $stream->getBuffer());
+		}else{
+			$this->db->delete($index . self::TAG_BLOCK_EXTRA_DATA);
+		}
+
+		//TODO: use this properly
+		$this->db->put($index . self::TAG_STATE_FINALISATION, chr(self::FINALISATION_DONE));
+
 		$this->writeTags($chunk->getTiles(), $index . self::TAG_BLOCK_ENTITY);
 		$this->writeTags($chunk->getEntities(), $index . self::TAG_ENTITY);
 
-		//TODO: clean up old data
+		$this->db->delete($index . self::TAG_DATA_2D_LEGACY);
+		$this->db->delete($index . self::TAG_LEGACY_TERRAIN);
 	}
 
+	/**
+	 * @param Entity[]|Tile[] $targets
+	 * @param string          $index
+	 */
 	private function writeTags(array $targets, string $index){
 		$nbt = new NBT(NBT::LITTLE_ENDIAN);
 		$out = [];

src/pocketmine/level/format/io/region/Anvil.php

@@ -24,7 +24,7 @@ declare(strict_types=1);
 namespace pocketmine\level\format\io\region;
 
 use pocketmine\level\format\Chunk;
-use pocketmine\level\format\io\ChunkException;
+use pocketmine\level\format\ChunkException;
 use pocketmine\level\format\io\ChunkUtils;
 use pocketmine\level\format\SubChunk;
 use pocketmine\nbt\NBT;

src/pocketmine/level/format/io/region/McRegion.php

@@ -25,7 +25,7 @@ namespace pocketmine\level\format\io\region;
 
 use pocketmine\level\format\Chunk;
 use pocketmine\level\format\io\BaseLevelProvider;
-use pocketmine\level\format\io\ChunkException;
+use pocketmine\level\format\ChunkException;
 use pocketmine\level\format\io\ChunkUtils;
 use pocketmine\level\format\SubChunk;
 use pocketmine\level\generator\Generator;

src/pocketmine/level/format/io/region/PMAnvil.php

@@ -24,7 +24,7 @@ declare(strict_types=1);
 namespace pocketmine\level\format\io\region;
 
 use pocketmine\level\format\Chunk;
-use pocketmine\level\format\io\ChunkException;
+use pocketmine\level\format\ChunkException;
 use pocketmine\level\format\SubChunk;
 use pocketmine\nbt\NBT;
 use pocketmine\nbt\tag\{

src/pocketmine/level/format/io/region/RegionLoader.php

@@ -24,7 +24,7 @@ declare(strict_types=1);
 namespace pocketmine\level\format\io\region;
 
 use pocketmine\level\format\Chunk;
-use pocketmine\level\format\io\ChunkException;
+use pocketmine\level\format\ChunkException;
 use pocketmine\utils\Binary;
 use pocketmine\utils\MainLogger;