Support paletted subchunks, drop all formats except leveldb

composer.json

@@ -8,6 +8,7 @@
       "php": ">=7.2.0",
       "php-64bit": "*",
       "ext-bcmath": "*",
+      "ext-chunkutils2": "^0.1.0",
       "ext-curl": "*",
       "ext-crypto": "^0.3.1",
       "ext-ctype": "*",
@@ -16,6 +17,7 @@
       "ext-gmp": "*",
       "ext-hash": "*",
       "ext-json": "*",
+      "ext-leveldb": "^0.2.1",
       "ext-mbstring": "*",
       "ext-openssl": "*",
       "ext-pcre": "*",

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "d865f72c482e34c96a7e46bc535d0c15",
+    "content-hash": "7d9bec9f6226ca3ec19b06f6ed406718",
     "packages": [
         {
             "name": "adhocore/json-comment",
@@ -532,6 +532,7 @@
         "php": ">=7.2.0",
         "php-64bit": "*",
         "ext-bcmath": "*",
+        "ext-chunkutils2": "^0.1.0",
         "ext-curl": "*",
         "ext-crypto": "^0.3.1",
         "ext-ctype": "*",
@@ -540,6 +541,7 @@
         "ext-gmp": "*",
         "ext-hash": "*",
         "ext-json": "*",
+        "ext-leveldb": "^0.2.1",
         "ext-mbstring": "*",
         "ext-openssl": "*",
         "ext-pcre": "*",

resources/legacy_id_map.json

@@ -247,5 +247,88 @@
     "minecraft:structure_block": 252,
     "minecraft:hard_glass": 253,
     "minecraft:hard_stained_glass": 254,
-    "minecraft:reserved6": 255
+    "minecraft:reserved6": 255,
-}
+    "minecraft:prismarine_stairs": 257,
+    "minecraft:dark_prismarine_stairs": 258,
+    "minecraft:prismarine_bricks_stairs": 259,
+    "minecraft:stripped_spruce_log": 260,
+    "minecraft:stripped_birch_log": 261,
+    "minecraft:stripped_jungle_log": 262,
+    "minecraft:stripped_acacia_log": 263,
+    "minecraft:stripped_dark_oak_log": 264,
+    "minecraft:stripped_oak_log": 265,
+    "minecraft:blue_ice": 266,
+    "minecraft:seagrass": 385,
+    "minecraft:coral": 386,
+    "minecraft:coral_block": 387,
+    "minecraft:coral_fan": 388,
+    "minecraft:coral_fan_dead": 389,
+    "minecraft:coral_fan_hang": 390,
+    "minecraft:coral_fan_hang2": 391,
+    "minecraft:coral_fan_hang3": 392,
+    "minecraft:kelp": 393,
+    "minecraft:dried_kelp_block": 394,
+    "minecraft:acacia_button": 395,
+    "minecraft:birch_button": 396,
+    "minecraft:dark_oak_button": 397,
+    "minecraft:jungle_button": 398,
+    "minecraft:spruce_button": 399,
+    "minecraft:acacia_trapdoor": 400,
+    "minecraft:birch_trapdoor": 401,
+    "minecraft:dark_oak_trapdoor": 402,
+    "minecraft:jungle_trapdoor": 403,
+    "minecraft:spruce_trapdoor": 404,
+    "minecraft:acacia_pressure_plate": 405,
+    "minecraft:birch_pressure_plate": 406,
+    "minecraft:dark_oak_pressure_plate": 407,
+    "minecraft:jungle_pressure_plate": 408,
+    "minecraft:spruce_pressure_plate": 409,
+    "minecraft:carved_pumpkin": 410,
+    "minecraft:sea_pickle": 411,
+    "minecraft:conduit": 412,
+    "minecraft:turtle_egg": 414,
+    "minecraft:bubble_column": 415,
+    "minecraft:barrier": 416,
+    "minecraft:stone_slab3": 417,
+    "minecraft:bamboo": 418,
+    "minecraft:bamboo_sapling": 419,
+    "minecraft:scaffolding": 420,
+    "minecraft:stone_slab4": 421,
+    "minecraft:double_stone_slab3": 422,
+    "minecraft:double_stone_slab4": 423,
+    "minecraft:granite_stairs": 424,
+    "minecraft:diorite_stairs": 425,
+    "minecraft:andesite_stairs": 426,
+    "minecraft:polished_granite_stairs": 427,
+    "minecraft:polished_diorite_stairs": 428,
+    "minecraft:polished_andesite_stairs": 429,
+    "minecraft:mossy_stone_brick_stairs": 430,
+    "minecraft:smooth_red_sandstone_stairs": 431,
+    "minecraft:smooth_sandstone_stairs": 432,
+    "minecraft:end_brick_stairs": 433,
+    "minecraft:mossy_cobblestone_stairs": 434,
+    "minecraft:normal_stone_stairs": 435,
+    "minecraft:spruce_standing_sign": 436,
+    "minecraft:spruce_wall_sign": 437,
+    "minecraft:smooth_stone": 438,
+    "minecraft:red_nether_brick_stairs": 439,
+    "minecraft:smooth_quartz_stairs": 440,
+    "minecraft:birch_standing_sign": 441,
+    "minecraft:birch_wall_sign": 442,
+    "minecraft:jungle_standing_sign": 443,
+    "minecraft:jungle_wall_sign": 444,
+    "minecraft:acacia_standing_sign": 445,
+    "minecraft:acacia_wall_sign": 446,
+    "minecraft:darkoak_standing_sign": 447,
+    "minecraft:darkoak_wall_sign": 448,
+    "minecraft:grindstone": 450,
+    "minecraft:blast_furnace": 451,
+    "minecraft:smoker": 453,
+    "minecraft:cartography_table": 455,
+    "minecraft:fletching_table": 456,
+    "minecraft:smithing_table": 457,
+    "minecraft:barrel": 458,
+    "minecraft:bell": 461,
+    "minecraft:lantern": 463,
+    "minecraft:lava_cauldron": 465
+}

src/pocketmine/PocketMine.php

@@ -76,6 +76,7 @@ namespace pocketmine {
 
 		$extensions = [
 			"bcmath" => "BC Math",
+			"chunkutils2" => "PocketMine ChunkUtils v2",
 			"curl" => "cURL",
 			"crypto" => "php-crypto",
 			"ctype" => "ctype",
@@ -84,6 +85,7 @@ namespace pocketmine {
 			"gmp" => "GMP",
 			"hash" => "Hash",
 			"json" => "JSON",
+			"leveldb" => "LevelDB",
 			"mbstring" => "Multibyte String",
 			"openssl" => "OpenSSL",
 			"pcre" => "PCRE",

src/pocketmine/level/Level.php

@@ -28,6 +28,7 @@ namespace pocketmine\level;
 
 use pocketmine\block\Block;
 use pocketmine\block\BlockFactory;
+use pocketmine\block\BlockIds;
 use pocketmine\block\UnknownBlock;
 use pocketmine\entity\Entity;
 use pocketmine\entity\EntityFactory;
@@ -380,7 +381,7 @@ class Level implements ChunkManager, Metadatable{
 
 		$dontTickBlocks = array_fill_keys($this->server->getProperty("chunk-ticking.disable-block-ticking", []), true);
 
-		$this->randomTickBlocks = new \SplFixedArray(4096);
+		$this->randomTickBlocks = new \SplFixedArray(16384);
 		foreach($this->randomTickBlocks as $i => $null){
 			$id = $i >> 4;
 			$meta = $i & 0xf;
@@ -2783,7 +2784,7 @@ class Level implements ChunkManager, Metadatable{
 		$z = (int) $v->z;
 		if($chunk !== null and $chunk->isGenerated()){
 			$y = (int) min($max - 2, $v->y);
-			$wasAir = ($chunk->getBlockId($x & 0x0f, $y - 1, $z & 0x0f) === 0);
+			$wasAir = $this->getBlockAt($x, $y - 1, $z)->getId() === BlockIds::AIR; //TODO: bad hack, clean up
 			for(; $y > 0; --$y){
 				if($this->isFullBlock($this->getBlockAt($x, $y, $z))){
 					if($wasAir){

src/pocketmine/level/format/Chunk.php

@@ -27,10 +27,12 @@ declare(strict_types=1);
 namespace pocketmine\level\format;
 
 use pocketmine\block\BlockFactory;
+use pocketmine\block\BlockIds;
 use pocketmine\entity\Entity;
 use pocketmine\entity\EntityFactory;
 use pocketmine\level\Level;
 use pocketmine\nbt\tag\CompoundTag;
+use pocketmine\network\mcpe\NetworkBinaryStream;
 use pocketmine\Player;
 use pocketmine\tile\Spawnable;
 use pocketmine\tile\Tile;
@@ -205,32 +207,6 @@ class Chunk{
 		return false;
 	}
 
-	/**
-	 * Returns the block ID at the specified chunk block coordinates
-	 *
-	 * @param int $x 0-15
-	 * @param int $y
-	 * @param int $z 0-15
-	 *
-	 * @return int 0-255
-	 */
-	public function getBlockId(int $x, int $y, int $z) : int{
-		return $this->getSubChunk($y >> 4)->getBlockId($x, $y & 0x0f, $z);
-	}
-
-	/**
-	 * Returns the block meta value at the specified chunk block coordinates
-	 *
-	 * @param int $x 0-15
-	 * @param int $y
-	 * @param int $z 0-15
-	 *
-	 * @return int 0-15
-	 */
-	public function getBlockData(int $x, int $y, int $z) : int{
-		return $this->getSubChunk($y >> 4)->getBlockData($x, $y & 0x0f, $z);
-	}
-
 	/**
 	 * Returns the sky light level at the specified chunk block coordinates
 	 *
@@ -698,7 +674,7 @@ class Chunk{
 		if($y < 0 or $y >= $this->height){
 			return $this->emptySubChunk;
 		}elseif($generateNew and $this->subChunks[$y] instanceof EmptySubChunk){
-			$this->subChunks[$y] = new SubChunk();
+			$this->subChunks[$y] = new SubChunk([new PalettedBlockArray(BlockIds::AIR << 4)]);
 		}
 
 		return $this->subChunks[$y];
@@ -765,10 +741,9 @@ class Chunk{
 	public function collectGarbage() : void{
 		foreach($this->subChunks as $y => $subChunk){
 			if($subChunk instanceof SubChunk){
+				$subChunk->collectGarbage();
 				if($subChunk->isEmpty()){
 					$this->subChunks[$y] = $this->emptySubChunk;
-				}else{
-					$subChunk->collectGarbage();
 				}
 			}
 		}
@@ -780,24 +755,42 @@ class Chunk{
 	 * @return string
 	 */
 	public function networkSerialize() : string{
-		$result = "";
+		$stream = new NetworkBinaryStream();
 		$subChunkCount = $this->getSubChunkSendCount();
-		$result .= chr($subChunkCount);
+		$stream->putByte($subChunkCount);
-		for($y = 0; $y < $subChunkCount; ++$y){
+
-			$result .= $this->subChunks[$y]->networkSerialize();
+		if(empty(BlockFactory::$staticRuntimeIdMap)){
+			BlockFactory::registerStaticRuntimeIdMappings();
 		}
-		$result .= pack("v*", ...$this->heightMap)
+
-			. $this->biomeIds
+		for($y = 0; $y < $subChunkCount; ++$y){
-			. chr(0); //border block array count
+			$layers = $this->subChunks[$y]->getBlockLayers();
+			$stream->putByte(8); //version
+
+			$stream->putByte(count($layers));
+
+			foreach($layers as $blocks){
+				$stream->putByte(($blocks->getBitsPerBlock() << 1) | 1); //last 1-bit means "network format", but seems pointless
+				$stream->put($blocks->getWordArray());
+				$palette = $blocks->getPalette();
+				$stream->putVarInt(count($palette)); //yes, this is intentionally zigzag
+				foreach($palette as $p){
+					$stream->putVarInt(BlockFactory::toStaticRuntimeId($p >> 4, $p & 0xf));
+				}
+			}
+		}
+		$stream->put(pack("v*", ...$this->heightMap));
+		$stream->put($this->biomeIds);
+		$stream->putByte(0); //border block array count
 		//Border block entry format: 1 byte (4 bits X, 4 bits Z). These are however useless since they crash the regular client.
 
 		foreach($this->tiles as $tile){
 			if($tile instanceof Spawnable){
-				$result .= $tile->getSerializedSpawnCompound();
+				$stream->put($tile->getSerializedSpawnCompound());
 			}
 		}
 
-		return $result;
+		return $stream->getBuffer();
 	}
 
 	/**
@@ -814,19 +807,35 @@ class Chunk{
 		if($this->terrainGenerated){
 			//subchunks
 			$count = 0;
-			$subChunks = "";
+			$subStream = new BinaryStream();
 			foreach($this->subChunks as $y => $subChunk){
 				if($subChunk instanceof EmptySubChunk){
 					continue;
 				}
 				++$count;
-				$subChunks .= chr($y) . $subChunk->getBlockIdArray() . $subChunk->getBlockDataArray();
+
+				$subStream->putByte($y);
+				$layers = $subChunk->getBlockLayers();
+				$subStream->putByte(count($subChunk->getBlockLayers()));
+				foreach($layers as $blocks){
+					$wordArray = $blocks->getWordArray();
+					$palette = $blocks->getPalette();
+
+					$subStream->putByte($blocks->getBitsPerBlock());
+					$subStream->put($wordArray);
+					$subStream->putInt(count($palette));
+					foreach($palette as $p){
+						$subStream->putInt($p);
+					}
+				}
+
 				if($this->lightPopulated){
-					$subChunks .= $subChunk->getBlockSkyLightArray() . $subChunk->getBlockLightArray();
+					$subStream->put($subChunk->getBlockSkyLightArray());
+					$subStream->put($subChunk->getBlockLightArray());
 				}
 			}
 			$stream->putByte($count);
-			$stream->put($subChunks);
+			$stream->put($subStream->getBuffer());
 
 			//biomes
 			$stream->put($this->biomeIds);
@@ -860,12 +869,23 @@ class Chunk{
 		$heightMap = [];
 		if($terrainGenerated){
 			$count = $stream->getByte();
-			for($y = 0; $y < $count; ++$y){
+			for($subCount = 0; $subCount < $count; ++$subCount){
-				$subChunks[$stream->getByte()] = new SubChunk(
+				$y = $stream->getByte();
-					$stream->get(4096), //blockids
+
-					$stream->get(2048), //blockdata
+				/** @var PalettedBlockArray[] $layers */
-					$lightPopulated ? $stream->get(2048) : "", //skylight
+				$layers = [];
-					$lightPopulated ? $stream->get(2048) : "" //blocklight
+				for($i = 0, $layerCount = $stream->getByte(); $i < $layerCount; ++$i){
+					$bitsPerBlock = $stream->getByte();
+					$words = $stream->get(PalettedBlockArray::getExpectedWordArraySize($bitsPerBlock));
+					$palette = [];
+					for($k = 0, $paletteSize = $stream->getInt(); $k < $paletteSize; ++$k){
+						$palette[] = $stream->getInt();
+					}
+
+					$layers[] = PalettedBlockArray::fromData($bitsPerBlock, $words, $palette);
+				}
+				$subChunks[$y] = new SubChunk(
+					$layers, $lightPopulated ? $stream->get(2048) : "", $lightPopulated ? $stream->get(2048) : "" //blocklight
 				);
 			}
 

src/pocketmine/level/format/EmptySubChunk.php

@@ -41,14 +41,6 @@ class EmptySubChunk implements SubChunkInterface{
 		return true;
 	}
 
-	public function getBlockId(int $x, int $y, int $z) : int{
-		return 0;
-	}
-
-	public function getBlockData(int $x, int $y, int $z) : int{
-		return 0;
-	}
-
 	public function getFullBlock(int $x, int $y, int $z) : int{
 		return 0;
 	}
@@ -57,6 +49,10 @@ class EmptySubChunk implements SubChunkInterface{
 		return false;
 	}
 
+	public function getBlockLayers() : array{
+		return [];
+	}
+
 	public function getBlockLight(int $x, int $y, int $z) : int{
 		return 0;
 	}
@@ -77,14 +73,6 @@ class EmptySubChunk implements SubChunkInterface{
 		return -1;
 	}
 
-	public function getBlockIdArray() : string{
-		return str_repeat("\x00", 4096);
-	}
-
-	public function getBlockDataArray() : string{
-		return str_repeat("\x00", 2048);
-	}
-
 	public function getBlockLightArray() : string{
 		return str_repeat("\x00", 2048);
 	}
@@ -100,8 +88,4 @@ class EmptySubChunk implements SubChunkInterface{
 	public function setBlockSkyLightArray(string $data) : void{
 
 	}
-
-	public function networkSerialize() : string{
-		return "\x00" . str_repeat("\x00", 6144);
-	}
 }

src/pocketmine/level/format/PalettedBlockArray.php

@@ -0,0 +1,53 @@
+<?php
+
+/*
+ *
+ *  ____            _        _   __  __ _                  __  __ ____
+ * |  _ \ ___   ___| | _____| |_|  \/  (_)_ __   ___      |  \/  |  _ \
+ * | |_) / _ \ / __| |/ / _ \ __| |\/| | | '_ \ / _ \_____| |\/| | |_) |
+ * |  __/ (_) | (__|   <  __/ |_| |  | | | | | |  __/_____| |  | |  __/
+ * |_|   \___/ \___|_|\_\___|\__|_|  |_|_|_| |_|\___|     |_|  |_|_|
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * @author PocketMine Team
+ * @link http://www.pocketmine.net/
+ *
+ *
+*/
+
+declare(strict_types=1);
+
+namespace pocketmine\level\format;
+
+die("This is a stub file for code completion purposes");
+
+class PalettedBlockArray{
+
+	public function __construct(int $fillEntry){}
+
+	public static function fromData(int $bitsPerBlock, string $wordArray, array $palette) : PalettedBlockArray{}
+
+	public function getWordArray() : string{}
+
+	public function getPalette() : array{}
+
+	public function getMaxPaletteSize() : int{}
+
+	public function getBitsPerBlock() : int{}
+
+	public function get(int $x, int $y, int $z) : int{}
+
+	public function set(int $x, int $y, int $z, int $val){}
+
+	public function replace(int $offset, int $val){}
+
+	public function replaceAll(int $oldVal, int $newVal){}
+
+	public function collectGarbage(bool $force = null){}
+
+	public static function getExpectedWordArraySize(int $bitsPerBlock) : int{}
+}

src/pocketmine/level/format/SubChunk.php

@@ -23,6 +23,8 @@ declare(strict_types=1);
 
 namespace pocketmine\level\format;
 
+use pocketmine\block\BlockIds;
+use function array_values;
 use function assert;
 use function chr;
 use function define;
@@ -37,9 +39,12 @@ if(!defined(__NAMESPACE__ . '\ZERO_NIBBLE_ARRAY')){
 }
 
 class SubChunk implements SubChunkInterface{
-	protected $ids;
+	/** @var PalettedBlockArray[] */
-	protected $data;
+	private $blockLayers;
+
+	/** @var string */
 	protected $blockLight;
+	/** @var string */
 	protected $skyLight;
 
 	private static function assignData(&$target, string $data, int $length, string $value = "\x00") : void{
@@ -51,57 +56,57 @@ class SubChunk implements SubChunkInterface{
 		}
 	}
 
-	public function __construct(string $ids = "", string $data = "", string $skyLight = "", string $blockLight = ""){
+	/**
-		self::assignData($this->ids, $ids, 4096);
+	 * SubChunk constructor.
-		self::assignData($this->data, $data, 2048);
+	 *
+	 * @param PalettedBlockArray[] $blocks
+	 * @param string               $skyLight
+	 * @param string               $blockLight
+	 */
+	public function __construct(array $blocks, string $skyLight = "", string $blockLight = ""){
+		$this->blockLayers = $blocks;
+
 		self::assignData($this->skyLight, $skyLight, 2048, "\xff");
 		self::assignData($this->blockLight, $blockLight, 2048);
-		$this->collectGarbage();
 	}
 
 	public function isEmpty(bool $checkLight = true) : bool{
-		return (
+		foreach($this->blockLayers as $layer){
-			substr_count($this->ids, "\x00") === 4096 and
+			$palette = $layer->getPalette();
+			foreach($palette as $p){
+				if(($p >> 4) !== BlockIds::AIR){
+					return false;
+				}
+			}
+		}
+		return
 			(!$checkLight or (
 				substr_count($this->skyLight, "\xff") === 2048 and
 				$this->blockLight === ZERO_NIBBLE_ARRAY
-			))
+			)
 		);
 	}
 
-	public function getBlockId(int $x, int $y, int $z) : int{
-		return ord($this->ids{($x << 8) | ($z << 4) | $y});
-	}
-
-	public function getBlockData(int $x, int $y, int $z) : int{
-		return (ord($this->data{($x << 7) | ($z << 3) | ($y >> 1)}) >> (($y & 1) << 2)) & 0xf;
-	}
-
 	public function getFullBlock(int $x, int $y, int $z) : int{
-		$i = ($x << 8) | ($z << 4) | $y;
+		if(empty($this->blockLayers)){
-		return (ord($this->ids{$i}) << 4) | ((ord($this->data{$i >> 1}) >> (($y & 1) << 2)) & 0xf);
+			return BlockIds::AIR << 4;
+		}
+		return $this->blockLayers[0]->get($x, $y, $z);
 	}
 
 	public function setBlock(int $x, int $y, int $z, int $id, int $data) : bool{
-		$i = ($x << 8) | ($z << 4) | $y;
+		if(empty($this->blockLayers)){
-		$changed = false;
+			$this->blockLayers[] = new PalettedBlockArray(BlockIds::AIR << 4);
-
-		$block = chr($id);
-		if($this->ids{$i} !== $block){
-			$this->ids{$i} = $block;
-			$changed = true;
 		}
+		$this->blockLayers[0]->set($x, $y, $z, ($id << 4) | $data);
+		return true;
+	}
 
-		$i >>= 1;
+	/**
-		$shift = ($y & 1) << 2;
+	 * @return PalettedBlockArray[]
-		$oldPair = ord($this->data{$i});
+	 */
-		$newPair = ($oldPair & ~(0xf << $shift)) | (($data & 0xf) << $shift);
+	public function getBlockLayers() : array{
-		if($newPair !== $oldPair){
+		return $this->blockLayers;
-			$this->data{$i} = chr($newPair);
-			$changed = true;
-		}
-
-		return $changed;
 	}
 
 	public function getBlockLight(int $x, int $y, int $z) : int{
@@ -136,7 +141,7 @@ class SubChunk implements SubChunkInterface{
 		$low = ($x << 8) | ($z << 4);
 		$i = $low | 0x0f;
 		for(; $i >= $low; --$i){
-			if($this->ids{$i} !== "\x00"){
+			if(($this->blockLayers[0]->get($x, $i, $z) >> 4) !== BlockIds::AIR){
 				return $i & 0x0f;
 			}
 		}
@@ -144,16 +149,6 @@ class SubChunk implements SubChunkInterface{
 		return -1; //highest block not in this subchunk
 	}
 
-	public function getBlockIdArray() : string{
-		assert(strlen($this->ids) === 4096, "Wrong length of ID array, expecting 4096 bytes, got " . strlen($this->ids));
-		return $this->ids;
-	}
-
-	public function getBlockDataArray() : string{
-		assert(strlen($this->data) === 2048, "Wrong length of data array, expecting 2048 bytes, got " . strlen($this->data));
-		return $this->data;
-	}
-
 	public function getBlockSkyLightArray() : string{
 		assert(strlen($this->skyLight) === 2048, "Wrong length of skylight array, expecting 2048 bytes, got " . strlen($this->skyLight));
 		return $this->skyLight;
@@ -174,23 +169,28 @@ class SubChunk implements SubChunkInterface{
 		$this->blockLight = $data;
 	}
 
-	public function networkSerialize() : string{
-		return "\x00" . $this->ids . $this->data;
-	}
-
 	public function __debugInfo(){
 		return [];
 	}
 
 	public function collectGarbage() : void{
+		foreach($this->blockLayers as $k => $layer){
+			$layer->collectGarbage();
+
+			foreach($layer->getPalette() as $p){
+				if(($p >> 4) !== BlockIds::AIR){
+					continue 2;
+				}
+			}
+			unset($this->blockLayers[$k]);
+		}
+		$this->blockLayers = array_values($this->blockLayers);
+
 		/*
 		 * This strange looking code is designed to exploit PHP's copy-on-write behaviour. Assigning will copy a
 		 * reference to the const instead of duplicating the whole string. The string will only be duplicated when
 		 * modified, which is perfect for this purpose.
 		 */
-		if($this->data === ZERO_NIBBLE_ARRAY){
-			$this->data = ZERO_NIBBLE_ARRAY;
-		}
 		if($this->skyLight === ZERO_NIBBLE_ARRAY){
 			$this->skyLight = ZERO_NIBBLE_ARRAY;
 		}

src/pocketmine/level/format/SubChunkInterface.php

@@ -32,24 +32,6 @@ interface SubChunkInterface{
 	 */
 	public function isEmpty(bool $checkLight = true) : bool;
 
-	/**
-	 * @param int $x
-	 * @param int $y
-	 * @param int $z
-	 *
-	 * @return int
-	 */
-	public function getBlockId(int $x, int $y, int $z) : int;
-
-	/**
-	 * @param int $x
-	 * @param int $y
-	 * @param int $z
-	 *
-	 * @return int
-	 */
-	public function getBlockData(int $x, int $y, int $z) : int;
-
 	/**
 	 * @param int $x
 	 * @param int $y
@@ -70,6 +52,11 @@ interface SubChunkInterface{
 	 */
 	public function setBlock(int $x, int $y, int $z, int $id, int $data) : bool;
 
+	/**
+	 * @return PalettedBlockArray[]
+	 */
+	public function getBlockLayers() : array;
+
 	/**
 	 * @param int $x
 	 * @param int $y
@@ -116,16 +103,6 @@ interface SubChunkInterface{
 	 */
 	public function getHighestBlockAt(int $x, int $z) : int;
 
-	/**
-	 * @return string
-	 */
-	public function getBlockIdArray() : string;
-
-	/**
-	 * @return string
-	 */
-	public function getBlockDataArray() : string;
-
 	/**
 	 * @return string
 	 */
@@ -145,9 +122,4 @@ interface SubChunkInterface{
 	 * @param string $data
 	 */
 	public function setBlockLightArray(string $data) : void;
-
-	/**
-	 * @return string
-	 */
-	public function networkSerialize() : string;
 }

src/pocketmine/level/format/io/LevelProviderManager.php

@@ -35,7 +35,7 @@ abstract class LevelProviderManager{
 	protected static $providers = [];
 
 	/** @var string|LevelProvider */
-	private static $default = PMAnvil::class;
+	private static $default = LevelDB::class;
 
 	public static function init() : void{
 		self::addProvider(Anvil::class, "anvil");

src/pocketmine/level/format/io/SubChunkConverter.php

@@ -0,0 +1,37 @@
+<?php
+
+/*
+ *
+ *  ____            _        _   __  __ _                  __  __ ____
+ * |  _ \ ___   ___| | _____| |_|  \/  (_)_ __   ___      |  \/  |  _ \
+ * | |_) / _ \ / __| |/ / _ \ __| |\/| | | '_ \ / _ \_____| |\/| | |_) |
+ * |  __/ (_) | (__|   <  __/ |_| |  | | | | | |  __/_____| |  | |  __/
+ * |_|   \___/ \___|_|\_\___|\__|_|  |_|_|_| |_|\___|     |_|  |_|_|
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * @author PocketMine Team
+ * @link http://www.pocketmine.net/
+ *
+ *
+*/
+
+declare(strict_types=1);
+
+namespace pocketmine\level\format\io;
+
+use pocketmine\level\format\PalettedBlockArray;
+
+die("This is a stub file for code completion purposes");
+
+class SubChunkConverter{
+	public static function convertSubChunkXZY(string $idArray, string $metaArray) : PalettedBlockArray{}
+
+	public static function convertSubChunkYZX(string $idArray, string $metaArray) : PalettedBlockArray{}
+
+	public static function convertSubChunkFromLegacyColumn(string $idArray, string $metaArray, int $yOffset) : PalettedBlockArray{}
+
+}

src/pocketmine/level/format/io/data/BedrockLevelData.php

@@ -46,7 +46,7 @@ use function time;
 
 class BedrockLevelData extends BaseNbtLevelData{
 
-	public const CURRENT_STORAGE_VERSION = 6;
+	public const CURRENT_STORAGE_VERSION = 8;
 
 	public const GENERATOR_LIMITED = 0;
 	public const GENERATOR_INFINITE = 1;

src/pocketmine/level/format/io/leveldb/LevelDB.php

@@ -23,6 +23,7 @@ declare(strict_types=1);
 
 namespace pocketmine\level\format\io\leveldb;
 
+use pocketmine\block\BlockIds;
 use pocketmine\level\format\Chunk;
 use pocketmine\level\format\io\BaseLevelProvider;
 use pocketmine\level\format\io\ChunkUtils;
@@ -31,22 +32,31 @@ use pocketmine\level\format\io\exception\CorruptedChunkException;
 use pocketmine\level\format\io\exception\UnsupportedChunkFormatException;
 use pocketmine\level\format\io\exception\UnsupportedLevelFormatException;
 use pocketmine\level\format\io\LevelData;
+use pocketmine\level\format\io\SubChunkConverter;
 use pocketmine\level\format\io\WritableLevelProvider;
+use pocketmine\level\format\PalettedBlockArray;
 use pocketmine\level\format\SubChunk;
 use pocketmine\level\generator\Generator;
 use pocketmine\nbt\LittleEndianNbtSerializer;
 use pocketmine\nbt\NbtDataException;
 use pocketmine\nbt\tag\CompoundTag;
+use pocketmine\nbt\tag\IntTag;
+use pocketmine\nbt\tag\ShortTag;
+use pocketmine\nbt\tag\StringTag;
 use pocketmine\utils\Binary;
 use pocketmine\utils\BinaryDataException;
 use pocketmine\utils\BinaryStream;
 use pocketmine\utils\Utils;
+use function array_flip;
 use function array_values;
 use function chr;
+use function count;
 use function defined;
 use function extension_loaded;
 use function file_exists;
+use function file_get_contents;
 use function is_dir;
+use function json_decode;
 use function mkdir;
 use function ord;
 use function str_repeat;
@@ -81,7 +91,7 @@ class LevelDB extends BaseLevelProvider implements WritableLevelProvider{
 	protected const ENTRY_FLAT_WORLD_LAYERS = "game_flatworldlayers";
 
 	protected const CURRENT_LEVEL_CHUNK_VERSION = 7;
-	protected const CURRENT_LEVEL_SUBCHUNK_VERSION = 0;
+	protected const CURRENT_LEVEL_SUBCHUNK_VERSION = 8;
 
 	/** @var \LevelDB */
 	protected $db;
@@ -132,6 +142,35 @@ class LevelDB extends BaseLevelProvider implements WritableLevelProvider{
 		BedrockLevelData::generate($path, $name, $seed, $generator, $options);
 	}
 
+	protected function deserializePaletted(BinaryStream $stream) : PalettedBlockArray{
+		static $stringToLegacyId = null;
+		if($stringToLegacyId === null){
+			$stringToLegacyId = json_decode(file_get_contents(\pocketmine\RESOURCE_PATH . '/legacy_id_map.json'), true);
+		}
+
+		$bitsPerBlock = $stream->getByte() >> 1;
+
+		try{
+			$words = $stream->get(PalettedBlockArray::getExpectedWordArraySize($bitsPerBlock));
+		}catch(\InvalidArgumentException $e){
+			throw new CorruptedChunkException("Failed to deserialize paletted storage: " . $e->getMessage(), 0, $e);
+		}
+		$nbt = new LittleEndianNbtSerializer();
+		$palette = [];
+		for($i = 0, $paletteSize = $stream->getLInt(); $i < $paletteSize; ++$i){
+			$offset = $stream->getOffset();
+			$tag = $nbt->read($stream->getBuffer(), $offset);
+			$stream->setOffset($offset);
+
+			$id = $stringToLegacyId[$tag->getString("name")] ?? BlockIds::INFO_UPDATE;
+			$data = $tag->getShort("val");
+			$palette[] = ($id << 4) | $data;
+		}
+
+		//TODO: exceptions
+		return PalettedBlockArray::fromData($bitsPerBlock, $words, $palette);
+	}
+
 	/**
 	 * @param int $chunkX
 	 * @param int $chunkZ
@@ -159,6 +198,8 @@ class LevelDB extends BaseLevelProvider implements WritableLevelProvider{
 		$binaryStream = new BinaryStream();
 
 		switch($chunkVersion){
+			case 10: //MCPE 1.9 (???)
+			case 9: //MCPE 1.8 (???)
 			case 7: //MCPE 1.2 (???)
 			case 4: //MCPE 1.1
 				//TODO: check beds
@@ -179,6 +220,12 @@ class LevelDB extends BaseLevelProvider implements WritableLevelProvider{
 
 					switch($subChunkVersion){
 						case 0:
+						case 2: //these are all identical to version 0, but vanilla respects these so we should also
+						case 3:
+						case 4:
+						case 5:
+						case 6:
+						case 7:
 							try{
 								$blocks = $binaryStream->get(4096);
 								$blockData = $binaryStream->get(2048);
@@ -191,7 +238,17 @@ class LevelDB extends BaseLevelProvider implements WritableLevelProvider{
 								throw new CorruptedChunkException($e->getMessage(), 0, $e);
 							}
 
-							$subChunks[$y] = new SubChunk($blocks, $blockData);
+							$subChunks[$y] = new SubChunk([SubChunkConverter::convertSubChunkXZY($blocks, $blockData)]);
+							break;
+						case 1: //paletted v1, has a single blockstorage
+							$subChunks[$y] = new SubChunk([$this->deserializePaletted($binaryStream)]);
+							break;
+						case 8:
+							$storages = [];
+							for($k = 0, $storageCount = $binaryStream->getByte(); $k < $storageCount; ++$k){
+								$storages[] = $this->deserializePaletted($binaryStream);
+							}
+							$subChunks[$y] = new SubChunk($storages);
 							break;
 						default:
 							//TODO: set chunks read-only so the version on disk doesn't get overwritten
@@ -225,20 +282,7 @@ class LevelDB extends BaseLevelProvider implements WritableLevelProvider{
 				}
 
 				for($yy = 0; $yy < 8; ++$yy){
-					$subOffset = ($yy << 4);
+					$subChunks[$yy] = new SubChunk([SubChunkConverter::convertSubChunkFromLegacyColumn($fullIds, $fullData, $yy)]);
-					$ids = "";
-					for($i = 0; $i < 256; ++$i){
-						$ids .= substr($fullIds, $subOffset, 16);
-						$subOffset += 128;
-					}
-					$data = "";
-					$subOffset = ($yy << 3);
-					for($i = 0; $i < 256; ++$i){
-						$data .= substr($fullData, $subOffset, 8);
-						$subOffset += 64;
-					}
-
-					$subChunks[$yy] = new SubChunk($ids, $data);
 				}
 
 				try{
@@ -307,6 +351,10 @@ class LevelDB extends BaseLevelProvider implements WritableLevelProvider{
 	}
 
 	protected function writeChunk(Chunk $chunk) : void{
+		static $idMap = null;
+		if($idMap === null){
+			$idMap = array_flip(json_decode(file_get_contents(\pocketmine\RESOURCE_PATH . '/legacy_id_map.json'), true));
+		}
 		$index = LevelDB::chunkIndex($chunk->getX(), $chunk->getZ());
 		$this->db->put($index . self::TAG_VERSION, chr(self::CURRENT_LEVEL_CHUNK_VERSION));
 
@@ -316,11 +364,30 @@ class LevelDB extends BaseLevelProvider implements WritableLevelProvider{
 			if($subChunk->isEmpty(false)){ //MCPE doesn't save light anymore as of 1.1
 				$this->db->delete($key);
 			}else{
-				$this->db->put($key,
+				$subStream = new BinaryStream();
-					chr(self::CURRENT_LEVEL_SUBCHUNK_VERSION) .
+				$subStream->putByte(self::CURRENT_LEVEL_SUBCHUNK_VERSION);
-					$subChunk->getBlockIdArray() .
+
-					$subChunk->getBlockDataArray()
+				$layers = $subChunk->getBlockLayers();
-				);
+				$subStream->putByte(count($layers));
+				foreach($layers as $blocks){
+					$subStream->putByte($blocks->getBitsPerBlock() << 1);
+					$subStream->put($blocks->getWordArray());
+
+					$palette = $blocks->getPalette();
+					$subStream->putLInt(count($palette));
+					$tags = [];
+					foreach($palette as $p){
+						$tags[] = new CompoundTag("", [
+							new StringTag("name", $idMap[$p >> 4] ?? "minecraft:info_update"),
+							new IntTag("oldid", $p >> 4), //PM only (debugging), vanilla doesn't have this
+							new ShortTag("val", $p & 0xf)
+						]);
+					}
+
+					$subStream->put((new LittleEndianNbtSerializer())->writeMultiple($tags));
+				}
+
+				$this->db->put($key, $subStream->getBuffer());
 			}
 		}
 

src/pocketmine/level/format/io/region/Anvil.php

@@ -23,31 +23,20 @@ declare(strict_types=1);
 
 namespace pocketmine\level\format\io\region;
 
-use pocketmine\level\format\io\ChunkUtils;
+use pocketmine\level\format\io\SubChunkConverter;
-use pocketmine\level\format\io\WritableLevelProvider;
 use pocketmine\level\format\SubChunk;
-use pocketmine\nbt\tag\ByteArrayTag;
 use pocketmine\nbt\tag\CompoundTag;
-use function str_repeat;
 
-class Anvil extends RegionLevelProvider implements WritableLevelProvider{
+class Anvil extends RegionLevelProvider{
 	use LegacyAnvilChunkTrait;
 
 	protected function serializeSubChunk(SubChunk $subChunk) : CompoundTag{
-		return new CompoundTag("", [
+		throw new \RuntimeException("Unsupported");
-			new ByteArrayTag("Blocks", ChunkUtils::reorderByteArray($subChunk->getBlockIdArray())), //Generic in-memory chunks are currently always XZY
-			new ByteArrayTag("Data", ChunkUtils::reorderNibbleArray($subChunk->getBlockDataArray())),
-			new ByteArrayTag("SkyLight", str_repeat("\x00", 2048)),
-			new ByteArrayTag("BlockLight", str_repeat("\x00", 2048))
-		]);
 	}
 
 	protected function deserializeSubChunk(CompoundTag $subChunk) : SubChunk{
-		return new SubChunk(
+		return new SubChunk([SubChunkConverter::convertSubChunkYZX($subChunk->getByteArray("Blocks"), $subChunk->getByteArray("Data"))]);
-			ChunkUtils::reorderByteArray($subChunk->getByteArray("Blocks")),
+		//ignore legacy light information
-			ChunkUtils::reorderNibbleArray($subChunk->getByteArray("Data"))
-			//ignore legacy light information
-		);
 	}
 
 	protected static function getRegionFileExtension() : string{

src/pocketmine/level/format/io/region/LegacyAnvilChunkTrait.php

@@ -28,12 +28,10 @@ use pocketmine\level\format\io\ChunkUtils;
 use pocketmine\level\format\io\exception\CorruptedChunkException;
 use pocketmine\level\format\SubChunk;
 use pocketmine\nbt\BigEndianNbtSerializer;
-use pocketmine\nbt\NBT;
 use pocketmine\nbt\NbtDataException;
 use pocketmine\nbt\tag\CompoundTag;
 use pocketmine\nbt\tag\IntArrayTag;
 use pocketmine\nbt\tag\ListTag;
-use function array_fill;
 
 /**
  * Trait containing I/O methods for handling legacy Anvil-style chunks.
@@ -49,42 +47,9 @@ use function array_fill;
 trait LegacyAnvilChunkTrait{
 
 	protected function serializeChunk(Chunk $chunk) : string{
-		$nbt = new CompoundTag("Level", []);
+		throw new \RuntimeException("Unsupported");
-		$nbt->setInt("xPos", $chunk->getX());
-		$nbt->setInt("zPos", $chunk->getZ());
-
-		$nbt->setByte("V", 1);
-		$nbt->setLong("LastUpdate", 0); //TODO
-		$nbt->setLong("InhabitedTime", 0); //TODO
-		$nbt->setByte("TerrainPopulated", $chunk->isPopulated() ? 1 : 0);
-		$nbt->setByte("LightPopulated", 0);
-
-		$subChunks = [];
-		foreach($chunk->getSubChunks() as $y => $subChunk){
-			if($subChunk->isEmpty()){
-				continue;
-			}
-
-			$tag = $this->serializeSubChunk($subChunk);
-			$tag->setByte("Y", $y);
-			$subChunks[] = $tag;
-		}
-		$nbt->setTag(new ListTag("Sections", $subChunks, NBT::TAG_Compound));
-
-		$nbt->setByteArray("Biomes", $chunk->getBiomeIdArray());
-		$nbt->setIntArray("HeightMap", array_fill(0, 256, 0));
-
-		$nbt->setTag(new ListTag("Entities", $chunk->getNBTentities(), NBT::TAG_Compound));
-		$nbt->setTag(new ListTag("TileEntities", $chunk->getNBTtiles(), NBT::TAG_Compound));
-
-		//TODO: TileTicks
-
-		$writer = new BigEndianNbtSerializer();
-		return $writer->writeCompressed(new CompoundTag("", [$nbt]), ZLIB_ENCODING_DEFLATE, RegionLoader::$COMPRESSION_LEVEL);
 	}
 
-	abstract protected function serializeSubChunk(SubChunk $subChunk) : CompoundTag;
-
 	/**
 	 * @param string $data
 	 *

src/pocketmine/level/format/io/region/McRegion.php

@@ -26,6 +26,7 @@ namespace pocketmine\level\format\io\region;
 use pocketmine\level\format\Chunk;
 use pocketmine\level\format\io\ChunkUtils;
 use pocketmine\level\format\io\exception\CorruptedChunkException;
+use pocketmine\level\format\io\SubChunkConverter;
 use pocketmine\level\format\SubChunk;
 use pocketmine\nbt\BigEndianNbtSerializer;
 use pocketmine\nbt\NbtDataException;
@@ -33,7 +34,6 @@ use pocketmine\nbt\tag\ByteArrayTag;
 use pocketmine\nbt\tag\IntArrayTag;
 use pocketmine\nbt\tag\ListTag;
 use function str_repeat;
-use function substr;
 
 class McRegion extends RegionLevelProvider{
 
@@ -71,19 +71,7 @@ class McRegion extends RegionLevelProvider{
 		$fullData = $chunk->hasTag("Data", ByteArrayTag::class) ? $chunk->getByteArray("Data") : str_repeat("\x00", 16384);
 
 		for($y = 0; $y < 8; ++$y){
-			$offset = ($y << 4);
+			$subChunks[$y] = new SubChunk([SubChunkConverter::convertSubChunkFromLegacyColumn($fullIds, $fullData, $y)]);
-			$ids = "";
-			for($i = 0; $i < 256; ++$i){
-				$ids .= substr($fullIds, $offset, 16);
-				$offset += 128;
-			}
-			$data = "";
-			$offset = ($y << 3);
-			for($i = 0; $i < 256; ++$i){
-				$data .= substr($fullData, $offset, 8);
-				$offset += 64;
-			}
-			$subChunks[$y] = new SubChunk($ids, $data);
 		}
 
 		if($chunk->hasTag("BiomeColors", IntArrayTag::class)){

src/pocketmine/level/format/io/region/PMAnvil.php

@@ -23,33 +23,19 @@ declare(strict_types=1);
 
 namespace pocketmine\level\format\io\region;
 
-use pocketmine\level\format\io\WritableLevelProvider;
+use pocketmine\level\format\io\SubChunkConverter;
 use pocketmine\level\format\SubChunk;
-use pocketmine\nbt\tag\ByteArrayTag;
 use pocketmine\nbt\tag\CompoundTag;
-use function str_repeat;
 
 /**
  * This format is exactly the same as the PC Anvil format, with the only difference being that the stored data order
  * is XZY instead of YZX for more performance loading and saving worlds.
  */
-class PMAnvil extends RegionLevelProvider implements WritableLevelProvider{
+class PMAnvil extends RegionLevelProvider{
 	use LegacyAnvilChunkTrait;
 
-	protected function serializeSubChunk(SubChunk $subChunk) : CompoundTag{
-		return new CompoundTag("", [
-			new ByteArrayTag("Blocks",     $subChunk->getBlockIdArray()),
-			new ByteArrayTag("Data",       $subChunk->getBlockDataArray()),
-			new ByteArrayTag("SkyLight",   str_repeat("\x00", 2048)),
-			new ByteArrayTag("BlockLight", str_repeat("\x00", 2048))
-		]);
-	}
-
 	protected function deserializeSubChunk(CompoundTag $subChunk) : SubChunk{
-		return new SubChunk(
+		return new SubChunk([SubChunkConverter::convertSubChunkXZY($subChunk->getByteArray("Blocks"), $subChunk->getByteArray("Data"))]);
-			$subChunk->getByteArray("Blocks"),
-			$subChunk->getByteArray("Data")
-		);
 	}
 
 	protected static function getRegionFileExtension() : string{

src/pocketmine/level/generator/populator/GroundCover.php

@@ -47,8 +47,7 @@ class GroundCover extends Populator{
 					}
 
 					for($y = 127; $y > 0; --$y){
-						$id = $chunk->getBlockId($x, $y, $z);
+						if(!BlockFactory::fromFullBlock($chunk->getFullBlock($x, $y, $z))->isTransparent()){
-						if($id !== Block::AIR and !BlockFactory::get($id)->isTransparent()){
 							break;
 						}
 					}
@@ -56,11 +55,11 @@ class GroundCover extends Populator{
 					$endY = $startY - count($cover);
 					for($y = $startY; $y > $endY and $y >= 0; --$y){
 						$b = $cover[$startY - $y];
-						$id = $chunk->getBlockId($x, $y, $z);
+						$id = BlockFactory::fromFullBlock($chunk->getFullBlock($x, $y, $z));
-						if($id === Block::AIR and $b->isSolid()){
+						if($id->getId() === Block::AIR and $b->isSolid()){
 							break;
 						}
-						if($b->canBeFlowedInto() and BlockFactory::get($id) instanceof Liquid){
+						if($b->canBeFlowedInto() and $id instanceof Liquid){
 							continue;
 						}