Improved chunk sending, moved chunk encoding and compression to another thread

src/pocketmine/Player.php

@@ -538,6 +538,22 @@ class Player extends Human implements CommandSender, InventoryHolder, IPlayer{
 		}
 	}
 
+	public function sendChunk($x, $z, $payload){
+		if($this->connected === false){
+			return;
+		}
+
+		$pk = new FullChunkDataPacket;
+		$pk->chunkX = $x;
+		$pk->chunkZ = $z;
+		$pk->data = $payload;
+		$cnt = $this->dataPacket($pk, true);
+		if($cnt === false or $cnt === true){
+			return;
+		}
+		$this->chunkACK[$cnt] = Level::chunkHash($x, $z);
+	}
+
 	/**
 	 * Sends, if available, the next ordered chunk to the client
 	 *
@@ -545,7 +561,6 @@ class Player extends Human implements CommandSender, InventoryHolder, IPlayer{
 	 * Changes to this function won't be recorded on the version.
 	 *
 	 */
-
 	public function sendNextChunk(){
 		if($this->connected === false or !isset($this->chunkLoadTask)){
 			return;
@@ -565,7 +580,7 @@ class Player extends Human implements CommandSender, InventoryHolder, IPlayer{
 				$Z = null;
 				Level::getXZ($index, $X, $Z);
 				if(!$this->getLevel()->isChunkPopulated($X, $Z)){
-					$this->chunkLoadTask->setNextRun($this->chunkLoadTask->getNextRun() + 30);
+					$this->chunkLoadTask->setNextRun($this->chunkLoadTask->getNextRun() + 5);
 					return;
 				}
 
@@ -573,22 +588,14 @@ class Player extends Human implements CommandSender, InventoryHolder, IPlayer{
 				$this->usedChunks[$index] = [false, 0];
 
 				$this->getLevel()->useChunk($X, $Z, $this);
-				$pk = new FullChunkDataPacket;
+				$this->getLevel()->requestChunk($X, $Z, $this);
-				$pk->chunkX = $X;
-				$pk->chunkZ = $Z;
-				$pk->data = $this->getLevel()->getNetworkChunk($X, $Z, 0xff);
-				$cnt = $this->dataPacket($pk, true);
-				if($cnt === false or $cnt === true){
-					return;
-				}
-				$this->chunkACK[$cnt] = $index;
 			}
 		}
 
-		if(count($this->usedChunks) < 8 and $this->spawned === true){
+		if(count($this->usedChunks) < 16 and $this->spawned === true){
 			$this->blocked = true;
 		}elseif($this->spawned === true){
-			$this->blocked = false;
+			$this->blocked = false; //TODO: reason of block to revert
 		}
 
 		if(count($this->usedChunks) >= 56 and $this->spawned === false){
@@ -599,7 +606,7 @@ class Player extends Human implements CommandSender, InventoryHolder, IPlayer{
 				}
 			}
 
-			if($spawned < 50){
+			if($spawned < 56){
 				return;
 			}
 

src/pocketmine/level/ChunkRequestTask.php

@@ -0,0 +1,148 @@
+<?php
+
+/*
+ *
+ *  ____            _        _   __  __ _                  __  __ ____
+ * |  _ \ ___   ___| | _____| |_|  \/  (_)_ __   ___      |  \/  |  _ \
+ * | |_) / _ \ / __| |/ / _ \ __| |\/| | | '_ \ / _ \_____| |\/| | |_) |
+ * |  __/ (_) | (__|   <  __/ |_| |  | | | | | |  __/_____| |  | |  __/
+ * |_|   \___/ \___|_|\_\___|\__|_|  |_|_|_| |_|\___|     |_|  |_|_|
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * @author PocketMine Team
+ * @link http://www.pocketmine.net/
+ *
+ *
+*/
+
+namespace pocketmine\level;
+
+use pocketmine\nbt\NBT;
+use pocketmine\scheduler\AsyncTask;
+use pocketmine\Server;
+use pocketmine\tile\Spawnable;
+use pocketmine\utils\Binary;
+
+class ChunkRequestTask extends AsyncTask{
+
+	protected $levelId;
+	protected $chunkX;
+	protected $chunkZ;
+	protected $compressionLevel;
+
+	/** @var string[4096] */
+	protected $ids;
+	/** @var string[2048] */
+	protected $meta;
+	/** @var string[2048] */
+	protected $blockLight;
+	/** @var string[2048] */
+	protected $skyLight;
+	/** @var string[256] */
+	protected $biomeIds;
+	/** @var int[] */
+	protected $biomeColors;
+
+	protected $tiles;
+
+	public function __construct(Level $level, $chunkX, $chunkZ){
+		$this->levelId = $level->getID();
+		$this->chunkX = $chunkX;
+		$this->chunkZ = $chunkZ;
+		$chunk = $level->getChunkAt($chunkX, $chunkZ);
+		$ids = "";
+		$meta = "";
+		$blockLight = "";
+		$skyLight = "";
+		$this->biomeIds = $chunk->getBiomeIdArray();
+		$biomeColors = "";
+		foreach($chunk->getBiomeColorArray() as $color){
+			$biomeColors .= Binary::writeInt($color);
+		}
+
+		$this->biomeColors = $biomeColors;
+
+		for($s = 0; $s < 8; ++$s){
+			$section = $chunk->getSection($s);
+			$ids .= $section->getIdArray();
+			$meta .= $section->getDataArray();
+			$blockLight .= $section->getLightArray();
+			$skyLight .= $section->getSkyLightArray();
+		}
+
+		$this->ids = $ids;
+		$this->meta = $meta;
+		$this->blockLight = $blockLight;
+		$this->skyLight = $skyLight;
+
+		$tiles = "";
+		$nbt = new NBT(NBT::LITTLE_ENDIAN);
+		foreach($chunk->getTiles() as $tile){
+			if($tile instanceof Spawnable){
+				$nbt->setData($tile->getSpawnCompound());
+				$tiles .= $nbt->write();
+			}
+		}
+
+		$this->tiles = $tiles;
+
+		$this->compressionLevel = Level::$COMPRESSION_LEVEL;
+
+	}
+
+	public function onRun(){
+		$orderedIds = "";
+		$orderedData = "";
+		$orderedSkyLight = "";
+		$orderedLight = "";
+
+		for($z = 0; $z < 16; ++$z){
+			for($x = 0; $x < 16; ++$x){
+				$orderedIds .= $this->getColumn($this->ids, $x, $z);
+				$orderedData .= $this->getHalfColumn($this->meta, $x, $z);
+				$orderedSkyLight .= $this->getHalfColumn($this->skyLight, $x, $z);
+				$orderedLight .= $this->getHalfColumn($this->blockLight, $x, $z);
+			}
+		}
+
+		$ordered = zlib_encode(Binary::writeLInt($this->chunkX) . Binary::writeLInt($this->chunkZ) . $orderedIds . $orderedData . $orderedSkyLight . $orderedLight . $this->biomeIds . $this->biomeColors . $this->tiles, ZLIB_ENCODING_DEFLATE, $this->compressionLevel);
+
+		$this->setResult($ordered);
+	}
+
+	public function getColumn($data, $x, $z){
+		$i = ($z << 4) + $x;
+		$column = "";
+		for($y = 0; $y < 128; ++$y){
+			$column .= $data{($y << 8) + $i};
+		}
+		return $column;
+	}
+
+	public function getHalfColumn($data, $x, $z){
+		$i = ($z << 3) + ($x >> 1);
+		$column = "";
+		if(($x & 1) === 0){
+			for($y = 0; $y < 128; $y += 2){
+				$column .= ($data{($y << 7) + $i} & "\x0f") | chr((ord($data{(($y + 1) << 7) + $i}) & 0x0f) << 4);
+			}
+		}else{
+			for($y = 0; $y < 128; $y += 2){
+				$column .= chr((ord($data{($y << 7) + $i}) & 0xf0) >> 4) | ($data{(($y + 1) << 7) + $i} & "\xf0");
+			}
+		}
+		return $column;
+	}
+
+	public function onCompletion(Server $server){
+		$level = $server->getLevel($this->levelId);
+		if($level instanceof Level and $this->hasResult()){
+			$level->chunkRequestCallback($this->chunkX, $this->chunkZ, $this->getResult());
+		}
+	}
+
+}

src/pocketmine/level/Level.php

@@ -118,6 +118,10 @@ class Level implements ChunkManager, Metadatable{
 	/** @var ReversePriorityQueue */
 	private $updateQueue;
 
+	/** @var Player[][] */
+	private $chunkSendQueue = [];
+	private $chunkSendTasks = [];
+
 	private $autoSave = true;
 
 	/** @var BlockMetadataStore */
@@ -427,11 +431,12 @@ class Level implements ChunkManager, Metadatable{
 		while($this->updateQueue->count() > 0 and $this->updateQueue->current()["priority"] <= $currentTick){
 			$block = $this->getBlock($this->updateQueue->extract()["data"]);
 			$block->onUpdate(self::BLOCK_UPDATE_SCHEDULED);
-
 		}
 
 		$this->tickChunks();
 
+		$this->processChunkRequest();
+
 		if($this->nextSave < microtime(true)){
 			$X = null;
 			$Z = null;
@@ -674,7 +679,7 @@ class Level implements ChunkManager, Metadatable{
 				$block->position($pos);
 				$index = Level::chunkHash($pos->x >> 4, $pos->z >> 4);
 				if(ADVANCED_CACHE == true){
-					Cache::remove("world:{$this->getName()}:{$index}");
+					Cache::remove("world:{$this->getID()}:{$index}");
 				}
 				if(!isset($this->changedBlocks[$index])){
 					$this->changedBlocks[$index] = [];
@@ -1315,66 +1320,57 @@ class Level implements ChunkManager, Metadatable{
 		$this->server->getPluginManager()->callEvent(new SpawnChangeEvent($this, $previousSpawn));
 	}
 
-	/**
+	public function requestChunk($x, $z, Player $player){
-	 * Gets a full chunk or parts of it for networking usage, allows cache usage
+		$index = Level::chunkHash($x, $z);
-	 *
+		if(!isset($this->chunkSendQueue[$index])){
-	 * @param int $X
+			$this->chunkSendQueue[$index] = [];
-	 * @param int $Z
-	 * @param int $Yndex bitmap of chunks to be returned
-	 *
-	 * @return bool|mixed|string
-	 */
-	public function getNetworkChunk($X, $Z, $Yndex){
-		if(ADVANCED_CACHE == true and $Yndex === 0xff){
-			$identifier = "world:".($this->getName()).":" . Level::chunkHash($X, $Z);
-			if(($cache = Cache::get($identifier)) !== false){
-				return $cache;
-			}
 		}
 
-		$orderedIds = "";
+		$this->chunkSendQueue[$index][spl_object_hash($player)] = $player;
-		$orderedData = "";
+	}
-		$orderedSkyLight = "";
-		$orderedLight = "";
-		$flag = chr($Yndex);
 
-		$chunk = $this->getChunkAt($X, $Z, true);
+	protected function processChunkRequest(){
-		$biomeIds = $chunk->getBiomeIdArray();
+		if(count($this->chunkSendQueue) > 0){
-		$biomeColors = implode(array_map("pocketmine\\utils\\Binary::writeInt", $chunk->getBiomeColorArray()));
+			$x = null;
-
+			$z = null;
-		/** @var \pocketmine\level\format\ChunkSection[] $sections */
+			foreach($this->chunkSendQueue as $index => $players){
-		$sections = [];
+				if(isset($this->chunkSendTasks[$index])){
-		foreach($chunk->getSections() as $section){
+					continue;
-			$sections[$section->getY()] = $section;
+				}
-		}
+				Level::getXZ($index, $x, $z);
-
+				if(ADVANCED_CACHE == true and ($cache = Cache::get("world:".$this->getID().":" . $index)) !== false){
-		for($x = 0; $x < 16; ++$x){
+					/** @var Player[] $players */
-			for($z = 0; $z < 16; ++$z){
+					foreach($players as $player){
-				for($Y = 0; $Y < 8; ++$Y){
+						if(isset($player->usedChunks[$index])){
-					$orderedIds .= $sections[$Y]->getBlockIdColumn($x, $z);
+							$player->sendChunk($x, $z, $cache);
-					$orderedData .= $sections[$Y]->getBlockDataColumn($x, $z);
+						}
-					$orderedSkyLight .= $sections[$Y]->getBlockSkyLightColumn($x, $z);
+					}
-					$orderedLight .= $sections[$Y]->getBlockLightColumn($x, $z);
+					unset($this->chunkSendQueue[$index]);
+				}else{
+					$task = new ChunkRequestTask($this, $x, $z);
+					$this->server->getScheduler()->scheduleAsyncTask($task);
+					$this->chunkSendTasks[$index] = $task;
 				}
 			}
 		}
+	}
 
-		$tiles = "";
+	public function chunkRequestCallback($x, $z, $payload){
-		$nbt = new NBT(NBT::LITTLE_ENDIAN);
+		$index = Level::chunkHash($x, $z);
-		foreach($chunk->getTiles() as $tile){
+		if(isset($this->chunkSendTasks[$index])){
-			if($tile instanceof Spawnable){
+
-				$nbt->setData($tile->getSpawnCompound());
+			if(ADVANCED_CACHE == true){
-				$tiles .= $nbt->write();
+				Cache::add("world:".$this->getID().":" . $index, $payload, 60);
 			}
+			foreach($this->chunkSendQueue[$index] as $player){
+				/** @var Player $player */
+				if(isset($player->usedChunks[$index])){
+					$player->sendChunk($x, $z, $payload);
+				}
+			}
+			unset($this->chunkSendQueue[$index]);
+			unset($this->chunkSendTasks[$index]);
 		}
-
-		$ordered = zlib_encode(Binary::writeLInt($X) . Binary::writeLInt($Z) . $orderedIds . $orderedData . $orderedSkyLight . $orderedLight . $biomeIds . $biomeColors . $tiles, ZLIB_ENCODING_DEFLATE, self::$COMPRESSION_LEVEL);
-
-		if(ADVANCED_CACHE == true and $Yndex === 0xff){
-			Cache::add($identifier, $ordered, 60);
-		}
-
-		return $ordered;
 	}
 
 	/**
@@ -1501,7 +1497,7 @@ class Level implements ChunkManager, Metadatable{
 		}
 
 		$this->provider->unloadChunk($x, $z);
-		Cache::remove("world:" . $this->getName() . ":$x:$z");
+		Cache::remove("world:" . $this->getID() . ":$x:$z");
 
 		return true;
 	}

src/pocketmine/level/format/generic/BaseChunk.php

@@ -95,7 +95,7 @@ abstract class BaseChunk implements Chunk{
 		if(count($biomeColors) === 256){
 			$this->biomeColors = $biomeColors;
 		}else{
-			$this->biomeColors = array_fill(0, 256, Binary::readInt("\x01\x85\xb2\x4a"));
+			$this->biomeColors = array_fill(0, 256, Binary::readInt("\x00\x85\xb2\x4a"));
 		}
 
 		foreach($entities as $nbt){

src/pocketmine/level/format/generic/EmptyChunkSection.php

@@ -76,7 +76,7 @@ class EmptyChunkSection implements ChunkSection{
 	}
 
 	public function getSkyLightArray(){
-		return str_repeat("\x00", 2048);
+		return str_repeat("\xff", 2048);
 	}
 
 	public function getLightArray(){

src/pocketmine/resources/pocketmine.yml

@@ -7,7 +7,7 @@ settings:
  advanced-cache: false
  upnp-forwarding: false
  send-usage: true
- async-workers: 3
+ async-workers: 4
 
 debug:
  #If > 1, it will show debug messages in the console
@@ -16,7 +16,7 @@ debug:
  commands: false
 
 chunk-sending:
- per-tick: 1 #may be safe to increase later
+ per-tick: 1
  compression-level: 7
 
 chunk-ticking:

src/pocketmine/scheduler/AsyncTask.php

@@ -29,36 +29,49 @@ use pocketmine\Server;
  */
 abstract class AsyncTask extends \Threaded{
 
-	private $complete;
+	private $complete = null;
-	private $finished;
+	private $finished = null;
-	private $result;
+	private $result = null;
 
 	public function run(){
+		$this->lock();
 		$this->finished = false;
 		$this->complete = false;
 		$this->result = null;
+		$this->unlock();
+
 		$this->onRun();
+
+		$this->lock();
 		$this->finished = true;
+		$this->unlock();
+
 	}
 
 	/**
 	 * @return bool
 	 */
 	public function isFinished(){
-		return $this->synchronized(function(){
+
-			return $this->finished === true;
+		return $this->finished === true;
-		});
+	}
+
+	/**
+	 * @return bool
+	 */
+	public function isCompleted(){
+		return $this->complete === true;
+	}
+
+	public function setCompleted(){
+		$this->complete = true;
 	}
 
 	/**
 	 * @return mixed
 	 */
 	public function getResult(){
-		return $this->synchronized(function (){
+		return @unserialize($this->result);
-			$this->finished = true;
-
-			return @unserialize($this->result);
-		});
 	}
 
 	/**

src/pocketmine/scheduler/ServerScheduler.php

@@ -215,12 +215,12 @@ class ServerScheduler{
 
 		if($this->asyncTasks > 0){ //Garbage collector
 			$this->asyncPool->collect(function (AsyncTask $task){
-				if($task->isFinished()){
+				if($task->isFinished() and !$task->isCompleted()){
 					--$this->asyncTasks;
 					$task->onCompletion(Server::getInstance());
+					$task->setCompleted();
 					return true;
 				}
-
 				return false;
 			});
 		}

src/spl/SplClassLoader.php

@@ -4,6 +4,10 @@ if(!interface_exists("SplAutoloader", false)){
 	require("SplAutoloader.php");
 }
 
+if(class_exists("SplClassLoader", false)){
+	return;
+}
+
 /**
  * SplClassLoader implementation that implements the technical interoperability
  * standards for PHP 5.3 namespaces and class names.