Implement send buffering and queuing for network sessions (#2358)

Async compression and broadcasts are now reliable and don't have race condition bugs.
This features improved performance and significantly reduced bandwidth wastage.

Reduce Level broadcast latency by ticking network after levels. This ensures that session buffers get flushed as soon as possible after level tick, if level broadcasts were done.

src/pocketmine/network/mcpe/ChunkRequestTask.php

@@ -24,7 +24,6 @@ declare(strict_types=1);
 namespace pocketmine\network\mcpe;
 
 use pocketmine\level\format\Chunk;
-use pocketmine\level\Level;
 use pocketmine\network\mcpe\protocol\FullChunkDataPacket;
 use pocketmine\scheduler\AsyncTask;
 use pocketmine\Server;
@@ -42,8 +41,7 @@ class ChunkRequestTask extends AsyncTask{
 	/** @var int */
 	protected $compressionLevel;
 
-	public function __construct(Level $level, int $chunkX, int $chunkZ, Chunk $chunk){
-		$this->storeLocal($level);
+	public function __construct(int $chunkX, int $chunkZ, Chunk $chunk, CompressBatchPromise $promise){
 		$this->compressionLevel = NetworkCompression::$LEVEL;
 
 		$this->chunk = $chunk->fastSerialize();
@@ -59,6 +57,8 @@ class ChunkRequestTask extends AsyncTask{
 		}
 
 		$this->tiles = $tiles;
+
+		$this->storeLocal($promise);
 	}
 
 	public function onRun() : void{
@@ -76,14 +76,8 @@ class ChunkRequestTask extends AsyncTask{
 	}
 
 	public function onCompletion(Server $server) : void{
-		/** @var Level $level */
-		$level = $this->fetchLocal();
-		if(!$level->isClosed()){
-			if($this->hasResult()){
-				$level->chunkRequestCallback($this->chunkX, $this->chunkZ, $this->getResult());
-			}else{
-				$level->getServer()->getLogger()->error("Chunk request for level " . $level->getName() . ", x=" . $this->chunkX . ", z=" . $this->chunkZ . " doesn't have any result data");
-			}
-		}
+		/** @var CompressBatchPromise $promise */
+		$promise = $this->fetchLocal();
+		$promise->resolve($this->getResult());
 	}
 }

src/pocketmine/network/mcpe/CompressBatchPromise.php

@@ -0,0 +1,62 @@
+<?php
+
+/*
+ *
+ *  ____            _        _   __  __ _                  __  __ ____
+ * |  _ \ ___   ___| | _____| |_|  \/  (_)_ __   ___      |  \/  |  _ \
+ * | |_) / _ \ / __| |/ / _ \ __| |\/| | | '_ \ / _ \_____| |\/| | |_) |
+ * |  __/ (_) | (__|   <  __/ |_| |  | | | | | |  __/_____| |  | |  __/
+ * |_|   \___/ \___|_|\_\___|\__|_|  |_|_|_| |_|\___|     |_|  |_|_|
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * @author PocketMine Team
+ * @link http://www.pocketmine.net/
+ *
+ *
+*/
+
+declare(strict_types=1);
+
+namespace pocketmine\network\mcpe;
+
+class CompressBatchPromise{
+	/** @var callable[] */
+	private $callbacks = [];
+
+	/** @var string|null */
+	private $result = null;
+
+	public function onResolve(callable $callback) : void{
+		if($this->result !== null){
+			$callback($this);
+		}else{
+			$this->callbacks[] = $callback;
+		}
+	}
+
+	public function resolve(string $result) : void{
+		if($this->result !== null){
+			throw new \InvalidStateException("Cannot resolve promise more than once");
+		}
+		$this->result = $result;
+		foreach($this->callbacks as $callback){
+			$callback($this);
+		}
+		$this->callbacks = [];
+	}
+
+	public function getResult() : string{
+		if($this->result === null){
+			throw new \InvalidStateException("Promise has not yet been resolved");
+		}
+		return $this->result;
+	}
+
+	public function hasResult() : bool{
+		return $this->result !== null;
+	}
+}

src/pocketmine/network/mcpe/CompressBatchTask.php

@@ -26,20 +26,20 @@ namespace pocketmine\network\mcpe;
 use pocketmine\scheduler\AsyncTask;
 use pocketmine\Server;
 
-class CompressBatchedTask extends AsyncTask{
+class CompressBatchTask extends AsyncTask{
 
 	private $level;
 	private $data;
 
 	/**
-	 * @param PacketStream     $stream
-	 * @param NetworkSession[] $targets
-	 * @param int              $compressionLevel
+	 * @param PacketStream         $stream
+	 * @param int                  $compressionLevel
+	 * @param CompressBatchPromise $promise
 	 */
-	public function __construct(PacketStream $stream, array $targets, int $compressionLevel){
+	public function __construct(PacketStream $stream, int $compressionLevel, CompressBatchPromise $promise){
 		$this->data = $stream->buffer;
 		$this->level = $compressionLevel;
-		$this->storeLocal($targets);
+		$this->storeLocal($promise);
 	}
 
 	public function onRun() : void{
@@ -47,9 +47,8 @@ class CompressBatchedTask extends AsyncTask{
 	}
 
 	public function onCompletion(Server $server) : void{
-		/** @var NetworkSession[] $targets */
-		$targets = $this->fetchLocal();
-
-		$server->broadcastPacketsCallback($this->getResult(), $targets);
+		/** @var CompressBatchPromise $promise */
+		$promise = $this->fetchLocal();
+		$promise->resolve($this->getResult());
 	}
 }

src/pocketmine/network/mcpe/NetworkSession.php

@@ -68,12 +68,20 @@ class NetworkSession{
 	/** @var NetworkCipher */
 	private $cipher;
 
+	/** @var PacketStream|null */
+	private $sendBuffer;
+
+	/** @var \SplQueue|CompressBatchPromise[] */
+	private $compressedQueue;
+
 	public function __construct(Server $server, NetworkInterface $interface, string $ip, int $port){
 		$this->server = $server;
 		$this->interface = $interface;
 		$this->ip = $ip;
 		$this->port = $port;
 
+		$this->compressedQueue = new \SplQueue();
+
 		$this->connectTime = time();
 		$this->server->getNetwork()->scheduleSessionTick($this);
 
@@ -206,10 +214,10 @@ class NetworkSession{
 				return false;
 			}
 
-			//TODO: implement buffering (this is just a quick fix)
-			$stream = new PacketStream();
-			$stream->putPacket($packet);
-			$this->server->batchPackets([$this], $stream, true, $immediate);
+			$this->addToSendBuffer($packet);
+			if($immediate){
+				$this->flushSendBuffer(true);
+			}
 
 			return true;
 		}finally{
@@ -217,7 +225,62 @@ class NetworkSession{
 		}
 	}
 
-	public function sendEncoded(string $payload, bool $immediate = false) : void{
+	/**
+	 * @internal
+	 * @param DataPacket $packet
+	 */
+	public function addToSendBuffer(DataPacket $packet) : void{
+		$timings = Timings::getSendDataPacketTimings($packet);
+		$timings->startTiming();
+		try{
+			if($this->sendBuffer === null){
+				$this->sendBuffer = new PacketStream();
+			}
+			$this->sendBuffer->putPacket($packet);
+			$this->server->getNetwork()->scheduleSessionTick($this);
+		}finally{
+			$timings->stopTiming();
+		}
+	}
+
+	private function flushSendBuffer(bool $immediate = false) : void{
+		if($this->sendBuffer !== null){
+			$promise = $this->server->prepareBatch($this->sendBuffer, $immediate);
+			$this->sendBuffer = null;
+			$this->queueCompressed($promise, $immediate);
+		}
+	}
+
+	public function queueCompressed(CompressBatchPromise $payload, bool $immediate = false) : void{
+		$this->flushSendBuffer($immediate); //Maintain ordering if possible
+		if($immediate){
+			//Skips all queues
+			$this->sendEncoded($payload->getResult(), true);
+		}else{
+			$this->compressedQueue->enqueue($payload);
+			$payload->onResolve(function(CompressBatchPromise $payload) : void{
+				if($this->connected and $this->compressedQueue->bottom() === $payload){
+					$this->compressedQueue->dequeue(); //result unused
+					$this->sendEncoded($payload->getResult());
+
+					while(!$this->compressedQueue->isEmpty()){
+						/** @var CompressBatchPromise $current */
+						$current = $this->compressedQueue->bottom();
+						if($current->hasResult()){
+							$this->compressedQueue->dequeue();
+
+							$this->sendEncoded($current->getResult());
+						}else{
+							//can't send any more queued until this one is ready
+							break;
+						}
+					}
+				}
+			});
+		}
+	}
+
+	private function sendEncoded(string $payload, bool $immediate = false) : void{
 		if($this->cipher !== null){
 			Timings::$playerNetworkSendEncryptTimer->startTiming();
 			$payload = $this->cipher->encrypt($payload);
@@ -289,6 +352,8 @@ class NetworkSession{
 		$this->handler = null;
 		$this->interface = null;
 		$this->player = null;
+		$this->sendBuffer = null;
+		$this->compressedQueue = null;
 	}
 
 	public function enableEncryption(string $encryptionKey, string $handshakeJwt) : void{
@@ -345,7 +410,10 @@ class NetworkSession{
 			return true; //keep ticking until timeout
 		}
 
-		//TODO: more stuff on tick
+		if($this->sendBuffer !== null){
+			$this->flushSendBuffer();
+		}
+
 		return false;
 	}
 }

src/pocketmine/network/mcpe/handler/PreSpawnSessionHandler.php

@@ -87,7 +87,7 @@ class PreSpawnSessionHandler extends SessionHandler{
 		$this->player->sendAllInventories();
 		$this->player->getInventory()->sendCreativeContents();
 		$this->player->getInventory()->sendHeldItem($this->player);
-		$this->session->sendEncoded($this->server->getCraftingManager()->getCraftingDataPacket());
+		$this->session->queueCompressed($this->server->getCraftingManager()->getCraftingDataPacket());
 
 		$this->server->sendFullPlayerListData($this->player);
 	}