From a0a8026cba6fca44aed8a8f6d43084e1f76d2f1a Mon Sep 17 00:00:00 2001 From: "Dylan K. Taylor" Date: Sun, 3 Mar 2019 17:37:20 +0000 Subject: [PATCH] Implemented automatic world format conversion --- .gitignore | 1 + src/pocketmine/Server.php | 11 +- src/pocketmine/level/Level.php | 14 +- src/pocketmine/level/LevelManager.php | 29 +++- .../level/format/io/FormatConverter.php | 146 ++++++++++++++++++ .../level/format/io/LevelProvider.php | 25 +-- .../level/format/io/LevelProviderManager.php | 6 +- .../level/format/io/WritableLevelProvider.php | 46 ++++++ .../level/format/io/leveldb/LevelDB.php | 18 ++- .../level/format/io/region/Anvil.php | 3 +- .../level/format/io/region/McRegion.php | 3 +- .../level/format/io/region/PMAnvil.php | 3 +- .../format/io/region/RegionLevelProvider.php | 17 +- 13 files changed, 274 insertions(+), 48 deletions(-) create mode 100644 src/pocketmine/level/format/io/FormatConverter.php create mode 100644 src/pocketmine/level/format/io/WritableLevelProvider.php diff --git a/.gitignore b/.gitignore index 9824e6038..afc4ebbca 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ players/* worlds/* +world_conversion_backups/* plugin_data/* plugins/* bin*/* diff --git a/src/pocketmine/Server.php b/src/pocketmine/Server.php index 93e5afc94..62ccae495 100644 --- a/src/pocketmine/Server.php +++ b/src/pocketmine/Server.php @@ -49,6 +49,7 @@ use pocketmine\lang\LanguageNotFoundException; use pocketmine\lang\TextContainer; use pocketmine\level\biome\Biome; use pocketmine\level\format\io\LevelProviderManager; +use pocketmine\level\format\io\WritableLevelProvider; use pocketmine\level\generator\Generator; use pocketmine\level\generator\GeneratorManager; use pocketmine\level\generator\normal\Normal; @@ -118,6 +119,7 @@ use function getopt; use function implode; use function ini_get; use function ini_set; +use function is_a; use function is_array; use function is_bool; use function is_string; @@ -1244,7 +1246,10 @@ class Server{ $this->pluginManager->registerInterface(new ScriptPluginLoader()); LevelProviderManager::init(); - if(($format = LevelProviderManager::getProviderByName($formatName = (string) $this->getProperty("level-settings.default-format"))) !== null){ + if( + ($format = LevelProviderManager::getProviderByName($formatName = (string) $this->getProperty("level-settings.default-format"))) !== null and + is_a($format, WritableLevelProvider::class, true) + ){ LevelProviderManager::setDefault($format); }elseif($formatName !== ""){ $this->logger->warning($this->language->translateString("pocketmine.level.badDefaultFormat", [$formatName])); @@ -1275,7 +1280,7 @@ class Server{ }elseif(!is_array($options)){ continue; } - if(!$this->levelManager->loadLevel($name)){ + if(!$this->levelManager->loadLevel($name, true)){ if(isset($options["generator"])){ $generatorOptions = explode(":", $options["generator"]); $generator = GeneratorManager::getGenerator(array_shift($generatorOptions)); @@ -1297,7 +1302,7 @@ class Server{ $default = "world"; $this->setConfigString("level-name", "world"); } - if(!$this->levelManager->loadLevel($default)){ + if(!$this->levelManager->loadLevel($default, true)){ $this->levelManager->generateLevel( $default, Generator::convertSeed($this->getConfigString("level-seed")), diff --git a/src/pocketmine/level/Level.php b/src/pocketmine/level/Level.php index b20e4b9d1..1c57ff5ca 100644 --- a/src/pocketmine/level/Level.php +++ b/src/pocketmine/level/Level.php @@ -51,7 +51,7 @@ use pocketmine\level\format\ChunkException; use pocketmine\level\format\EmptySubChunk; use pocketmine\level\format\io\exception\CorruptedChunkException; use pocketmine\level\format\io\exception\UnsupportedChunkFormatException; -use pocketmine\level\format\io\LevelProvider; +use pocketmine\level\format\io\WritableLevelProvider; use pocketmine\level\generator\Generator; use pocketmine\level\generator\GeneratorManager; use pocketmine\level\generator\GeneratorRegisterTask; @@ -159,7 +159,7 @@ class Level implements ChunkManager, Metadatable{ /** @var int */ private $levelId; - /** @var LevelProvider */ + /** @var WritableLevelProvider */ private $provider; /** @var int */ private $providerGarbageCollectionTicker = 0; @@ -347,11 +347,11 @@ class Level implements ChunkManager, Metadatable{ /** * Init the default level data * - * @param Server $server - * @param string $name - * @param LevelProvider $provider + * @param Server $server + * @param string $name + * @param WritableLevelProvider $provider */ - public function __construct(Server $server, string $name, LevelProvider $provider){ + public function __construct(Server $server, string $name, WritableLevelProvider $provider){ $this->levelId = static::$levelIdCounter++; $this->blockMetadata = new BlockMetadataStore($this); $this->server = $server; @@ -426,7 +426,7 @@ class Level implements ChunkManager, Metadatable{ return $this->server; } - final public function getProvider() : LevelProvider{ + final public function getProvider() : WritableLevelProvider{ return $this->provider; } diff --git a/src/pocketmine/level/LevelManager.php b/src/pocketmine/level/LevelManager.php index f4d29b4c4..7f0eebe32 100644 --- a/src/pocketmine/level/LevelManager.php +++ b/src/pocketmine/level/LevelManager.php @@ -28,8 +28,10 @@ use pocketmine\event\level\LevelInitEvent; use pocketmine\event\level\LevelLoadEvent; use pocketmine\event\level\LevelUnloadEvent; use pocketmine\level\format\io\exception\UnsupportedLevelFormatException; +use pocketmine\level\format\io\FormatConverter; use pocketmine\level\format\io\LevelProvider; use pocketmine\level\format\io\LevelProviderManager; +use pocketmine\level\format\io\WritableLevelProvider; use pocketmine\level\generator\Generator; use pocketmine\level\generator\normal\Normal; use pocketmine\Server; @@ -184,12 +186,13 @@ class LevelManager{ * Loads a level from the data directory * * @param string $name + * @param bool $autoUpgrade Converts worlds to the default format if the world's format is not writable / deprecated * * @return bool * * @throws LevelException */ - public function loadLevel(string $name) : bool{ + public function loadLevel(string $name, bool $autoUpgrade = false) : bool{ if(trim($name) === ""){ throw new LevelException("Invalid empty world name"); } @@ -213,9 +216,25 @@ class LevelManager{ } $providerClass = array_shift($providers); + /** + * @var LevelProvider + * @see LevelProvider::__construct() + */ + $provider = new $providerClass($path); + if(!($provider instanceof WritableLevelProvider)){ + if(!$autoUpgrade){ + throw new LevelException("World \"$name\" is in an unsupported format and needs to be upgraded"); + } + $this->server->getLogger()->notice("Upgrading world \"$name\" to new format. This may take a while."); + + $converter = new FormatConverter($provider, LevelProviderManager::getDefault(), $this->server->getDataPath() . "world_conversion_backups", $this->server->getLogger()); + $provider = $converter->execute(); + + $this->server->getLogger()->notice("Upgraded world \"$name\" to new format successfully. Backed up pre-conversion world at " . $converter->getBackupPath()); + } + try{ - /** @see LevelProvider::__construct() */ - $level = new Level($this->server, $name, new $providerClass($path)); + $level = new Level($this->server, $name, $provider); }catch(UnsupportedLevelFormatException $e){ $this->server->getLogger()->error($this->server->getLanguage()->translateString("pocketmine.level.loadError", [$name, $e->getMessage()])); return false; @@ -253,10 +272,10 @@ class LevelManager{ $providerClass = LevelProviderManager::getDefault(); $path = $this->server->getDataPath() . "worlds/" . $name . "/"; - /** @var LevelProvider $providerClass */ + /** @var WritableLevelProvider $providerClass */ $providerClass::generate($path, $name, $seed, $generator, $options); - /** @see LevelProvider::__construct() */ + /** @see WritableLevelProvider::__construct() */ $level = new Level($this->server, $name, new $providerClass($path)); $this->levels[$level->getId()] = $level; diff --git a/src/pocketmine/level/format/io/FormatConverter.php b/src/pocketmine/level/format/io/FormatConverter.php new file mode 100644 index 000000000..b9bce909f --- /dev/null +++ b/src/pocketmine/level/format/io/FormatConverter.php @@ -0,0 +1,146 @@ +oldProvider = $oldProvider; + Utils::testValidInstance($newProvider, WritableLevelProvider::class); + $this->newProvider = $newProvider; + + $this->backupPath = $backupPath . DIRECTORY_SEPARATOR . basename($this->oldProvider->getPath()); + if(!file_exists($backupPath)){ + @mkdir($backupPath); + }elseif(!is_dir($backupPath)){ + throw new \RuntimeException("Backup path $backupPath exists and is not a directory"); + } + + $this->logger = new \PrefixedLogger($logger, "World Converter - " . $this->oldProvider->getLevelData()->getName()); + } + + public function getBackupPath() : string{ + return $this->backupPath; + } + + public function execute() : WritableLevelProvider{ + $new = $this->generateNew(); + + $this->populateLevelData($new->getLevelData()); + $this->convertTerrain($new); + + $path = $this->oldProvider->getPath(); + $this->oldProvider->close(); + $new->close(); + + $this->logger->info("Backing up pre-conversion world to " . $this->backupPath); + rename($path, $this->backupPath); + rename($new->getPath(), $path); + + $this->logger->info("Conversion completed"); + /** + * @see WritableLevelProvider::__construct() + */ + return new $this->newProvider($path); + } + + private function generateNew() : WritableLevelProvider{ + $this->logger->info("Generating new world"); + $data = $this->oldProvider->getLevelData(); + + $convertedOutput = rtrim($this->oldProvider->getPath(), "/\\") . "_converted" . DIRECTORY_SEPARATOR; + if(file_exists($convertedOutput)){ + $this->logger->info("Found previous conversion attempt, deleting..."); + Utils::recursiveUnlink($convertedOutput); + } + $this->newProvider::generate($convertedOutput, $data->getName(), $data->getSeed(), $data->getGenerator(), $data->getGeneratorOptions()); + + /** + * @see WritableLevelProvider::__construct() + */ + return new $this->newProvider($convertedOutput); + } + + private function populateLevelData(LevelData $data) : void{ + $this->logger->info("Converting world manifest"); + $oldData = $this->oldProvider->getLevelData(); + $data->setDifficulty($oldData->getDifficulty()); + $data->setLightningLevel($oldData->getLightningLevel()); + $data->setLightningTime($oldData->getLightningTime()); + $data->setRainLevel($oldData->getRainLevel()); + $data->setRainTime($oldData->getRainTime()); + $data->setSpawn($oldData->getSpawn()); + $data->setTime($oldData->getTime()); + + $this->logger->info("Finished converting manifest"); + //TODO: add more properties as-needed + } + + private function convertTerrain(WritableLevelProvider $new) : void{ + $this->logger->info("Calculating chunk count"); + $count = $this->oldProvider->calculateChunkCount(); + $this->logger->info("Discovered $count chunks"); + + $counter = 0; + + $start = microtime(true); + $thisRound = $start; + static $reportInterval = 256; + foreach($this->oldProvider->getAllChunks(true) as $chunk){ + $new->saveChunk($chunk); + $counter++; + if(($counter % $reportInterval) === 0){ + $time = microtime(true); + $diff = $time - $thisRound; + $thisRound = $time; + $this->logger->info("Converted $counter / $count chunks (" . floor($reportInterval / $diff) . " chunks/sec)"); + } + } + $total = microtime(true) - $start; + $this->logger->info("Converted $counter / $counter chunks in " . round($total, 3) . " seconds (" . floor($counter / $total) . " chunks/sec)"); + } +} diff --git a/src/pocketmine/level/format/io/LevelProvider.php b/src/pocketmine/level/format/io/LevelProvider.php index 4a7e089e9..e70bf2cc3 100644 --- a/src/pocketmine/level/format/io/LevelProvider.php +++ b/src/pocketmine/level/format/io/LevelProvider.php @@ -56,24 +56,6 @@ interface LevelProvider{ */ public static function isValid(string $path) : bool; - /** - * Generate the needed files in the path given - * - * @param string $path - * @param string $name - * @param int $seed - * @param string $generator - * @param array[] $options - */ - public static function generate(string $path, string $name, int $seed, string $generator, array $options = []) : void; - - /** - * Saves a chunk (usually to disk). - * - * @param Chunk $chunk - */ - public function saveChunk(Chunk $chunk) : void; - /** * Loads a chunk (usually from disk storage) and returns it. If the chunk does not exist, null is returned. * @@ -107,9 +89,14 @@ interface LevelProvider{ /** * Returns a generator which yields all the chunks in this level. * + * @param bool $skipCorrupted + * + * @param \Logger|null $logger + * * @return \Generator|Chunk[] + * @throws CorruptedChunkException */ - public function getAllChunks() : \Generator; + public function getAllChunks(bool $skipCorrupted = false, ?\Logger $logger = null) : \Generator; /** * Returns the number of chunks in the provider. Used for world conversion time estimations. diff --git a/src/pocketmine/level/format/io/LevelProviderManager.php b/src/pocketmine/level/format/io/LevelProviderManager.php index 5c9b33f5d..7585f077e 100644 --- a/src/pocketmine/level/format/io/LevelProviderManager.php +++ b/src/pocketmine/level/format/io/LevelProviderManager.php @@ -47,7 +47,7 @@ abstract class LevelProviderManager{ /** * Returns the default format used to generate new levels. * - * @return string + * @return string|WritableLevelProvider */ public static function getDefault() : string{ return self::$default; @@ -56,12 +56,12 @@ abstract class LevelProviderManager{ /** * Sets the default format. * - * @param string $class Class extending LevelProvider + * @param string $class Class implementing WritableLevelProvider * * @throws \InvalidArgumentException */ public static function setDefault(string $class) : void{ - Utils::testValidInstance($class, LevelProvider::class); + Utils::testValidInstance($class, WritableLevelProvider::class); self::$default = $class; } diff --git a/src/pocketmine/level/format/io/WritableLevelProvider.php b/src/pocketmine/level/format/io/WritableLevelProvider.php new file mode 100644 index 000000000..fca2024d6 --- /dev/null +++ b/src/pocketmine/level/format/io/WritableLevelProvider.php @@ -0,0 +1,46 @@ +db->close(); } - public function getAllChunks() : \Generator{ + public function getAllChunks(bool $skipCorrupted = false, ?\Logger $logger = null) : \Generator{ foreach($this->db->getIterator() as $key => $_){ if(strlen($key) === 9 and substr($key, -1) === self::TAG_VERSION){ $chunkX = Binary::readLInt(substr($key, 0, 4)); $chunkZ = Binary::readLInt(substr($key, 4, 4)); - if(($chunk = $this->loadChunk($chunkX, $chunkZ)) !== null){ - yield $chunk; + try{ + if(($chunk = $this->loadChunk($chunkX, $chunkZ)) !== null){ + yield $chunk; + } + }catch(CorruptedChunkException $e){ + if(!$skipCorrupted){ + throw $e; + } + if($logger !== null){ + $logger->error("Skipped corrupted chunk $chunkX $chunkZ (" . $e->getMessage() . ")"); + } } } } diff --git a/src/pocketmine/level/format/io/region/Anvil.php b/src/pocketmine/level/format/io/region/Anvil.php index 998b41840..abeef950d 100644 --- a/src/pocketmine/level/format/io/region/Anvil.php +++ b/src/pocketmine/level/format/io/region/Anvil.php @@ -24,12 +24,13 @@ declare(strict_types=1); namespace pocketmine\level\format\io\region; use pocketmine\level\format\io\ChunkUtils; +use pocketmine\level\format\io\WritableLevelProvider; use pocketmine\level\format\SubChunk; use pocketmine\nbt\tag\ByteArrayTag; use pocketmine\nbt\tag\CompoundTag; use function str_repeat; -class Anvil extends RegionLevelProvider{ +class Anvil extends RegionLevelProvider implements WritableLevelProvider{ use LegacyAnvilChunkTrait; protected function serializeSubChunk(SubChunk $subChunk) : CompoundTag{ diff --git a/src/pocketmine/level/format/io/region/McRegion.php b/src/pocketmine/level/format/io/region/McRegion.php index 11aef6279..194662a84 100644 --- a/src/pocketmine/level/format/io/region/McRegion.php +++ b/src/pocketmine/level/format/io/region/McRegion.php @@ -26,6 +26,7 @@ namespace pocketmine\level\format\io\region; use pocketmine\level\format\Chunk; use pocketmine\level\format\io\ChunkUtils; use pocketmine\level\format\io\exception\CorruptedChunkException; +use pocketmine\level\format\io\WritableLevelProvider; use pocketmine\level\format\SubChunk; use pocketmine\nbt\BigEndianNbtSerializer; use pocketmine\nbt\NBT; @@ -37,7 +38,7 @@ use pocketmine\nbt\tag\ListTag; use function str_repeat; use function substr; -class McRegion extends RegionLevelProvider{ +class McRegion extends RegionLevelProvider implements WritableLevelProvider{ /** * @param Chunk $chunk diff --git a/src/pocketmine/level/format/io/region/PMAnvil.php b/src/pocketmine/level/format/io/region/PMAnvil.php index 566a5b82e..a1b1f46e4 100644 --- a/src/pocketmine/level/format/io/region/PMAnvil.php +++ b/src/pocketmine/level/format/io/region/PMAnvil.php @@ -23,6 +23,7 @@ declare(strict_types=1); namespace pocketmine\level\format\io\region; +use pocketmine\level\format\io\WritableLevelProvider; use pocketmine\level\format\SubChunk; use pocketmine\nbt\tag\ByteArrayTag; use pocketmine\nbt\tag\CompoundTag; @@ -32,7 +33,7 @@ use function str_repeat; * This format is exactly the same as the PC Anvil format, with the only difference being that the stored data order * is XZY instead of YZX for more performance loading and saving worlds. */ -class PMAnvil extends RegionLevelProvider{ +class PMAnvil extends RegionLevelProvider implements WritableLevelProvider{ use LegacyAnvilChunkTrait; protected function serializeSubChunk(SubChunk $subChunk) : CompoundTag{ diff --git a/src/pocketmine/level/format/io/region/RegionLevelProvider.php b/src/pocketmine/level/format/io/region/RegionLevelProvider.php index 2a356ec13..56f8f1ef5 100644 --- a/src/pocketmine/level/format/io/region/RegionLevelProvider.php +++ b/src/pocketmine/level/format/io/region/RegionLevelProvider.php @@ -222,7 +222,7 @@ abstract class RegionLevelProvider extends BaseLevelProvider{ ); } - public function getAllChunks() : \Generator{ + public function getAllChunks(bool $skipCorrupted = false, ?\Logger $logger = null) : \Generator{ $iterator = $this->createRegionIterator(); foreach($iterator as $region){ @@ -231,9 +231,18 @@ abstract class RegionLevelProvider extends BaseLevelProvider{ for($chunkX = $rX; $chunkX < $rX + 32; ++$chunkX){ for($chunkZ = $rZ; $chunkZ < $rZ + 32; ++$chunkZ){ - $chunk = $this->loadChunk($chunkX, $chunkZ); - if($chunk !== null){ - yield $chunk; + try{ + $chunk = $this->loadChunk($chunkX, $chunkZ); + if($chunk !== null){ + yield $chunk; + } + }catch(CorruptedChunkException $e){ + if(!$skipCorrupted){ + throw $e; + } + if($logger !== null){ + $logger->error("Skipped corrupted chunk $chunkX $chunkZ (" . $e->getMessage() . ")"); + } } } }