diff --git a/resources/pocketmine.yml b/resources/pocketmine.yml index 408b5b95b..531924973 100644 --- a/resources/pocketmine.yml +++ b/resources/pocketmine.yml @@ -54,12 +54,6 @@ memory: #This only affects the main thread. Other threads should fire their own collections period: 36000 - #Fire asynchronous tasks to collect garbage from workers - collect-async-worker: true - - #Trigger on low memory - low-memory-trigger: true - #Settings controlling memory dump handling. memory-dump: #Dump memory from async workers as well as the main thread. If you have issues with segfaults when dumping memory, disable this setting. @@ -69,16 +63,6 @@ memory: #Cap maximum render distance per player when low memory is triggered. Set to 0 to disable cap. chunk-radius: 4 - #Do chunk garbage collection on trigger - trigger-chunk-collect: true - - world-caches: - #Disallow adding to world chunk-packet caches when memory is low - disable-chunk-cache: true - #Clear world caches when memory is low - low-memory-trigger: true - - network: #Threshold for batching packets, in bytes. Only these packets will be compressed #Set to 0 to compress everything, -1 to disable. diff --git a/src/GarbageCollectorManager.php b/src/GarbageCollectorManager.php new file mode 100644 index 000000000..a8912a90d --- /dev/null +++ b/src/GarbageCollectorManager.php @@ -0,0 +1,103 @@ +logger = new \PrefixedLogger($logger, "Cyclic Garbage Collector"); + $this->timings = new TimingsHandler("Cyclic Garbage Collector", $parentTimings); + } + + private function adjustGcThreshold(int $cyclesCollected, int $rootsAfterGC) : void{ + //TODO Very simple heuristic for dynamic GC buffer resizing: + //If there are "too few" collections, increase the collection threshold + //by a fixed step + //Adapted from zend_gc.c/gc_adjust_threshold() as of PHP 8.3.14 + if($cyclesCollected < self::GC_THRESHOLD_TRIGGER || $rootsAfterGC >= $this->threshold){ + $this->threshold = min(self::GC_THRESHOLD_MAX, $this->threshold + self::GC_THRESHOLD_STEP); + }elseif($this->threshold > self::GC_THRESHOLD_DEFAULT){ + $this->threshold = max(self::GC_THRESHOLD_DEFAULT, $this->threshold - self::GC_THRESHOLD_STEP); + } + } + + public function getThreshold() : int{ return $this->threshold; } + + public function getCollectionTimeTotalNs() : int{ return $this->collectionTimeTotalNs; } + + public function maybeCollectCycles() : int{ + $rootsBefore = gc_status()["roots"]; + if($rootsBefore < $this->threshold){ + return 0; + } + + $this->timings->startTiming(); + + $start = hrtime(true); + $cycles = gc_collect_cycles(); + $end = hrtime(true); + + $rootsAfter = gc_status()["roots"]; + $this->adjustGcThreshold($cycles, $rootsAfter); + + $this->timings->stopTiming(); + + $time = $end - $start; + $this->collectionTimeTotalNs += $time; + $this->logger->debug("gc_collect_cycles: " . number_format($time) . " ns ($rootsBefore -> $rootsAfter roots, $cycles cycles collected) - total GC time: " . number_format($this->collectionTimeTotalNs) . " ns"); + + return $cycles; + } +} diff --git a/src/MemoryDump.php b/src/MemoryDump.php new file mode 100644 index 000000000..5a9c274cb --- /dev/null +++ b/src/MemoryDump.php @@ -0,0 +1,305 @@ +getProperties() as $property){ + if(!$property->isStatic() || $property->getDeclaringClass()->getName() !== $className){ + continue; + } + + if(!$property->isInitialized()){ + continue; + } + + $staticCount++; + $staticProperties[$className][$property->getName()] = self::continueDump($property->getValue(), $objects, $refCounts, 0, $maxNesting, $maxStringSize); + } + + if(count($staticProperties[$className]) === 0){ + unset($staticProperties[$className]); + } + + foreach($reflection->getMethods() as $method){ + if($method->getDeclaringClass()->getName() !== $reflection->getName()){ + continue; + } + $methodStatics = []; + foreach(Utils::promoteKeys($method->getStaticVariables()) as $name => $variable){ + $methodStatics[$name] = self::continueDump($variable, $objects, $refCounts, 0, $maxNesting, $maxStringSize); + } + if(count($methodStatics) > 0){ + $functionStaticVars[$className . "::" . $method->getName()] = $methodStatics; + $functionStaticVarsCount += count($functionStaticVars); + } + } + } + + file_put_contents(Path::join($outputFolder, "staticProperties.js"), json_encode($staticProperties, JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR)); + $logger->info("Wrote $staticCount static properties"); + + $globalVariables = []; + $globalCount = 0; + + $ignoredGlobals = [ + 'GLOBALS' => true, + '_SERVER' => true, + '_REQUEST' => true, + '_POST' => true, + '_GET' => true, + '_FILES' => true, + '_ENV' => true, + '_COOKIE' => true, + '_SESSION' => true + ]; + + foreach(Utils::promoteKeys($GLOBALS) as $varName => $value){ + if(isset($ignoredGlobals[$varName])){ + continue; + } + + $globalCount++; + $globalVariables[$varName] = self::continueDump($value, $objects, $refCounts, 0, $maxNesting, $maxStringSize); + } + + file_put_contents(Path::join($outputFolder, "globalVariables.js"), json_encode($globalVariables, JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR)); + $logger->info("Wrote $globalCount global variables"); + + foreach(get_defined_functions()["user"] as $function){ + $reflect = new \ReflectionFunction($function); + + $vars = []; + foreach(Utils::promoteKeys($reflect->getStaticVariables()) as $varName => $variable){ + $vars[$varName] = self::continueDump($variable, $objects, $refCounts, 0, $maxNesting, $maxStringSize); + } + if(count($vars) > 0){ + $functionStaticVars[$function] = $vars; + $functionStaticVarsCount += count($vars); + } + } + file_put_contents(Path::join($outputFolder, 'functionStaticVars.js'), json_encode($functionStaticVars, JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR)); + $logger->info("Wrote $functionStaticVarsCount function static variables"); + + $data = self::continueDump($startingObject, $objects, $refCounts, 0, $maxNesting, $maxStringSize); + + do{ + $continue = false; + foreach(Utils::stringifyKeys($objects) as $hash => $object){ + if(!is_object($object)){ + continue; + } + $continue = true; + + $className = get_class($object); + if(!isset($instanceCounts[$className])){ + $instanceCounts[$className] = 1; + }else{ + $instanceCounts[$className]++; + } + + $objects[$hash] = true; + $info = [ + "information" => "$hash@$className", + ]; + if($object instanceof \Closure){ + $info["definition"] = Utils::getNiceClosureName($object); + $info["referencedVars"] = []; + $reflect = new \ReflectionFunction($object); + if(($closureThis = $reflect->getClosureThis()) !== null){ + $info["this"] = self::continueDump($closureThis, $objects, $refCounts, 0, $maxNesting, $maxStringSize); + } + + foreach(Utils::promoteKeys($reflect->getStaticVariables()) as $name => $variable){ + $info["referencedVars"][$name] = self::continueDump($variable, $objects, $refCounts, 0, $maxNesting, $maxStringSize); + } + }else{ + $reflection = new \ReflectionObject($object); + + $info["properties"] = []; + + for($original = $reflection; $reflection !== false; $reflection = $reflection->getParentClass()){ + foreach($reflection->getProperties() as $property){ + if($property->isStatic()){ + continue; + } + + $name = $property->getName(); + if($reflection !== $original){ + if($property->isPrivate()){ + $name = $reflection->getName() . ":" . $name; + }else{ + continue; + } + } + if(!$property->isInitialized($object)){ + continue; + } + + $info["properties"][$name] = self::continueDump($property->getValue($object), $objects, $refCounts, 0, $maxNesting, $maxStringSize); + } + } + } + + fwrite($obData, json_encode($info, JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR) . "\n"); + } + + }while($continue); + + $logger->info("Wrote " . count($objects) . " objects"); + + fclose($obData); + + file_put_contents(Path::join($outputFolder, "serverEntry.js"), json_encode($data, JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR)); + file_put_contents(Path::join($outputFolder, "referenceCounts.js"), json_encode($refCounts, JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR)); + + arsort($instanceCounts, SORT_NUMERIC); + file_put_contents(Path::join($outputFolder, "instanceCounts.js"), json_encode($instanceCounts, JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR)); + + $logger->info("Finished!"); + + ini_set('memory_limit', $hardLimit); + if($gcEnabled){ + gc_enable(); + } + } + + /** + * @param object[] $objects reference parameter + * @param int[] $refCounts reference parameter + * + * @phpstan-param array $objects + * @phpstan-param array $refCounts + * @phpstan-param-out array $objects + * @phpstan-param-out array $refCounts + */ + private static function continueDump(mixed $from, array &$objects, array &$refCounts, int $recursion, int $maxNesting, int $maxStringSize) : mixed{ + if($maxNesting <= 0){ + return "(error) NESTING LIMIT REACHED"; + } + + --$maxNesting; + + if(is_object($from)){ + if(!isset($objects[$hash = spl_object_hash($from)])){ + $objects[$hash] = $from; + $refCounts[$hash] = 0; + } + + ++$refCounts[$hash]; + + $data = "(object) $hash"; + }elseif(is_array($from)){ + if($recursion >= 5){ + return "(error) ARRAY RECURSION LIMIT REACHED"; + } + $data = []; + $numeric = 0; + foreach(Utils::promoteKeys($from) as $key => $value){ + $data[$numeric] = [ + "k" => self::continueDump($key, $objects, $refCounts, $recursion + 1, $maxNesting, $maxStringSize), + "v" => self::continueDump($value, $objects, $refCounts, $recursion + 1, $maxNesting, $maxStringSize), + ]; + $numeric++; + } + }elseif(is_string($from)){ + $data = "(string) len(" . strlen($from) . ") " . substr(Utils::printable($from), 0, $maxStringSize); + }elseif(is_resource($from)){ + $data = "(resource) " . print_r($from, true); + }elseif(is_float($from)){ + $data = "(float) $from"; + }else{ + $data = $from; + } + + return $data; + } +} diff --git a/src/MemoryManager.php b/src/MemoryManager.php index 4308167d3..2b7f5f1d3 100644 --- a/src/MemoryManager.php +++ b/src/MemoryManager.php @@ -29,52 +29,24 @@ use pocketmine\scheduler\DumpWorkerMemoryTask; use pocketmine\scheduler\GarbageCollectionTask; use pocketmine\timings\Timings; use pocketmine\utils\Process; -use pocketmine\utils\Utils; use pocketmine\YmlServerProperties as Yml; -use Symfony\Component\Filesystem\Path; -use function arsort; -use function count; -use function fclose; -use function file_exists; -use function file_put_contents; -use function fopen; -use function fwrite; use function gc_collect_cycles; -use function gc_disable; -use function gc_enable; use function gc_mem_caches; -use function get_class; -use function get_declared_classes; -use function get_defined_functions; -use function ini_get; use function ini_set; use function intdiv; -use function is_array; -use function is_float; -use function is_object; -use function is_resource; -use function is_string; -use function json_encode; use function mb_strtoupper; use function min; -use function mkdir; use function preg_match; -use function print_r; use function round; -use function spl_object_hash; use function sprintf; -use function strlen; -use function substr; -use const JSON_PRETTY_PRINT; -use const JSON_THROW_ON_ERROR; -use const JSON_UNESCAPED_SLASHES; -use const SORT_NUMERIC; class MemoryManager{ private const DEFAULT_CHECK_RATE = Server::TARGET_TICKS_PER_SECOND; private const DEFAULT_CONTINUOUS_TRIGGER_RATE = Server::TARGET_TICKS_PER_SECOND * 2; private const DEFAULT_TICKS_PER_GC = 30 * 60 * Server::TARGET_TICKS_PER_SECOND; + private GarbageCollectorManager $cycleGcManager; + private int $memoryLimit; private int $globalMemoryLimit; private int $checkRate; @@ -88,14 +60,8 @@ class MemoryManager{ private int $garbageCollectionPeriod; private int $garbageCollectionTicker = 0; - private bool $garbageCollectionTrigger; - private bool $garbageCollectionAsync; private int $lowMemChunkRadiusOverride; - private bool $lowMemChunkGC; - - private bool $lowMemDisableChunkCache; - private bool $lowMemClearWorldCache; private bool $dumpWorkers = true; @@ -105,6 +71,7 @@ class MemoryManager{ private Server $server ){ $this->logger = new \PrefixedLogger($server->getLogger(), "Memory Manager"); + $this->cycleGcManager = new GarbageCollectorManager($this->logger, Timings::$memoryManager); $this->init($server->getConfigGroup()); } @@ -142,17 +109,10 @@ class MemoryManager{ $this->continuousTriggerRate = $config->getPropertyInt(Yml::MEMORY_CONTINUOUS_TRIGGER_RATE, self::DEFAULT_CONTINUOUS_TRIGGER_RATE); $this->garbageCollectionPeriod = $config->getPropertyInt(Yml::MEMORY_GARBAGE_COLLECTION_PERIOD, self::DEFAULT_TICKS_PER_GC); - $this->garbageCollectionTrigger = $config->getPropertyBool(Yml::MEMORY_GARBAGE_COLLECTION_LOW_MEMORY_TRIGGER, true); - $this->garbageCollectionAsync = $config->getPropertyBool(Yml::MEMORY_GARBAGE_COLLECTION_COLLECT_ASYNC_WORKER, true); $this->lowMemChunkRadiusOverride = $config->getPropertyInt(Yml::MEMORY_MAX_CHUNKS_CHUNK_RADIUS, 4); - $this->lowMemChunkGC = $config->getPropertyBool(Yml::MEMORY_MAX_CHUNKS_TRIGGER_CHUNK_COLLECT, true); - - $this->lowMemDisableChunkCache = $config->getPropertyBool(Yml::MEMORY_WORLD_CACHES_DISABLE_CHUNK_CACHE, true); - $this->lowMemClearWorldCache = $config->getPropertyBool(Yml::MEMORY_WORLD_CACHES_LOW_MEMORY_TRIGGER, true); $this->dumpWorkers = $config->getPropertyBool(Yml::MEMORY_MEMORY_DUMP_DUMP_ASYNC_WORKER, true); - gc_enable(); } public function isLowMemory() : bool{ @@ -163,8 +123,11 @@ class MemoryManager{ return $this->globalMemoryLimit; } + /** + * @deprecated + */ public function canUseChunkCache() : bool{ - return !$this->lowMemory || !$this->lowMemDisableChunkCache; + return !$this->lowMemory; } /** @@ -180,26 +143,19 @@ class MemoryManager{ public function trigger(int $memory, int $limit, bool $global = false, int $triggerCount = 0) : void{ $this->logger->debug(sprintf("%sLow memory triggered, limit %gMB, using %gMB", $global ? "Global " : "", round(($limit / 1024) / 1024, 2), round(($memory / 1024) / 1024, 2))); - if($this->lowMemClearWorldCache){ - foreach($this->server->getWorldManager()->getWorlds() as $world){ - $world->clearCache(true); - } - ChunkCache::pruneCaches(); + foreach($this->server->getWorldManager()->getWorlds() as $world){ + $world->clearCache(true); } + ChunkCache::pruneCaches(); - if($this->lowMemChunkGC){ - foreach($this->server->getWorldManager()->getWorlds() as $world){ - $world->doChunkGarbageCollection(); - } + foreach($this->server->getWorldManager()->getWorlds() as $world){ + $world->doChunkGarbageCollection(); } $ev = new LowMemoryEvent($memory, $limit, $global, $triggerCount); $ev->call(); - $cycles = 0; - if($this->garbageCollectionTrigger){ - $cycles = $this->triggerGarbageCollector(); - } + $cycles = $this->triggerGarbageCollector(); $this->logger->debug(sprintf("Freed %gMB, $cycles cycles", round(($ev->getMemoryFreed() / 1024) / 1024, 2))); } @@ -239,6 +195,8 @@ class MemoryManager{ if($this->garbageCollectionPeriod > 0 && ++$this->garbageCollectionTicker >= $this->garbageCollectionPeriod){ $this->garbageCollectionTicker = 0; $this->triggerGarbageCollector(); + }else{ + $this->cycleGcManager->maybeCollectCycles(); } Timings::$memoryManager->stopTiming(); @@ -247,14 +205,12 @@ class MemoryManager{ public function triggerGarbageCollector() : int{ Timings::$garbageCollector->startTiming(); - if($this->garbageCollectionAsync){ - $pool = $this->server->getAsyncPool(); - if(($w = $pool->shutdownUnusedWorkers()) > 0){ - $this->logger->debug("Shut down $w idle async pool workers"); - } - foreach($pool->getRunningWorkers() as $i){ - $pool->submitTaskToWorker(new GarbageCollectionTask(), $i); - } + $pool = $this->server->getAsyncPool(); + if(($w = $pool->shutdownUnusedWorkers()) > 0){ + $this->logger->debug("Shut down $w idle async pool workers"); + } + foreach($pool->getRunningWorkers() as $i){ + $pool->submitTaskToWorker(new GarbageCollectionTask(), $i); } $cycles = gc_collect_cycles(); @@ -271,7 +227,7 @@ class MemoryManager{ public function dumpServerMemory(string $outputFolder, int $maxNesting, int $maxStringSize) : void{ $logger = new \PrefixedLogger($this->server->getLogger(), "Memory Dump"); $logger->notice("After the memory dump is done, the server might crash"); - self::dumpMemory($this->server, $outputFolder, $maxNesting, $maxStringSize, $logger); + MemoryDump::dumpMemory($this->server, $outputFolder, $maxNesting, $maxStringSize, $logger); if($this->dumpWorkers){ $pool = $this->server->getAsyncPool(); @@ -283,239 +239,10 @@ class MemoryManager{ /** * Static memory dumper accessible from any thread. + * @deprecated + * @see MemoryDump */ public static function dumpMemory(mixed $startingObject, string $outputFolder, int $maxNesting, int $maxStringSize, \Logger $logger) : void{ - $hardLimit = Utils::assumeNotFalse(ini_get('memory_limit'), "memory_limit INI directive should always exist"); - ini_set('memory_limit', '-1'); - gc_disable(); - - if(!file_exists($outputFolder)){ - mkdir($outputFolder, 0777, true); - } - - $obData = Utils::assumeNotFalse(fopen(Path::join($outputFolder, "objects.js"), "wb+")); - - $objects = []; - - $refCounts = []; - - $instanceCounts = []; - - $staticProperties = []; - $staticCount = 0; - - $functionStaticVars = []; - $functionStaticVarsCount = 0; - - foreach(get_declared_classes() as $className){ - $reflection = new \ReflectionClass($className); - $staticProperties[$className] = []; - foreach($reflection->getProperties() as $property){ - if(!$property->isStatic() || $property->getDeclaringClass()->getName() !== $className){ - continue; - } - - if(!$property->isInitialized()){ - continue; - } - - $staticCount++; - $staticProperties[$className][$property->getName()] = self::continueDump($property->getValue(), $objects, $refCounts, 0, $maxNesting, $maxStringSize); - } - - if(count($staticProperties[$className]) === 0){ - unset($staticProperties[$className]); - } - - foreach($reflection->getMethods() as $method){ - if($method->getDeclaringClass()->getName() !== $reflection->getName()){ - continue; - } - $methodStatics = []; - foreach(Utils::promoteKeys($method->getStaticVariables()) as $name => $variable){ - $methodStatics[$name] = self::continueDump($variable, $objects, $refCounts, 0, $maxNesting, $maxStringSize); - } - if(count($methodStatics) > 0){ - $functionStaticVars[$className . "::" . $method->getName()] = $methodStatics; - $functionStaticVarsCount += count($functionStaticVars); - } - } - } - - file_put_contents(Path::join($outputFolder, "staticProperties.js"), json_encode($staticProperties, JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR)); - $logger->info("Wrote $staticCount static properties"); - - $globalVariables = []; - $globalCount = 0; - - $ignoredGlobals = [ - 'GLOBALS' => true, - '_SERVER' => true, - '_REQUEST' => true, - '_POST' => true, - '_GET' => true, - '_FILES' => true, - '_ENV' => true, - '_COOKIE' => true, - '_SESSION' => true - ]; - - foreach(Utils::promoteKeys($GLOBALS) as $varName => $value){ - if(isset($ignoredGlobals[$varName])){ - continue; - } - - $globalCount++; - $globalVariables[$varName] = self::continueDump($value, $objects, $refCounts, 0, $maxNesting, $maxStringSize); - } - - file_put_contents(Path::join($outputFolder, "globalVariables.js"), json_encode($globalVariables, JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR)); - $logger->info("Wrote $globalCount global variables"); - - foreach(get_defined_functions()["user"] as $function){ - $reflect = new \ReflectionFunction($function); - - $vars = []; - foreach(Utils::promoteKeys($reflect->getStaticVariables()) as $varName => $variable){ - $vars[$varName] = self::continueDump($variable, $objects, $refCounts, 0, $maxNesting, $maxStringSize); - } - if(count($vars) > 0){ - $functionStaticVars[$function] = $vars; - $functionStaticVarsCount += count($vars); - } - } - file_put_contents(Path::join($outputFolder, 'functionStaticVars.js'), json_encode($functionStaticVars, JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR)); - $logger->info("Wrote $functionStaticVarsCount function static variables"); - - $data = self::continueDump($startingObject, $objects, $refCounts, 0, $maxNesting, $maxStringSize); - - do{ - $continue = false; - foreach(Utils::stringifyKeys($objects) as $hash => $object){ - if(!is_object($object)){ - continue; - } - $continue = true; - - $className = get_class($object); - if(!isset($instanceCounts[$className])){ - $instanceCounts[$className] = 1; - }else{ - $instanceCounts[$className]++; - } - - $objects[$hash] = true; - $info = [ - "information" => "$hash@$className", - ]; - if($object instanceof \Closure){ - $info["definition"] = Utils::getNiceClosureName($object); - $info["referencedVars"] = []; - $reflect = new \ReflectionFunction($object); - if(($closureThis = $reflect->getClosureThis()) !== null){ - $info["this"] = self::continueDump($closureThis, $objects, $refCounts, 0, $maxNesting, $maxStringSize); - } - - foreach(Utils::promoteKeys($reflect->getStaticVariables()) as $name => $variable){ - $info["referencedVars"][$name] = self::continueDump($variable, $objects, $refCounts, 0, $maxNesting, $maxStringSize); - } - }else{ - $reflection = new \ReflectionObject($object); - - $info["properties"] = []; - - for($original = $reflection; $reflection !== false; $reflection = $reflection->getParentClass()){ - foreach($reflection->getProperties() as $property){ - if($property->isStatic()){ - continue; - } - - $name = $property->getName(); - if($reflection !== $original){ - if($property->isPrivate()){ - $name = $reflection->getName() . ":" . $name; - }else{ - continue; - } - } - if(!$property->isInitialized($object)){ - continue; - } - - $info["properties"][$name] = self::continueDump($property->getValue($object), $objects, $refCounts, 0, $maxNesting, $maxStringSize); - } - } - } - - fwrite($obData, json_encode($info, JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR) . "\n"); - } - - }while($continue); - - $logger->info("Wrote " . count($objects) . " objects"); - - fclose($obData); - - file_put_contents(Path::join($outputFolder, "serverEntry.js"), json_encode($data, JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR)); - file_put_contents(Path::join($outputFolder, "referenceCounts.js"), json_encode($refCounts, JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR)); - - arsort($instanceCounts, SORT_NUMERIC); - file_put_contents(Path::join($outputFolder, "instanceCounts.js"), json_encode($instanceCounts, JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR)); - - $logger->info("Finished!"); - - ini_set('memory_limit', $hardLimit); - gc_enable(); - } - - /** - * @param object[] $objects reference parameter - * @param int[] $refCounts reference parameter - * - * @phpstan-param array $objects - * @phpstan-param array $refCounts - * @phpstan-param-out array $objects - * @phpstan-param-out array $refCounts - */ - private static function continueDump(mixed $from, array &$objects, array &$refCounts, int $recursion, int $maxNesting, int $maxStringSize) : mixed{ - if($maxNesting <= 0){ - return "(error) NESTING LIMIT REACHED"; - } - - --$maxNesting; - - if(is_object($from)){ - if(!isset($objects[$hash = spl_object_hash($from)])){ - $objects[$hash] = $from; - $refCounts[$hash] = 0; - } - - ++$refCounts[$hash]; - - $data = "(object) $hash"; - }elseif(is_array($from)){ - if($recursion >= 5){ - return "(error) ARRAY RECURSION LIMIT REACHED"; - } - $data = []; - $numeric = 0; - foreach(Utils::promoteKeys($from) as $key => $value){ - $data[$numeric] = [ - "k" => self::continueDump($key, $objects, $refCounts, $recursion + 1, $maxNesting, $maxStringSize), - "v" => self::continueDump($value, $objects, $refCounts, $recursion + 1, $maxNesting, $maxStringSize), - ]; - $numeric++; - } - }elseif(is_string($from)){ - $data = "(string) len(" . strlen($from) . ") " . substr(Utils::printable($from), 0, $maxStringSize); - }elseif(is_resource($from)){ - $data = "(resource) " . print_r($from, true); - }elseif(is_float($from)){ - $data = "(float) $from"; - }else{ - $data = $from; - } - - return $data; + MemoryDump::dumpMemory($startingObject, $outputFolder, $maxNesting, $maxStringSize, $logger); } } diff --git a/src/YmlServerProperties.php b/src/YmlServerProperties.php index 9bd203eef..282b0b3cd 100644 --- a/src/YmlServerProperties.php +++ b/src/YmlServerProperties.php @@ -75,20 +75,14 @@ final class YmlServerProperties{ public const MEMORY_CONTINUOUS_TRIGGER = 'memory.continuous-trigger'; public const MEMORY_CONTINUOUS_TRIGGER_RATE = 'memory.continuous-trigger-rate'; public const MEMORY_GARBAGE_COLLECTION = 'memory.garbage-collection'; - public const MEMORY_GARBAGE_COLLECTION_COLLECT_ASYNC_WORKER = 'memory.garbage-collection.collect-async-worker'; - public const MEMORY_GARBAGE_COLLECTION_LOW_MEMORY_TRIGGER = 'memory.garbage-collection.low-memory-trigger'; public const MEMORY_GARBAGE_COLLECTION_PERIOD = 'memory.garbage-collection.period'; public const MEMORY_GLOBAL_LIMIT = 'memory.global-limit'; public const MEMORY_MAIN_HARD_LIMIT = 'memory.main-hard-limit'; public const MEMORY_MAIN_LIMIT = 'memory.main-limit'; public const MEMORY_MAX_CHUNKS = 'memory.max-chunks'; public const MEMORY_MAX_CHUNKS_CHUNK_RADIUS = 'memory.max-chunks.chunk-radius'; - public const MEMORY_MAX_CHUNKS_TRIGGER_CHUNK_COLLECT = 'memory.max-chunks.trigger-chunk-collect'; public const MEMORY_MEMORY_DUMP = 'memory.memory-dump'; public const MEMORY_MEMORY_DUMP_DUMP_ASYNC_WORKER = 'memory.memory-dump.dump-async-worker'; - public const MEMORY_WORLD_CACHES = 'memory.world-caches'; - public const MEMORY_WORLD_CACHES_DISABLE_CHUNK_CACHE = 'memory.world-caches.disable-chunk-cache'; - public const MEMORY_WORLD_CACHES_LOW_MEMORY_TRIGGER = 'memory.world-caches.low-memory-trigger'; public const NETWORK = 'network'; public const NETWORK_ASYNC_COMPRESSION = 'network.async-compression'; public const NETWORK_ASYNC_COMPRESSION_THRESHOLD = 'network.async-compression-threshold'; diff --git a/src/network/mcpe/raklib/RakLibServer.php b/src/network/mcpe/raklib/RakLibServer.php index 5137b94ba..3b4b8da74 100644 --- a/src/network/mcpe/raklib/RakLibServer.php +++ b/src/network/mcpe/raklib/RakLibServer.php @@ -82,7 +82,11 @@ class RakLibServer extends Thread{ } protected function onRun() : void{ + //TODO: switch to manually triggered GC + //the best time to do it is between ticks when the server would otherwise be sleeping, but RakLib's current + //design doesn't allow this as of 1.1.1 gc_enable(); + ini_set("display_errors", '1'); ini_set("display_startup_errors", '1'); \GlobalLogger::set($this->logger); diff --git a/src/scheduler/AsyncTask.php b/src/scheduler/AsyncTask.php index eb75130b6..0e9b06d7b 100644 --- a/src/scheduler/AsyncTask.php +++ b/src/scheduler/AsyncTask.php @@ -93,6 +93,7 @@ abstract class AsyncTask extends Runnable{ $this->finished = true; AsyncWorker::getNotifier()->wakeupSleeper(); + AsyncWorker::maybeCollectCycles(); } /** diff --git a/src/scheduler/AsyncWorker.php b/src/scheduler/AsyncWorker.php index 5fdfb1ebb..e208d427c 100644 --- a/src/scheduler/AsyncWorker.php +++ b/src/scheduler/AsyncWorker.php @@ -24,12 +24,13 @@ declare(strict_types=1); namespace pocketmine\scheduler; use pmmp\thread\Thread as NativeThread; +use pocketmine\GarbageCollectorManager; use pocketmine\snooze\SleeperHandlerEntry; use pocketmine\snooze\SleeperNotifier; use pocketmine\thread\log\ThreadSafeLogger; use pocketmine\thread\Worker; +use pocketmine\timings\Timings; use pocketmine\utils\AssumptionFailedError; -use function gc_enable; use function ini_set; class AsyncWorker extends Worker{ @@ -37,6 +38,7 @@ class AsyncWorker extends Worker{ private static array $store = []; private static ?SleeperNotifier $notifier = null; + private static ?GarbageCollectorManager $cycleGcManager = null; public function __construct( private ThreadSafeLogger $logger, @@ -52,11 +54,16 @@ class AsyncWorker extends Worker{ throw new AssumptionFailedError("SleeperNotifier not found in thread-local storage"); } + public static function maybeCollectCycles() : void{ + if(self::$cycleGcManager === null){ + throw new AssumptionFailedError("GarbageCollectorManager not found in thread-local storage"); + } + self::$cycleGcManager->maybeCollectCycles(); + } + protected function onRun() : void{ \GlobalLogger::set($this->logger); - gc_enable(); - if($this->memoryLimit > 0){ ini_set('memory_limit', $this->memoryLimit . 'M'); $this->logger->debug("Set memory limit to " . $this->memoryLimit . " MB"); @@ -66,6 +73,7 @@ class AsyncWorker extends Worker{ } self::$notifier = $this->sleeperEntry->createNotifier(); + self::$cycleGcManager = new GarbageCollectorManager($this->logger, Timings::$asyncTaskWorkers); } public function getLogger() : ThreadSafeLogger{ diff --git a/src/scheduler/DumpWorkerMemoryTask.php b/src/scheduler/DumpWorkerMemoryTask.php index 5ef787b5b..fac8d6368 100644 --- a/src/scheduler/DumpWorkerMemoryTask.php +++ b/src/scheduler/DumpWorkerMemoryTask.php @@ -24,7 +24,7 @@ declare(strict_types=1); namespace pocketmine\scheduler; use pmmp\thread\Thread as NativeThread; -use pocketmine\MemoryManager; +use pocketmine\MemoryDump; use Symfony\Component\Filesystem\Path; use function assert; @@ -41,7 +41,7 @@ class DumpWorkerMemoryTask extends AsyncTask{ public function onRun() : void{ $worker = NativeThread::getCurrentThread(); assert($worker instanceof AsyncWorker); - MemoryManager::dumpMemory( + MemoryDump::dumpMemory( $worker, Path::join($this->outputFolder, "AsyncWorker#" . $worker->getAsyncWorkerId()), $this->maxNesting, diff --git a/src/timings/Timings.php b/src/timings/Timings.php index e71700023..ee737f537 100644 --- a/src/timings/Timings.php +++ b/src/timings/Timings.php @@ -131,7 +131,7 @@ abstract class Timings{ /** @var TimingsHandler[] */ private static array $asyncTaskError = []; - private static TimingsHandler $asyncTaskWorkers; + public static TimingsHandler $asyncTaskWorkers; /** @var TimingsHandler[] */ private static array $asyncTaskRun = []; diff --git a/src/world/World.php b/src/world/World.php index ff65377c0..c2d153921 100644 --- a/src/world/World.php +++ b/src/world/World.php @@ -167,6 +167,9 @@ class World implements ChunkManager{ public const DEFAULT_TICKED_BLOCKS_PER_SUBCHUNK_PER_TICK = 3; + //TODO: this could probably do with being a lot bigger + private const BLOCK_CACHE_SIZE_CAP = 2048; + /** * @var Player[] entity runtime ID => Player * @phpstan-var array @@ -202,6 +205,7 @@ class World implements ChunkManager{ * @phpstan-var array> */ private array $blockCache = []; + private int $blockCacheSize = 0; /** * @var AxisAlignedBB[][][] chunkHash => [relativeBlockHash => AxisAlignedBB[]] * @phpstan-var array>> @@ -653,6 +657,7 @@ class World implements ChunkManager{ $this->provider->close(); $this->blockCache = []; + $this->blockCacheSize = 0; $this->blockCollisionBoxCache = []; $this->unloaded = true; @@ -1138,13 +1143,16 @@ class World implements ChunkManager{ public function clearCache(bool $force = false) : void{ if($force){ $this->blockCache = []; + $this->blockCacheSize = 0; $this->blockCollisionBoxCache = []; }else{ - $count = 0; + //Recalculate this when we're asked - blockCacheSize may be higher than the real size + $this->blockCacheSize = 0; foreach($this->blockCache as $list){ - $count += count($list); - if($count > 2048){ + $this->blockCacheSize += count($list); + if($this->blockCacheSize > self::BLOCK_CACHE_SIZE_CAP){ $this->blockCache = []; + $this->blockCacheSize = 0; break; } } @@ -1152,7 +1160,7 @@ class World implements ChunkManager{ $count = 0; foreach($this->blockCollisionBoxCache as $list){ $count += count($list); - if($count > 2048){ + if($count > self::BLOCK_CACHE_SIZE_CAP){ //TODO: Is this really the best logic? $this->blockCollisionBoxCache = []; break; @@ -1161,6 +1169,19 @@ class World implements ChunkManager{ } } + private function trimBlockCache() : void{ + $before = $this->blockCacheSize; + //Since PHP maintains key order, earliest in foreach should be the oldest entries + //Older entries are less likely to be hot, so destroying these should usually have the lowest impact on performance + foreach($this->blockCache as $chunkHash => $blocks){ + unset($this->blockCache[$chunkHash]); + $this->blockCacheSize -= count($blocks); + if($this->blockCacheSize < self::BLOCK_CACHE_SIZE_CAP){ + break; + } + } + } + /** * @return true[] fullID => dummy * @phpstan-return array @@ -1921,6 +1942,10 @@ class World implements ChunkManager{ if($addToCache && $relativeBlockHash !== null){ $this->blockCache[$chunkHash][$relativeBlockHash] = $block; + + if(++$this->blockCacheSize >= self::BLOCK_CACHE_SIZE_CAP){ + $this->trimBlockCache(); + } } return $block; @@ -1967,6 +1992,7 @@ class World implements ChunkManager{ $relativeBlockHash = World::chunkBlockHash($x, $y, $z); unset($this->blockCache[$chunkHash][$relativeBlockHash]); + $this->blockCacheSize--; unset($this->blockCollisionBoxCache[$chunkHash][$relativeBlockHash]); //blocks like fences have collision boxes that reach into neighbouring blocks, so we need to invalidate the //caches for those blocks as well @@ -2570,6 +2596,7 @@ class World implements ChunkManager{ $this->chunks[$chunkHash] = $chunk; + $this->blockCacheSize -= count($this->blockCache[$chunkHash] ?? []); unset($this->blockCache[$chunkHash]); unset($this->blockCollisionBoxCache[$chunkHash]); unset($this->changedBlocks[$chunkHash]); @@ -2854,6 +2881,8 @@ class World implements ChunkManager{ $this->logger->debug("Chunk $x $z has been upgraded, will be saved at the next autosave opportunity"); } $this->chunks[$chunkHash] = $chunk; + + $this->blockCacheSize -= count($this->blockCache[$chunkHash] ?? []); unset($this->blockCache[$chunkHash]); unset($this->blockCollisionBoxCache[$chunkHash]); @@ -3013,6 +3042,7 @@ class World implements ChunkManager{ } unset($this->chunks[$chunkHash]); + $this->blockCacheSize -= count($this->blockCache[$chunkHash] ?? []); unset($this->blockCache[$chunkHash]); unset($this->blockCollisionBoxCache[$chunkHash]); unset($this->changedBlocks[$chunkHash]);