Better cURL API, use async in timings (#834)

* Improved cURL functions
* Created BulkCurlTask
* Use asynchronous cURL posting in /timings paste

Closes #509
This commit is contained in:
SOFe
2017-04-25 18:52:18 +08:00
committed by Dylan K. Taylor
parent beed94dfb9
commit 5a9b5db103
3 changed files with 187 additions and 75 deletions

View File

@ -22,6 +22,7 @@
/**
* Various Utilities used around the code
*/
namespace pocketmine\utils;
use pocketmine\ThreadManager;
@ -350,71 +351,113 @@ class Utils{
* GETs an URL using cURL
* NOTE: This is a blocking operation and can take a significant amount of time. It is inadvisable to use this method on the main thread.
*
* @param $page
* @param int $timeout default 10
* @param array $extraHeaders
* @param string &$err Will be set to the output of curl_error(). Use this to retrieve errors that occured during the operation.
* @param $page
* @param int $timeout default 10
* @param array $extraHeaders
* @param string &$err Will be set to the output of curl_error(). Use this to retrieve errors that occured during the operation.
* @param array[] &$headers
* @param int &$httpCode
*
* @return bool|mixed false if an error occurred, mixed data if successful.
*/
public static function getURL($page, $timeout = 10, array $extraHeaders = [], &$err = null){
if(Utils::$online === false){
public static function getURL($page, $timeout = 10, array $extraHeaders = [], &$err = null, &$headers = null, &$httpCode = null){
try{
list($ret, $headers, $httpCode) = self::simpleCurl($page, $timeout, $extraHeaders);
return $ret;
}catch(\RuntimeException $ex){
$err = $ex->getMessage();
return false;
}
$ch = curl_init($page);
curl_setopt($ch, CURLOPT_HTTPHEADER, array_merge(["User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20100101 Firefox/12.0 PocketMine-MP"], $extraHeaders));
curl_setopt($ch, CURLOPT_AUTOREFERER, true);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 2);
curl_setopt($ch, CURLOPT_FORBID_REUSE, 1);
curl_setopt($ch, CURLOPT_FRESH_CONNECT, 1);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, (int) $timeout);
curl_setopt($ch, CURLOPT_TIMEOUT, (int) $timeout);
$ret = curl_exec($ch);
$err = curl_error($ch);
curl_close($ch);
return $ret;
}
/**
* POSTs data to an URL
* NOTE: This is a blocking operation and can take a significant amount of time. It is inadvisable to use this method on the main thread.
*
* @param $page
* @param string $page
* @param array|string $args
* @param int $timeout
* @param array $extraHeaders
* @param string &$err Will be set to the output of curl_error(). Use this to retrieve errors that occured during the operation.
* @param array[] &$headers
* @param int &$httpCode
*
* @return bool|mixed false if an error occurred, mixed data if successful.
*/
public static function postURL($page, $args, $timeout = 10, array $extraHeaders = [], &$err = null){
if(Utils::$online === false){
public static function postURL($page, $args, $timeout = 10, array $extraHeaders = [], &$err = null, &$headers = null, &$httpCode = null){
try{
list($ret, $headers, $httpCode) = self::simpleCurl($page, $timeout, $extraHeaders, [
CURLOPT_POST => 1,
CURLOPT_POSTFIELDS => $args
]);
return $ret;
}catch(\RuntimeException $ex){
$err = $ex->getMessage();
return false;
}
$ch = curl_init($page);
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 2);
curl_setopt($ch, CURLOPT_FORBID_REUSE, 1);
curl_setopt($ch, CURLOPT_FRESH_CONNECT, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, $args);
curl_setopt($ch, CURLOPT_AUTOREFERER, true);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_HTTPHEADER, array_merge(["User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20100101 Firefox/12.0 PocketMine-MP"], $extraHeaders));
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, (int) $timeout);
curl_setopt($ch, CURLOPT_TIMEOUT, (int) $timeout);
$ret = curl_exec($ch);
$err = curl_error($ch);
curl_close($ch);
}
return $ret;
/**
* General cURL shorthand function.
* NOTE: This is a blocking operation and can take a significant amount of time. It is inadvisable to use this method on the main thread.
*
* @param string $page
* @param float|int $timeout The maximum connect timeout and timeout in seconds, correct to ms.
* @param string[] $extraHeaders extra headers to send as a plain string array
* @param array $extraOpts extra CURLOPT_* to set as an [opt => value] map
* @param callable|null $onSuccess function to be called if there is no error. Accepts a resource argument as the cURL handle.
*
* @return array a plain array of three [result body : string, headers : array[], HTTP response code : int]. Headers are grouped by requests with strtolower(header name) as keys and header value as values
*
* @throws \RuntimeException if a cURL error occurs
*/
public static function simpleCurl(string $page, $timeout = 10, array $extraHeaders = [], array $extraOpts = [], callable $onSuccess = null){
if(Utils::$online === false){
throw new \RuntimeException("Server is offline");
}
$ch = curl_init($page);
curl_setopt_array($ch, $extraOpts + [
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_SSL_VERIFYHOST => 2,
CURLOPT_FORBID_REUSE => 1,
CURLOPT_FRESH_CONNECT => 1,
CURLOPT_AUTOREFERER => true,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_CONNECTTIMEOUT_MS => (int) ($timeout * 1000),
CURLOPT_TIMEOUT_MS => (int) ($timeout * 1000),
CURLOPT_HTTPHEADER => array_merge(["User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20100101 Firefox/12.0 PocketMine-MP"], $extraHeaders),
CURLOPT_HEADER => true
]);
try{
$raw = curl_exec($ch);
$error = curl_error($ch);
if($error !== ""){
throw new \RuntimeException($error);
}
$httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
$headerSize = curl_getinfo($ch, CURLINFO_HEADER_SIZE);
$rawHeaders = substr($raw, 0, $headerSize);
$body = substr($raw, $headerSize);
$headers = [];
foreach(explode("\r\n\r\n", $rawHeaders) as $rawHeaderGroup){
$headerGroup = [];
foreach(explode("\r\n", $rawHeaderGroup) as $line){
$nameValue = explode(":", $line, 2);
if(isset($nameValue[1])) $headerGroup[trim(strtolower($nameValue[0]))] = trim($nameValue[1]);
}
$headers[] = $headerGroup;
}
if($onSuccess !== null){
$onSuccess($ch);
}
return [$body, $headers, $httpCode];
}finally{
curl_close($ch);
}
}
public static function javaStringHash($string){