TextFormat: Fixed tokenize() not being Unicode-aware

This commit is contained in:
Dylan K. Taylor 2019-04-08 15:45:26 +01:00
parent 95313e0a90
commit 383ec8a8e3

View File

@ -74,7 +74,7 @@ abstract class TextFormat{
* @return array
*/
public static function tokenize(string $string) : array{
return preg_split("/(" . TextFormat::ESCAPE . "[0-9a-fk-or])/", $string, -1, PREG_SPLIT_NO_EMPTY | PREG_SPLIT_DELIM_CAPTURE);
return preg_split("/(" . TextFormat::ESCAPE . "[0-9a-fk-or])/u", $string, -1, PREG_SPLIT_NO_EMPTY | PREG_SPLIT_DELIM_CAPTURE);
}
/**