"Fossies" - the Fresh Open Source Software Archive

Member "drupal-9.4.5/vendor/doctrine/lexer/lib/Doctrine/Common/Lexer/AbstractLexer.php" (28 Feb 2022, 7684 Bytes) of package /linux/www/drupal-9.4.5.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) PHP source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. See also the last Fossies "Diffs" side-by-side code changes report for "AbstractLexer.php": 9.3.16_vs_9.4.0.

    1 <?php
    2 
    3 declare(strict_types=1);
    4 
    5 namespace Doctrine\Common\Lexer;
    6 
    7 use ReflectionClass;
    8 
    9 use function implode;
   10 use function in_array;
   11 use function preg_split;
   12 use function sprintf;
   13 use function substr;
   14 
   15 use const PREG_SPLIT_DELIM_CAPTURE;
   16 use const PREG_SPLIT_NO_EMPTY;
   17 use const PREG_SPLIT_OFFSET_CAPTURE;
   18 
   19 /**
   20  * Base class for writing simple lexers, i.e. for creating small DSLs.
   21  *
   22  * @psalm-type Token = array{value: int|string, type:string|int|null, position:int}
   23  */
   24 abstract class AbstractLexer
   25 {
   26     /**
   27      * Lexer original input string.
   28      *
   29      * @var string
   30      */
   31     private $input;
   32 
   33     /**
   34      * Array of scanned tokens.
   35      *
   36      * Each token is an associative array containing three items:
   37      *  - 'value'    : the string value of the token in the input string
   38      *  - 'type'     : the type of the token (identifier, numeric, string, input
   39      *                 parameter, none)
   40      *  - 'position' : the position of the token in the input string
   41      *
   42      * @var mixed[][]
   43      * @psalm-var list<Token>
   44      */
   45     private $tokens = [];
   46 
   47     /**
   48      * Current lexer position in input string.
   49      *
   50      * @var int
   51      */
   52     private $position = 0;
   53 
   54     /**
   55      * Current peek of current lexer position.
   56      *
   57      * @var int
   58      */
   59     private $peek = 0;
   60 
   61     /**
   62      * The next token in the input.
   63      *
   64      * @var mixed[]|null
   65      * @psalm-var Token|null
   66      */
   67     public $lookahead;
   68 
   69     /**
   70      * The last matched/seen token.
   71      *
   72      * @var mixed[]|null
   73      * @psalm-var Token|null
   74      */
   75     public $token;
   76 
   77     /**
   78      * Composed regex for input parsing.
   79      *
   80      * @var string|null
   81      */
   82     private $regex;
   83 
   84     /**
   85      * Sets the input data to be tokenized.
   86      *
   87      * The Lexer is immediately reset and the new input tokenized.
   88      * Any unprocessed tokens from any previous input are lost.
   89      *
   90      * @param string $input The input to be tokenized.
   91      *
   92      * @return void
   93      */
   94     public function setInput($input)
   95     {
   96         $this->input  = $input;
   97         $this->tokens = [];
   98 
   99         $this->reset();
  100         $this->scan($input);
  101     }
  102 
  103     /**
  104      * Resets the lexer.
  105      *
  106      * @return void
  107      */
  108     public function reset()
  109     {
  110         $this->lookahead = null;
  111         $this->token     = null;
  112         $this->peek      = 0;
  113         $this->position  = 0;
  114     }
  115 
  116     /**
  117      * Resets the peek pointer to 0.
  118      *
  119      * @return void
  120      */
  121     public function resetPeek()
  122     {
  123         $this->peek = 0;
  124     }
  125 
  126     /**
  127      * Resets the lexer position on the input to the given position.
  128      *
  129      * @param int $position Position to place the lexical scanner.
  130      *
  131      * @return void
  132      */
  133     public function resetPosition($position = 0)
  134     {
  135         $this->position = $position;
  136     }
  137 
  138     /**
  139      * Retrieve the original lexer's input until a given position.
  140      *
  141      * @param int $position
  142      *
  143      * @return string
  144      */
  145     public function getInputUntilPosition($position)
  146     {
  147         return substr($this->input, 0, $position);
  148     }
  149 
  150     /**
  151      * Checks whether a given token matches the current lookahead.
  152      *
  153      * @param int|string $type
  154      *
  155      * @return bool
  156      */
  157     public function isNextToken($type)
  158     {
  159         return $this->lookahead !== null && $this->lookahead['type'] === $type;
  160     }
  161 
  162     /**
  163      * Checks whether any of the given tokens matches the current lookahead.
  164      *
  165      * @param list<int|string> $types
  166      *
  167      * @return bool
  168      */
  169     public function isNextTokenAny(array $types)
  170     {
  171         return $this->lookahead !== null && in_array($this->lookahead['type'], $types, true);
  172     }
  173 
  174     /**
  175      * Moves to the next token in the input string.
  176      *
  177      * @return bool
  178      */
  179     public function moveNext()
  180     {
  181         $this->peek      = 0;
  182         $this->token     = $this->lookahead;
  183         $this->lookahead = isset($this->tokens[$this->position])
  184             ? $this->tokens[$this->position++] : null;
  185 
  186         return $this->lookahead !== null;
  187     }
  188 
  189     /**
  190      * Tells the lexer to skip input tokens until it sees a token with the given value.
  191      *
  192      * @param string $type The token type to skip until.
  193      *
  194      * @return void
  195      */
  196     public function skipUntil($type)
  197     {
  198         while ($this->lookahead !== null && $this->lookahead['type'] !== $type) {
  199             $this->moveNext();
  200         }
  201     }
  202 
  203     /**
  204      * Checks if given value is identical to the given token.
  205      *
  206      * @param mixed      $value
  207      * @param int|string $token
  208      *
  209      * @return bool
  210      */
  211     public function isA($value, $token)
  212     {
  213         return $this->getType($value) === $token;
  214     }
  215 
  216     /**
  217      * Moves the lookahead token forward.
  218      *
  219      * @return mixed[]|null The next token or NULL if there are no more tokens ahead.
  220      * @psalm-return Token|null
  221      */
  222     public function peek()
  223     {
  224         if (isset($this->tokens[$this->position + $this->peek])) {
  225             return $this->tokens[$this->position + $this->peek++];
  226         }
  227 
  228         return null;
  229     }
  230 
  231     /**
  232      * Peeks at the next token, returns it and immediately resets the peek.
  233      *
  234      * @return mixed[]|null The next token or NULL if there are no more tokens ahead.
  235      * @psalm-return Token|null
  236      */
  237     public function glimpse()
  238     {
  239         $peek       = $this->peek();
  240         $this->peek = 0;
  241 
  242         return $peek;
  243     }
  244 
  245     /**
  246      * Scans the input string for tokens.
  247      *
  248      * @param string $input A query string.
  249      *
  250      * @return void
  251      */
  252     protected function scan($input)
  253     {
  254         if (! isset($this->regex)) {
  255             $this->regex = sprintf(
  256                 '/(%s)|%s/%s',
  257                 implode(')|(', $this->getCatchablePatterns()),
  258                 implode('|', $this->getNonCatchablePatterns()),
  259                 $this->getModifiers()
  260             );
  261         }
  262 
  263         $flags   = PREG_SPLIT_NO_EMPTY | PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_OFFSET_CAPTURE;
  264         $matches = preg_split($this->regex, $input, -1, $flags);
  265 
  266         if ($matches === false) {
  267             // Work around https://bugs.php.net/78122
  268             $matches = [[$input, 0]];
  269         }
  270 
  271         foreach ($matches as $match) {
  272             // Must remain before 'value' assignment since it can change content
  273             $type = $this->getType($match[0]);
  274 
  275             $this->tokens[] = [
  276                 'value' => $match[0],
  277                 'type'  => $type,
  278                 'position' => $match[1],
  279             ];
  280         }
  281     }
  282 
  283     /**
  284      * Gets the literal for a given token.
  285      *
  286      * @param int|string $token
  287      *
  288      * @return int|string
  289      */
  290     public function getLiteral($token)
  291     {
  292         $className = static::class;
  293         $reflClass = new ReflectionClass($className);
  294         $constants = $reflClass->getConstants();
  295 
  296         foreach ($constants as $name => $value) {
  297             if ($value === $token) {
  298                 return $className . '::' . $name;
  299             }
  300         }
  301 
  302         return $token;
  303     }
  304 
  305     /**
  306      * Regex modifiers
  307      *
  308      * @return string
  309      */
  310     protected function getModifiers()
  311     {
  312         return 'iu';
  313     }
  314 
  315     /**
  316      * Lexical catchable patterns.
  317      *
  318      * @return string[]
  319      */
  320     abstract protected function getCatchablePatterns();
  321 
  322     /**
  323      * Lexical non-catchable patterns.
  324      *
  325      * @return string[]
  326      */
  327     abstract protected function getNonCatchablePatterns();
  328 
  329     /**
  330      * Retrieve token type. Also processes the token value if necessary.
  331      *
  332      * @param string $value
  333      *
  334      * @return int|string|null
  335      */
  336     abstract protected function getType(&$value);
  337 }