| 148 |
lars |
1 |
<?php declare(strict_types=1);
|
|
|
2 |
|
|
|
3 |
namespace PhpParser;
|
|
|
4 |
|
|
|
5 |
use PhpParser\Parser\Tokens;
|
|
|
6 |
|
|
|
7 |
class Lexer
|
|
|
8 |
{
|
|
|
9 |
protected $code;
|
|
|
10 |
protected $tokens;
|
|
|
11 |
protected $pos;
|
|
|
12 |
protected $line;
|
|
|
13 |
protected $filePos;
|
|
|
14 |
protected $prevCloseTagHasNewline;
|
|
|
15 |
|
|
|
16 |
protected $tokenMap;
|
|
|
17 |
protected $dropTokens;
|
|
|
18 |
protected $identifierTokens;
|
|
|
19 |
|
|
|
20 |
private $attributeStartLineUsed;
|
|
|
21 |
private $attributeEndLineUsed;
|
|
|
22 |
private $attributeStartTokenPosUsed;
|
|
|
23 |
private $attributeEndTokenPosUsed;
|
|
|
24 |
private $attributeStartFilePosUsed;
|
|
|
25 |
private $attributeEndFilePosUsed;
|
|
|
26 |
private $attributeCommentsUsed;
|
|
|
27 |
|
|
|
28 |
/**
|
|
|
29 |
* Creates a Lexer.
|
|
|
30 |
*
|
|
|
31 |
* @param array $options Options array. Currently only the 'usedAttributes' option is supported,
|
|
|
32 |
* which is an array of attributes to add to the AST nodes. Possible
|
|
|
33 |
* attributes are: 'comments', 'startLine', 'endLine', 'startTokenPos',
|
|
|
34 |
* 'endTokenPos', 'startFilePos', 'endFilePos'. The option defaults to the
|
|
|
35 |
* first three. For more info see getNextToken() docs.
|
|
|
36 |
*/
|
|
|
37 |
public function __construct(array $options = []) {
|
|
|
38 |
// Create Map from internal tokens to PhpParser tokens.
|
|
|
39 |
$this->defineCompatibilityTokens();
|
|
|
40 |
$this->tokenMap = $this->createTokenMap();
|
|
|
41 |
$this->identifierTokens = $this->createIdentifierTokenMap();
|
|
|
42 |
|
|
|
43 |
// map of tokens to drop while lexing (the map is only used for isset lookup,
|
|
|
44 |
// that's why the value is simply set to 1; the value is never actually used.)
|
|
|
45 |
$this->dropTokens = array_fill_keys(
|
|
|
46 |
[\T_WHITESPACE, \T_OPEN_TAG, \T_COMMENT, \T_DOC_COMMENT, \T_BAD_CHARACTER], 1
|
|
|
47 |
);
|
|
|
48 |
|
|
|
49 |
$defaultAttributes = ['comments', 'startLine', 'endLine'];
|
|
|
50 |
$usedAttributes = array_fill_keys($options['usedAttributes'] ?? $defaultAttributes, true);
|
|
|
51 |
|
|
|
52 |
// Create individual boolean properties to make these checks faster.
|
|
|
53 |
$this->attributeStartLineUsed = isset($usedAttributes['startLine']);
|
|
|
54 |
$this->attributeEndLineUsed = isset($usedAttributes['endLine']);
|
|
|
55 |
$this->attributeStartTokenPosUsed = isset($usedAttributes['startTokenPos']);
|
|
|
56 |
$this->attributeEndTokenPosUsed = isset($usedAttributes['endTokenPos']);
|
|
|
57 |
$this->attributeStartFilePosUsed = isset($usedAttributes['startFilePos']);
|
|
|
58 |
$this->attributeEndFilePosUsed = isset($usedAttributes['endFilePos']);
|
|
|
59 |
$this->attributeCommentsUsed = isset($usedAttributes['comments']);
|
|
|
60 |
}
|
|
|
61 |
|
|
|
62 |
/**
|
|
|
63 |
* Initializes the lexer for lexing the provided source code.
|
|
|
64 |
*
|
|
|
65 |
* This function does not throw if lexing errors occur. Instead, errors may be retrieved using
|
|
|
66 |
* the getErrors() method.
|
|
|
67 |
*
|
|
|
68 |
* @param string $code The source code to lex
|
|
|
69 |
* @param ErrorHandler|null $errorHandler Error handler to use for lexing errors. Defaults to
|
|
|
70 |
* ErrorHandler\Throwing
|
|
|
71 |
*/
|
|
|
72 |
public function startLexing(string $code, ErrorHandler $errorHandler = null) {
|
|
|
73 |
if (null === $errorHandler) {
|
|
|
74 |
$errorHandler = new ErrorHandler\Throwing();
|
|
|
75 |
}
|
|
|
76 |
|
|
|
77 |
$this->code = $code; // keep the code around for __halt_compiler() handling
|
|
|
78 |
$this->pos = -1;
|
|
|
79 |
$this->line = 1;
|
|
|
80 |
$this->filePos = 0;
|
|
|
81 |
|
|
|
82 |
// If inline HTML occurs without preceding code, treat it as if it had a leading newline.
|
|
|
83 |
// This ensures proper composability, because having a newline is the "safe" assumption.
|
|
|
84 |
$this->prevCloseTagHasNewline = true;
|
|
|
85 |
|
|
|
86 |
$scream = ini_set('xdebug.scream', '0');
|
|
|
87 |
|
|
|
88 |
$this->tokens = @token_get_all($code);
|
|
|
89 |
$this->postprocessTokens($errorHandler);
|
|
|
90 |
|
|
|
91 |
if (false !== $scream) {
|
|
|
92 |
ini_set('xdebug.scream', $scream);
|
|
|
93 |
}
|
|
|
94 |
}
|
|
|
95 |
|
|
|
96 |
private function handleInvalidCharacterRange($start, $end, $line, ErrorHandler $errorHandler) {
|
|
|
97 |
$tokens = [];
|
|
|
98 |
for ($i = $start; $i < $end; $i++) {
|
|
|
99 |
$chr = $this->code[$i];
|
|
|
100 |
if ($chr === "\0") {
|
|
|
101 |
// PHP cuts error message after null byte, so need special case
|
|
|
102 |
$errorMsg = 'Unexpected null byte';
|
|
|
103 |
} else {
|
|
|
104 |
$errorMsg = sprintf(
|
|
|
105 |
'Unexpected character "%s" (ASCII %d)', $chr, ord($chr)
|
|
|
106 |
);
|
|
|
107 |
}
|
|
|
108 |
|
|
|
109 |
$tokens[] = [\T_BAD_CHARACTER, $chr, $line];
|
|
|
110 |
$errorHandler->handleError(new Error($errorMsg, [
|
|
|
111 |
'startLine' => $line,
|
|
|
112 |
'endLine' => $line,
|
|
|
113 |
'startFilePos' => $i,
|
|
|
114 |
'endFilePos' => $i,
|
|
|
115 |
]));
|
|
|
116 |
}
|
|
|
117 |
return $tokens;
|
|
|
118 |
}
|
|
|
119 |
|
|
|
120 |
/**
|
|
|
121 |
* Check whether comment token is unterminated.
|
|
|
122 |
*
|
|
|
123 |
* @return bool
|
|
|
124 |
*/
|
|
|
125 |
private function isUnterminatedComment($token) : bool {
|
|
|
126 |
return ($token[0] === \T_COMMENT || $token[0] === \T_DOC_COMMENT)
|
|
|
127 |
&& substr($token[1], 0, 2) === '/*'
|
|
|
128 |
&& substr($token[1], -2) !== '*/';
|
|
|
129 |
}
|
|
|
130 |
|
|
|
131 |
protected function postprocessTokens(ErrorHandler $errorHandler) {
|
|
|
132 |
// PHP's error handling for token_get_all() is rather bad, so if we want detailed
|
|
|
133 |
// error information we need to compute it ourselves. Invalid character errors are
|
|
|
134 |
// detected by finding "gaps" in the token array. Unterminated comments are detected
|
|
|
135 |
// by checking if a trailing comment has a "*/" at the end.
|
|
|
136 |
//
|
|
|
137 |
// Additionally, we perform a number of canonicalizations here:
|
|
|
138 |
// * Use the PHP 8.0 comment format, which does not include trailing whitespace anymore.
|
|
|
139 |
// * Use PHP 8.0 T_NAME_* tokens.
|
|
|
140 |
// * Use PHP 8.1 T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG and
|
|
|
141 |
// T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG tokens used to disambiguate intersection types.
|
|
|
142 |
|
|
|
143 |
$filePos = 0;
|
|
|
144 |
$line = 1;
|
|
|
145 |
$numTokens = \count($this->tokens);
|
|
|
146 |
for ($i = 0; $i < $numTokens; $i++) {
|
|
|
147 |
$token = $this->tokens[$i];
|
|
|
148 |
|
|
|
149 |
// Since PHP 7.4 invalid characters are represented by a T_BAD_CHARACTER token.
|
|
|
150 |
// In this case we only need to emit an error.
|
|
|
151 |
if ($token[0] === \T_BAD_CHARACTER) {
|
|
|
152 |
$this->handleInvalidCharacterRange($filePos, $filePos + 1, $line, $errorHandler);
|
|
|
153 |
}
|
|
|
154 |
|
|
|
155 |
if ($token[0] === \T_COMMENT && substr($token[1], 0, 2) !== '/*'
|
|
|
156 |
&& preg_match('/(\r\n|\n|\r)$/D', $token[1], $matches)) {
|
|
|
157 |
$trailingNewline = $matches[0];
|
|
|
158 |
$token[1] = substr($token[1], 0, -strlen($trailingNewline));
|
|
|
159 |
$this->tokens[$i] = $token;
|
|
|
160 |
if (isset($this->tokens[$i + 1]) && $this->tokens[$i + 1][0] === \T_WHITESPACE) {
|
|
|
161 |
// Move trailing newline into following T_WHITESPACE token, if it already exists.
|
|
|
162 |
$this->tokens[$i + 1][1] = $trailingNewline . $this->tokens[$i + 1][1];
|
|
|
163 |
$this->tokens[$i + 1][2]--;
|
|
|
164 |
} else {
|
|
|
165 |
// Otherwise, we need to create a new T_WHITESPACE token.
|
|
|
166 |
array_splice($this->tokens, $i + 1, 0, [
|
|
|
167 |
[\T_WHITESPACE, $trailingNewline, $line],
|
|
|
168 |
]);
|
|
|
169 |
$numTokens++;
|
|
|
170 |
}
|
|
|
171 |
}
|
|
|
172 |
|
|
|
173 |
// Emulate PHP 8 T_NAME_* tokens, by combining sequences of T_NS_SEPARATOR and T_STRING
|
|
|
174 |
// into a single token.
|
|
|
175 |
if (\is_array($token)
|
|
|
176 |
&& ($token[0] === \T_NS_SEPARATOR || isset($this->identifierTokens[$token[0]]))) {
|
|
|
177 |
$lastWasSeparator = $token[0] === \T_NS_SEPARATOR;
|
|
|
178 |
$text = $token[1];
|
|
|
179 |
for ($j = $i + 1; isset($this->tokens[$j]); $j++) {
|
|
|
180 |
if ($lastWasSeparator) {
|
|
|
181 |
if (!isset($this->identifierTokens[$this->tokens[$j][0]])) {
|
|
|
182 |
break;
|
|
|
183 |
}
|
|
|
184 |
$lastWasSeparator = false;
|
|
|
185 |
} else {
|
|
|
186 |
if ($this->tokens[$j][0] !== \T_NS_SEPARATOR) {
|
|
|
187 |
break;
|
|
|
188 |
}
|
|
|
189 |
$lastWasSeparator = true;
|
|
|
190 |
}
|
|
|
191 |
$text .= $this->tokens[$j][1];
|
|
|
192 |
}
|
|
|
193 |
if ($lastWasSeparator) {
|
|
|
194 |
// Trailing separator is not part of the name.
|
|
|
195 |
$j--;
|
|
|
196 |
$text = substr($text, 0, -1);
|
|
|
197 |
}
|
|
|
198 |
if ($j > $i + 1) {
|
|
|
199 |
if ($token[0] === \T_NS_SEPARATOR) {
|
|
|
200 |
$type = \T_NAME_FULLY_QUALIFIED;
|
|
|
201 |
} else if ($token[0] === \T_NAMESPACE) {
|
|
|
202 |
$type = \T_NAME_RELATIVE;
|
|
|
203 |
} else {
|
|
|
204 |
$type = \T_NAME_QUALIFIED;
|
|
|
205 |
}
|
|
|
206 |
$token = [$type, $text, $line];
|
|
|
207 |
array_splice($this->tokens, $i, $j - $i, [$token]);
|
|
|
208 |
$numTokens -= $j - $i - 1;
|
|
|
209 |
}
|
|
|
210 |
}
|
|
|
211 |
|
|
|
212 |
if ($token === '&') {
|
|
|
213 |
$next = $i + 1;
|
|
|
214 |
while (isset($this->tokens[$next]) && $this->tokens[$next][0] === \T_WHITESPACE) {
|
|
|
215 |
$next++;
|
|
|
216 |
}
|
|
|
217 |
$followedByVarOrVarArg = isset($this->tokens[$next]) &&
|
|
|
218 |
($this->tokens[$next][0] === \T_VARIABLE || $this->tokens[$next][0] === \T_ELLIPSIS);
|
|
|
219 |
$this->tokens[$i] = $token = [
|
|
|
220 |
$followedByVarOrVarArg
|
|
|
221 |
? \T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG
|
|
|
222 |
: \T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG,
|
|
|
223 |
'&',
|
|
|
224 |
$line,
|
|
|
225 |
];
|
|
|
226 |
}
|
|
|
227 |
|
|
|
228 |
$tokenValue = \is_string($token) ? $token : $token[1];
|
|
|
229 |
$tokenLen = \strlen($tokenValue);
|
|
|
230 |
|
|
|
231 |
if (substr($this->code, $filePos, $tokenLen) !== $tokenValue) {
|
|
|
232 |
// Something is missing, must be an invalid character
|
|
|
233 |
$nextFilePos = strpos($this->code, $tokenValue, $filePos);
|
|
|
234 |
$badCharTokens = $this->handleInvalidCharacterRange(
|
|
|
235 |
$filePos, $nextFilePos, $line, $errorHandler);
|
|
|
236 |
$filePos = (int) $nextFilePos;
|
|
|
237 |
|
|
|
238 |
array_splice($this->tokens, $i, 0, $badCharTokens);
|
|
|
239 |
$numTokens += \count($badCharTokens);
|
|
|
240 |
$i += \count($badCharTokens);
|
|
|
241 |
}
|
|
|
242 |
|
|
|
243 |
$filePos += $tokenLen;
|
|
|
244 |
$line += substr_count($tokenValue, "\n");
|
|
|
245 |
}
|
|
|
246 |
|
|
|
247 |
if ($filePos !== \strlen($this->code)) {
|
|
|
248 |
if (substr($this->code, $filePos, 2) === '/*') {
|
|
|
249 |
// Unlike PHP, HHVM will drop unterminated comments entirely
|
|
|
250 |
$comment = substr($this->code, $filePos);
|
|
|
251 |
$errorHandler->handleError(new Error('Unterminated comment', [
|
|
|
252 |
'startLine' => $line,
|
|
|
253 |
'endLine' => $line + substr_count($comment, "\n"),
|
|
|
254 |
'startFilePos' => $filePos,
|
|
|
255 |
'endFilePos' => $filePos + \strlen($comment),
|
|
|
256 |
]));
|
|
|
257 |
|
|
|
258 |
// Emulate the PHP behavior
|
|
|
259 |
$isDocComment = isset($comment[3]) && $comment[3] === '*';
|
|
|
260 |
$this->tokens[] = [$isDocComment ? \T_DOC_COMMENT : \T_COMMENT, $comment, $line];
|
|
|
261 |
} else {
|
|
|
262 |
// Invalid characters at the end of the input
|
|
|
263 |
$badCharTokens = $this->handleInvalidCharacterRange(
|
|
|
264 |
$filePos, \strlen($this->code), $line, $errorHandler);
|
|
|
265 |
$this->tokens = array_merge($this->tokens, $badCharTokens);
|
|
|
266 |
}
|
|
|
267 |
return;
|
|
|
268 |
}
|
|
|
269 |
|
|
|
270 |
if (count($this->tokens) > 0) {
|
|
|
271 |
// Check for unterminated comment
|
|
|
272 |
$lastToken = $this->tokens[count($this->tokens) - 1];
|
|
|
273 |
if ($this->isUnterminatedComment($lastToken)) {
|
|
|
274 |
$errorHandler->handleError(new Error('Unterminated comment', [
|
|
|
275 |
'startLine' => $line - substr_count($lastToken[1], "\n"),
|
|
|
276 |
'endLine' => $line,
|
|
|
277 |
'startFilePos' => $filePos - \strlen($lastToken[1]),
|
|
|
278 |
'endFilePos' => $filePos,
|
|
|
279 |
]));
|
|
|
280 |
}
|
|
|
281 |
}
|
|
|
282 |
}
|
|
|
283 |
|
|
|
284 |
/**
|
|
|
285 |
* Fetches the next token.
|
|
|
286 |
*
|
|
|
287 |
* The available attributes are determined by the 'usedAttributes' option, which can
|
|
|
288 |
* be specified in the constructor. The following attributes are supported:
|
|
|
289 |
*
|
|
|
290 |
* * 'comments' => Array of PhpParser\Comment or PhpParser\Comment\Doc instances,
|
|
|
291 |
* representing all comments that occurred between the previous
|
|
|
292 |
* non-discarded token and the current one.
|
|
|
293 |
* * 'startLine' => Line in which the node starts.
|
|
|
294 |
* * 'endLine' => Line in which the node ends.
|
|
|
295 |
* * 'startTokenPos' => Offset into the token array of the first token in the node.
|
|
|
296 |
* * 'endTokenPos' => Offset into the token array of the last token in the node.
|
|
|
297 |
* * 'startFilePos' => Offset into the code string of the first character that is part of the node.
|
|
|
298 |
* * 'endFilePos' => Offset into the code string of the last character that is part of the node.
|
|
|
299 |
*
|
|
|
300 |
* @param mixed $value Variable to store token content in
|
|
|
301 |
* @param mixed $startAttributes Variable to store start attributes in
|
|
|
302 |
* @param mixed $endAttributes Variable to store end attributes in
|
|
|
303 |
*
|
|
|
304 |
* @return int Token id
|
|
|
305 |
*/
|
|
|
306 |
public function getNextToken(&$value = null, &$startAttributes = null, &$endAttributes = null) : int {
|
|
|
307 |
$startAttributes = [];
|
|
|
308 |
$endAttributes = [];
|
|
|
309 |
|
|
|
310 |
while (1) {
|
|
|
311 |
if (isset($this->tokens[++$this->pos])) {
|
|
|
312 |
$token = $this->tokens[$this->pos];
|
|
|
313 |
} else {
|
|
|
314 |
// EOF token with ID 0
|
|
|
315 |
$token = "\0";
|
|
|
316 |
}
|
|
|
317 |
|
|
|
318 |
if ($this->attributeStartLineUsed) {
|
|
|
319 |
$startAttributes['startLine'] = $this->line;
|
|
|
320 |
}
|
|
|
321 |
if ($this->attributeStartTokenPosUsed) {
|
|
|
322 |
$startAttributes['startTokenPos'] = $this->pos;
|
|
|
323 |
}
|
|
|
324 |
if ($this->attributeStartFilePosUsed) {
|
|
|
325 |
$startAttributes['startFilePos'] = $this->filePos;
|
|
|
326 |
}
|
|
|
327 |
|
|
|
328 |
if (\is_string($token)) {
|
|
|
329 |
$value = $token;
|
|
|
330 |
if (isset($token[1])) {
|
|
|
331 |
// bug in token_get_all
|
|
|
332 |
$this->filePos += 2;
|
|
|
333 |
$id = ord('"');
|
|
|
334 |
} else {
|
|
|
335 |
$this->filePos += 1;
|
|
|
336 |
$id = ord($token);
|
|
|
337 |
}
|
|
|
338 |
} elseif (!isset($this->dropTokens[$token[0]])) {
|
|
|
339 |
$value = $token[1];
|
|
|
340 |
$id = $this->tokenMap[$token[0]];
|
|
|
341 |
if (\T_CLOSE_TAG === $token[0]) {
|
|
|
342 |
$this->prevCloseTagHasNewline = false !== strpos($token[1], "\n")
|
|
|
343 |
|| false !== strpos($token[1], "\r");
|
|
|
344 |
} elseif (\T_INLINE_HTML === $token[0]) {
|
|
|
345 |
$startAttributes['hasLeadingNewline'] = $this->prevCloseTagHasNewline;
|
|
|
346 |
}
|
|
|
347 |
|
|
|
348 |
$this->line += substr_count($value, "\n");
|
|
|
349 |
$this->filePos += \strlen($value);
|
|
|
350 |
} else {
|
|
|
351 |
$origLine = $this->line;
|
|
|
352 |
$origFilePos = $this->filePos;
|
|
|
353 |
$this->line += substr_count($token[1], "\n");
|
|
|
354 |
$this->filePos += \strlen($token[1]);
|
|
|
355 |
|
|
|
356 |
if (\T_COMMENT === $token[0] || \T_DOC_COMMENT === $token[0]) {
|
|
|
357 |
if ($this->attributeCommentsUsed) {
|
|
|
358 |
$comment = \T_DOC_COMMENT === $token[0]
|
|
|
359 |
? new Comment\Doc($token[1],
|
|
|
360 |
$origLine, $origFilePos, $this->pos,
|
|
|
361 |
$this->line, $this->filePos - 1, $this->pos)
|
|
|
362 |
: new Comment($token[1],
|
|
|
363 |
$origLine, $origFilePos, $this->pos,
|
|
|
364 |
$this->line, $this->filePos - 1, $this->pos);
|
|
|
365 |
$startAttributes['comments'][] = $comment;
|
|
|
366 |
}
|
|
|
367 |
}
|
|
|
368 |
continue;
|
|
|
369 |
}
|
|
|
370 |
|
|
|
371 |
if ($this->attributeEndLineUsed) {
|
|
|
372 |
$endAttributes['endLine'] = $this->line;
|
|
|
373 |
}
|
|
|
374 |
if ($this->attributeEndTokenPosUsed) {
|
|
|
375 |
$endAttributes['endTokenPos'] = $this->pos;
|
|
|
376 |
}
|
|
|
377 |
if ($this->attributeEndFilePosUsed) {
|
|
|
378 |
$endAttributes['endFilePos'] = $this->filePos - 1;
|
|
|
379 |
}
|
|
|
380 |
|
|
|
381 |
return $id;
|
|
|
382 |
}
|
|
|
383 |
|
|
|
384 |
throw new \RuntimeException('Reached end of lexer loop');
|
|
|
385 |
}
|
|
|
386 |
|
|
|
387 |
/**
|
|
|
388 |
* Returns the token array for current code.
|
|
|
389 |
*
|
|
|
390 |
* The token array is in the same format as provided by the
|
|
|
391 |
* token_get_all() function and does not discard tokens (i.e.
|
|
|
392 |
* whitespace and comments are included). The token position
|
|
|
393 |
* attributes are against this token array.
|
|
|
394 |
*
|
|
|
395 |
* @return array Array of tokens in token_get_all() format
|
|
|
396 |
*/
|
|
|
397 |
public function getTokens() : array {
|
|
|
398 |
return $this->tokens;
|
|
|
399 |
}
|
|
|
400 |
|
|
|
401 |
/**
|
|
|
402 |
* Handles __halt_compiler() by returning the text after it.
|
|
|
403 |
*
|
|
|
404 |
* @return string Remaining text
|
|
|
405 |
*/
|
|
|
406 |
public function handleHaltCompiler() : string {
|
|
|
407 |
// text after T_HALT_COMPILER, still including ();
|
|
|
408 |
$textAfter = substr($this->code, $this->filePos);
|
|
|
409 |
|
|
|
410 |
// ensure that it is followed by ();
|
|
|
411 |
// this simplifies the situation, by not allowing any comments
|
|
|
412 |
// in between of the tokens.
|
|
|
413 |
if (!preg_match('~^\s*\(\s*\)\s*(?:;|\?>\r?\n?)~', $textAfter, $matches)) {
|
|
|
414 |
throw new Error('__HALT_COMPILER must be followed by "();"');
|
|
|
415 |
}
|
|
|
416 |
|
|
|
417 |
// prevent the lexer from returning any further tokens
|
|
|
418 |
$this->pos = count($this->tokens);
|
|
|
419 |
|
|
|
420 |
// return with (); removed
|
|
|
421 |
return substr($textAfter, strlen($matches[0]));
|
|
|
422 |
}
|
|
|
423 |
|
|
|
424 |
private function defineCompatibilityTokens() {
|
|
|
425 |
static $compatTokensDefined = false;
|
|
|
426 |
if ($compatTokensDefined) {
|
|
|
427 |
return;
|
|
|
428 |
}
|
|
|
429 |
|
|
|
430 |
$compatTokens = [
|
|
|
431 |
// PHP 7.4
|
|
|
432 |
'T_BAD_CHARACTER',
|
|
|
433 |
'T_FN',
|
|
|
434 |
'T_COALESCE_EQUAL',
|
|
|
435 |
// PHP 8.0
|
|
|
436 |
'T_NAME_QUALIFIED',
|
|
|
437 |
'T_NAME_FULLY_QUALIFIED',
|
|
|
438 |
'T_NAME_RELATIVE',
|
|
|
439 |
'T_MATCH',
|
|
|
440 |
'T_NULLSAFE_OBJECT_OPERATOR',
|
|
|
441 |
'T_ATTRIBUTE',
|
|
|
442 |
// PHP 8.1
|
|
|
443 |
'T_ENUM',
|
|
|
444 |
'T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG',
|
|
|
445 |
'T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG',
|
|
|
446 |
'T_READONLY',
|
|
|
447 |
];
|
|
|
448 |
|
|
|
449 |
// PHP-Parser might be used together with another library that also emulates some or all
|
|
|
450 |
// of these tokens. Perform a sanity-check that all already defined tokens have been
|
|
|
451 |
// assigned a unique ID.
|
|
|
452 |
$usedTokenIds = [];
|
|
|
453 |
foreach ($compatTokens as $token) {
|
|
|
454 |
if (\defined($token)) {
|
|
|
455 |
$tokenId = \constant($token);
|
|
|
456 |
$clashingToken = $usedTokenIds[$tokenId] ?? null;
|
|
|
457 |
if ($clashingToken !== null) {
|
|
|
458 |
throw new \Error(sprintf(
|
|
|
459 |
'Token %s has same ID as token %s, ' .
|
|
|
460 |
'you may be using a library with broken token emulation',
|
|
|
461 |
$token, $clashingToken
|
|
|
462 |
));
|
|
|
463 |
}
|
|
|
464 |
$usedTokenIds[$tokenId] = $token;
|
|
|
465 |
}
|
|
|
466 |
}
|
|
|
467 |
|
|
|
468 |
// Now define any tokens that have not yet been emulated. Try to assign IDs from -1
|
|
|
469 |
// downwards, but skip any IDs that may already be in use.
|
|
|
470 |
$newTokenId = -1;
|
|
|
471 |
foreach ($compatTokens as $token) {
|
|
|
472 |
if (!\defined($token)) {
|
|
|
473 |
while (isset($usedTokenIds[$newTokenId])) {
|
|
|
474 |
$newTokenId--;
|
|
|
475 |
}
|
|
|
476 |
\define($token, $newTokenId);
|
|
|
477 |
$newTokenId--;
|
|
|
478 |
}
|
|
|
479 |
}
|
|
|
480 |
|
|
|
481 |
$compatTokensDefined = true;
|
|
|
482 |
}
|
|
|
483 |
|
|
|
484 |
/**
|
|
|
485 |
* Creates the token map.
|
|
|
486 |
*
|
|
|
487 |
* The token map maps the PHP internal token identifiers
|
|
|
488 |
* to the identifiers used by the Parser. Additionally it
|
|
|
489 |
* maps T_OPEN_TAG_WITH_ECHO to T_ECHO and T_CLOSE_TAG to ';'.
|
|
|
490 |
*
|
|
|
491 |
* @return array The token map
|
|
|
492 |
*/
|
|
|
493 |
protected function createTokenMap() : array {
|
|
|
494 |
$tokenMap = [];
|
|
|
495 |
|
|
|
496 |
// 256 is the minimum possible token number, as everything below
|
|
|
497 |
// it is an ASCII value
|
|
|
498 |
for ($i = 256; $i < 1000; ++$i) {
|
|
|
499 |
if (\T_DOUBLE_COLON === $i) {
|
|
|
500 |
// T_DOUBLE_COLON is equivalent to T_PAAMAYIM_NEKUDOTAYIM
|
|
|
501 |
$tokenMap[$i] = Tokens::T_PAAMAYIM_NEKUDOTAYIM;
|
|
|
502 |
} elseif(\T_OPEN_TAG_WITH_ECHO === $i) {
|
|
|
503 |
// T_OPEN_TAG_WITH_ECHO with dropped T_OPEN_TAG results in T_ECHO
|
|
|
504 |
$tokenMap[$i] = Tokens::T_ECHO;
|
|
|
505 |
} elseif(\T_CLOSE_TAG === $i) {
|
|
|
506 |
// T_CLOSE_TAG is equivalent to ';'
|
|
|
507 |
$tokenMap[$i] = ord(';');
|
|
|
508 |
} elseif ('UNKNOWN' !== $name = token_name($i)) {
|
|
|
509 |
if ('T_HASHBANG' === $name) {
|
|
|
510 |
// HHVM uses a special token for #! hashbang lines
|
|
|
511 |
$tokenMap[$i] = Tokens::T_INLINE_HTML;
|
|
|
512 |
} elseif (defined($name = Tokens::class . '::' . $name)) {
|
|
|
513 |
// Other tokens can be mapped directly
|
|
|
514 |
$tokenMap[$i] = constant($name);
|
|
|
515 |
}
|
|
|
516 |
}
|
|
|
517 |
}
|
|
|
518 |
|
|
|
519 |
// HHVM uses a special token for numbers that overflow to double
|
|
|
520 |
if (defined('T_ONUMBER')) {
|
|
|
521 |
$tokenMap[\T_ONUMBER] = Tokens::T_DNUMBER;
|
|
|
522 |
}
|
|
|
523 |
// HHVM also has a separate token for the __COMPILER_HALT_OFFSET__ constant
|
|
|
524 |
if (defined('T_COMPILER_HALT_OFFSET')) {
|
|
|
525 |
$tokenMap[\T_COMPILER_HALT_OFFSET] = Tokens::T_STRING;
|
|
|
526 |
}
|
|
|
527 |
|
|
|
528 |
// Assign tokens for which we define compatibility constants, as token_name() does not know them.
|
|
|
529 |
$tokenMap[\T_FN] = Tokens::T_FN;
|
|
|
530 |
$tokenMap[\T_COALESCE_EQUAL] = Tokens::T_COALESCE_EQUAL;
|
|
|
531 |
$tokenMap[\T_NAME_QUALIFIED] = Tokens::T_NAME_QUALIFIED;
|
|
|
532 |
$tokenMap[\T_NAME_FULLY_QUALIFIED] = Tokens::T_NAME_FULLY_QUALIFIED;
|
|
|
533 |
$tokenMap[\T_NAME_RELATIVE] = Tokens::T_NAME_RELATIVE;
|
|
|
534 |
$tokenMap[\T_MATCH] = Tokens::T_MATCH;
|
|
|
535 |
$tokenMap[\T_NULLSAFE_OBJECT_OPERATOR] = Tokens::T_NULLSAFE_OBJECT_OPERATOR;
|
|
|
536 |
$tokenMap[\T_ATTRIBUTE] = Tokens::T_ATTRIBUTE;
|
|
|
537 |
$tokenMap[\T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG] = Tokens::T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG;
|
|
|
538 |
$tokenMap[\T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG] = Tokens::T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG;
|
|
|
539 |
$tokenMap[\T_ENUM] = Tokens::T_ENUM;
|
|
|
540 |
$tokenMap[\T_READONLY] = Tokens::T_READONLY;
|
|
|
541 |
|
|
|
542 |
return $tokenMap;
|
|
|
543 |
}
|
|
|
544 |
|
|
|
545 |
private function createIdentifierTokenMap(): array {
|
|
|
546 |
// Based on semi_reserved production.
|
|
|
547 |
return array_fill_keys([
|
|
|
548 |
\T_STRING,
|
|
|
549 |
\T_STATIC, \T_ABSTRACT, \T_FINAL, \T_PRIVATE, \T_PROTECTED, \T_PUBLIC, \T_READONLY,
|
|
|
550 |
\T_INCLUDE, \T_INCLUDE_ONCE, \T_EVAL, \T_REQUIRE, \T_REQUIRE_ONCE, \T_LOGICAL_OR, \T_LOGICAL_XOR, \T_LOGICAL_AND,
|
|
|
551 |
\T_INSTANCEOF, \T_NEW, \T_CLONE, \T_EXIT, \T_IF, \T_ELSEIF, \T_ELSE, \T_ENDIF, \T_ECHO, \T_DO, \T_WHILE,
|
|
|
552 |
\T_ENDWHILE, \T_FOR, \T_ENDFOR, \T_FOREACH, \T_ENDFOREACH, \T_DECLARE, \T_ENDDECLARE, \T_AS, \T_TRY, \T_CATCH,
|
|
|
553 |
\T_FINALLY, \T_THROW, \T_USE, \T_INSTEADOF, \T_GLOBAL, \T_VAR, \T_UNSET, \T_ISSET, \T_EMPTY, \T_CONTINUE, \T_GOTO,
|
|
|
554 |
\T_FUNCTION, \T_CONST, \T_RETURN, \T_PRINT, \T_YIELD, \T_LIST, \T_SWITCH, \T_ENDSWITCH, \T_CASE, \T_DEFAULT,
|
|
|
555 |
\T_BREAK, \T_ARRAY, \T_CALLABLE, \T_EXTENDS, \T_IMPLEMENTS, \T_NAMESPACE, \T_TRAIT, \T_INTERFACE, \T_CLASS,
|
|
|
556 |
\T_CLASS_C, \T_TRAIT_C, \T_FUNC_C, \T_METHOD_C, \T_LINE, \T_FILE, \T_DIR, \T_NS_C, \T_HALT_COMPILER, \T_FN,
|
|
|
557 |
\T_MATCH,
|
|
|
558 |
], true);
|
|
|
559 |
}
|
|
|
560 |
}
|