Graphite: Refactor lexer and parser (#17958)

This commit is contained in:
Tobias Skarhed
2019-07-05 16:47:21 +02:00
committed by Torkel Ödegaard
parent 9b468caea4
commit 88a2c6fce7
2 changed files with 82 additions and 73 deletions
+38 -34
View File
@@ -898,7 +898,7 @@ const unicodeLetterTable = [
195101, 195101,
]; ];
const identifierStartTable = []; const identifierStartTable: any[] = [];
for (let i = 0; i < 128; i++) { for (let i = 0; i < 128; i++) {
identifierStartTable[i] = identifierStartTable[i] =
@@ -922,24 +922,28 @@ for (let i = 0; i < 128; i++) {
const identifierPartTable = identifierStartTable; const identifierPartTable = identifierStartTable;
export function Lexer(this: any, expression) { export class Lexer {
this.input = expression; input: any;
this.char = 1; char: number;
this.from = 1; from: number;
}
Lexer.prototype = { constructor(expression: any) {
peek: function(i) { this.input = expression;
this.char = 1;
this.from = 1;
}
peek(i?: number) {
return this.input.charAt(i || 0); return this.input.charAt(i || 0);
}, }
skip: function(i) { skip(i?: number) {
i = i || 1; i = i || 1;
this.char += i; this.char += i;
this.input = this.input.slice(i); this.input = this.input.slice(i);
}, }
tokenize: function() { tokenize() {
const list = []; const list = [];
let token = this.next(); let token = this.next();
while (token) { while (token) {
@@ -947,9 +951,9 @@ Lexer.prototype = {
token = this.next(); token = this.next();
} }
return list; return list;
}, }
next: function() { next() {
this.from = this.char; this.from = this.char;
// Move to the next non-space character. // Move to the next non-space character.
@@ -979,9 +983,9 @@ Lexer.prototype = {
// No token could be matched, give up. // No token could be matched, give up.
return null; return null;
}, }
scanTemplateSequence: function() { scanTemplateSequence() {
if (this.peek() === '[' && this.peek(1) === '[') { if (this.peek() === '[' && this.peek(1) === '[') {
return { return {
type: 'templateStart', type: 'templateStart',
@@ -999,7 +1003,7 @@ Lexer.prototype = {
} }
return null; return null;
}, }
/* /*
* Extract a JavaScript identifier out of the next sequence of * Extract a JavaScript identifier out of the next sequence of
@@ -1007,7 +1011,7 @@ Lexer.prototype = {
* to Identifier this method can also produce BooleanLiteral * to Identifier this method can also produce BooleanLiteral
* (true/false) and NullLiteral (null). * (true/false) and NullLiteral (null).
*/ */
scanIdentifier: function() { scanIdentifier() {
let id = ''; let id = '';
let index = 0; let index = 0;
let type, char; let type, char;
@@ -1020,7 +1024,7 @@ Lexer.prototype = {
// Both approach and unicodeLetterTable were borrowed from // Both approach and unicodeLetterTable were borrowed from
// Google's Traceur. // Google's Traceur.
function isUnicodeLetter(code) { function isUnicodeLetter(code: number) {
for (let i = 0; i < unicodeLetterTable.length; ) { for (let i = 0; i < unicodeLetterTable.length; ) {
if (code < unicodeLetterTable[i++]) { if (code < unicodeLetterTable[i++]) {
return false; return false;
@@ -1034,7 +1038,7 @@ Lexer.prototype = {
return false; return false;
} }
function isHexDigit(str) { function isHexDigit(str: string) {
return /^[0-9a-fA-F]$/.test(str); return /^[0-9a-fA-F]$/.test(str);
} }
@@ -1157,7 +1161,7 @@ Lexer.prototype = {
value: id, value: id,
pos: this.char, pos: this.char,
}; };
}, }
/* /*
* Extract a numeric literal out of the next sequence of * Extract a numeric literal out of the next sequence of
@@ -1168,26 +1172,26 @@ Lexer.prototype = {
* This method's implementation was heavily influenced by the * This method's implementation was heavily influenced by the
* scanNumericLiteral function in the Esprima parser's source code. * scanNumericLiteral function in the Esprima parser's source code.
*/ */
scanNumericLiteral: function(): any { scanNumericLiteral(): any {
let index = 0; let index = 0;
let value = ''; let value = '';
const length = this.input.length; const length = this.input.length;
let char = this.peek(index); let char = this.peek(index);
let bad; let bad;
function isDecimalDigit(str) { function isDecimalDigit(str: string) {
return /^[0-9]$/.test(str); return /^[0-9]$/.test(str);
} }
function isOctalDigit(str) { function isOctalDigit(str: string) {
return /^[0-7]$/.test(str); return /^[0-7]$/.test(str);
} }
function isHexDigit(str) { function isHexDigit(str: string) {
return /^[0-9a-fA-F]$/.test(str); return /^[0-9a-fA-F]$/.test(str);
} }
function isIdentifierStart(ch) { function isIdentifierStart(ch: string) {
return ch === '$' || ch === '_' || ch === '\\' || (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z'); return ch === '$' || ch === '_' || ch === '\\' || (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z');
} }
@@ -1369,9 +1373,9 @@ Lexer.prototype = {
pos: this.char, pos: this.char,
isMalformed: !isFinite(+value), isMalformed: !isFinite(+value),
}; };
}, }
isPunctuator: ch1 => { isPunctuator(ch1: string) {
switch (ch1) { switch (ch1) {
case '.': case '.':
case '(': case '(':
@@ -1383,9 +1387,9 @@ Lexer.prototype = {
} }
return false; return false;
}, }
scanPunctuator: function() { scanPunctuator() {
const ch1 = this.peek(); const ch1 = this.peek();
if (this.isPunctuator(ch1)) { if (this.isPunctuator(ch1)) {
@@ -1397,7 +1401,7 @@ Lexer.prototype = {
} }
return null; return null;
}, }
/* /*
* Extract a string out of the next sequence of characters and/or * Extract a string out of the next sequence of characters and/or
@@ -1410,7 +1414,7 @@ Lexer.prototype = {
* var str = "hello\ * var str = "hello\
* world"; * world";
*/ */
scanStringLiteral: function() { scanStringLiteral() {
/*jshint loopfunc:true */ /*jshint loopfunc:true */
const quote = this.peek(); const quote = this.peek();
@@ -1451,5 +1455,5 @@ Lexer.prototype = {
quote: quote, quote: quote,
pos: this.char, pos: this.char,
}; };
}, }
}; }
@@ -1,18 +1,23 @@
import { Lexer } from './lexer'; import { Lexer } from './lexer';
export function Parser(this: any, expression) { export class Parser {
this.expression = expression; expression: any;
this.lexer = new Lexer(expression); lexer: Lexer;
this.tokens = this.lexer.tokenize(); tokens: any;
this.index = 0; index: number;
}
Parser.prototype = { constructor(expression: any) {
getAst: function() { this.expression = expression;
this.lexer = new Lexer(expression);
this.tokens = this.lexer.tokenize();
this.index = 0;
}
getAst() {
return this.start(); return this.start();
}, }
start: function() { start() {
try { try {
return this.functionCall() || this.metricExpression(); return this.functionCall() || this.metricExpression();
} catch (e) { } catch (e) {
@@ -22,9 +27,9 @@ Parser.prototype = {
pos: e.pos, pos: e.pos,
}; };
} }
}, }
curlyBraceSegment: function() { curlyBraceSegment() {
if (this.match('identifier', '{') || this.match('{')) { if (this.match('identifier', '{') || this.match('{')) {
let curlySegment = ''; let curlySegment = '';
@@ -51,9 +56,9 @@ Parser.prototype = {
} else { } else {
return null; return null;
} }
}, }
metricSegment: function() { metricSegment() {
const curly = this.curlyBraceSegment(); const curly = this.curlyBraceSegment();
if (curly) { if (curly) {
return curly; return curly;
@@ -97,14 +102,14 @@ Parser.prototype = {
this.consumeToken(); this.consumeToken();
return node; return node;
}, }
metricExpression: function() { metricExpression() {
if (!this.match('templateStart') && !this.match('identifier') && !this.match('number') && !this.match('{')) { if (!this.match('templateStart') && !this.match('identifier') && !this.match('number') && !this.match('{')) {
return null; return null;
} }
const node = { const node: any = {
type: 'metric', type: 'metric',
segments: [], segments: [],
}; };
@@ -123,9 +128,9 @@ Parser.prototype = {
} }
return node; return node;
}, }
functionCall: function() { functionCall() {
if (!this.match('identifier', '(')) { if (!this.match('identifier', '(')) {
return null; return null;
} }
@@ -147,9 +152,9 @@ Parser.prototype = {
this.consumeToken(); this.consumeToken();
return node; return node;
}, }
boolExpression: function() { boolExpression() {
if (!this.match('bool')) { if (!this.match('bool')) {
return null; return null;
} }
@@ -158,9 +163,9 @@ Parser.prototype = {
type: 'bool', type: 'bool',
value: this.consumeToken().value === 'true', value: this.consumeToken().value === 'true',
}; };
}, }
functionParameters: function() { functionParameters(): any {
if (this.match(')') || this.match('')) { if (this.match(')') || this.match('')) {
return []; return [];
} }
@@ -179,9 +184,9 @@ Parser.prototype = {
this.consumeToken(); this.consumeToken();
return [param].concat(this.functionParameters()); return [param].concat(this.functionParameters());
}, }
seriesRefExpression: function() { seriesRefExpression() {
if (!this.match('identifier')) { if (!this.match('identifier')) {
return null; return null;
} }
@@ -197,9 +202,9 @@ Parser.prototype = {
type: 'series-ref', type: 'series-ref',
value: token.value, value: token.value,
}; };
}, }
numericLiteral: function() { numericLiteral() {
if (!this.match('number')) { if (!this.match('number')) {
return null; return null;
} }
@@ -208,9 +213,9 @@ Parser.prototype = {
type: 'number', type: 'number',
value: parseFloat(this.consumeToken().value), value: parseFloat(this.consumeToken().value),
}; };
}, }
stringLiteral: function() { stringLiteral() {
if (!this.match('string')) { if (!this.match('string')) {
return null; return null;
} }
@@ -224,29 +229,29 @@ Parser.prototype = {
type: 'string', type: 'string',
value: token.value, value: token.value,
}; };
}, }
errorMark: function(text) { errorMark(text: string) {
const currentToken = this.tokens[this.index]; const currentToken = this.tokens[this.index];
const type = currentToken ? currentToken.type : 'end of string'; const type = currentToken ? currentToken.type : 'end of string';
throw { throw {
message: text + ' instead found ' + type, message: text + ' instead found ' + type,
pos: currentToken ? currentToken.pos : this.lexer.char, pos: currentToken ? currentToken.pos : this.lexer.char,
}; };
}, }
// returns token value and incre // returns token value and incre
consumeToken: function() { consumeToken() {
this.index++; this.index++;
return this.tokens[this.index - 1]; return this.tokens[this.index - 1];
}, }
matchToken: function(type, index) { matchToken(type: any, index: number) {
const token = this.tokens[this.index + index]; const token = this.tokens[this.index + index];
return (token === undefined && type === '') || (token && token.type === type); return (token === undefined && type === '') || (token && token.type === type);
}, }
match: function(token1, token2) { match(token1: any, token2?: any) {
return this.matchToken(token1, 0) && (!token2 || this.matchToken(token2, 1)); return this.matchToken(token1, 0) && (!token2 || this.matchToken(token2, 1));
}, }
}; }