File size: 5,086 Bytes
bc20498 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 |
// @flow
/**
* The Lexer class handles tokenizing the input in various ways. Since our
* parser expects us to be able to backtrack, the lexer allows lexing from any
* given starting point.
*
* Its main exposed function is the `lex` function, which takes a position to
* lex from and a type of token to lex. It defers to the appropriate `_innerLex`
* function.
*
* The various `_innerLex` functions perform the actual lexing of different
* kinds.
*/
import ParseError from "./ParseError";
import SourceLocation from "./SourceLocation";
import {Token} from "./Token";
import type {LexerInterface} from "./Token";
import type Settings from "./Settings";
/* The following tokenRegex
* - matches typical whitespace (but not NBSP etc.) using its first group
* - does not match any control character \x00-\x1f except whitespace
* - does not match a bare backslash
* - matches any ASCII character except those just mentioned
* - does not match the BMP private use area \uE000-\uF8FF
* - does not match bare surrogate code units
* - matches any BMP character except for those just described
* - matches any valid Unicode surrogate pair
* - matches a backslash followed by one or more whitespace characters
* - matches a backslash followed by one or more letters then whitespace
* - matches a backslash followed by any BMP character
* Capturing groups:
* [1] regular whitespace
* [2] backslash followed by whitespace
* [3] anything else, which may include:
* [4] left character of \verb*
* [5] left character of \verb
* [6] backslash followed by word, excluding any trailing whitespace
* Just because the Lexer matches something doesn't mean it's valid input:
* If there is no matching function or symbol definition, the Parser will
* still reject the input.
*/
const spaceRegexString = "[ \r\n\t]";
const controlWordRegexString = "\\\\[a-zA-Z@]+";
const controlSymbolRegexString = "\\\\[^\uD800-\uDFFF]";
const controlWordWhitespaceRegexString =
`(${controlWordRegexString})${spaceRegexString}*`;
const controlSpaceRegexString = "\\\\(\n|[ \r\t]+\n?)[ \r\t]*";
const combiningDiacriticalMarkString = "[\u0300-\u036f]";
export const combiningDiacriticalMarksEndRegex: RegExp =
new RegExp(`${combiningDiacriticalMarkString}+$`);
const tokenRegexString = `(${spaceRegexString}+)|` + // whitespace
`${controlSpaceRegexString}|` + // \whitespace
"([!-\\[\\]-\u2027\u202A-\uD7FF\uF900-\uFFFF]" + // single codepoint
`${combiningDiacriticalMarkString}*` + // ...plus accents
"|[\uD800-\uDBFF][\uDC00-\uDFFF]" + // surrogate pair
`${combiningDiacriticalMarkString}*` + // ...plus accents
"|\\\\verb\\*([^]).*?\\4" + // \verb*
"|\\\\verb([^*a-zA-Z]).*?\\5" + // \verb unstarred
`|${controlWordWhitespaceRegexString}` + // \macroName + spaces
`|${controlSymbolRegexString})`; // \\, \', etc.
/** Main Lexer class */
export default class Lexer implements LexerInterface {
input: string;
settings: Settings;
tokenRegex: RegExp;
// Category codes. The lexer only supports comment characters (14) for now.
// MacroExpander additionally distinguishes active (13).
catcodes: {[string]: number};
constructor(input: string, settings: Settings) {
// Separate accents from characters
this.input = input;
this.settings = settings;
this.tokenRegex = new RegExp(tokenRegexString, 'g');
this.catcodes = {
"%": 14, // comment character
"~": 13, // active character
};
}
setCatcode(char: string, code: number) {
this.catcodes[char] = code;
}
/**
* This function lexes a single token.
*/
lex(): Token {
const input = this.input;
const pos = this.tokenRegex.lastIndex;
if (pos === input.length) {
return new Token("EOF", new SourceLocation(this, pos, pos));
}
const match = this.tokenRegex.exec(input);
if (match === null || match.index !== pos) {
throw new ParseError(
`Unexpected character: '${input[pos]}'`,
new Token(input[pos], new SourceLocation(this, pos, pos + 1)));
}
const text = match[6] || match[3] || (match[2] ? "\\ " : " ");
if (this.catcodes[text] === 14) { // comment character
const nlIndex = input.indexOf('\n', this.tokenRegex.lastIndex);
if (nlIndex === -1) {
this.tokenRegex.lastIndex = input.length; // EOF
this.settings.reportNonstrict("commentAtEnd",
"% comment has no terminating newline; LaTeX would " +
"fail because of commenting the end of math mode (e.g. $)");
} else {
this.tokenRegex.lastIndex = nlIndex + 1;
}
return this.lex();
}
return new Token(text, new SourceLocation(this, pos,
this.tokenRegex.lastIndex));
}
}
|