File size: 3,042 Bytes
bc20498 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 |
/* Hand-written tokenizers for JavaScript tokens that can't be
expressed by lezer's built-in tokenizer. */
import {ExternalTokenizer, ContextTracker} from "@lezer/lr"
import {insertSemi, noSemi, incdec, incdecPrefix, questionDot,
spaces, newline, BlockComment, LineComment,
JSXStartTag, Dialect_jsx} from "./parser.terms.js"
const space = [9, 10, 11, 12, 13, 32, 133, 160, 5760, 8192, 8193, 8194, 8195, 8196, 8197, 8198, 8199, 8200,
8201, 8202, 8232, 8233, 8239, 8287, 12288]
const braceR = 125, semicolon = 59, slash = 47, star = 42, plus = 43, minus = 45, lt = 60, comma = 44,
question = 63, dot = 46
export const trackNewline = new ContextTracker({
start: false,
shift(context, term) {
return term == LineComment || term == BlockComment || term == spaces ? context : term == newline
},
strict: false
})
export const insertSemicolon = new ExternalTokenizer((input, stack) => {
let {next} = input
if (next == braceR || next == -1 || stack.context)
input.acceptToken(insertSemi)
}, {contextual: true, fallback: true})
export const noSemicolon = new ExternalTokenizer((input, stack) => {
let {next} = input, after
if (space.indexOf(next) > -1) return
if (next == slash && ((after = input.peek(1)) == slash || after == star)) return
if (next != braceR && next != semicolon && next != -1 && !stack.context)
input.acceptToken(noSemi)
}, {contextual: true})
export const operatorToken = new ExternalTokenizer((input, stack) => {
let {next} = input
if (next == plus || next == minus) {
input.advance()
if (next == input.next) {
input.advance()
let mayPostfix = !stack.context && stack.canShift(incdec)
input.acceptToken(mayPostfix ? incdec : incdecPrefix)
}
} else if (next == question && input.peek(1) == dot) {
input.advance(); input.advance()
if (input.next < 48 || input.next > 57) // No digit after
input.acceptToken(questionDot)
}
}, {contextual: true})
function identifierChar(ch, start) {
return ch >= 65 && ch <= 90 || ch >= 97 && ch <= 122 || ch == 95 || ch >= 192 ||
!start && ch >= 48 && ch <= 57
}
export const jsx = new ExternalTokenizer((input, stack) => {
if (input.next != lt || !stack.dialectEnabled(Dialect_jsx)) return
input.advance()
if (input.next == slash) return
// Scan for an identifier followed by a comma or 'extends', don't
// treat this as a start tag if present.
let back = 0
while (space.indexOf(input.next) > -1) { input.advance(); back++ }
if (identifierChar(input.next, true)) {
input.advance()
back++
while (identifierChar(input.next, false)) { input.advance(); back++ }
while (space.indexOf(input.next) > -1) { input.advance(); back++ }
if (input.next == comma) return
for (let i = 0;; i++) {
if (i == 7) {
if (!identifierChar(input.next, true)) return
break
}
if (input.next != "extends".charCodeAt(i)) break
input.advance()
back++
}
}
input.acceptToken(JSXStartTag, -back)
})
|