1 // ## Token types
3 // The assignment of fine-grained, information-carrying type objects
4 // allows the tokenizer to store the information it has about a
5 // token in a way that is very cheap for the parser to look up.
7 // All token type variables start with an underscore, to make them
8 // easy to recognize.
10 // The `beforeExpr` property is used to disambiguate between regular
11 // expressions and divisions. It is set on all token types that can
12 // be followed by an expression (thus, a slash after them would be a
13 // regular expression).
14 //
15 // `isLoop` marks a keyword as starting a loop, which is important
16 // to know when parsing a label, in order to allow or disallow
17 // continue jumps to that label.
19 export class TokenType {
20 constructor(label, conf = {}) {
21 this.label = label
22 this.keyword = conf.keyword
23 this.beforeExpr = !!conf.beforeExpr
24 this.startsExpr = !!conf.startsExpr
25 this.isLoop = !!conf.isLoop
26 this.isAssign = !!conf.isAssign
27 this.prefix = !!conf.prefix
28 this.postfix = !!conf.postfix
29 this.binop = conf.binop || null
30 this.updateContext = null
31 }
32 }
34 function binop(name, prec) {
35 return new TokenType(name, {beforeExpr: true, binop: prec})
36 }
37 const beforeExpr = {beforeExpr: true}, startsExpr = {startsExpr: true}
39 export const types = {
40 num: new TokenType("num", startsExpr),
41 regexp: new TokenType("regexp", startsExpr),
42 string: new TokenType("string", startsExpr),
43 name: new TokenType("name", startsExpr),
44 eof: new TokenType("eof"),
46 // Punctuation token types.
47 bracketL: new TokenType("[", {beforeExpr: true, startsExpr: true}),
48 bracketR: new TokenType("]"),
49 braceL: new TokenType("{", {beforeExpr: true, startsExpr: true}),
50 braceR: new TokenType("}"),
51 parenL: new TokenType("(", {beforeExpr: true, startsExpr: true}),
52 parenR: new TokenType(")"),
53 comma: new TokenType(",", beforeExpr),
54 semi: new TokenType(";", beforeExpr),
55 colon: new TokenType(":", beforeExpr),
56 dot: new TokenType("."),
57 question: new TokenType("?", beforeExpr),
58 arrow: new TokenType("=>", beforeExpr),
59 template: new TokenType("template"),
60 ellipsis: new TokenType("...", beforeExpr),
61 backQuote: new TokenType("`", startsExpr),
62 dollarBraceL: new TokenType("${", {beforeExpr: true, startsExpr: true}),
64 // Operators. These carry several kinds of properties to help the
65 // parser use them properly (the presence of these properties is
66 // what categorizes them as operators).
67 //
68 // `binop`, when present, specifies that this operator is a binary
69 // operator, and will refer to its precedence.
70 //
71 // `prefix` and `postfix` mark the operator as a prefix or postfix
72 // unary operator.
73 //
74 // `isAssign` marks all of `=`, `+=`, `-=` etcetera, which act as
75 // binary operators with a very low precedence, that should result
76 // in AssignmentExpression nodes.
78 eq: new TokenType("=", {beforeExpr: true, isAssign: true}),
79 assign: new TokenType("_=", {beforeExpr: true, isAssign: true}),
80 incDec: new TokenType("++/--", {prefix: true, postfix: true, startsExpr: true}),
81 prefix: new TokenType("prefix", {beforeExpr: true, prefix: true, startsExpr: true}),
82 logicalOR: binop("||", 1),
83 logicalAND: binop("&&", 2),
84 bitwiseOR: binop("|", 3),
85 bitwiseXOR: binop("^", 4),
86 bitwiseAND: binop("&", 5),
87 equality: binop("==/!=", 6),
88 relational: binop("</>", 7),
89 bitShift: binop("<</>>", 8),
90 plusMin: new TokenType("+/-", {beforeExpr: true, binop: 9, prefix: true, startsExpr: true}),
91 modulo: binop("%", 10),
92 star: binop("*", 10),
93 slash: binop("/", 10)
94 }
96 // Map keyword names to token types.
98 export const keywords = {}
100 // Succinct definitions of keyword token types
101 function kw(name, options = {}) {
102 options.keyword = name
103 keywords[name] = types["_" + name] = new TokenType(name, options)
104 }
106 kw("break")
107 kw("case", beforeExpr)
108 kw("catch")
109 kw("continue")
110 kw("debugger")
111 kw("default")
112 kw("do", {isLoop: true})
113 kw("else", beforeExpr)
114 kw("finally")
115 kw("for", {isLoop: true})
116 kw("function", startsExpr)
117 kw("if")
118 kw("return", beforeExpr)
119 kw("switch")
120 kw("throw", beforeExpr)
121 kw("try")
122 kw("var")
123 kw("let")
124 kw("const")
125 kw("while", {isLoop: true})
126 kw("with")
127 kw("new", {beforeExpr: true, startsExpr: true})
128 kw("this", startsExpr)
129 kw("super", startsExpr)
130 kw("class")
131 kw("extends", beforeExpr)
132 kw("export")
133 kw("import")
134 kw("yield", {beforeExpr: true, startsExpr: true})
135 kw("null", startsExpr)
136 kw("true", startsExpr)
137 kw("false", startsExpr)
138 kw("in", {beforeExpr: true, binop: 7})
139 kw("instanceof", {beforeExpr: true, binop: 7})
140 kw("typeof", {beforeExpr: true, prefix: true, startsExpr: true})
141 kw("void", {beforeExpr: true, prefix: true, startsExpr: true})
142 kw("delete", {beforeExpr: true, prefix: true, startsExpr: true})