1
2
3
4
5 package parse
6
7 import (
8 "fmt"
9 "testing"
10 )
11
12
13 var itemName = map[itemType]string{
14 itemError: "error",
15 itemBool: "bool",
16 itemChar: "char",
17 itemCharConstant: "charconst",
18 itemComment: "comment",
19 itemComplex: "complex",
20 itemDeclare: ":=",
21 itemEOF: "EOF",
22 itemField: "field",
23 itemIdentifier: "identifier",
24 itemLeftDelim: "left delim",
25 itemLeftParen: "(",
26 itemNumber: "number",
27 itemPipe: "pipe",
28 itemRawString: "raw string",
29 itemRightDelim: "right delim",
30 itemRightParen: ")",
31 itemSpace: "space",
32 itemString: "string",
33 itemVariable: "variable",
34
35
36 itemDot: ".",
37 itemBlock: "block",
38 itemBreak: "break",
39 itemContinue: "continue",
40 itemDefine: "define",
41 itemElse: "else",
42 itemIf: "if",
43 itemEnd: "end",
44 itemNil: "nil",
45 itemRange: "range",
46 itemTemplate: "template",
47 itemWith: "with",
48 }
49
50 func (i itemType) String() string {
51 s := itemName[i]
52 if s == "" {
53 return fmt.Sprintf("item%d", int(i))
54 }
55 return s
56 }
57
58 type lexTest struct {
59 name string
60 input string
61 items []item
62 }
63
64 func mkItem(typ itemType, text string) item {
65 return item{
66 typ: typ,
67 val: text,
68 }
69 }
70
71 var (
72 tDot = mkItem(itemDot, ".")
73 tBlock = mkItem(itemBlock, "block")
74 tEOF = mkItem(itemEOF, "")
75 tFor = mkItem(itemIdentifier, "for")
76 tLeft = mkItem(itemLeftDelim, "{{")
77 tLpar = mkItem(itemLeftParen, "(")
78 tPipe = mkItem(itemPipe, "|")
79 tQuote = mkItem(itemString, `"abc \n\t\" "`)
80 tRange = mkItem(itemRange, "range")
81 tRight = mkItem(itemRightDelim, "}}")
82 tRpar = mkItem(itemRightParen, ")")
83 tSpace = mkItem(itemSpace, " ")
84 raw = "`" + `abc\n\t\" ` + "`"
85 rawNL = "`now is{{\n}}the time`"
86 tRawQuote = mkItem(itemRawString, raw)
87 tRawQuoteNL = mkItem(itemRawString, rawNL)
88 )
89
90 var lexTests = []lexTest{
91 {"empty", "", []item{tEOF}},
92 {"spaces", " \t\n", []item{mkItem(itemText, " \t\n"), tEOF}},
93 {"text", `now is the time`, []item{mkItem(itemText, "now is the time"), tEOF}},
94 {"text with comment", "hello-{{/* this is a comment */}}-world", []item{
95 mkItem(itemText, "hello-"),
96 mkItem(itemComment, "/* this is a comment */"),
97 mkItem(itemText, "-world"),
98 tEOF,
99 }},
100 {"punctuation", "{{,@% }}", []item{
101 tLeft,
102 mkItem(itemChar, ","),
103 mkItem(itemChar, "@"),
104 mkItem(itemChar, "%"),
105 tSpace,
106 tRight,
107 tEOF,
108 }},
109 {"parens", "{{((3))}}", []item{
110 tLeft,
111 tLpar,
112 tLpar,
113 mkItem(itemNumber, "3"),
114 tRpar,
115 tRpar,
116 tRight,
117 tEOF,
118 }},
119 {"empty action", `{{}}`, []item{tLeft, tRight, tEOF}},
120 {"for", `{{for}}`, []item{tLeft, tFor, tRight, tEOF}},
121 {"block", `{{block "foo" .}}`, []item{
122 tLeft, tBlock, tSpace, mkItem(itemString, `"foo"`), tSpace, tDot, tRight, tEOF,
123 }},
124 {"quote", `{{"abc \n\t\" "}}`, []item{tLeft, tQuote, tRight, tEOF}},
125 {"raw quote", "{{" + raw + "}}", []item{tLeft, tRawQuote, tRight, tEOF}},
126 {"raw quote with newline", "{{" + rawNL + "}}", []item{tLeft, tRawQuoteNL, tRight, tEOF}},
127 {"numbers", "{{1 02 0x14 0X14 -7.2i 1e3 1E3 +1.2e-4 4.2i 1+2i 1_2 0x1.e_fp4 0X1.E_FP4}}", []item{
128 tLeft,
129 mkItem(itemNumber, "1"),
130 tSpace,
131 mkItem(itemNumber, "02"),
132 tSpace,
133 mkItem(itemNumber, "0x14"),
134 tSpace,
135 mkItem(itemNumber, "0X14"),
136 tSpace,
137 mkItem(itemNumber, "-7.2i"),
138 tSpace,
139 mkItem(itemNumber, "1e3"),
140 tSpace,
141 mkItem(itemNumber, "1E3"),
142 tSpace,
143 mkItem(itemNumber, "+1.2e-4"),
144 tSpace,
145 mkItem(itemNumber, "4.2i"),
146 tSpace,
147 mkItem(itemComplex, "1+2i"),
148 tSpace,
149 mkItem(itemNumber, "1_2"),
150 tSpace,
151 mkItem(itemNumber, "0x1.e_fp4"),
152 tSpace,
153 mkItem(itemNumber, "0X1.E_FP4"),
154 tRight,
155 tEOF,
156 }},
157 {"characters", `{{'a' '\n' '\'' '\\' '\u00FF' '\xFF' '本'}}`, []item{
158 tLeft,
159 mkItem(itemCharConstant, `'a'`),
160 tSpace,
161 mkItem(itemCharConstant, `'\n'`),
162 tSpace,
163 mkItem(itemCharConstant, `'\''`),
164 tSpace,
165 mkItem(itemCharConstant, `'\\'`),
166 tSpace,
167 mkItem(itemCharConstant, `'\u00FF'`),
168 tSpace,
169 mkItem(itemCharConstant, `'\xFF'`),
170 tSpace,
171 mkItem(itemCharConstant, `'本'`),
172 tRight,
173 tEOF,
174 }},
175 {"bools", "{{true false}}", []item{
176 tLeft,
177 mkItem(itemBool, "true"),
178 tSpace,
179 mkItem(itemBool, "false"),
180 tRight,
181 tEOF,
182 }},
183 {"dot", "{{.}}", []item{
184 tLeft,
185 tDot,
186 tRight,
187 tEOF,
188 }},
189 {"nil", "{{nil}}", []item{
190 tLeft,
191 mkItem(itemNil, "nil"),
192 tRight,
193 tEOF,
194 }},
195 {"dots", "{{.x . .2 .x.y.z}}", []item{
196 tLeft,
197 mkItem(itemField, ".x"),
198 tSpace,
199 tDot,
200 tSpace,
201 mkItem(itemNumber, ".2"),
202 tSpace,
203 mkItem(itemField, ".x"),
204 mkItem(itemField, ".y"),
205 mkItem(itemField, ".z"),
206 tRight,
207 tEOF,
208 }},
209 {"keywords", "{{range if else end with}}", []item{
210 tLeft,
211 mkItem(itemRange, "range"),
212 tSpace,
213 mkItem(itemIf, "if"),
214 tSpace,
215 mkItem(itemElse, "else"),
216 tSpace,
217 mkItem(itemEnd, "end"),
218 tSpace,
219 mkItem(itemWith, "with"),
220 tRight,
221 tEOF,
222 }},
223 {"variables", "{{$c := printf $ $hello $23 $ $var.Field .Method}}", []item{
224 tLeft,
225 mkItem(itemVariable, "$c"),
226 tSpace,
227 mkItem(itemDeclare, ":="),
228 tSpace,
229 mkItem(itemIdentifier, "printf"),
230 tSpace,
231 mkItem(itemVariable, "$"),
232 tSpace,
233 mkItem(itemVariable, "$hello"),
234 tSpace,
235 mkItem(itemVariable, "$23"),
236 tSpace,
237 mkItem(itemVariable, "$"),
238 tSpace,
239 mkItem(itemVariable, "$var"),
240 mkItem(itemField, ".Field"),
241 tSpace,
242 mkItem(itemField, ".Method"),
243 tRight,
244 tEOF,
245 }},
246 {"variable invocation", "{{$x 23}}", []item{
247 tLeft,
248 mkItem(itemVariable, "$x"),
249 tSpace,
250 mkItem(itemNumber, "23"),
251 tRight,
252 tEOF,
253 }},
254 {"pipeline", `intro {{echo hi 1.2 |noargs|args 1 "hi"}} outro`, []item{
255 mkItem(itemText, "intro "),
256 tLeft,
257 mkItem(itemIdentifier, "echo"),
258 tSpace,
259 mkItem(itemIdentifier, "hi"),
260 tSpace,
261 mkItem(itemNumber, "1.2"),
262 tSpace,
263 tPipe,
264 mkItem(itemIdentifier, "noargs"),
265 tPipe,
266 mkItem(itemIdentifier, "args"),
267 tSpace,
268 mkItem(itemNumber, "1"),
269 tSpace,
270 mkItem(itemString, `"hi"`),
271 tRight,
272 mkItem(itemText, " outro"),
273 tEOF,
274 }},
275 {"declaration", "{{$v := 3}}", []item{
276 tLeft,
277 mkItem(itemVariable, "$v"),
278 tSpace,
279 mkItem(itemDeclare, ":="),
280 tSpace,
281 mkItem(itemNumber, "3"),
282 tRight,
283 tEOF,
284 }},
285 {"2 declarations", "{{$v , $w := 3}}", []item{
286 tLeft,
287 mkItem(itemVariable, "$v"),
288 tSpace,
289 mkItem(itemChar, ","),
290 tSpace,
291 mkItem(itemVariable, "$w"),
292 tSpace,
293 mkItem(itemDeclare, ":="),
294 tSpace,
295 mkItem(itemNumber, "3"),
296 tRight,
297 tEOF,
298 }},
299 {"field of parenthesized expression", "{{(.X).Y}}", []item{
300 tLeft,
301 tLpar,
302 mkItem(itemField, ".X"),
303 tRpar,
304 mkItem(itemField, ".Y"),
305 tRight,
306 tEOF,
307 }},
308 {"trimming spaces before and after", "hello- {{- 3 -}} -world", []item{
309 mkItem(itemText, "hello-"),
310 tLeft,
311 mkItem(itemNumber, "3"),
312 tRight,
313 mkItem(itemText, "-world"),
314 tEOF,
315 }},
316 {"trimming spaces before and after comment", "hello- {{- /* hello */ -}} -world", []item{
317 mkItem(itemText, "hello-"),
318 mkItem(itemComment, "/* hello */"),
319 mkItem(itemText, "-world"),
320 tEOF,
321 }},
322
323 {"badchar", "#{{\x01}}", []item{
324 mkItem(itemText, "#"),
325 tLeft,
326 mkItem(itemError, "unrecognized character in action: U+0001"),
327 }},
328 {"unclosed action", "{{", []item{
329 tLeft,
330 mkItem(itemError, "unclosed action"),
331 }},
332 {"EOF in action", "{{range", []item{
333 tLeft,
334 tRange,
335 mkItem(itemError, "unclosed action"),
336 }},
337 {"unclosed quote", "{{\"\n\"}}", []item{
338 tLeft,
339 mkItem(itemError, "unterminated quoted string"),
340 }},
341 {"unclosed raw quote", "{{`xx}}", []item{
342 tLeft,
343 mkItem(itemError, "unterminated raw quoted string"),
344 }},
345 {"unclosed char constant", "{{'\n}}", []item{
346 tLeft,
347 mkItem(itemError, "unterminated character constant"),
348 }},
349 {"bad number", "{{3k}}", []item{
350 tLeft,
351 mkItem(itemError, `bad number syntax: "3k"`),
352 }},
353 {"unclosed paren", "{{(3}}", []item{
354 tLeft,
355 tLpar,
356 mkItem(itemNumber, "3"),
357 mkItem(itemError, `unclosed left paren`),
358 }},
359 {"extra right paren", "{{3)}}", []item{
360 tLeft,
361 mkItem(itemNumber, "3"),
362 mkItem(itemError, "unexpected right paren"),
363 }},
364
365
366
367
368 {"long pipeline deadlock", "{{|||||}}", []item{
369 tLeft,
370 tPipe,
371 tPipe,
372 tPipe,
373 tPipe,
374 tPipe,
375 tRight,
376 tEOF,
377 }},
378 {"text with bad comment", "hello-{{/*/}}-world", []item{
379 mkItem(itemText, "hello-"),
380 mkItem(itemError, `unclosed comment`),
381 }},
382 {"text with comment close separated from delim", "hello-{{/* */ }}-world", []item{
383 mkItem(itemText, "hello-"),
384 mkItem(itemError, `comment ends before closing delimiter`),
385 }},
386
387
388 {"unmatched right delimiter", "hello-{.}}-world", []item{
389 mkItem(itemText, "hello-{.}}-world"),
390 tEOF,
391 }},
392 }
393
394
395 func collect(t *lexTest, left, right string) (items []item) {
396 l := lex(t.name, t.input, left, right)
397 l.options = lexOptions{
398 emitComment: true,
399 breakOK: true,
400 continueOK: true,
401 }
402 for {
403 item := l.nextItem()
404 items = append(items, item)
405 if item.typ == itemEOF || item.typ == itemError {
406 break
407 }
408 }
409 return
410 }
411
412 func equal(i1, i2 []item, checkPos bool) bool {
413 if len(i1) != len(i2) {
414 return false
415 }
416 for k := range i1 {
417 if i1[k].typ != i2[k].typ {
418 return false
419 }
420 if i1[k].val != i2[k].val {
421 return false
422 }
423 if checkPos && i1[k].pos != i2[k].pos {
424 return false
425 }
426 if checkPos && i1[k].line != i2[k].line {
427 return false
428 }
429 }
430 return true
431 }
432
433 func TestLex(t *testing.T) {
434 for _, test := range lexTests {
435 items := collect(&test, "", "")
436 if !equal(items, test.items, false) {
437 t.Errorf("%s: got\n\t%+v\nexpected\n\t%v", test.name, items, test.items)
438 return
439 }
440 t.Log(test.name, "OK")
441 }
442 }
443
444
445 var lexDelimTests = []lexTest{
446 {"punctuation", "$$,@%{{}}@@", []item{
447 tLeftDelim,
448 mkItem(itemChar, ","),
449 mkItem(itemChar, "@"),
450 mkItem(itemChar, "%"),
451 mkItem(itemChar, "{"),
452 mkItem(itemChar, "{"),
453 mkItem(itemChar, "}"),
454 mkItem(itemChar, "}"),
455 tRightDelim,
456 tEOF,
457 }},
458 {"empty action", `$$@@`, []item{tLeftDelim, tRightDelim, tEOF}},
459 {"for", `$$for@@`, []item{tLeftDelim, tFor, tRightDelim, tEOF}},
460 {"quote", `$$"abc \n\t\" "@@`, []item{tLeftDelim, tQuote, tRightDelim, tEOF}},
461 {"raw quote", "$$" + raw + "@@", []item{tLeftDelim, tRawQuote, tRightDelim, tEOF}},
462 }
463
464 var (
465 tLeftDelim = mkItem(itemLeftDelim, "$$")
466 tRightDelim = mkItem(itemRightDelim, "@@")
467 )
468
469 func TestDelims(t *testing.T) {
470 for _, test := range lexDelimTests {
471 items := collect(&test, "$$", "@@")
472 if !equal(items, test.items, false) {
473 t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
474 }
475 }
476 }
477
478 func TestDelimsAlphaNumeric(t *testing.T) {
479 test := lexTest{"right delimiter with alphanumeric start", "{{hub .host hub}}", []item{
480 mkItem(itemLeftDelim, "{{hub"),
481 mkItem(itemSpace, " "),
482 mkItem(itemField, ".host"),
483 mkItem(itemSpace, " "),
484 mkItem(itemRightDelim, "hub}}"),
485 tEOF,
486 }}
487 items := collect(&test, "{{hub", "hub}}")
488
489 if !equal(items, test.items, false) {
490 t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
491 }
492 }
493
494 func TestDelimsAndMarkers(t *testing.T) {
495 test := lexTest{"delims that look like markers", "{{- .x -}} {{- - .x - -}}", []item{
496 mkItem(itemLeftDelim, "{{- "),
497 mkItem(itemField, ".x"),
498 mkItem(itemRightDelim, " -}}"),
499 mkItem(itemLeftDelim, "{{- "),
500 mkItem(itemField, ".x"),
501 mkItem(itemRightDelim, " -}}"),
502 tEOF,
503 }}
504 items := collect(&test, "{{- ", " -}}")
505
506 if !equal(items, test.items, false) {
507 t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
508 }
509 }
510
511 var lexPosTests = []lexTest{
512 {"empty", "", []item{{itemEOF, 0, "", 1}}},
513 {"punctuation", "{{,@%#}}", []item{
514 {itemLeftDelim, 0, "{{", 1},
515 {itemChar, 2, ",", 1},
516 {itemChar, 3, "@", 1},
517 {itemChar, 4, "%", 1},
518 {itemChar, 5, "#", 1},
519 {itemRightDelim, 6, "}}", 1},
520 {itemEOF, 8, "", 1},
521 }},
522 {"sample", "0123{{hello}}xyz", []item{
523 {itemText, 0, "0123", 1},
524 {itemLeftDelim, 4, "{{", 1},
525 {itemIdentifier, 6, "hello", 1},
526 {itemRightDelim, 11, "}}", 1},
527 {itemText, 13, "xyz", 1},
528 {itemEOF, 16, "", 1},
529 }},
530 {"trimafter", "{{x -}}\n{{y}}", []item{
531 {itemLeftDelim, 0, "{{", 1},
532 {itemIdentifier, 2, "x", 1},
533 {itemRightDelim, 5, "}}", 1},
534 {itemLeftDelim, 8, "{{", 2},
535 {itemIdentifier, 10, "y", 2},
536 {itemRightDelim, 11, "}}", 2},
537 {itemEOF, 13, "", 2},
538 }},
539 {"trimbefore", "{{x}}\n{{- y}}", []item{
540 {itemLeftDelim, 0, "{{", 1},
541 {itemIdentifier, 2, "x", 1},
542 {itemRightDelim, 3, "}}", 1},
543 {itemLeftDelim, 6, "{{", 2},
544 {itemIdentifier, 10, "y", 2},
545 {itemRightDelim, 11, "}}", 2},
546 {itemEOF, 13, "", 2},
547 }},
548 {"longcomment", "{{/*\n*/}}\n{{undefinedFunction \"test\"}}", []item{
549 {itemComment, 2, "/*\n*/", 1},
550 {itemText, 9, "\n", 2},
551 {itemLeftDelim, 10, "{{", 3},
552 {itemIdentifier, 12, "undefinedFunction", 3},
553 {itemSpace, 29, " ", 3},
554 {itemString, 30, "\"test\"", 3},
555 {itemRightDelim, 36, "}}", 3},
556 {itemEOF, 38, "", 3},
557 }},
558 }
559
560
561
562 func TestPos(t *testing.T) {
563 for _, test := range lexPosTests {
564 items := collect(&test, "", "")
565 if !equal(items, test.items, true) {
566 t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
567 if len(items) == len(test.items) {
568
569 for i := range items {
570 if !equal(items[i:i+1], test.items[i:i+1], true) {
571 i1 := items[i]
572 i2 := test.items[i]
573 t.Errorf("\t#%d: got {%v %d %q %d} expected {%v %d %q %d}",
574 i, i1.typ, i1.pos, i1.val, i1.line, i2.typ, i2.pos, i2.val, i2.line)
575 }
576 }
577 }
578 }
579 }
580 }
581
582
583
584 func (t *Tree) parseLexer(lex *lexer) (tree *Tree, err error) {
585 defer t.recover(&err)
586 t.ParseName = t.Name
587 t.startParse(nil, lex, map[string]*Tree{})
588 t.parse()
589 t.add()
590 t.stopParse()
591 return t, nil
592 }
593
View as plain text