1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
16 var fset = token.NewFileSet()
27 func tokenclass(tok token.Token) int {
31 case tok.IsOperator():
47 var tokens = [...]elt{
49 {token.COMMENT, "/* a comment */", special},
50 {token.COMMENT, "// a comment \n", special},
52 // Identifiers and basic type literals
53 {token.IDENT, "foobar", literal},
54 {token.IDENT, "a۰۱۸", literal},
55 {token.IDENT, "foo६४", literal},
56 {token.IDENT, "bar9876", literal},
57 {token.INT, "0", literal},
58 {token.INT, "1", literal},
59 {token.INT, "123456789012345678890", literal},
60 {token.INT, "01234567", literal},
61 {token.INT, "0xcafebabe", literal},
62 {token.FLOAT, "0.", literal},
63 {token.FLOAT, ".0", literal},
64 {token.FLOAT, "3.14159265", literal},
65 {token.FLOAT, "1e0", literal},
66 {token.FLOAT, "1e+100", literal},
67 {token.FLOAT, "1e-100", literal},
68 {token.FLOAT, "2.71828e-1000", literal},
69 {token.IMAG, "0i", literal},
70 {token.IMAG, "1i", literal},
71 {token.IMAG, "012345678901234567889i", literal},
72 {token.IMAG, "123456789012345678890i", literal},
73 {token.IMAG, "0.i", literal},
74 {token.IMAG, ".0i", literal},
75 {token.IMAG, "3.14159265i", literal},
76 {token.IMAG, "1e0i", literal},
77 {token.IMAG, "1e+100i", literal},
78 {token.IMAG, "1e-100i", literal},
79 {token.IMAG, "2.71828e-1000i", literal},
80 {token.CHAR, "'a'", literal},
81 {token.CHAR, "'\\000'", literal},
82 {token.CHAR, "'\\xFF'", literal},
83 {token.CHAR, "'\\uff16'", literal},
84 {token.CHAR, "'\\U0000ff16'", literal},
85 {token.STRING, "`foobar`", literal},
86 {token.STRING, "`" + `foo
92 // Operators and delimitors
93 {token.ADD, "+", operator},
94 {token.SUB, "-", operator},
95 {token.MUL, "*", operator},
96 {token.QUO, "/", operator},
97 {token.REM, "%", operator},
99 {token.AND, "&", operator},
100 {token.OR, "|", operator},
101 {token.XOR, "^", operator},
102 {token.SHL, "<<", operator},
103 {token.SHR, ">>", operator},
104 {token.AND_NOT, "&^", operator},
106 {token.ADD_ASSIGN, "+=", operator},
107 {token.SUB_ASSIGN, "-=", operator},
108 {token.MUL_ASSIGN, "*=", operator},
109 {token.QUO_ASSIGN, "/=", operator},
110 {token.REM_ASSIGN, "%=", operator},
112 {token.AND_ASSIGN, "&=", operator},
113 {token.OR_ASSIGN, "|=", operator},
114 {token.XOR_ASSIGN, "^=", operator},
115 {token.SHL_ASSIGN, "<<=", operator},
116 {token.SHR_ASSIGN, ">>=", operator},
117 {token.AND_NOT_ASSIGN, "&^=", operator},
119 {token.LAND, "&&", operator},
120 {token.LOR, "||", operator},
121 {token.ARROW, "<-", operator},
122 {token.INC, "++", operator},
123 {token.DEC, "--", operator},
125 {token.EQL, "==", operator},
126 {token.LSS, "<", operator},
127 {token.GTR, ">", operator},
128 {token.ASSIGN, "=", operator},
129 {token.NOT, "!", operator},
131 {token.NEQ, "!=", operator},
132 {token.LEQ, "<=", operator},
133 {token.GEQ, ">=", operator},
134 {token.DEFINE, ":=", operator},
135 {token.ELLIPSIS, "...", operator},
137 {token.LPAREN, "(", operator},
138 {token.LBRACK, "[", operator},
139 {token.LBRACE, "{", operator},
140 {token.COMMA, ",", operator},
141 {token.PERIOD, ".", operator},
143 {token.RPAREN, ")", operator},
144 {token.RBRACK, "]", operator},
145 {token.RBRACE, "}", operator},
146 {token.SEMICOLON, ";", operator},
147 {token.COLON, ":", operator},
150 {token.BREAK, "break", keyword},
151 {token.CASE, "case", keyword},
152 {token.CHAN, "chan", keyword},
153 {token.CONST, "const", keyword},
154 {token.CONTINUE, "continue", keyword},
156 {token.DEFAULT, "default", keyword},
157 {token.DEFER, "defer", keyword},
158 {token.ELSE, "else", keyword},
159 {token.FALLTHROUGH, "fallthrough", keyword},
160 {token.FOR, "for", keyword},
162 {token.FUNC, "func", keyword},
163 {token.GO, "go", keyword},
164 {token.GOTO, "goto", keyword},
165 {token.IF, "if", keyword},
166 {token.IMPORT, "import", keyword},
168 {token.INTERFACE, "interface", keyword},
169 {token.MAP, "map", keyword},
170 {token.PACKAGE, "package", keyword},
171 {token.RANGE, "range", keyword},
172 {token.RETURN, "return", keyword},
174 {token.SELECT, "select", keyword},
175 {token.STRUCT, "struct", keyword},
176 {token.SWITCH, "switch", keyword},
177 {token.TYPE, "type", keyword},
178 {token.VAR, "var", keyword},
182 const whitespace = " \t \n\n\n" // to separate tokens
184 type testErrorHandler struct {
188 func (h *testErrorHandler) Error(pos token.Position, msg string) {
189 h.t.Errorf("Error() called (msg = %s)", msg)
193 func newlineCount(s string) int {
195 for i := 0; i < len(s); i++ {
204 func checkPos(t *testing.T, lit string, p token.Pos, expected token.Position) {
205 pos := fset.Position(p)
206 if pos.Filename != expected.Filename {
207 t.Errorf("bad filename for %q: got %s, expected %s", lit, pos.Filename, expected.Filename)
209 if pos.Offset != expected.Offset {
210 t.Errorf("bad position for %q: got %d, expected %d", lit, pos.Offset, expected.Offset)
212 if pos.Line != expected.Line {
213 t.Errorf("bad line for %q: got %d, expected %d", lit, pos.Line, expected.Line)
215 if pos.Column != expected.Column {
216 t.Errorf("bad column for %q: got %d, expected %d", lit, pos.Column, expected.Column)
221 // Verify that calling Scan() provides the correct results.
222 func TestScan(t *testing.T) {
225 for _, e := range tokens {
226 src += e.lit + whitespace
228 src_linecount := newlineCount(src)
229 whitespace_linecount := newlineCount(whitespace)
233 s.Init(fset.AddFile("", fset.Base(), len(src)), []byte(src), &testErrorHandler{t}, ScanComments)
235 epos := token.Position{"", 0, 1, 1} // expected position
237 pos, tok, lit := s.Scan()
238 e := elt{token.EOF, "", special}
239 if index < len(tokens) {
242 if tok == token.EOF {
244 epos.Line = src_linecount
247 checkPos(t, lit, pos, epos)
249 t.Errorf("bad token for %q: got %s, expected %s", lit, tok.String(), e.tok.String())
251 if e.tok.IsLiteral() && lit != e.lit {
252 t.Errorf("bad literal for %q: got %q, expected %q", lit, lit, e.lit)
254 if tokenclass(tok) != e.class {
255 t.Errorf("bad class for %q: got %d, expected %d", lit, tokenclass(tok), e.class)
257 epos.Offset += len(lit) + len(whitespace)
258 epos.Line += newlineCount(lit) + whitespace_linecount
259 if tok == token.COMMENT && lit[1] == '/' {
260 // correct for unaccounted '/n' in //-style comment
265 if tok == token.EOF {
269 if s.ErrorCount != 0 {
270 t.Errorf("found %d errors", s.ErrorCount)
275 func checkSemi(t *testing.T, line string, mode uint) {
277 file := fset.AddFile("TestSemis", fset.Base(), len(line))
278 S.Init(file, []byte(line), nil, mode)
279 pos, tok, lit := S.Scan()
280 for tok != token.EOF {
281 if tok == token.ILLEGAL {
282 // the illegal token literal indicates what
283 // kind of semicolon literal to expect
288 // next token must be a semicolon
289 semiPos := file.Position(pos)
292 pos, tok, lit = S.Scan()
293 if tok == token.SEMICOLON {
295 t.Errorf(`bad literal for %q: got %q, expected %q`, line, lit, semiLit)
297 checkPos(t, line, pos, semiPos)
299 t.Errorf("bad token for %q: got %s, expected ;", line, tok.String())
301 } else if tok == token.SEMICOLON {
302 t.Errorf("bad token for %q: got ;, expected no ;", line)
304 pos, tok, lit = S.Scan()
309 var lines = []string{
310 // # indicates a semicolon present in the source
311 // $ indicates an automatically inserted semicolon
411 "foo$/*comment*/ \n",
416 "foo $/*comment*/\n",
418 "foo $/* */ /* \n */ bar$/**/\n",
419 "foo $/*0*/ /*1*/ /*2*/\n",
421 "foo $/*comment*/ \n",
422 "foo $/*0*/ /*1*/ /*2*/ \n",
423 "foo $/**/ /*-------------*/ /*----\n*/bar $/* \n*/baa$\n",
424 "foo $/* an EOF terminates a line */",
425 "foo $/* an EOF terminates a line */ /*",
426 "foo $/* an EOF terminates a line */ //",
428 "package main$\n\nfunc main() {\n\tif {\n\t\treturn /* */ }$\n}$\n",
433 func TestSemis(t *testing.T) {
434 for _, line := range lines {
435 checkSemi(t, line, AllowIllegalChars|InsertSemis)
436 checkSemi(t, line, AllowIllegalChars|InsertSemis|ScanComments)
438 // if the input ended in newlines, the input must tokenize the
439 // same with or without those newlines
440 for i := len(line) - 1; i >= 0 && line[i] == '\n'; i-- {
441 checkSemi(t, line[0:i], AllowIllegalChars|InsertSemis)
442 checkSemi(t, line[0:i], AllowIllegalChars|InsertSemis|ScanComments)
447 type segment struct {
448 srcline string // a line of source text
449 filename string // filename for current token
450 line int // line number for current token
453 var segments = []segment{
454 // exactly one token per line since the test consumes one token per segment
455 {" line1", filepath.Join("dir", "TestLineComments"), 1},
456 {"\nline2", filepath.Join("dir", "TestLineComments"), 2},
457 {"\nline3 //line File1.go:100", filepath.Join("dir", "TestLineComments"), 3}, // bad line comment, ignored
458 {"\nline4", filepath.Join("dir", "TestLineComments"), 4},
459 {"\n//line File1.go:100\n line100", filepath.Join("dir", "File1.go"), 100},
460 {"\n//line File2.go:200\n line200", filepath.Join("dir", "File2.go"), 200},
461 {"\n//line :1\n line1", "dir", 1},
462 {"\n//line foo:42\n line42", filepath.Join("dir", "foo"), 42},
463 {"\n //line foo:42\n line44", filepath.Join("dir", "foo"), 44}, // bad line comment, ignored
464 {"\n//line foo 42\n line46", filepath.Join("dir", "foo"), 46}, // bad line comment, ignored
465 {"\n//line foo:42 extra text\n line48", filepath.Join("dir", "foo"), 48}, // bad line comment, ignored
466 {"\n//line /bar:42\n line42", string(filepath.Separator) + "bar", 42},
467 {"\n//line ./foo:42\n line42", filepath.Join("dir", "foo"), 42},
468 {"\n//line a/b/c/File1.go:100\n line100", filepath.Join("dir", "a", "b", "c", "File1.go"), 100},
471 var winsegments = []segment{
472 {"\n//line c:\\dir\\File1.go:100\n line100", "c:\\dir\\File1.go", 100},
476 // Verify that comments of the form "//line filename:line" are interpreted correctly.
477 func TestLineComments(t *testing.T) {
478 if runtime.GOOS == "windows" {
479 segments = append(segments, winsegments...)
484 for _, e := range segments {
490 file := fset.AddFile(filepath.Join("dir", "TestLineComments"), fset.Base(), len(src))
491 S.Init(file, []byte(src), nil, 0)
492 for _, s := range segments {
493 p, _, lit := S.Scan()
494 pos := file.Position(p)
495 checkPos(t, lit, p, token.Position{s.filename, pos.Offset, s.line, pos.Column})
498 if S.ErrorCount != 0 {
499 t.Errorf("found %d errors", S.ErrorCount)
504 // Verify that initializing the same scanner more then once works correctly.
505 func TestInit(t *testing.T) {
509 src1 := "if true { }"
510 f1 := fset.AddFile("src1", fset.Base(), len(src1))
511 s.Init(f1, []byte(src1), nil, 0)
512 if f1.Size() != len(src1) {
513 t.Errorf("bad file size: got %d, expected %d", f1.Size(), len(src1))
517 _, tok, _ := s.Scan() // {
518 if tok != token.LBRACE {
519 t.Errorf("bad token: got %s, expected %s", tok.String(), token.LBRACE)
523 src2 := "go true { ]"
524 f2 := fset.AddFile("src2", fset.Base(), len(src2))
525 s.Init(f2, []byte(src2), nil, 0)
526 if f2.Size() != len(src2) {
527 t.Errorf("bad file size: got %d, expected %d", f2.Size(), len(src2))
529 _, tok, _ = s.Scan() // go
531 t.Errorf("bad token: got %s, expected %s", tok.String(), token.GO)
534 if s.ErrorCount != 0 {
535 t.Errorf("found %d errors", s.ErrorCount)
540 func TestIllegalChars(t *testing.T) {
543 const src = "*?*$*@*"
544 file := fset.AddFile("", fset.Base(), len(src))
545 s.Init(file, []byte(src), &testErrorHandler{t}, AllowIllegalChars)
546 for offs, ch := range src {
547 pos, tok, lit := s.Scan()
548 if poffs := file.Offset(pos); poffs != offs {
549 t.Errorf("bad position for %s: got %d, expected %d", lit, poffs, offs)
551 if tok == token.ILLEGAL && lit != string(ch) {
552 t.Errorf("bad token: got %s, expected %s", lit, string(ch))
556 if s.ErrorCount != 0 {
557 t.Errorf("found %d errors", s.ErrorCount)
562 func TestStdErrorHander(t *testing.T) {
563 const src = "@\n" + // illegal character, cause an error
564 "@ @\n" + // two errors on the same line
565 "//line File2:20\n" +
566 "@\n" + // different file, but same line
568 "@ @\n" + // same file, decreasing line number
570 "@ @ @" // original file, line 1 again
572 v := new(ErrorVector)
574 s.Init(fset.AddFile("File1", fset.Base(), len(src)), []byte(src), v, 0)
576 if _, tok, _ := s.Scan(); tok == token.EOF {
581 list := v.GetErrorList(Raw)
583 t.Errorf("found %d raw errors, expected 9", len(list))
584 PrintError(os.Stderr, list)
587 list = v.GetErrorList(Sorted)
589 t.Errorf("found %d sorted errors, expected 9", len(list))
590 PrintError(os.Stderr, list)
593 list = v.GetErrorList(NoMultiples)
595 t.Errorf("found %d one-per-line errors, expected 4", len(list))
596 PrintError(os.Stderr, list)
599 if v.ErrorCount() != s.ErrorCount {
600 t.Errorf("found %d errors, expected %d", v.ErrorCount(), s.ErrorCount)
605 type errorCollector struct {
606 cnt int // number of errors encountered
607 msg string // last error message encountered
608 pos token.Position // last error position encountered
612 func (h *errorCollector) Error(pos token.Position, msg string) {
619 func checkError(t *testing.T, src string, tok token.Token, pos int, err string) {
622 s.Init(fset.AddFile("", fset.Base(), len(src)), []byte(src), &h, ScanComments)
623 _, tok0, _ := s.Scan()
624 _, tok1, _ := s.Scan()
626 t.Errorf("%q: got %s, expected %s", src, tok0, tok)
628 if tok1 != token.EOF {
629 t.Errorf("%q: got %s, expected EOF", src, tok1)
636 t.Errorf("%q: got cnt %d, expected %d", src, h.cnt, cnt)
639 t.Errorf("%q: got msg %q, expected %q", src, h.msg, err)
641 if h.pos.Offset != pos {
642 t.Errorf("%q: got offset %d, expected %d", src, h.pos.Offset, pos)
647 var errors = []struct {
653 {`#`, token.ILLEGAL, 0, "illegal character '#' (U+23)"},
654 {`' '`, token.CHAR, 0, ""},
655 {`''`, token.CHAR, 0, "illegal character literal"},
656 {`'\8'`, token.CHAR, 2, "unknown escape sequence"},
657 {`'\08'`, token.CHAR, 3, "illegal character in escape sequence"},
658 {`'\x0g'`, token.CHAR, 4, "illegal character in escape sequence"},
659 {`'\Uffffffff'`, token.CHAR, 2, "escape sequence is invalid Unicode code point"},
660 {`'`, token.CHAR, 0, "character literal not terminated"},
661 {`""`, token.STRING, 0, ""},
662 {`"`, token.STRING, 0, "string not terminated"},
663 {"``", token.STRING, 0, ""},
664 {"`", token.STRING, 0, "string not terminated"},
665 {"/**/", token.COMMENT, 0, ""},
666 {"/*", token.COMMENT, 0, "comment not terminated"},
667 {"077", token.INT, 0, ""},
668 {"078.", token.FLOAT, 0, ""},
669 {"07801234567.", token.FLOAT, 0, ""},
670 {"078e0", token.FLOAT, 0, ""},
671 {"078", token.INT, 0, "illegal octal number"},
672 {"07800000009", token.INT, 0, "illegal octal number"},
673 {"\"abc\x00def\"", token.STRING, 4, "illegal character NUL"},
674 {"\"abc\x80def\"", token.STRING, 4, "illegal UTF-8 encoding"},
678 func TestScanErrors(t *testing.T) {
679 for _, e := range errors {
680 checkError(t, e.src, e.tok, e.pos, e.err)