Source file src/go/parser/parser_test.go

Documentation: go/parser

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package parser
     6  
     7  import (
     8  	"bytes"
     9  	"fmt"
    10  	"go/ast"
    11  	"go/token"
    12  	"os"
    13  	"strings"
    14  	"testing"
    15  )
    16  
    17  var validFiles = []string{
    18  	"parser.go",
    19  	"parser_test.go",
    20  	"error_test.go",
    21  	"short_test.go",
    22  }
    23  
    24  func TestParse(t *testing.T) {
    25  	for _, filename := range validFiles {
    26  		_, err := ParseFile(token.NewFileSet(), filename, nil, DeclarationErrors)
    27  		if err != nil {
    28  			t.Fatalf("ParseFile(%s): %v", filename, err)
    29  		}
    30  	}
    31  }
    32  
    33  func nameFilter(filename string) bool {
    34  	switch filename {
    35  	case "parser.go", "interface.go", "parser_test.go":
    36  		return true
    37  	case "parser.go.orig":
    38  		return true // permit but should be ignored by ParseDir
    39  	}
    40  	return false
    41  }
    42  
    43  func dirFilter(f os.FileInfo) bool { return nameFilter(f.Name()) }
    44  
    45  func TestParseFile(t *testing.T) {
    46  	src := "package p\nvar _=s[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]"
    47  	_, err := ParseFile(token.NewFileSet(), "", src, 0)
    48  	if err == nil {
    49  		t.Errorf("ParseFile(%s) succeeded unexpectedly", src)
    50  	}
    51  }
    52  
    53  func TestParseExprFrom(t *testing.T) {
    54  	src := "s[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]"
    55  	_, err := ParseExprFrom(token.NewFileSet(), "", src, 0)
    56  	if err == nil {
    57  		t.Errorf("ParseExprFrom(%s) succeeded unexpectedly", src)
    58  	}
    59  }
    60  
    61  func TestParseDir(t *testing.T) {
    62  	path := "."
    63  	pkgs, err := ParseDir(token.NewFileSet(), path, dirFilter, 0)
    64  	if err != nil {
    65  		t.Fatalf("ParseDir(%s): %v", path, err)
    66  	}
    67  	if n := len(pkgs); n != 1 {
    68  		t.Errorf("got %d packages; want 1", n)
    69  	}
    70  	pkg := pkgs["parser"]
    71  	if pkg == nil {
    72  		t.Errorf(`package "parser" not found`)
    73  		return
    74  	}
    75  	if n := len(pkg.Files); n != 3 {
    76  		t.Errorf("got %d package files; want 3", n)
    77  	}
    78  	for filename := range pkg.Files {
    79  		if !nameFilter(filename) {
    80  			t.Errorf("unexpected package file: %s", filename)
    81  		}
    82  	}
    83  }
    84  
    85  func TestParseExpr(t *testing.T) {
    86  	// just kicking the tires:
    87  	// a valid arithmetic expression
    88  	src := "a + b"
    89  	x, err := ParseExpr(src)
    90  	if err != nil {
    91  		t.Errorf("ParseExpr(%q): %v", src, err)
    92  	}
    93  	// sanity check
    94  	if _, ok := x.(*ast.BinaryExpr); !ok {
    95  		t.Errorf("ParseExpr(%q): got %T, want *ast.BinaryExpr", src, x)
    96  	}
    97  
    98  	// a valid type expression
    99  	src = "struct{x *int}"
   100  	x, err = ParseExpr(src)
   101  	if err != nil {
   102  		t.Errorf("ParseExpr(%q): %v", src, err)
   103  	}
   104  	// sanity check
   105  	if _, ok := x.(*ast.StructType); !ok {
   106  		t.Errorf("ParseExpr(%q): got %T, want *ast.StructType", src, x)
   107  	}
   108  
   109  	// an invalid expression
   110  	src = "a + *"
   111  	x, err = ParseExpr(src)
   112  	if err == nil {
   113  		t.Errorf("ParseExpr(%q): got no error", src)
   114  	}
   115  	if x == nil {
   116  		t.Errorf("ParseExpr(%q): got no (partial) result", src)
   117  	}
   118  	if _, ok := x.(*ast.BinaryExpr); !ok {
   119  		t.Errorf("ParseExpr(%q): got %T, want *ast.BinaryExpr", src, x)
   120  	}
   121  
   122  	// a valid expression followed by extra tokens is invalid
   123  	src = "a[i] := x"
   124  	if _, err := ParseExpr(src); err == nil {
   125  		t.Errorf("ParseExpr(%q): got no error", src)
   126  	}
   127  
   128  	// a semicolon is not permitted unless automatically inserted
   129  	src = "a + b\n"
   130  	if _, err := ParseExpr(src); err != nil {
   131  		t.Errorf("ParseExpr(%q): got error %s", src, err)
   132  	}
   133  	src = "a + b;"
   134  	if _, err := ParseExpr(src); err == nil {
   135  		t.Errorf("ParseExpr(%q): got no error", src)
   136  	}
   137  
   138  	// various other stuff following a valid expression
   139  	const validExpr = "a + b"
   140  	const anything = "dh3*#D)#_"
   141  	for _, c := range "!)]};," {
   142  		src := validExpr + string(c) + anything
   143  		if _, err := ParseExpr(src); err == nil {
   144  			t.Errorf("ParseExpr(%q): got no error", src)
   145  		}
   146  	}
   147  
   148  	// ParseExpr must not crash
   149  	for _, src := range valids {
   150  		ParseExpr(src)
   151  	}
   152  }
   153  
   154  func TestColonEqualsScope(t *testing.T) {
   155  	f, err := ParseFile(token.NewFileSet(), "", `package p; func f() { x, y, z := x, y, z }`, 0)
   156  	if err != nil {
   157  		t.Fatal(err)
   158  	}
   159  
   160  	// RHS refers to undefined globals; LHS does not.
   161  	as := f.Decls[0].(*ast.FuncDecl).Body.List[0].(*ast.AssignStmt)
   162  	for _, v := range as.Rhs {
   163  		id := v.(*ast.Ident)
   164  		if id.Obj != nil {
   165  			t.Errorf("rhs %s has Obj, should not", id.Name)
   166  		}
   167  	}
   168  	for _, v := range as.Lhs {
   169  		id := v.(*ast.Ident)
   170  		if id.Obj == nil {
   171  			t.Errorf("lhs %s does not have Obj, should", id.Name)
   172  		}
   173  	}
   174  }
   175  
   176  func TestVarScope(t *testing.T) {
   177  	f, err := ParseFile(token.NewFileSet(), "", `package p; func f() { var x, y, z = x, y, z }`, 0)
   178  	if err != nil {
   179  		t.Fatal(err)
   180  	}
   181  
   182  	// RHS refers to undefined globals; LHS does not.
   183  	as := f.Decls[0].(*ast.FuncDecl).Body.List[0].(*ast.DeclStmt).Decl.(*ast.GenDecl).Specs[0].(*ast.ValueSpec)
   184  	for _, v := range as.Values {
   185  		id := v.(*ast.Ident)
   186  		if id.Obj != nil {
   187  			t.Errorf("rhs %s has Obj, should not", id.Name)
   188  		}
   189  	}
   190  	for _, id := range as.Names {
   191  		if id.Obj == nil {
   192  			t.Errorf("lhs %s does not have Obj, should", id.Name)
   193  		}
   194  	}
   195  }
   196  
   197  func TestObjects(t *testing.T) {
   198  	const src = `
   199  package p
   200  import fmt "fmt"
   201  const pi = 3.14
   202  type T struct{}
   203  var x int
   204  func f() { L: }
   205  `
   206  
   207  	f, err := ParseFile(token.NewFileSet(), "", src, 0)
   208  	if err != nil {
   209  		t.Fatal(err)
   210  	}
   211  
   212  	objects := map[string]ast.ObjKind{
   213  		"p":   ast.Bad, // not in a scope
   214  		"fmt": ast.Bad, // not resolved yet
   215  		"pi":  ast.Con,
   216  		"T":   ast.Typ,
   217  		"x":   ast.Var,
   218  		"int": ast.Bad, // not resolved yet
   219  		"f":   ast.Fun,
   220  		"L":   ast.Lbl,
   221  	}
   222  
   223  	ast.Inspect(f, func(n ast.Node) bool {
   224  		if ident, ok := n.(*ast.Ident); ok {
   225  			obj := ident.Obj
   226  			if obj == nil {
   227  				if objects[ident.Name] != ast.Bad {
   228  					t.Errorf("no object for %s", ident.Name)
   229  				}
   230  				return true
   231  			}
   232  			if obj.Name != ident.Name {
   233  				t.Errorf("names don't match: obj.Name = %s, ident.Name = %s", obj.Name, ident.Name)
   234  			}
   235  			kind := objects[ident.Name]
   236  			if obj.Kind != kind {
   237  				t.Errorf("%s: obj.Kind = %s; want %s", ident.Name, obj.Kind, kind)
   238  			}
   239  		}
   240  		return true
   241  	})
   242  }
   243  
   244  func TestUnresolved(t *testing.T) {
   245  	f, err := ParseFile(token.NewFileSet(), "", `
   246  package p
   247  //
   248  func f1a(int)
   249  func f2a(byte, int, float)
   250  func f3a(a, b int, c float)
   251  func f4a(...complex)
   252  func f5a(a s1a, b ...complex)
   253  //
   254  func f1b(*int)
   255  func f2b([]byte, (int), *float)
   256  func f3b(a, b *int, c []float)
   257  func f4b(...*complex)
   258  func f5b(a s1a, b ...[]complex)
   259  //
   260  type s1a struct { int }
   261  type s2a struct { byte; int; s1a }
   262  type s3a struct { a, b int; c float }
   263  //
   264  type s1b struct { *int }
   265  type s2b struct { byte; int; *float }
   266  type s3b struct { a, b *s3b; c []float }
   267  `, 0)
   268  	if err != nil {
   269  		t.Fatal(err)
   270  	}
   271  
   272  	want := "int " + // f1a
   273  		"byte int float " + // f2a
   274  		"int float " + // f3a
   275  		"complex " + // f4a
   276  		"complex " + // f5a
   277  		//
   278  		"int " + // f1b
   279  		"byte int float " + // f2b
   280  		"int float " + // f3b
   281  		"complex " + // f4b
   282  		"complex " + // f5b
   283  		//
   284  		"int " + // s1a
   285  		"byte int " + // s2a
   286  		"int float " + // s3a
   287  		//
   288  		"int " + // s1a
   289  		"byte int float " + // s2a
   290  		"float " // s3a
   291  
   292  	// collect unresolved identifiers
   293  	var buf bytes.Buffer
   294  	for _, u := range f.Unresolved {
   295  		buf.WriteString(u.Name)
   296  		buf.WriteByte(' ')
   297  	}
   298  	got := buf.String()
   299  
   300  	if got != want {
   301  		t.Errorf("\ngot:  %s\nwant: %s", got, want)
   302  	}
   303  }
   304  
   305  var imports = map[string]bool{
   306  	`"a"`:        true,
   307  	"`a`":        true,
   308  	`"a/b"`:      true,
   309  	`"a.b"`:      true,
   310  	`"m\x61th"`:  true,
   311  	`"greek/αβ"`: true,
   312  	`""`:         false,
   313  
   314  	// Each of these pairs tests both `` vs "" strings
   315  	// and also use of invalid characters spelled out as
   316  	// escape sequences and written directly.
   317  	// For example `"\x00"` tests import "\x00"
   318  	// while "`\x00`" tests import `<actual-NUL-byte>`.
   319  	`"\x00"`:     false,
   320  	"`\x00`":     false,
   321  	`"\x7f"`:     false,
   322  	"`\x7f`":     false,
   323  	`"a!"`:       false,
   324  	"`a!`":       false,
   325  	`"a b"`:      false,
   326  	"`a b`":      false,
   327  	`"a\\b"`:     false,
   328  	"`a\\b`":     false,
   329  	"\"`a`\"":    false,
   330  	"`\"a\"`":    false,
   331  	`"\x80\x80"`: false,
   332  	"`\x80\x80`": false,
   333  	`"\xFFFD"`:   false,
   334  	"`\xFFFD`":   false,
   335  }
   336  
   337  func TestImports(t *testing.T) {
   338  	for path, isValid := range imports {
   339  		src := fmt.Sprintf("package p; import %s", path)
   340  		_, err := ParseFile(token.NewFileSet(), "", src, 0)
   341  		switch {
   342  		case err != nil && isValid:
   343  			t.Errorf("ParseFile(%s): got %v; expected no error", src, err)
   344  		case err == nil && !isValid:
   345  			t.Errorf("ParseFile(%s): got no error; expected one", src)
   346  		}
   347  	}
   348  }
   349  
   350  func TestCommentGroups(t *testing.T) {
   351  	f, err := ParseFile(token.NewFileSet(), "", `
   352  package p /* 1a */ /* 1b */      /* 1c */ // 1d
   353  /* 2a
   354  */
   355  // 2b
   356  const pi = 3.1415
   357  /* 3a */ // 3b
   358  /* 3c */ const e = 2.7182
   359  
   360  // Example from issue 3139
   361  func ExampleCount() {
   362  	fmt.Println(strings.Count("cheese", "e"))
   363  	fmt.Println(strings.Count("five", "")) // before & after each rune
   364  	// Output:
   365  	// 3
   366  	// 5
   367  }
   368  `, ParseComments)
   369  	if err != nil {
   370  		t.Fatal(err)
   371  	}
   372  	expected := [][]string{
   373  		{"/* 1a */", "/* 1b */", "/* 1c */", "// 1d"},
   374  		{"/* 2a\n*/", "// 2b"},
   375  		{"/* 3a */", "// 3b", "/* 3c */"},
   376  		{"// Example from issue 3139"},
   377  		{"// before & after each rune"},
   378  		{"// Output:", "// 3", "// 5"},
   379  	}
   380  	if len(f.Comments) != len(expected) {
   381  		t.Fatalf("got %d comment groups; expected %d", len(f.Comments), len(expected))
   382  	}
   383  	for i, exp := range expected {
   384  		got := f.Comments[i].List
   385  		if len(got) != len(exp) {
   386  			t.Errorf("got %d comments in group %d; expected %d", len(got), i, len(exp))
   387  			continue
   388  		}
   389  		for j, exp := range exp {
   390  			got := got[j].Text
   391  			if got != exp {
   392  				t.Errorf("got %q in group %d; expected %q", got, i, exp)
   393  			}
   394  		}
   395  	}
   396  }
   397  
   398  func getField(file *ast.File, fieldname string) *ast.Field {
   399  	parts := strings.Split(fieldname, ".")
   400  	for _, d := range file.Decls {
   401  		if d, ok := d.(*ast.GenDecl); ok && d.Tok == token.TYPE {
   402  			for _, s := range d.Specs {
   403  				if s, ok := s.(*ast.TypeSpec); ok && s.Name.Name == parts[0] {
   404  					if s, ok := s.Type.(*ast.StructType); ok {
   405  						for _, f := range s.Fields.List {
   406  							for _, name := range f.Names {
   407  								if name.Name == parts[1] {
   408  									return f
   409  								}
   410  							}
   411  						}
   412  					}
   413  				}
   414  			}
   415  		}
   416  	}
   417  	return nil
   418  }
   419  
   420  // Don't use ast.CommentGroup.Text() - we want to see exact comment text.
   421  func commentText(c *ast.CommentGroup) string {
   422  	var buf bytes.Buffer
   423  	if c != nil {
   424  		for _, c := range c.List {
   425  			buf.WriteString(c.Text)
   426  		}
   427  	}
   428  	return buf.String()
   429  }
   430  
   431  func checkFieldComments(t *testing.T, file *ast.File, fieldname, lead, line string) {
   432  	f := getField(file, fieldname)
   433  	if f == nil {
   434  		t.Fatalf("field not found: %s", fieldname)
   435  	}
   436  	if got := commentText(f.Doc); got != lead {
   437  		t.Errorf("got lead comment %q; expected %q", got, lead)
   438  	}
   439  	if got := commentText(f.Comment); got != line {
   440  		t.Errorf("got line comment %q; expected %q", got, line)
   441  	}
   442  }
   443  
   444  func TestLeadAndLineComments(t *testing.T) {
   445  	f, err := ParseFile(token.NewFileSet(), "", `
   446  package p
   447  type T struct {
   448  	/* F1 lead comment */
   449  	//
   450  	F1 int  /* F1 */ // line comment
   451  	// F2 lead
   452  	// comment
   453  	F2 int  // F2 line comment
   454  	// f3 lead comment
   455  	f3 int  // f3 line comment
   456  }
   457  `, ParseComments)
   458  	if err != nil {
   459  		t.Fatal(err)
   460  	}
   461  	checkFieldComments(t, f, "T.F1", "/* F1 lead comment *///", "/* F1 */// line comment")
   462  	checkFieldComments(t, f, "T.F2", "// F2 lead// comment", "// F2 line comment")
   463  	checkFieldComments(t, f, "T.f3", "// f3 lead comment", "// f3 line comment")
   464  	ast.FileExports(f)
   465  	checkFieldComments(t, f, "T.F1", "/* F1 lead comment *///", "/* F1 */// line comment")
   466  	checkFieldComments(t, f, "T.F2", "// F2 lead// comment", "// F2 line comment")
   467  	if getField(f, "T.f3") != nil {
   468  		t.Error("not expected to find T.f3")
   469  	}
   470  }
   471  
   472  // TestIssue9979 verifies that empty statements are contained within their enclosing blocks.
   473  func TestIssue9979(t *testing.T) {
   474  	for _, src := range []string{
   475  		"package p; func f() {;}",
   476  		"package p; func f() {L:}",
   477  		"package p; func f() {L:;}",
   478  		"package p; func f() {L:\n}",
   479  		"package p; func f() {L:\n;}",
   480  		"package p; func f() { ; }",
   481  		"package p; func f() { L: }",
   482  		"package p; func f() { L: ; }",
   483  		"package p; func f() { L: \n}",
   484  		"package p; func f() { L: \n; }",
   485  	} {
   486  		fset := token.NewFileSet()
   487  		f, err := ParseFile(fset, "", src, 0)
   488  		if err != nil {
   489  			t.Fatal(err)
   490  		}
   491  
   492  		var pos, end token.Pos
   493  		ast.Inspect(f, func(x ast.Node) bool {
   494  			switch s := x.(type) {
   495  			case *ast.BlockStmt:
   496  				pos, end = s.Pos()+1, s.End()-1 // exclude "{", "}"
   497  			case *ast.LabeledStmt:
   498  				pos, end = s.Pos()+2, s.End() // exclude "L:"
   499  			case *ast.EmptyStmt:
   500  				// check containment
   501  				if s.Pos() < pos || s.End() > end {
   502  					t.Errorf("%s: %T[%d, %d] not inside [%d, %d]", src, s, s.Pos(), s.End(), pos, end)
   503  				}
   504  				// check semicolon
   505  				offs := fset.Position(s.Pos()).Offset
   506  				if ch := src[offs]; ch != ';' != s.Implicit {
   507  					want := "want ';'"
   508  					if s.Implicit {
   509  						want = "but ';' is implicit"
   510  					}
   511  					t.Errorf("%s: found %q at offset %d; %s", src, ch, offs, want)
   512  				}
   513  			}
   514  			return true
   515  		})
   516  	}
   517  }
   518  
   519  // TestIncompleteSelection ensures that an incomplete selector
   520  // expression is parsed as a (blank) *ast.SelectorExpr, not a
   521  // *ast.BadExpr.
   522  func TestIncompleteSelection(t *testing.T) {
   523  	for _, src := range []string{
   524  		"package p; var _ = fmt.",             // at EOF
   525  		"package p; var _ = fmt.\ntype X int", // not at EOF
   526  	} {
   527  		fset := token.NewFileSet()
   528  		f, err := ParseFile(fset, "", src, 0)
   529  		if err == nil {
   530  			t.Errorf("ParseFile(%s) succeeded unexpectedly", src)
   531  			continue
   532  		}
   533  
   534  		const wantErr = "expected selector or type assertion"
   535  		if !strings.Contains(err.Error(), wantErr) {
   536  			t.Errorf("ParseFile returned wrong error %q, want %q", err, wantErr)
   537  		}
   538  
   539  		var sel *ast.SelectorExpr
   540  		ast.Inspect(f, func(n ast.Node) bool {
   541  			if n, ok := n.(*ast.SelectorExpr); ok {
   542  				sel = n
   543  			}
   544  			return true
   545  		})
   546  		if sel == nil {
   547  			t.Error("found no *ast.SelectorExpr")
   548  			continue
   549  		}
   550  		const wantSel = "&{fmt _}"
   551  		if fmt.Sprint(sel) != wantSel {
   552  			t.Errorf("found selector %s, want %s", sel, wantSel)
   553  			continue
   554  		}
   555  	}
   556  }
   557  
   558  func TestLastLineComment(t *testing.T) {
   559  	const src = `package main
   560  type x int // comment
   561  `
   562  	fset := token.NewFileSet()
   563  	f, err := ParseFile(fset, "", src, ParseComments)
   564  	if err != nil {
   565  		t.Fatal(err)
   566  	}
   567  	comment := f.Decls[0].(*ast.GenDecl).Specs[0].(*ast.TypeSpec).Comment.List[0].Text
   568  	if comment != "// comment" {
   569  		t.Errorf("got %q, want %q", comment, "// comment")
   570  	}
   571  }
   572  

View as plain text