...
Run Format

Source file src/text/template/parse/lex_test.go

Documentation: text/template/parse

     1  // Copyright 2011 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package parse
     6  
     7  import (
     8  	"fmt"
     9  	"testing"
    10  )
    11  
    12  // Make the types prettyprint.
    13  var itemName = map[itemType]string{
    14  	itemError:        "error",
    15  	itemBool:         "bool",
    16  	itemChar:         "char",
    17  	itemCharConstant: "charconst",
    18  	itemComplex:      "complex",
    19  	itemColonEquals:  ":=",
    20  	itemEOF:          "EOF",
    21  	itemField:        "field",
    22  	itemIdentifier:   "identifier",
    23  	itemLeftDelim:    "left delim",
    24  	itemLeftParen:    "(",
    25  	itemNumber:       "number",
    26  	itemPipe:         "pipe",
    27  	itemRawString:    "raw string",
    28  	itemRightDelim:   "right delim",
    29  	itemRightParen:   ")",
    30  	itemSpace:        "space",
    31  	itemString:       "string",
    32  	itemVariable:     "variable",
    33  
    34  	// keywords
    35  	itemDot:      ".",
    36  	itemBlock:    "block",
    37  	itemDefine:   "define",
    38  	itemElse:     "else",
    39  	itemIf:       "if",
    40  	itemEnd:      "end",
    41  	itemNil:      "nil",
    42  	itemRange:    "range",
    43  	itemTemplate: "template",
    44  	itemWith:     "with",
    45  }
    46  
    47  func (i itemType) String() string {
    48  	s := itemName[i]
    49  	if s == "" {
    50  		return fmt.Sprintf("item%d", int(i))
    51  	}
    52  	return s
    53  }
    54  
    55  type lexTest struct {
    56  	name  string
    57  	input string
    58  	items []item
    59  }
    60  
    61  func mkItem(typ itemType, text string) item {
    62  	return item{
    63  		typ: typ,
    64  		val: text,
    65  	}
    66  }
    67  
    68  var (
    69  	tDot        = mkItem(itemDot, ".")
    70  	tBlock      = mkItem(itemBlock, "block")
    71  	tEOF        = mkItem(itemEOF, "")
    72  	tFor        = mkItem(itemIdentifier, "for")
    73  	tLeft       = mkItem(itemLeftDelim, "{{")
    74  	tLpar       = mkItem(itemLeftParen, "(")
    75  	tPipe       = mkItem(itemPipe, "|")
    76  	tQuote      = mkItem(itemString, `"abc \n\t\" "`)
    77  	tRange      = mkItem(itemRange, "range")
    78  	tRight      = mkItem(itemRightDelim, "}}")
    79  	tRpar       = mkItem(itemRightParen, ")")
    80  	tSpace      = mkItem(itemSpace, " ")
    81  	raw         = "`" + `abc\n\t\" ` + "`"
    82  	rawNL       = "`now is{{\n}}the time`" // Contains newline inside raw quote.
    83  	tRawQuote   = mkItem(itemRawString, raw)
    84  	tRawQuoteNL = mkItem(itemRawString, rawNL)
    85  )
    86  
    87  var lexTests = []lexTest{
    88  	{"empty", "", []item{tEOF}},
    89  	{"spaces", " \t\n", []item{mkItem(itemText, " \t\n"), tEOF}},
    90  	{"text", `now is the time`, []item{mkItem(itemText, "now is the time"), tEOF}},
    91  	{"text with comment", "hello-{{/* this is a comment */}}-world", []item{
    92  		mkItem(itemText, "hello-"),
    93  		mkItem(itemText, "-world"),
    94  		tEOF,
    95  	}},
    96  	{"punctuation", "{{,@% }}", []item{
    97  		tLeft,
    98  		mkItem(itemChar, ","),
    99  		mkItem(itemChar, "@"),
   100  		mkItem(itemChar, "%"),
   101  		tSpace,
   102  		tRight,
   103  		tEOF,
   104  	}},
   105  	{"parens", "{{((3))}}", []item{
   106  		tLeft,
   107  		tLpar,
   108  		tLpar,
   109  		mkItem(itemNumber, "3"),
   110  		tRpar,
   111  		tRpar,
   112  		tRight,
   113  		tEOF,
   114  	}},
   115  	{"empty action", `{{}}`, []item{tLeft, tRight, tEOF}},
   116  	{"for", `{{for}}`, []item{tLeft, tFor, tRight, tEOF}},
   117  	{"block", `{{block "foo" .}}`, []item{
   118  		tLeft, tBlock, tSpace, mkItem(itemString, `"foo"`), tSpace, tDot, tRight, tEOF,
   119  	}},
   120  	{"quote", `{{"abc \n\t\" "}}`, []item{tLeft, tQuote, tRight, tEOF}},
   121  	{"raw quote", "{{" + raw + "}}", []item{tLeft, tRawQuote, tRight, tEOF}},
   122  	{"raw quote with newline", "{{" + rawNL + "}}", []item{tLeft, tRawQuoteNL, tRight, tEOF}},
   123  	{"numbers", "{{1 02 0x14 -7.2i 1e3 +1.2e-4 4.2i 1+2i}}", []item{
   124  		tLeft,
   125  		mkItem(itemNumber, "1"),
   126  		tSpace,
   127  		mkItem(itemNumber, "02"),
   128  		tSpace,
   129  		mkItem(itemNumber, "0x14"),
   130  		tSpace,
   131  		mkItem(itemNumber, "-7.2i"),
   132  		tSpace,
   133  		mkItem(itemNumber, "1e3"),
   134  		tSpace,
   135  		mkItem(itemNumber, "+1.2e-4"),
   136  		tSpace,
   137  		mkItem(itemNumber, "4.2i"),
   138  		tSpace,
   139  		mkItem(itemComplex, "1+2i"),
   140  		tRight,
   141  		tEOF,
   142  	}},
   143  	{"characters", `{{'a' '\n' '\'' '\\' '\u00FF' '\xFF' '本'}}`, []item{
   144  		tLeft,
   145  		mkItem(itemCharConstant, `'a'`),
   146  		tSpace,
   147  		mkItem(itemCharConstant, `'\n'`),
   148  		tSpace,
   149  		mkItem(itemCharConstant, `'\''`),
   150  		tSpace,
   151  		mkItem(itemCharConstant, `'\\'`),
   152  		tSpace,
   153  		mkItem(itemCharConstant, `'\u00FF'`),
   154  		tSpace,
   155  		mkItem(itemCharConstant, `'\xFF'`),
   156  		tSpace,
   157  		mkItem(itemCharConstant, `'本'`),
   158  		tRight,
   159  		tEOF,
   160  	}},
   161  	{"bools", "{{true false}}", []item{
   162  		tLeft,
   163  		mkItem(itemBool, "true"),
   164  		tSpace,
   165  		mkItem(itemBool, "false"),
   166  		tRight,
   167  		tEOF,
   168  	}},
   169  	{"dot", "{{.}}", []item{
   170  		tLeft,
   171  		tDot,
   172  		tRight,
   173  		tEOF,
   174  	}},
   175  	{"nil", "{{nil}}", []item{
   176  		tLeft,
   177  		mkItem(itemNil, "nil"),
   178  		tRight,
   179  		tEOF,
   180  	}},
   181  	{"dots", "{{.x . .2 .x.y.z}}", []item{
   182  		tLeft,
   183  		mkItem(itemField, ".x"),
   184  		tSpace,
   185  		tDot,
   186  		tSpace,
   187  		mkItem(itemNumber, ".2"),
   188  		tSpace,
   189  		mkItem(itemField, ".x"),
   190  		mkItem(itemField, ".y"),
   191  		mkItem(itemField, ".z"),
   192  		tRight,
   193  		tEOF,
   194  	}},
   195  	{"keywords", "{{range if else end with}}", []item{
   196  		tLeft,
   197  		mkItem(itemRange, "range"),
   198  		tSpace,
   199  		mkItem(itemIf, "if"),
   200  		tSpace,
   201  		mkItem(itemElse, "else"),
   202  		tSpace,
   203  		mkItem(itemEnd, "end"),
   204  		tSpace,
   205  		mkItem(itemWith, "with"),
   206  		tRight,
   207  		tEOF,
   208  	}},
   209  	{"variables", "{{$c := printf $ $hello $23 $ $var.Field .Method}}", []item{
   210  		tLeft,
   211  		mkItem(itemVariable, "$c"),
   212  		tSpace,
   213  		mkItem(itemColonEquals, ":="),
   214  		tSpace,
   215  		mkItem(itemIdentifier, "printf"),
   216  		tSpace,
   217  		mkItem(itemVariable, "$"),
   218  		tSpace,
   219  		mkItem(itemVariable, "$hello"),
   220  		tSpace,
   221  		mkItem(itemVariable, "$23"),
   222  		tSpace,
   223  		mkItem(itemVariable, "$"),
   224  		tSpace,
   225  		mkItem(itemVariable, "$var"),
   226  		mkItem(itemField, ".Field"),
   227  		tSpace,
   228  		mkItem(itemField, ".Method"),
   229  		tRight,
   230  		tEOF,
   231  	}},
   232  	{"variable invocation", "{{$x 23}}", []item{
   233  		tLeft,
   234  		mkItem(itemVariable, "$x"),
   235  		tSpace,
   236  		mkItem(itemNumber, "23"),
   237  		tRight,
   238  		tEOF,
   239  	}},
   240  	{"pipeline", `intro {{echo hi 1.2 |noargs|args 1 "hi"}} outro`, []item{
   241  		mkItem(itemText, "intro "),
   242  		tLeft,
   243  		mkItem(itemIdentifier, "echo"),
   244  		tSpace,
   245  		mkItem(itemIdentifier, "hi"),
   246  		tSpace,
   247  		mkItem(itemNumber, "1.2"),
   248  		tSpace,
   249  		tPipe,
   250  		mkItem(itemIdentifier, "noargs"),
   251  		tPipe,
   252  		mkItem(itemIdentifier, "args"),
   253  		tSpace,
   254  		mkItem(itemNumber, "1"),
   255  		tSpace,
   256  		mkItem(itemString, `"hi"`),
   257  		tRight,
   258  		mkItem(itemText, " outro"),
   259  		tEOF,
   260  	}},
   261  	{"declaration", "{{$v := 3}}", []item{
   262  		tLeft,
   263  		mkItem(itemVariable, "$v"),
   264  		tSpace,
   265  		mkItem(itemColonEquals, ":="),
   266  		tSpace,
   267  		mkItem(itemNumber, "3"),
   268  		tRight,
   269  		tEOF,
   270  	}},
   271  	{"2 declarations", "{{$v , $w := 3}}", []item{
   272  		tLeft,
   273  		mkItem(itemVariable, "$v"),
   274  		tSpace,
   275  		mkItem(itemChar, ","),
   276  		tSpace,
   277  		mkItem(itemVariable, "$w"),
   278  		tSpace,
   279  		mkItem(itemColonEquals, ":="),
   280  		tSpace,
   281  		mkItem(itemNumber, "3"),
   282  		tRight,
   283  		tEOF,
   284  	}},
   285  	{"field of parenthesized expression", "{{(.X).Y}}", []item{
   286  		tLeft,
   287  		tLpar,
   288  		mkItem(itemField, ".X"),
   289  		tRpar,
   290  		mkItem(itemField, ".Y"),
   291  		tRight,
   292  		tEOF,
   293  	}},
   294  	{"trimming spaces before and after", "hello- {{- 3 -}} -world", []item{
   295  		mkItem(itemText, "hello-"),
   296  		tLeft,
   297  		mkItem(itemNumber, "3"),
   298  		tRight,
   299  		mkItem(itemText, "-world"),
   300  		tEOF,
   301  	}},
   302  	{"trimming spaces before and after comment", "hello- {{- /* hello */ -}} -world", []item{
   303  		mkItem(itemText, "hello-"),
   304  		mkItem(itemText, "-world"),
   305  		tEOF,
   306  	}},
   307  	// errors
   308  	{"badchar", "#{{\x01}}", []item{
   309  		mkItem(itemText, "#"),
   310  		tLeft,
   311  		mkItem(itemError, "unrecognized character in action: U+0001"),
   312  	}},
   313  	{"unclosed action", "{{\n}}", []item{
   314  		tLeft,
   315  		mkItem(itemError, "unclosed action"),
   316  	}},
   317  	{"EOF in action", "{{range", []item{
   318  		tLeft,
   319  		tRange,
   320  		mkItem(itemError, "unclosed action"),
   321  	}},
   322  	{"unclosed quote", "{{\"\n\"}}", []item{
   323  		tLeft,
   324  		mkItem(itemError, "unterminated quoted string"),
   325  	}},
   326  	{"unclosed raw quote", "{{`xx}}", []item{
   327  		tLeft,
   328  		mkItem(itemError, "unterminated raw quoted string"),
   329  	}},
   330  	{"unclosed char constant", "{{'\n}}", []item{
   331  		tLeft,
   332  		mkItem(itemError, "unterminated character constant"),
   333  	}},
   334  	{"bad number", "{{3k}}", []item{
   335  		tLeft,
   336  		mkItem(itemError, `bad number syntax: "3k"`),
   337  	}},
   338  	{"unclosed paren", "{{(3}}", []item{
   339  		tLeft,
   340  		tLpar,
   341  		mkItem(itemNumber, "3"),
   342  		mkItem(itemError, `unclosed left paren`),
   343  	}},
   344  	{"extra right paren", "{{3)}}", []item{
   345  		tLeft,
   346  		mkItem(itemNumber, "3"),
   347  		tRpar,
   348  		mkItem(itemError, `unexpected right paren U+0029 ')'`),
   349  	}},
   350  
   351  	// Fixed bugs
   352  	// Many elements in an action blew the lookahead until
   353  	// we made lexInsideAction not loop.
   354  	{"long pipeline deadlock", "{{|||||}}", []item{
   355  		tLeft,
   356  		tPipe,
   357  		tPipe,
   358  		tPipe,
   359  		tPipe,
   360  		tPipe,
   361  		tRight,
   362  		tEOF,
   363  	}},
   364  	{"text with bad comment", "hello-{{/*/}}-world", []item{
   365  		mkItem(itemText, "hello-"),
   366  		mkItem(itemError, `unclosed comment`),
   367  	}},
   368  	{"text with comment close separated from delim", "hello-{{/* */ }}-world", []item{
   369  		mkItem(itemText, "hello-"),
   370  		mkItem(itemError, `comment ends before closing delimiter`),
   371  	}},
   372  	// This one is an error that we can't catch because it breaks templates with
   373  	// minimized JavaScript. Should have fixed it before Go 1.1.
   374  	{"unmatched right delimiter", "hello-{.}}-world", []item{
   375  		mkItem(itemText, "hello-{.}}-world"),
   376  		tEOF,
   377  	}},
   378  }
   379  
   380  // collect gathers the emitted items into a slice.
   381  func collect(t *lexTest, left, right string) (items []item) {
   382  	l := lex(t.name, t.input, left, right)
   383  	for {
   384  		item := l.nextItem()
   385  		items = append(items, item)
   386  		if item.typ == itemEOF || item.typ == itemError {
   387  			break
   388  		}
   389  	}
   390  	return
   391  }
   392  
   393  func equal(i1, i2 []item, checkPos bool) bool {
   394  	if len(i1) != len(i2) {
   395  		return false
   396  	}
   397  	for k := range i1 {
   398  		if i1[k].typ != i2[k].typ {
   399  			return false
   400  		}
   401  		if i1[k].val != i2[k].val {
   402  			return false
   403  		}
   404  		if checkPos && i1[k].pos != i2[k].pos {
   405  			return false
   406  		}
   407  		if checkPos && i1[k].line != i2[k].line {
   408  			return false
   409  		}
   410  	}
   411  	return true
   412  }
   413  
   414  func TestLex(t *testing.T) {
   415  	for _, test := range lexTests {
   416  		items := collect(&test, "", "")
   417  		if !equal(items, test.items, false) {
   418  			t.Errorf("%s: got\n\t%+v\nexpected\n\t%v", test.name, items, test.items)
   419  		}
   420  	}
   421  }
   422  
   423  // Some easy cases from above, but with delimiters $$ and @@
   424  var lexDelimTests = []lexTest{
   425  	{"punctuation", "$$,@%{{}}@@", []item{
   426  		tLeftDelim,
   427  		mkItem(itemChar, ","),
   428  		mkItem(itemChar, "@"),
   429  		mkItem(itemChar, "%"),
   430  		mkItem(itemChar, "{"),
   431  		mkItem(itemChar, "{"),
   432  		mkItem(itemChar, "}"),
   433  		mkItem(itemChar, "}"),
   434  		tRightDelim,
   435  		tEOF,
   436  	}},
   437  	{"empty action", `$$@@`, []item{tLeftDelim, tRightDelim, tEOF}},
   438  	{"for", `$$for@@`, []item{tLeftDelim, tFor, tRightDelim, tEOF}},
   439  	{"quote", `$$"abc \n\t\" "@@`, []item{tLeftDelim, tQuote, tRightDelim, tEOF}},
   440  	{"raw quote", "$$" + raw + "@@", []item{tLeftDelim, tRawQuote, tRightDelim, tEOF}},
   441  }
   442  
   443  var (
   444  	tLeftDelim  = mkItem(itemLeftDelim, "$$")
   445  	tRightDelim = mkItem(itemRightDelim, "@@")
   446  )
   447  
   448  func TestDelims(t *testing.T) {
   449  	for _, test := range lexDelimTests {
   450  		items := collect(&test, "$$", "@@")
   451  		if !equal(items, test.items, false) {
   452  			t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
   453  		}
   454  	}
   455  }
   456  
   457  var lexPosTests = []lexTest{
   458  	{"empty", "", []item{{itemEOF, 0, "", 1}}},
   459  	{"punctuation", "{{,@%#}}", []item{
   460  		{itemLeftDelim, 0, "{{", 1},
   461  		{itemChar, 2, ",", 1},
   462  		{itemChar, 3, "@", 1},
   463  		{itemChar, 4, "%", 1},
   464  		{itemChar, 5, "#", 1},
   465  		{itemRightDelim, 6, "}}", 1},
   466  		{itemEOF, 8, "", 1},
   467  	}},
   468  	{"sample", "0123{{hello}}xyz", []item{
   469  		{itemText, 0, "0123", 1},
   470  		{itemLeftDelim, 4, "{{", 1},
   471  		{itemIdentifier, 6, "hello", 1},
   472  		{itemRightDelim, 11, "}}", 1},
   473  		{itemText, 13, "xyz", 1},
   474  		{itemEOF, 16, "", 1},
   475  	}},
   476  	{"trimafter", "{{x -}}\n{{y}}", []item{
   477  		{itemLeftDelim, 0, "{{", 1},
   478  		{itemIdentifier, 2, "x", 1},
   479  		{itemRightDelim, 5, "}}", 1},
   480  		{itemLeftDelim, 8, "{{", 2},
   481  		{itemIdentifier, 10, "y", 2},
   482  		{itemRightDelim, 11, "}}", 2},
   483  		{itemEOF, 13, "", 2},
   484  	}},
   485  	{"trimbefore", "{{x}}\n{{- y}}", []item{
   486  		{itemLeftDelim, 0, "{{", 1},
   487  		{itemIdentifier, 2, "x", 1},
   488  		{itemRightDelim, 3, "}}", 1},
   489  		{itemLeftDelim, 6, "{{", 2},
   490  		{itemIdentifier, 10, "y", 2},
   491  		{itemRightDelim, 11, "}}", 2},
   492  		{itemEOF, 13, "", 2},
   493  	}},
   494  }
   495  
   496  // The other tests don't check position, to make the test cases easier to construct.
   497  // This one does.
   498  func TestPos(t *testing.T) {
   499  	for _, test := range lexPosTests {
   500  		items := collect(&test, "", "")
   501  		if !equal(items, test.items, true) {
   502  			t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
   503  			if len(items) == len(test.items) {
   504  				// Detailed print; avoid item.String() to expose the position value.
   505  				for i := range items {
   506  					if !equal(items[i:i+1], test.items[i:i+1], true) {
   507  						i1 := items[i]
   508  						i2 := test.items[i]
   509  						t.Errorf("\t#%d: got {%v %d %q %d} expected {%v %d %q %d}",
   510  							i, i1.typ, i1.pos, i1.val, i1.line, i2.typ, i2.pos, i2.val, i2.line)
   511  					}
   512  				}
   513  			}
   514  		}
   515  	}
   516  }
   517  
   518  // Test that an error shuts down the lexing goroutine.
   519  func TestShutdown(t *testing.T) {
   520  	// We need to duplicate template.Parse here to hold on to the lexer.
   521  	const text = "erroneous{{define}}{{else}}1234"
   522  	lexer := lex("foo", text, "{{", "}}")
   523  	_, err := New("root").parseLexer(lexer)
   524  	if err == nil {
   525  		t.Fatalf("expected error")
   526  	}
   527  	// The error should have drained the input. Therefore, the lexer should be shut down.
   528  	token, ok := <-lexer.items
   529  	if ok {
   530  		t.Fatalf("input was not drained; got %v", token)
   531  	}
   532  }
   533  
   534  // parseLexer is a local version of parse that lets us pass in the lexer instead of building it.
   535  // We expect an error, so the tree set and funcs list are explicitly nil.
   536  func (t *Tree) parseLexer(lex *lexer) (tree *Tree, err error) {
   537  	defer t.recover(&err)
   538  	t.ParseName = t.Name
   539  	t.startParse(nil, lex, map[string]*Tree{})
   540  	t.parse()
   541  	t.add()
   542  	t.stopParse()
   543  	return t, nil
   544  }
   545  

View as plain text