...
Run Format

Source file src/encoding/csv/reader_test.go

Documentation: encoding/csv

  // Copyright 2011 The Go Authors. All rights reserved.
  // Use of this source code is governed by a BSD-style
  // license that can be found in the LICENSE file.
  
  package csv
  
  import (
  	"io"
  	"reflect"
  	"strings"
  	"testing"
  	"unicode/utf8"
  )
  
  func TestRead(t *testing.T) {
  	tests := []struct {
  		Name   string
  		Input  string
  		Output [][]string
  		Error  error
  
  		// These fields are copied into the Reader
  		Comma              rune
  		Comment            rune
  		UseFieldsPerRecord bool // false (default) means FieldsPerRecord is -1
  		FieldsPerRecord    int
  		LazyQuotes         bool
  		TrimLeadingSpace   bool
  		ReuseRecord        bool
  	}{{
  		Name:   "Simple",
  		Input:  "a,b,c\n",
  		Output: [][]string{{"a", "b", "c"}},
  	}, {
  		Name:   "CRLF",
  		Input:  "a,b\r\nc,d\r\n",
  		Output: [][]string{{"a", "b"}, {"c", "d"}},
  	}, {
  		Name:   "BareCR",
  		Input:  "a,b\rc,d\r\n",
  		Output: [][]string{{"a", "b\rc", "d"}},
  	}, {
  		Name: "RFC4180test",
  		Input: `#field1,field2,field3
  "aaa","bb
  b","ccc"
  "a,a","b""bb","ccc"
  zzz,yyy,xxx
  `,
  		Output: [][]string{
  			{"#field1", "field2", "field3"},
  			{"aaa", "bb\nb", "ccc"},
  			{"a,a", `b"bb`, "ccc"},
  			{"zzz", "yyy", "xxx"},
  		},
  		UseFieldsPerRecord: true,
  		FieldsPerRecord:    0,
  	}, {
  		Name:   "NoEOLTest",
  		Input:  "a,b,c",
  		Output: [][]string{{"a", "b", "c"}},
  	}, {
  		Name:   "Semicolon",
  		Input:  "a;b;c\n",
  		Output: [][]string{{"a", "b", "c"}},
  		Comma:  ';',
  	}, {
  		Name: "MultiLine",
  		Input: `"two
  line","one line","three
  line
  field"`,
  		Output: [][]string{{"two\nline", "one line", "three\nline\nfield"}},
  	}, {
  		Name:  "BlankLine",
  		Input: "a,b,c\n\nd,e,f\n\n",
  		Output: [][]string{
  			{"a", "b", "c"},
  			{"d", "e", "f"},
  		},
  	}, {
  		Name:  "BlankLineFieldCount",
  		Input: "a,b,c\n\nd,e,f\n\n",
  		Output: [][]string{
  			{"a", "b", "c"},
  			{"d", "e", "f"},
  		},
  		UseFieldsPerRecord: true,
  		FieldsPerRecord:    0,
  	}, {
  		Name:             "TrimSpace",
  		Input:            " a,  b,   c\n",
  		Output:           [][]string{{"a", "b", "c"}},
  		TrimLeadingSpace: true,
  	}, {
  		Name:   "LeadingSpace",
  		Input:  " a,  b,   c\n",
  		Output: [][]string{{" a", "  b", "   c"}},
  	}, {
  		Name:    "Comment",
  		Input:   "#1,2,3\na,b,c\n#comment",
  		Output:  [][]string{{"a", "b", "c"}},
  		Comment: '#',
  	}, {
  		Name:   "NoComment",
  		Input:  "#1,2,3\na,b,c",
  		Output: [][]string{{"#1", "2", "3"}, {"a", "b", "c"}},
  	}, {
  		Name:       "LazyQuotes",
  		Input:      `a "word","1"2",a","b`,
  		Output:     [][]string{{`a "word"`, `1"2`, `a"`, `b`}},
  		LazyQuotes: true,
  	}, {
  		Name:       "BareQuotes",
  		Input:      `a "word","1"2",a"`,
  		Output:     [][]string{{`a "word"`, `1"2`, `a"`}},
  		LazyQuotes: true,
  	}, {
  		Name:       "BareDoubleQuotes",
  		Input:      `a""b,c`,
  		Output:     [][]string{{`a""b`, `c`}},
  		LazyQuotes: true,
  	}, {
  		Name:  "BadDoubleQuotes",
  		Input: `a""b,c`,
  		Error: &ParseError{StartLine: 1, Line: 1, Column: 1, Err: ErrBareQuote},
  	}, {
  		Name:             "TrimQuote",
  		Input:            ` "a"," b",c`,
  		Output:           [][]string{{"a", " b", "c"}},
  		TrimLeadingSpace: true,
  	}, {
  		Name:  "BadBareQuote",
  		Input: `a "word","b"`,
  		Error: &ParseError{StartLine: 1, Line: 1, Column: 2, Err: ErrBareQuote},
  	}, {
  		Name:  "BadTrailingQuote",
  		Input: `"a word",b"`,
  		Error: &ParseError{StartLine: 1, Line: 1, Column: 10, Err: ErrBareQuote},
  	}, {
  		Name:  "ExtraneousQuote",
  		Input: `"a "word","b"`,
  		Error: &ParseError{StartLine: 1, Line: 1, Column: 3, Err: ErrQuote},
  	}, {
  		Name:               "BadFieldCount",
  		Input:              "a,b,c\nd,e",
  		Error:              &ParseError{StartLine: 2, Line: 2, Err: ErrFieldCount},
  		UseFieldsPerRecord: true,
  		FieldsPerRecord:    0,
  	}, {
  		Name:               "BadFieldCount1",
  		Input:              `a,b,c`,
  		Error:              &ParseError{StartLine: 1, Line: 1, Err: ErrFieldCount},
  		UseFieldsPerRecord: true,
  		FieldsPerRecord:    2,
  	}, {
  		Name:   "FieldCount",
  		Input:  "a,b,c\nd,e",
  		Output: [][]string{{"a", "b", "c"}, {"d", "e"}},
  	}, {
  		Name:   "TrailingCommaEOF",
  		Input:  "a,b,c,",
  		Output: [][]string{{"a", "b", "c", ""}},
  	}, {
  		Name:   "TrailingCommaEOL",
  		Input:  "a,b,c,\n",
  		Output: [][]string{{"a", "b", "c", ""}},
  	}, {
  		Name:             "TrailingCommaSpaceEOF",
  		Input:            "a,b,c, ",
  		Output:           [][]string{{"a", "b", "c", ""}},
  		TrimLeadingSpace: true,
  	}, {
  		Name:             "TrailingCommaSpaceEOL",
  		Input:            "a,b,c, \n",
  		Output:           [][]string{{"a", "b", "c", ""}},
  		TrimLeadingSpace: true,
  	}, {
  		Name:             "TrailingCommaLine3",
  		Input:            "a,b,c\nd,e,f\ng,hi,",
  		Output:           [][]string{{"a", "b", "c"}, {"d", "e", "f"}, {"g", "hi", ""}},
  		TrimLeadingSpace: true,
  	}, {
  		Name:   "NotTrailingComma3",
  		Input:  "a,b,c, \n",
  		Output: [][]string{{"a", "b", "c", " "}},
  	}, {
  		Name: "CommaFieldTest",
  		Input: `x,y,z,w
  x,y,z,
  x,y,,
  x,,,
  ,,,
  "x","y","z","w"
  "x","y","z",""
  "x","y","",""
  "x","","",""
  "","","",""
  `,
  		Output: [][]string{
  			{"x", "y", "z", "w"},
  			{"x", "y", "z", ""},
  			{"x", "y", "", ""},
  			{"x", "", "", ""},
  			{"", "", "", ""},
  			{"x", "y", "z", "w"},
  			{"x", "y", "z", ""},
  			{"x", "y", "", ""},
  			{"x", "", "", ""},
  			{"", "", "", ""},
  		},
  	}, {
  		Name:  "TrailingCommaIneffective1",
  		Input: "a,b,\nc,d,e",
  		Output: [][]string{
  			{"a", "b", ""},
  			{"c", "d", "e"},
  		},
  		TrimLeadingSpace: true,
  	}, {
  		Name:  "ReadAllReuseRecord",
  		Input: "a,b\nc,d",
  		Output: [][]string{
  			{"a", "b"},
  			{"c", "d"},
  		},
  		ReuseRecord: true,
  	}, {
  		Name:  "StartLine1", // Issue 19019
  		Input: "a,\"b\nc\"d,e",
  		Error: &ParseError{StartLine: 1, Line: 2, Column: 1, Err: ErrQuote},
  	}, {
  		Name:  "StartLine2",
  		Input: "a,b\n\"d\n\n,e",
  		Error: &ParseError{StartLine: 2, Line: 5, Column: 0, Err: ErrQuote},
  	}, {
  		Name:  "CRLFInQuotedField", // Issue 21201
  		Input: "A,\"Hello\r\nHi\",B\r\n",
  		Output: [][]string{
  			{"A", "Hello\nHi", "B"},
  		},
  	}, {
  		Name:   "BinaryBlobField", // Issue 19410
  		Input:  "x09\x41\xb4\x1c,aktau",
  		Output: [][]string{{"x09A\xb4\x1c", "aktau"}},
  	}, {
  		Name:   "TrailingCR",
  		Input:  "field1,field2\r",
  		Output: [][]string{{"field1", "field2"}},
  	}, {
  		Name:   "QuotedTrailingCR",
  		Input:  "\"field\"\r",
  		Output: [][]string{{"field"}},
  	}, {
  		Name:  "QuotedTrailingCRCR",
  		Input: "\"field\"\r\r",
  		Error: &ParseError{StartLine: 1, Line: 1, Column: 6, Err: ErrQuote},
  	}, {
  		Name:   "FieldCR",
  		Input:  "field\rfield\r",
  		Output: [][]string{{"field\rfield"}},
  	}, {
  		Name:   "FieldCRCR",
  		Input:  "field\r\rfield\r\r",
  		Output: [][]string{{"field\r\rfield\r"}},
  	}, {
  		Name:   "FieldCRCRLF",
  		Input:  "field\r\r\nfield\r\r\n",
  		Output: [][]string{{"field\r"}, {"field\r"}},
  	}, {
  		Name:   "FieldCRCRLFCR",
  		Input:  "field\r\r\n\rfield\r\r\n\r",
  		Output: [][]string{{"field\r"}, {"\rfield\r"}},
  	}, {
  		Name:   "FieldCRCRLFCRCR",
  		Input:  "field\r\r\n\r\rfield\r\r\n\r\r",
  		Output: [][]string{{"field\r"}, {"\r\rfield\r"}, {"\r"}},
  	}, {
  		Name:  "MultiFieldCRCRLFCRCR",
  		Input: "field1,field2\r\r\n\r\rfield1,field2\r\r\n\r\r,",
  		Output: [][]string{
  			{"field1", "field2\r"},
  			{"\r\rfield1", "field2\r"},
  			{"\r\r", ""},
  		},
  	}, {
  		Name:             "NonASCIICommaAndComment",
  		Input:            "a£b,c£ \td,e\n€ comment\n",
  		Output:           [][]string{{"a", "b,c", "d,e"}},
  		TrimLeadingSpace: true,
  		Comma:            '£',
  		Comment:          '€',
  	}, {
  		Name:    "NonASCIICommaAndCommentWithQuotes",
  		Input:   "a€\"  b,\"€ c\nλ comment\n",
  		Output:  [][]string{{"a", "  b,", " c"}},
  		Comma:   '€',
  		Comment: 'λ',
  	}, {
  		// λ and θ start with the same byte.
  		// This tests that the parser doesn't confuse such characters.
  		Name:    "NonASCIICommaConfusion",
  		Input:   "\"abθcd\"λefθgh",
  		Output:  [][]string{{"abθcd", "efθgh"}},
  		Comma:   'λ',
  		Comment: '€',
  	}, {
  		Name:    "NonASCIICommentConfusion",
  		Input:   "λ\nλ\nθ\nλ\n",
  		Output:  [][]string{{"λ"}, {"λ"}, {"λ"}},
  		Comment: 'θ',
  	}, {
  		Name:   "QuotedFieldMultipleLF",
  		Input:  "\"\n\n\n\n\"",
  		Output: [][]string{{"\n\n\n\n"}},
  	}, {
  		Name:  "MultipleCRLF",
  		Input: "\r\n\r\n\r\n\r\n",
  	}, {
  		// The implementation may read each line in several chunks if it doesn't fit entirely
  		// in the read buffer, so we should test the code to handle that condition.
  		Name:    "HugeLines",
  		Input:   strings.Repeat("#ignore\n", 10000) + strings.Repeat("@", 5000) + "," + strings.Repeat("*", 5000),
  		Output:  [][]string{{strings.Repeat("@", 5000), strings.Repeat("*", 5000)}},
  		Comment: '#',
  	}, {
  		Name:  "QuoteWithTrailingCRLF",
  		Input: "\"foo\"bar\"\r\n",
  		Error: &ParseError{StartLine: 1, Line: 1, Column: 4, Err: ErrQuote},
  	}, {
  		Name:       "LazyQuoteWithTrailingCRLF",
  		Input:      "\"foo\"bar\"\r\n",
  		Output:     [][]string{{`foo"bar`}},
  		LazyQuotes: true,
  	}, {
  		Name:   "DoubleQuoteWithTrailingCRLF",
  		Input:  "\"foo\"\"bar\"\r\n",
  		Output: [][]string{{`foo"bar`}},
  	}, {
  		Name:   "EvenQuotes",
  		Input:  `""""""""`,
  		Output: [][]string{{`"""`}},
  	}, {
  		Name:  "OddQuotes",
  		Input: `"""""""`,
  		Error: &ParseError{StartLine: 1, Line: 1, Column: 7, Err: ErrQuote},
  	}, {
  		Name:       "LazyOddQuotes",
  		Input:      `"""""""`,
  		Output:     [][]string{{`"""`}},
  		LazyQuotes: true,
  	}, {
  		Name:  "BadComma1",
  		Comma: '\n',
  		Error: errInvalidDelim,
  	}, {
  		Name:  "BadComma2",
  		Comma: '\r',
  		Error: errInvalidDelim,
  	}, {
  		Name:  "BadComma3",
  		Comma: '"',
  		Error: errInvalidDelim,
  	}, {
  		Name:  "BadComma4",
  		Comma: utf8.RuneError,
  		Error: errInvalidDelim,
  	}, {
  		Name:    "BadComment1",
  		Comment: '\n',
  		Error:   errInvalidDelim,
  	}, {
  		Name:    "BadComment2",
  		Comment: '\r',
  		Error:   errInvalidDelim,
  	}, {
  		Name:    "BadComment3",
  		Comment: utf8.RuneError,
  		Error:   errInvalidDelim,
  	}, {
  		Name:    "BadCommaComment",
  		Comma:   'X',
  		Comment: 'X',
  		Error:   errInvalidDelim,
  	}}
  
  	for _, tt := range tests {
  		t.Run(tt.Name, func(t *testing.T) {
  			r := NewReader(strings.NewReader(tt.Input))
  
  			if tt.Comma != 0 {
  				r.Comma = tt.Comma
  			}
  			r.Comment = tt.Comment
  			if tt.UseFieldsPerRecord {
  				r.FieldsPerRecord = tt.FieldsPerRecord
  			} else {
  				r.FieldsPerRecord = -1
  			}
  			r.LazyQuotes = tt.LazyQuotes
  			r.TrimLeadingSpace = tt.TrimLeadingSpace
  			r.ReuseRecord = tt.ReuseRecord
  
  			out, err := r.ReadAll()
  			if !reflect.DeepEqual(err, tt.Error) {
  				t.Errorf("ReadAll() error:\ngot  %v\nwant %v", err, tt.Error)
  			} else if !reflect.DeepEqual(out, tt.Output) {
  				t.Errorf("ReadAll() output:\ngot  %q\nwant %q", out, tt.Output)
  			}
  		})
  	}
  }
  
  // nTimes is an io.Reader which yields the string s n times.
  type nTimes struct {
  	s   string
  	n   int
  	off int
  }
  
  func (r *nTimes) Read(p []byte) (n int, err error) {
  	for {
  		if r.n <= 0 || r.s == "" {
  			return n, io.EOF
  		}
  		n0 := copy(p, r.s[r.off:])
  		p = p[n0:]
  		n += n0
  		r.off += n0
  		if r.off == len(r.s) {
  			r.off = 0
  			r.n--
  		}
  		if len(p) == 0 {
  			return
  		}
  	}
  }
  
  // benchmarkRead measures reading the provided CSV rows data.
  // initReader, if non-nil, modifies the Reader before it's used.
  func benchmarkRead(b *testing.B, initReader func(*Reader), rows string) {
  	b.ReportAllocs()
  	r := NewReader(&nTimes{s: rows, n: b.N})
  	if initReader != nil {
  		initReader(r)
  	}
  	for {
  		_, err := r.Read()
  		if err == io.EOF {
  			break
  		}
  		if err != nil {
  			b.Fatal(err)
  		}
  	}
  }
  
  const benchmarkCSVData = `x,y,z,w
  x,y,z,
  x,y,,
  x,,,
  ,,,
  "x","y","z","w"
  "x","y","z",""
  "x","y","",""
  "x","","",""
  "","","",""
  `
  
  func BenchmarkRead(b *testing.B) {
  	benchmarkRead(b, nil, benchmarkCSVData)
  }
  
  func BenchmarkReadWithFieldsPerRecord(b *testing.B) {
  	benchmarkRead(b, func(r *Reader) { r.FieldsPerRecord = 4 }, benchmarkCSVData)
  }
  
  func BenchmarkReadWithoutFieldsPerRecord(b *testing.B) {
  	benchmarkRead(b, func(r *Reader) { r.FieldsPerRecord = -1 }, benchmarkCSVData)
  }
  
  func BenchmarkReadLargeFields(b *testing.B) {
  	benchmarkRead(b, nil, strings.Repeat(`xxxxxxxxxxxxxxxx,yyyyyyyyyyyyyyyy,zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
  xxxxxxxxxxxxxxxxxxxxxxxx,yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy,zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvv
  ,,zzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
  xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx,yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy,zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
  `, 3))
  }
  
  func BenchmarkReadReuseRecord(b *testing.B) {
  	benchmarkRead(b, func(r *Reader) { r.ReuseRecord = true }, benchmarkCSVData)
  }
  
  func BenchmarkReadReuseRecordWithFieldsPerRecord(b *testing.B) {
  	benchmarkRead(b, func(r *Reader) { r.ReuseRecord = true; r.FieldsPerRecord = 4 }, benchmarkCSVData)
  }
  
  func BenchmarkReadReuseRecordWithoutFieldsPerRecord(b *testing.B) {
  	benchmarkRead(b, func(r *Reader) { r.ReuseRecord = true; r.FieldsPerRecord = -1 }, benchmarkCSVData)
  }
  
  func BenchmarkReadReuseRecordLargeFields(b *testing.B) {
  	benchmarkRead(b, func(r *Reader) { r.ReuseRecord = true }, strings.Repeat(`xxxxxxxxxxxxxxxx,yyyyyyyyyyyyyyyy,zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
  xxxxxxxxxxxxxxxxxxxxxxxx,yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy,zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvv
  ,,zzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
  xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx,yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy,zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
  `, 3))
  }
  

View as plain text