...
Run Format

Source file src/unicode/maketables.go

Documentation: unicode

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // +build ignore
     6  
     7  // Unicode table generator.
     8  // Data read from the web.
     9  
    10  package main
    11  
    12  import (
    13  	"bufio"
    14  	"flag"
    15  	"fmt"
    16  	"io"
    17  	"log"
    18  	"net/http"
    19  	"os"
    20  	"os/exec"
    21  	"path/filepath"
    22  	"regexp"
    23  	"sort"
    24  	"strconv"
    25  	"strings"
    26  	"unicode"
    27  )
    28  
    29  func main() {
    30  	flag.Parse()
    31  	setupOutput()
    32  	loadChars() // always needed
    33  	loadCasefold()
    34  	printCategories()
    35  	printScriptOrProperty(false)
    36  	printScriptOrProperty(true)
    37  	printCases()
    38  	printLatinProperties()
    39  	printCasefold()
    40  	printSizes()
    41  	flushOutput()
    42  }
    43  
    44  func defaultVersion() string {
    45  	if v := os.Getenv("UNICODE_VERSION"); v != "" {
    46  		return v
    47  	}
    48  	return unicode.Version
    49  }
    50  
    51  var dataURL = flag.String("data", "", "full URL for UnicodeData.txt; defaults to --url/UnicodeData.txt")
    52  var casefoldingURL = flag.String("casefolding", "", "full URL for CaseFolding.txt; defaults to --url/CaseFolding.txt")
    53  var url = flag.String("url",
    54  	"http://www.unicode.org/Public/"+defaultVersion()+"/ucd/",
    55  	"URL of Unicode database directory")
    56  var tablelist = flag.String("tables",
    57  	"all",
    58  	"comma-separated list of which tables to generate; can be letter")
    59  var scriptlist = flag.String("scripts",
    60  	"all",
    61  	"comma-separated list of which script tables to generate")
    62  var proplist = flag.String("props",
    63  	"all",
    64  	"comma-separated list of which property tables to generate")
    65  var cases = flag.Bool("cases",
    66  	true,
    67  	"generate case tables")
    68  var test = flag.Bool("test",
    69  	false,
    70  	"test existing tables; can be used to compare web data with package data")
    71  var localFiles = flag.Bool("local",
    72  	false,
    73  	"data files have been copied to current directory; for debugging only")
    74  var outputFile = flag.String("output",
    75  	"",
    76  	"output file for generated tables; default stdout")
    77  
    78  var scriptRe = regexp.MustCompile(`^([0-9A-F]+)(\.\.[0-9A-F]+)? *; ([A-Za-z_]+)$`)
    79  var logger = log.New(os.Stderr, "", log.Lshortfile)
    80  
    81  var output *bufio.Writer // points to os.Stdout or to "gofmt > outputFile"
    82  
    83  func setupOutput() {
    84  	output = bufio.NewWriter(startGofmt())
    85  }
    86  
    87  // startGofmt connects output to a gofmt process if -output is set.
    88  func startGofmt() io.Writer {
    89  	if *outputFile == "" {
    90  		return os.Stdout
    91  	}
    92  	stdout, err := os.Create(*outputFile)
    93  	if err != nil {
    94  		logger.Fatal(err)
    95  	}
    96  	// Pipe output to gofmt.
    97  	gofmt := exec.Command("gofmt")
    98  	fd, err := gofmt.StdinPipe()
    99  	if err != nil {
   100  		logger.Fatal(err)
   101  	}
   102  	gofmt.Stdout = stdout
   103  	gofmt.Stderr = os.Stderr
   104  	err = gofmt.Start()
   105  	if err != nil {
   106  		logger.Fatal(err)
   107  	}
   108  	return fd
   109  }
   110  
   111  func flushOutput() {
   112  	err := output.Flush()
   113  	if err != nil {
   114  		logger.Fatal(err)
   115  	}
   116  }
   117  
   118  func printf(format string, args ...interface{}) {
   119  	fmt.Fprintf(output, format, args...)
   120  }
   121  
   122  func print(args ...interface{}) {
   123  	fmt.Fprint(output, args...)
   124  }
   125  
   126  func println(args ...interface{}) {
   127  	fmt.Fprintln(output, args...)
   128  }
   129  
   130  type reader struct {
   131  	*bufio.Reader
   132  	fd   *os.File
   133  	resp *http.Response
   134  }
   135  
   136  func open(url string) *reader {
   137  	file := filepath.Base(url)
   138  	if *localFiles {
   139  		fd, err := os.Open(file)
   140  		if err != nil {
   141  			logger.Fatal(err)
   142  		}
   143  		return &reader{bufio.NewReader(fd), fd, nil}
   144  	}
   145  	resp, err := http.Get(url)
   146  	if err != nil {
   147  		logger.Fatal(err)
   148  	}
   149  	if resp.StatusCode != 200 {
   150  		logger.Fatalf("bad GET status for %s: %d", file, resp.Status)
   151  	}
   152  	return &reader{bufio.NewReader(resp.Body), nil, resp}
   153  
   154  }
   155  
   156  func (r *reader) close() {
   157  	if r.fd != nil {
   158  		r.fd.Close()
   159  	} else {
   160  		r.resp.Body.Close()
   161  	}
   162  }
   163  
   164  var category = map[string]bool{
   165  	// Nd Lu etc.
   166  	// We use one-character names to identify merged categories
   167  	"L": true, // Lu Ll Lt Lm Lo
   168  	"P": true, // Pc Pd Ps Pe Pu Pf Po
   169  	"M": true, // Mn Mc Me
   170  	"N": true, // Nd Nl No
   171  	"S": true, // Sm Sc Sk So
   172  	"Z": true, // Zs Zl Zp
   173  	"C": true, // Cc Cf Cs Co Cn
   174  }
   175  
   176  // UnicodeData.txt has form:
   177  //	0037;DIGIT SEVEN;Nd;0;EN;;7;7;7;N;;;;;
   178  //	007A;LATIN SMALL LETTER Z;Ll;0;L;;;;;N;;;005A;;005A
   179  // See https://www.unicode.org/reports/tr44/ for a full explanation
   180  // The fields:
   181  const (
   182  	FCodePoint = iota
   183  	FName
   184  	FGeneralCategory
   185  	FCanonicalCombiningClass
   186  	FBidiClass
   187  	FDecompositionTypeAndMapping
   188  	FNumericType
   189  	FNumericDigit // If a decimal digit.
   190  	FNumericValue // Includes non-decimal, e.g. U+2155=1/5
   191  	FBidiMirrored
   192  	FUnicode1Name
   193  	FISOComment
   194  	FSimpleUppercaseMapping
   195  	FSimpleLowercaseMapping
   196  	FSimpleTitlecaseMapping
   197  	NumField
   198  
   199  	MaxChar = 0x10FFFF // anything above this shouldn't exist
   200  )
   201  
   202  var fieldName = []string{
   203  	FCodePoint:                   "CodePoint",
   204  	FName:                        "Name",
   205  	FGeneralCategory:             "GeneralCategory",
   206  	FCanonicalCombiningClass:     "CanonicalCombiningClass",
   207  	FBidiClass:                   "BidiClass",
   208  	FDecompositionTypeAndMapping: "DecompositionTypeAndMapping",
   209  	FNumericType:                 "NumericType",
   210  	FNumericDigit:                "NumericDigit",
   211  	FNumericValue:                "NumericValue",
   212  	FBidiMirrored:                "BidiMirrored",
   213  	FUnicode1Name:                "Unicode1Name",
   214  	FISOComment:                  "ISOComment",
   215  	FSimpleUppercaseMapping:      "SimpleUppercaseMapping",
   216  	FSimpleLowercaseMapping:      "SimpleLowercaseMapping",
   217  	FSimpleTitlecaseMapping:      "SimpleTitlecaseMapping",
   218  }
   219  
   220  // This contains only the properties we're interested in.
   221  type Char struct {
   222  	field     []string // debugging only; could be deleted if we take out char.dump()
   223  	codePoint rune     // if zero, this index is not a valid code point.
   224  	category  string
   225  	upperCase rune
   226  	lowerCase rune
   227  	titleCase rune
   228  	foldCase  rune // simple case folding
   229  	caseOrbit rune // next in simple case folding orbit
   230  }
   231  
   232  // Scripts.txt has form:
   233  //	A673          ; Cyrillic # Po       SLAVONIC ASTERISK
   234  //	A67C..A67D    ; Cyrillic # Mn   [2] COMBINING CYRILLIC KAVYKA..COMBINING CYRILLIC PAYEROK
   235  // See https://www.unicode.org/Public/5.1.0/ucd/UCD.html for full explanation
   236  
   237  type Script struct {
   238  	lo, hi uint32 // range of code points
   239  	script string
   240  }
   241  
   242  var chars = make([]Char, MaxChar+1)
   243  var scripts = make(map[string][]Script)
   244  var props = make(map[string][]Script) // a property looks like a script; can share the format
   245  
   246  var lastChar rune = 0
   247  
   248  // In UnicodeData.txt, some ranges are marked like this:
   249  //	3400;<CJK Ideograph Extension A, First>;Lo;0;L;;;;;N;;;;;
   250  //	4DB5;<CJK Ideograph Extension A, Last>;Lo;0;L;;;;;N;;;;;
   251  // parseCategory returns a state variable indicating the weirdness.
   252  type State int
   253  
   254  const (
   255  	SNormal State = iota // known to be zero for the type
   256  	SFirst
   257  	SLast
   258  	SMissing
   259  )
   260  
   261  func parseCategory(line string) (state State) {
   262  	field := strings.Split(line, ";")
   263  	if len(field) != NumField {
   264  		logger.Fatalf("%5s: %d fields (expected %d)\n", line, len(field), NumField)
   265  	}
   266  	point, err := strconv.ParseUint(field[FCodePoint], 16, 64)
   267  	if err != nil {
   268  		logger.Fatalf("%.5s...: %s", line, err)
   269  	}
   270  	lastChar = rune(point)
   271  	if point > MaxChar {
   272  		return
   273  	}
   274  	char := &chars[point]
   275  	char.field = field
   276  	if char.codePoint != 0 {
   277  		logger.Fatalf("point %U reused", point)
   278  	}
   279  	char.codePoint = lastChar
   280  	char.category = field[FGeneralCategory]
   281  	category[char.category] = true
   282  	switch char.category {
   283  	case "Nd":
   284  		// Decimal digit
   285  		_, err := strconv.Atoi(field[FNumericValue])
   286  		if err != nil {
   287  			logger.Fatalf("%U: bad numeric field: %s", point, err)
   288  		}
   289  	case "Lu":
   290  		char.letter(field[FCodePoint], field[FSimpleLowercaseMapping], field[FSimpleTitlecaseMapping])
   291  	case "Ll":
   292  		char.letter(field[FSimpleUppercaseMapping], field[FCodePoint], field[FSimpleTitlecaseMapping])
   293  	case "Lt":
   294  		char.letter(field[FSimpleUppercaseMapping], field[FSimpleLowercaseMapping], field[FCodePoint])
   295  	default:
   296  		char.letter(field[FSimpleUppercaseMapping], field[FSimpleLowercaseMapping], field[FSimpleTitlecaseMapping])
   297  	}
   298  	switch {
   299  	case strings.Index(field[FName], ", First>") > 0:
   300  		state = SFirst
   301  	case strings.Index(field[FName], ", Last>") > 0:
   302  		state = SLast
   303  	}
   304  	return
   305  }
   306  
   307  func (char *Char) dump(s string) {
   308  	print(s, " ")
   309  	for i := 0; i < len(char.field); i++ {
   310  		printf("%s:%q ", fieldName[i], char.field[i])
   311  	}
   312  	print("\n")
   313  }
   314  
   315  func (char *Char) letter(u, l, t string) {
   316  	char.upperCase = char.letterValue(u, "U")
   317  	char.lowerCase = char.letterValue(l, "L")
   318  	char.titleCase = char.letterValue(t, "T")
   319  }
   320  
   321  func (char *Char) letterValue(s string, cas string) rune {
   322  	if s == "" {
   323  		return 0
   324  	}
   325  	v, err := strconv.ParseUint(s, 16, 64)
   326  	if err != nil {
   327  		char.dump(cas)
   328  		logger.Fatalf("%U: bad letter(%s): %s", char.codePoint, s, err)
   329  	}
   330  	return rune(v)
   331  }
   332  
   333  func allCategories() []string {
   334  	a := make([]string, 0, len(category))
   335  	for k := range category {
   336  		a = append(a, k)
   337  	}
   338  	sort.Strings(a)
   339  	return a
   340  }
   341  
   342  func all(scripts map[string][]Script) []string {
   343  	a := make([]string, 0, len(scripts))
   344  	for k := range scripts {
   345  		a = append(a, k)
   346  	}
   347  	sort.Strings(a)
   348  	return a
   349  }
   350  
   351  func allCatFold(m map[string]map[rune]bool) []string {
   352  	a := make([]string, 0, len(m))
   353  	for k := range m {
   354  		a = append(a, k)
   355  	}
   356  	sort.Strings(a)
   357  	return a
   358  }
   359  
   360  // Extract the version number from the URL
   361  func version() string {
   362  	// Break on slashes and look for the first numeric field
   363  	fields := strings.Split(*url, "/")
   364  	for _, f := range fields {
   365  		if len(f) > 0 && '0' <= f[0] && f[0] <= '9' {
   366  			return f
   367  		}
   368  	}
   369  	logger.Fatal("unknown version")
   370  	return "Unknown"
   371  }
   372  
   373  func categoryOp(code rune, class uint8) bool {
   374  	category := chars[code].category
   375  	return len(category) > 0 && category[0] == class
   376  }
   377  
   378  func loadChars() {
   379  	if *dataURL == "" {
   380  		flag.Set("data", *url+"UnicodeData.txt")
   381  	}
   382  	input := open(*dataURL)
   383  	defer input.close()
   384  	scanner := bufio.NewScanner(input)
   385  	var first rune = 0
   386  	for scanner.Scan() {
   387  		switch parseCategory(scanner.Text()) {
   388  		case SNormal:
   389  			if first != 0 {
   390  				logger.Fatalf("bad state normal at %U", lastChar)
   391  			}
   392  		case SFirst:
   393  			if first != 0 {
   394  				logger.Fatalf("bad state first at %U", lastChar)
   395  			}
   396  			first = lastChar
   397  		case SLast:
   398  			if first == 0 {
   399  				logger.Fatalf("bad state last at %U", lastChar)
   400  			}
   401  			for i := first + 1; i <= lastChar; i++ {
   402  				chars[i] = chars[first]
   403  				chars[i].codePoint = i
   404  			}
   405  			first = 0
   406  		}
   407  	}
   408  	if scanner.Err() != nil {
   409  		logger.Fatal(scanner.Err())
   410  	}
   411  }
   412  
   413  func loadCasefold() {
   414  	if *casefoldingURL == "" {
   415  		flag.Set("casefolding", *url+"CaseFolding.txt")
   416  	}
   417  	input := open(*casefoldingURL)
   418  	defer input.close()
   419  	scanner := bufio.NewScanner(input)
   420  	for scanner.Scan() {
   421  		line := scanner.Text()
   422  		if len(line) == 0 || line[0] == '#' || len(strings.TrimSpace(line)) == 0 {
   423  			continue
   424  		}
   425  		field := strings.Split(line, "; ")
   426  		if len(field) != 4 {
   427  			logger.Fatalf("CaseFolding.txt %.5s...: %d fields (expected %d)\n", line, len(field), 4)
   428  		}
   429  		kind := field[1]
   430  		if kind != "C" && kind != "S" {
   431  			// Only care about 'common' and 'simple' foldings.
   432  			continue
   433  		}
   434  		p1, err := strconv.ParseUint(field[0], 16, 64)
   435  		if err != nil {
   436  			logger.Fatalf("CaseFolding.txt %.5s...: %s", line, err)
   437  		}
   438  		p2, err := strconv.ParseUint(field[2], 16, 64)
   439  		if err != nil {
   440  			logger.Fatalf("CaseFolding.txt %.5s...: %s", line, err)
   441  		}
   442  		chars[p1].foldCase = rune(p2)
   443  	}
   444  	if scanner.Err() != nil {
   445  		logger.Fatal(scanner.Err())
   446  	}
   447  }
   448  
   449  const progHeader = `// Copyright 2013 The Go Authors. All rights reserved.
   450  // Use of this source code is governed by a BSD-style
   451  // license that can be found in the LICENSE file.
   452  
   453  // Code generated by maketables; DO NOT EDIT.
   454  // To regenerate, run:
   455  //	maketables --tables=%s --data=%s --casefolding=%s
   456  
   457  package unicode
   458  
   459  `
   460  
   461  var categoryMapping = map[string]string{
   462  	"Lu": "Letter, uppercase",
   463  	"Ll": "Letter, lowercase",
   464  	"Lt": "Letter, titlecase",
   465  	"Lm": "Letter, modifier",
   466  	"Lo": "Letter, other",
   467  	"Mn": "Mark, nonspacing",
   468  	"Mc": "Mark, spacing combining",
   469  	"Me": "Mark, enclosing",
   470  	"Nd": "Number, decimal digit",
   471  	"Nl": "Number, letter",
   472  	"No": "Number, other",
   473  	"Pc": "Punctuation, connector",
   474  	"Pd": "Punctuation, dash",
   475  	"Ps": "Punctuation, open",
   476  	"Pe": "Punctuation, close",
   477  	"Pi": "Punctuation, initial quote",
   478  	"Pf": "Punctuation, final quote",
   479  	"Po": "Punctuation, other",
   480  	"Sm": "Symbol, math",
   481  	"Sc": "Symbol, currency",
   482  	"Sk": "Symbol, modifier",
   483  	"So": "Symbol, other",
   484  	"Zs": "Separator, space",
   485  	"Zl": "Separator, line",
   486  	"Zp": "Separator, paragraph",
   487  	"Cc": "Other, control",
   488  	"Cf": "Other, format",
   489  	"Cs": "Other, surrogate",
   490  	"Co": "Other, private use",
   491  	"Cn": "Other, not assigned",
   492  }
   493  
   494  func printCategories() {
   495  	if *tablelist == "" {
   496  		return
   497  	}
   498  	// Find out which categories to dump
   499  	list := strings.Split(*tablelist, ",")
   500  	if *tablelist == "all" {
   501  		list = allCategories()
   502  	}
   503  	if *test {
   504  		fullCategoryTest(list)
   505  		return
   506  	}
   507  	printf(progHeader, *tablelist, *dataURL, *casefoldingURL)
   508  
   509  	println("// Version is the Unicode edition from which the tables are derived.")
   510  	printf("const Version = %q\n\n", version())
   511  
   512  	if *tablelist == "all" {
   513  		println("// Categories is the set of Unicode category tables.")
   514  		println("var Categories = map[string] *RangeTable {")
   515  		for _, k := range allCategories() {
   516  			printf("\t%q: %s,\n", k, k)
   517  		}
   518  		print("}\n\n")
   519  	}
   520  
   521  	decl := make(sort.StringSlice, len(list))
   522  	ndecl := 0
   523  	for _, name := range list {
   524  		if _, ok := category[name]; !ok {
   525  			logger.Fatal("unknown category", name)
   526  		}
   527  		// We generate an UpperCase name to serve as concise documentation and an _UnderScored
   528  		// name to store the data. This stops godoc dumping all the tables but keeps them
   529  		// available to clients.
   530  		// Cases deserving special comments
   531  		varDecl := ""
   532  		switch name {
   533  		case "C":
   534  			varDecl = "\tOther = _C;	// Other/C is the set of Unicode control and special characters, category C.\n"
   535  			varDecl += "\tC = _C\n"
   536  		case "L":
   537  			varDecl = "\tLetter = _L;	// Letter/L is the set of Unicode letters, category L.\n"
   538  			varDecl += "\tL = _L\n"
   539  		case "M":
   540  			varDecl = "\tMark = _M;	// Mark/M is the set of Unicode mark characters, category M.\n"
   541  			varDecl += "\tM = _M\n"
   542  		case "N":
   543  			varDecl = "\tNumber = _N;	// Number/N is the set of Unicode number characters, category N.\n"
   544  			varDecl += "\tN = _N\n"
   545  		case "P":
   546  			varDecl = "\tPunct = _P;	// Punct/P is the set of Unicode punctuation characters, category P.\n"
   547  			varDecl += "\tP = _P\n"
   548  		case "S":
   549  			varDecl = "\tSymbol = _S;	// Symbol/S is the set of Unicode symbol characters, category S.\n"
   550  			varDecl += "\tS = _S\n"
   551  		case "Z":
   552  			varDecl = "\tSpace = _Z;	// Space/Z is the set of Unicode space characters, category Z.\n"
   553  			varDecl += "\tZ = _Z\n"
   554  		case "Nd":
   555  			varDecl = "\tDigit = _Nd;	// Digit is the set of Unicode characters with the \"decimal digit\" property.\n"
   556  		case "Lu":
   557  			varDecl = "\tUpper = _Lu;	// Upper is the set of Unicode upper case letters.\n"
   558  		case "Ll":
   559  			varDecl = "\tLower = _Ll;	// Lower is the set of Unicode lower case letters.\n"
   560  		case "Lt":
   561  			varDecl = "\tTitle = _Lt;	// Title is the set of Unicode title case letters.\n"
   562  		}
   563  		if len(name) > 1 {
   564  			desc, ok := categoryMapping[name]
   565  			if ok {
   566  				varDecl += fmt.Sprintf(
   567  					"\t%s = _%s;	// %s is the set of Unicode characters in category %s (%s).\n",
   568  					name, name, name, name, desc)
   569  			} else {
   570  				varDecl += fmt.Sprintf(
   571  					"\t%s = _%s;	// %s is the set of Unicode characters in category %s.\n",
   572  					name, name, name, name)
   573  			}
   574  		}
   575  		decl[ndecl] = varDecl
   576  		ndecl++
   577  		if len(name) == 1 { // unified categories
   578  			decl := fmt.Sprintf("var _%s = &RangeTable{\n", name)
   579  			dumpRange(
   580  				decl,
   581  				func(code rune) bool { return categoryOp(code, name[0]) })
   582  			continue
   583  		}
   584  		dumpRange(
   585  			fmt.Sprintf("var _%s = &RangeTable{\n", name),
   586  			func(code rune) bool { return chars[code].category == name })
   587  	}
   588  	decl.Sort()
   589  	println("// These variables have type *RangeTable.")
   590  	println("var (")
   591  	for _, d := range decl {
   592  		print(d)
   593  	}
   594  	print(")\n\n")
   595  }
   596  
   597  type Op func(code rune) bool
   598  
   599  const format = "\t\t{0x%04x, 0x%04x, %d},\n"
   600  
   601  func dumpRange(header string, inCategory Op) {
   602  	print(header)
   603  	next := rune(0)
   604  	latinOffset := 0
   605  	print("\tR16: []Range16{\n")
   606  	// one Range for each iteration
   607  	count := &range16Count
   608  	size := 16
   609  	for {
   610  		// look for start of range
   611  		for next < rune(len(chars)) && !inCategory(next) {
   612  			next++
   613  		}
   614  		if next >= rune(len(chars)) {
   615  			// no characters remain
   616  			break
   617  		}
   618  
   619  		// start of range
   620  		lo := next
   621  		hi := next
   622  		stride := rune(1)
   623  		// accept lo
   624  		next++
   625  		// look for another character to set the stride
   626  		for next < rune(len(chars)) && !inCategory(next) {
   627  			next++
   628  		}
   629  		if next >= rune(len(chars)) {
   630  			// no more characters
   631  			printf(format, lo, hi, stride)
   632  			break
   633  		}
   634  		// set stride
   635  		stride = next - lo
   636  		// check for length of run. next points to first jump in stride
   637  		for i := next; i < rune(len(chars)); i++ {
   638  			if inCategory(i) == (((i - lo) % stride) == 0) {
   639  				// accept
   640  				if inCategory(i) {
   641  					hi = i
   642  				}
   643  			} else {
   644  				// no more characters in this run
   645  				break
   646  			}
   647  		}
   648  		if uint32(hi) <= unicode.MaxLatin1 {
   649  			latinOffset++
   650  		}
   651  		size, count = printRange(uint32(lo), uint32(hi), uint32(stride), size, count)
   652  		// next range: start looking where this range ends
   653  		next = hi + 1
   654  	}
   655  	print("\t},\n")
   656  	if latinOffset > 0 {
   657  		printf("\tLatinOffset: %d,\n", latinOffset)
   658  	}
   659  	print("}\n\n")
   660  }
   661  
   662  func printRange(lo, hi, stride uint32, size int, count *int) (int, *int) {
   663  	if size == 16 && hi >= 1<<16 {
   664  		if lo < 1<<16 {
   665  			if lo+stride != hi {
   666  				logger.Fatalf("unexpected straddle: %U %U %d", lo, hi, stride)
   667  			}
   668  			// No range contains U+FFFF as an instance, so split
   669  			// the range into two entries. That way we can maintain
   670  			// the invariant that R32 contains only >= 1<<16.
   671  			printf(format, lo, lo, 1)
   672  			lo = hi
   673  			stride = 1
   674  			*count++
   675  		}
   676  		print("\t},\n")
   677  		print("\tR32: []Range32{\n")
   678  		size = 32
   679  		count = &range32Count
   680  	}
   681  	printf(format, lo, hi, stride)
   682  	*count++
   683  	return size, count
   684  }
   685  
   686  func fullCategoryTest(list []string) {
   687  	for _, name := range list {
   688  		if _, ok := category[name]; !ok {
   689  			logger.Fatal("unknown category", name)
   690  		}
   691  		r, ok := unicode.Categories[name]
   692  		if !ok && len(name) > 1 {
   693  			logger.Fatalf("unknown table %q", name)
   694  		}
   695  		if len(name) == 1 {
   696  			verifyRange(name, func(code rune) bool { return categoryOp(code, name[0]) }, r)
   697  		} else {
   698  			verifyRange(
   699  				name,
   700  				func(code rune) bool { return chars[code].category == name },
   701  				r)
   702  		}
   703  	}
   704  }
   705  
   706  func verifyRange(name string, inCategory Op, table *unicode.RangeTable) {
   707  	count := 0
   708  	for j := range chars {
   709  		i := rune(j)
   710  		web := inCategory(i)
   711  		pkg := unicode.Is(table, i)
   712  		if web != pkg {
   713  			fmt.Fprintf(os.Stderr, "%s: %U: web=%t pkg=%t\n", name, i, web, pkg)
   714  			count++
   715  			if count > 10 {
   716  				break
   717  			}
   718  		}
   719  	}
   720  }
   721  
   722  func parseScript(line string, scripts map[string][]Script) {
   723  	comment := strings.Index(line, "#")
   724  	if comment >= 0 {
   725  		line = line[0:comment]
   726  	}
   727  	line = strings.TrimSpace(line)
   728  	if len(line) == 0 {
   729  		return
   730  	}
   731  	field := strings.Split(line, ";")
   732  	if len(field) != 2 {
   733  		logger.Fatalf("%s: %d fields (expected 2)\n", line, len(field))
   734  	}
   735  	matches := scriptRe.FindStringSubmatch(line)
   736  	if len(matches) != 4 {
   737  		logger.Fatalf("%s: %d matches (expected 3)\n", line, len(matches))
   738  	}
   739  	lo, err := strconv.ParseUint(matches[1], 16, 64)
   740  	if err != nil {
   741  		logger.Fatalf("%.5s...: %s", line, err)
   742  	}
   743  	hi := lo
   744  	if len(matches[2]) > 2 { // ignore leading ..
   745  		hi, err = strconv.ParseUint(matches[2][2:], 16, 64)
   746  		if err != nil {
   747  			logger.Fatalf("%.5s...: %s", line, err)
   748  		}
   749  	}
   750  	name := matches[3]
   751  	scripts[name] = append(scripts[name], Script{uint32(lo), uint32(hi), name})
   752  }
   753  
   754  // The script tables have a lot of adjacent elements. Fold them together.
   755  func foldAdjacent(r []Script) []unicode.Range32 {
   756  	s := make([]unicode.Range32, 0, len(r))
   757  	j := 0
   758  	for i := 0; i < len(r); i++ {
   759  		if j > 0 && r[i].lo == s[j-1].Hi+1 {
   760  			s[j-1].Hi = r[i].hi
   761  		} else {
   762  			s = s[0 : j+1]
   763  			s[j] = unicode.Range32{
   764  				Lo:     uint32(r[i].lo),
   765  				Hi:     uint32(r[i].hi),
   766  				Stride: 1,
   767  			}
   768  			j++
   769  		}
   770  	}
   771  	return s
   772  }
   773  
   774  func fullScriptTest(list []string, installed map[string]*unicode.RangeTable, scripts map[string][]Script) {
   775  	for _, name := range list {
   776  		if _, ok := scripts[name]; !ok {
   777  			logger.Fatal("unknown script", name)
   778  		}
   779  		_, ok := installed[name]
   780  		if !ok {
   781  			logger.Fatal("unknown table", name)
   782  		}
   783  		for _, script := range scripts[name] {
   784  			for r := script.lo; r <= script.hi; r++ {
   785  				if !unicode.Is(installed[name], rune(r)) {
   786  					fmt.Fprintf(os.Stderr, "%U: not in script %s\n", r, name)
   787  				}
   788  			}
   789  		}
   790  	}
   791  }
   792  
   793  var deprecatedAliases = map[string]string{
   794  	"Sentence_Terminal": "STerm",
   795  }
   796  
   797  // PropList.txt has the same format as Scripts.txt so we can share its parser.
   798  func printScriptOrProperty(doProps bool) {
   799  	flag := "scripts"
   800  	flaglist := *scriptlist
   801  	file := "Scripts.txt"
   802  	table := scripts
   803  	installed := unicode.Scripts
   804  	if doProps {
   805  		flag = "props"
   806  		flaglist = *proplist
   807  		file = "PropList.txt"
   808  		table = props
   809  		installed = unicode.Properties
   810  	}
   811  	if flaglist == "" {
   812  		return
   813  	}
   814  	input := open(*url + file)
   815  	scanner := bufio.NewScanner(input)
   816  	for scanner.Scan() {
   817  		parseScript(scanner.Text(), table)
   818  	}
   819  	if scanner.Err() != nil {
   820  		logger.Fatal(scanner.Err())
   821  	}
   822  	input.close()
   823  
   824  	// Find out which scripts to dump
   825  	list := strings.Split(flaglist, ",")
   826  	if flaglist == "all" {
   827  		list = all(table)
   828  	}
   829  	if *test {
   830  		fullScriptTest(list, installed, table)
   831  		return
   832  	}
   833  
   834  	printf(
   835  		"// Generated by running\n"+
   836  			"//	maketables --%s=%s --url=%s\n"+
   837  			"// DO NOT EDIT\n\n",
   838  		flag,
   839  		flaglist,
   840  		*url)
   841  	if flaglist == "all" {
   842  		if doProps {
   843  			println("// Properties is the set of Unicode property tables.")
   844  			println("var Properties = map[string] *RangeTable{")
   845  		} else {
   846  			println("// Scripts is the set of Unicode script tables.")
   847  			println("var Scripts = map[string] *RangeTable{")
   848  		}
   849  		for _, k := range all(table) {
   850  			printf("\t%q: %s,\n", k, k)
   851  			if alias, ok := deprecatedAliases[k]; ok {
   852  				printf("\t%q: %s,\n", alias, k)
   853  			}
   854  		}
   855  		print("}\n\n")
   856  	}
   857  
   858  	decl := make(sort.StringSlice, len(list)+len(deprecatedAliases))
   859  	ndecl := 0
   860  	for _, name := range list {
   861  		if doProps {
   862  			decl[ndecl] = fmt.Sprintf(
   863  				"\t%s = _%s;\t// %s is the set of Unicode characters with property %s.\n",
   864  				name, name, name, name)
   865  		} else {
   866  			decl[ndecl] = fmt.Sprintf(
   867  				"\t%s = _%s;\t// %s is the set of Unicode characters in script %s.\n",
   868  				name, name, name, name)
   869  		}
   870  		ndecl++
   871  		if alias, ok := deprecatedAliases[name]; ok {
   872  			decl[ndecl] = fmt.Sprintf(
   873  				"\t%[1]s = _%[2]s;\t// %[1]s is an alias for %[2]s.\n",
   874  				alias, name)
   875  			ndecl++
   876  		}
   877  		printf("var _%s = &RangeTable {\n", name)
   878  		ranges := foldAdjacent(table[name])
   879  		print("\tR16: []Range16{\n")
   880  		size := 16
   881  		count := &range16Count
   882  		for _, s := range ranges {
   883  			size, count = printRange(s.Lo, s.Hi, s.Stride, size, count)
   884  		}
   885  		print("\t},\n")
   886  		if off := findLatinOffset(ranges); off > 0 {
   887  			printf("\tLatinOffset: %d,\n", off)
   888  		}
   889  		print("}\n\n")
   890  	}
   891  	decl.Sort()
   892  	println("// These variables have type *RangeTable.")
   893  	println("var (")
   894  	for _, d := range decl {
   895  		print(d)
   896  	}
   897  	print(")\n\n")
   898  }
   899  
   900  func findLatinOffset(ranges []unicode.Range32) int {
   901  	i := 0
   902  	for i < len(ranges) && ranges[i].Hi <= unicode.MaxLatin1 {
   903  		i++
   904  	}
   905  	return i
   906  }
   907  
   908  const (
   909  	CaseUpper = 1 << iota
   910  	CaseLower
   911  	CaseTitle
   912  	CaseNone    = 0  // must be zero
   913  	CaseMissing = -1 // character not present; not a valid case state
   914  )
   915  
   916  type caseState struct {
   917  	point        rune
   918  	_case        int
   919  	deltaToUpper rune
   920  	deltaToLower rune
   921  	deltaToTitle rune
   922  }
   923  
   924  // Is d a continuation of the state of c?
   925  func (c *caseState) adjacent(d *caseState) bool {
   926  	if d.point < c.point {
   927  		c, d = d, c
   928  	}
   929  	switch {
   930  	case d.point != c.point+1: // code points not adjacent (shouldn't happen)
   931  		return false
   932  	case d._case != c._case: // different cases
   933  		return c.upperLowerAdjacent(d)
   934  	case c._case == CaseNone:
   935  		return false
   936  	case c._case == CaseMissing:
   937  		return false
   938  	case d.deltaToUpper != c.deltaToUpper:
   939  		return false
   940  	case d.deltaToLower != c.deltaToLower:
   941  		return false
   942  	case d.deltaToTitle != c.deltaToTitle:
   943  		return false
   944  	}
   945  	return true
   946  }
   947  
   948  // Is d the same as c, but opposite in upper/lower case? this would make it
   949  // an element of an UpperLower sequence.
   950  func (c *caseState) upperLowerAdjacent(d *caseState) bool {
   951  	// check they're a matched case pair.  we know they have adjacent values
   952  	switch {
   953  	case c._case == CaseUpper && d._case != CaseLower:
   954  		return false
   955  	case c._case == CaseLower && d._case != CaseUpper:
   956  		return false
   957  	}
   958  	// matched pair (at least in upper/lower).  make the order Upper Lower
   959  	if c._case == CaseLower {
   960  		c, d = d, c
   961  	}
   962  	// for an Upper Lower sequence the deltas have to be in order
   963  	//	c: 0 1 0
   964  	//	d: -1 0 -1
   965  	switch {
   966  	case c.deltaToUpper != 0:
   967  		return false
   968  	case c.deltaToLower != 1:
   969  		return false
   970  	case c.deltaToTitle != 0:
   971  		return false
   972  	case d.deltaToUpper != -1:
   973  		return false
   974  	case d.deltaToLower != 0:
   975  		return false
   976  	case d.deltaToTitle != -1:
   977  		return false
   978  	}
   979  	return true
   980  }
   981  
   982  // Does this character start an UpperLower sequence?
   983  func (c *caseState) isUpperLower() bool {
   984  	// for an Upper Lower sequence the deltas have to be in order
   985  	//	c: 0 1 0
   986  	switch {
   987  	case c.deltaToUpper != 0:
   988  		return false
   989  	case c.deltaToLower != 1:
   990  		return false
   991  	case c.deltaToTitle != 0:
   992  		return false
   993  	}
   994  	return true
   995  }
   996  
   997  // Does this character start a LowerUpper sequence?
   998  func (c *caseState) isLowerUpper() bool {
   999  	// for an Upper Lower sequence the deltas have to be in order
  1000  	//	c: -1 0 -1
  1001  	switch {
  1002  	case c.deltaToUpper != -1:
  1003  		return false
  1004  	case c.deltaToLower != 0:
  1005  		return false
  1006  	case c.deltaToTitle != -1:
  1007  		return false
  1008  	}
  1009  	return true
  1010  }
  1011  
  1012  func getCaseState(i rune) (c *caseState) {
  1013  	c = &caseState{point: i, _case: CaseNone}
  1014  	ch := &chars[i]
  1015  	switch ch.codePoint {
  1016  	case 0:
  1017  		c._case = CaseMissing // Will get NUL wrong but that doesn't matter
  1018  		return
  1019  	case ch.upperCase:
  1020  		c._case = CaseUpper
  1021  	case ch.lowerCase:
  1022  		c._case = CaseLower
  1023  	case ch.titleCase:
  1024  		c._case = CaseTitle
  1025  	}
  1026  	// Some things such as roman numeral U+2161 don't describe themselves
  1027  	// as upper case, but have a lower case. Second-guess them.
  1028  	if c._case == CaseNone && ch.lowerCase != 0 {
  1029  		c._case = CaseUpper
  1030  	}
  1031  	// Same in the other direction.
  1032  	if c._case == CaseNone && ch.upperCase != 0 {
  1033  		c._case = CaseLower
  1034  	}
  1035  
  1036  	if ch.upperCase != 0 {
  1037  		c.deltaToUpper = ch.upperCase - i
  1038  	}
  1039  	if ch.lowerCase != 0 {
  1040  		c.deltaToLower = ch.lowerCase - i
  1041  	}
  1042  	if ch.titleCase != 0 {
  1043  		c.deltaToTitle = ch.titleCase - i
  1044  	}
  1045  	return
  1046  }
  1047  
  1048  func printCases() {
  1049  	if !*cases {
  1050  		return
  1051  	}
  1052  	if *test {
  1053  		fullCaseTest()
  1054  		return
  1055  	}
  1056  	printf(
  1057  		"// Generated by running\n"+
  1058  			"//	maketables --data=%s --casefolding=%s\n"+
  1059  			"// DO NOT EDIT\n\n"+
  1060  			"// CaseRanges is the table describing case mappings for all letters with\n"+
  1061  			"// non-self mappings.\n"+
  1062  			"var CaseRanges = _CaseRanges\n"+
  1063  			"var _CaseRanges = []CaseRange {\n",
  1064  		*dataURL, *casefoldingURL)
  1065  
  1066  	var startState *caseState    // the start of a run; nil for not active
  1067  	var prevState = &caseState{} // the state of the previous character
  1068  	for i := range chars {
  1069  		state := getCaseState(rune(i))
  1070  		if state.adjacent(prevState) {
  1071  			prevState = state
  1072  			continue
  1073  		}
  1074  		// end of run (possibly)
  1075  		printCaseRange(startState, prevState)
  1076  		startState = nil
  1077  		if state._case != CaseMissing && state._case != CaseNone {
  1078  			startState = state
  1079  		}
  1080  		prevState = state
  1081  	}
  1082  	print("}\n")
  1083  }
  1084  
  1085  func printCaseRange(lo, hi *caseState) {
  1086  	if lo == nil {
  1087  		return
  1088  	}
  1089  	if lo.deltaToUpper == 0 && lo.deltaToLower == 0 && lo.deltaToTitle == 0 {
  1090  		// character represents itself in all cases - no need to mention it
  1091  		return
  1092  	}
  1093  	switch {
  1094  	case hi.point > lo.point && lo.isUpperLower():
  1095  		printf("\t{0x%04X, 0x%04X, d{UpperLower, UpperLower, UpperLower}},\n",
  1096  			lo.point, hi.point)
  1097  	case hi.point > lo.point && lo.isLowerUpper():
  1098  		logger.Fatalf("LowerUpper sequence: should not happen: %U.  If it's real, need to fix To()", lo.point)
  1099  		printf("\t{0x%04X, 0x%04X, d{LowerUpper, LowerUpper, LowerUpper}},\n",
  1100  			lo.point, hi.point)
  1101  	default:
  1102  		printf("\t{0x%04X, 0x%04X, d{%d, %d, %d}},\n",
  1103  			lo.point, hi.point,
  1104  			lo.deltaToUpper, lo.deltaToLower, lo.deltaToTitle)
  1105  	}
  1106  }
  1107  
  1108  // If the cased value in the Char is 0, it means use the rune itself.
  1109  func caseIt(r, cased rune) rune {
  1110  	if cased == 0 {
  1111  		return r
  1112  	}
  1113  	return cased
  1114  }
  1115  
  1116  func fullCaseTest() {
  1117  	for j, c := range chars {
  1118  		i := rune(j)
  1119  		lower := unicode.ToLower(i)
  1120  		want := caseIt(i, c.lowerCase)
  1121  		if lower != want {
  1122  			fmt.Fprintf(os.Stderr, "lower %U should be %U is %U\n", i, want, lower)
  1123  		}
  1124  		upper := unicode.ToUpper(i)
  1125  		want = caseIt(i, c.upperCase)
  1126  		if upper != want {
  1127  			fmt.Fprintf(os.Stderr, "upper %U should be %U is %U\n", i, want, upper)
  1128  		}
  1129  		title := unicode.ToTitle(i)
  1130  		want = caseIt(i, c.titleCase)
  1131  		if title != want {
  1132  			fmt.Fprintf(os.Stderr, "title %U should be %U is %U\n", i, want, title)
  1133  		}
  1134  	}
  1135  }
  1136  
  1137  func printLatinProperties() {
  1138  	if *test {
  1139  		return
  1140  	}
  1141  	println("var properties = [MaxLatin1+1]uint8{")
  1142  	for code := 0; code <= unicode.MaxLatin1; code++ {
  1143  		var property string
  1144  		switch chars[code].category {
  1145  		case "Cc", "": // NUL has no category.
  1146  			property = "pC"
  1147  		case "Cf": // soft hyphen, unique category, not printable.
  1148  			property = "0"
  1149  		case "Ll":
  1150  			property = "pLl | pp"
  1151  		case "Lo":
  1152  			property = "pLo | pp"
  1153  		case "Lu":
  1154  			property = "pLu | pp"
  1155  		case "Nd", "No":
  1156  			property = "pN | pp"
  1157  		case "Pc", "Pd", "Pe", "Pf", "Pi", "Po", "Ps":
  1158  			property = "pP | pp"
  1159  		case "Sc", "Sk", "Sm", "So":
  1160  			property = "pS | pp"
  1161  		case "Zs":
  1162  			property = "pZ"
  1163  		default:
  1164  			logger.Fatalf("%U has unknown category %q", code, chars[code].category)
  1165  		}
  1166  		// Special case
  1167  		if code == ' ' {
  1168  			property = "pZ | pp"
  1169  		}
  1170  		printf("\t0x%02X: %s, // %q\n", code, property, code)
  1171  	}
  1172  	printf("}\n\n")
  1173  }
  1174  
  1175  func printCasefold() {
  1176  	// Build list of case-folding groups attached to each canonical folded char (typically lower case).
  1177  	var caseOrbit = make([][]rune, MaxChar+1)
  1178  	for j := range chars {
  1179  		i := rune(j)
  1180  		c := &chars[i]
  1181  		if c.foldCase == 0 {
  1182  			continue
  1183  		}
  1184  		orb := caseOrbit[c.foldCase]
  1185  		if orb == nil {
  1186  			orb = append(orb, c.foldCase)
  1187  		}
  1188  		caseOrbit[c.foldCase] = append(orb, i)
  1189  	}
  1190  
  1191  	// Insert explicit 1-element groups when assuming [lower, upper] would be wrong.
  1192  	for j := range chars {
  1193  		i := rune(j)
  1194  		c := &chars[i]
  1195  		f := c.foldCase
  1196  		if f == 0 {
  1197  			f = i
  1198  		}
  1199  		orb := caseOrbit[f]
  1200  		if orb == nil && (c.upperCase != 0 && c.upperCase != i || c.lowerCase != 0 && c.lowerCase != i) {
  1201  			// Default assumption of [upper, lower] is wrong.
  1202  			caseOrbit[i] = []rune{i}
  1203  		}
  1204  	}
  1205  
  1206  	// Delete the groups for which assuming [lower, upper] or [upper, lower] is right.
  1207  	for i, orb := range caseOrbit {
  1208  		if len(orb) == 2 && chars[orb[0]].upperCase == orb[1] && chars[orb[1]].lowerCase == orb[0] {
  1209  			caseOrbit[i] = nil
  1210  		}
  1211  		if len(orb) == 2 && chars[orb[1]].upperCase == orb[0] && chars[orb[0]].lowerCase == orb[1] {
  1212  			caseOrbit[i] = nil
  1213  		}
  1214  	}
  1215  
  1216  	// Record orbit information in chars.
  1217  	for _, orb := range caseOrbit {
  1218  		if orb == nil {
  1219  			continue
  1220  		}
  1221  		sort.Slice(orb, func(i, j int) bool {
  1222  			return orb[i] < orb[j]
  1223  		})
  1224  		c := orb[len(orb)-1]
  1225  		for _, d := range orb {
  1226  			chars[c].caseOrbit = d
  1227  			c = d
  1228  		}
  1229  	}
  1230  
  1231  	printAsciiFold()
  1232  	printCaseOrbit()
  1233  
  1234  	// Tables of category and script folding exceptions: code points
  1235  	// that must be added when interpreting a particular category/script
  1236  	// in a case-folding context.
  1237  	cat := make(map[string]map[rune]bool)
  1238  	for name := range category {
  1239  		if x := foldExceptions(inCategory(name)); len(x) > 0 {
  1240  			cat[name] = x
  1241  		}
  1242  	}
  1243  
  1244  	scr := make(map[string]map[rune]bool)
  1245  	for name := range scripts {
  1246  		if x := foldExceptions(inScript(name)); len(x) > 0 {
  1247  			scr[name] = x
  1248  		}
  1249  	}
  1250  
  1251  	printCatFold("FoldCategory", cat)
  1252  	printCatFold("FoldScript", scr)
  1253  }
  1254  
  1255  // inCategory returns a list of all the runes in the category.
  1256  func inCategory(name string) []rune {
  1257  	var x []rune
  1258  	for j := range chars {
  1259  		i := rune(j)
  1260  		c := &chars[i]
  1261  		if c.category == name || len(name) == 1 && len(c.category) > 1 && c.category[0] == name[0] {
  1262  			x = append(x, i)
  1263  		}
  1264  	}
  1265  	return x
  1266  }
  1267  
  1268  // inScript returns a list of all the runes in the script.
  1269  func inScript(name string) []rune {
  1270  	var x []rune
  1271  	for _, s := range scripts[name] {
  1272  		for c := s.lo; c <= s.hi; c++ {
  1273  			x = append(x, rune(c))
  1274  		}
  1275  	}
  1276  	return x
  1277  }
  1278  
  1279  // foldExceptions returns a list of all the runes fold-equivalent
  1280  // to runes in class but not in class themselves.
  1281  func foldExceptions(class []rune) map[rune]bool {
  1282  	// Create map containing class and all fold-equivalent chars.
  1283  	m := make(map[rune]bool)
  1284  	for _, r := range class {
  1285  		c := &chars[r]
  1286  		if c.caseOrbit == 0 {
  1287  			// Just upper and lower.
  1288  			if u := c.upperCase; u != 0 {
  1289  				m[u] = true
  1290  			}
  1291  			if l := c.lowerCase; l != 0 {
  1292  				m[l] = true
  1293  			}
  1294  			m[r] = true
  1295  			continue
  1296  		}
  1297  		// Otherwise walk orbit.
  1298  		r0 := r
  1299  		for {
  1300  			m[r] = true
  1301  			r = chars[r].caseOrbit
  1302  			if r == r0 {
  1303  				break
  1304  			}
  1305  		}
  1306  	}
  1307  
  1308  	// Remove class itself.
  1309  	for _, r := range class {
  1310  		delete(m, r)
  1311  	}
  1312  
  1313  	// What's left is the exceptions.
  1314  	return m
  1315  }
  1316  
  1317  var comment = map[string]string{
  1318  	"FoldCategory": "// FoldCategory maps a category name to a table of\n" +
  1319  		"// code points outside the category that are equivalent under\n" +
  1320  		"// simple case folding to code points inside the category.\n" +
  1321  		"// If there is no entry for a category name, there are no such points.\n",
  1322  
  1323  	"FoldScript": "// FoldScript maps a script name to a table of\n" +
  1324  		"// code points outside the script that are equivalent under\n" +
  1325  		"// simple case folding to code points inside the script.\n" +
  1326  		"// If there is no entry for a script name, there are no such points.\n",
  1327  }
  1328  
  1329  func printAsciiFold() {
  1330  	printf("var asciiFold = [MaxASCII + 1]uint16{\n")
  1331  	for i := rune(0); i <= unicode.MaxASCII; i++ {
  1332  		c := chars[i]
  1333  		f := c.caseOrbit
  1334  		if f == 0 {
  1335  			if c.lowerCase != i && c.lowerCase != 0 {
  1336  				f = c.lowerCase
  1337  			} else if c.upperCase != i && c.upperCase != 0 {
  1338  				f = c.upperCase
  1339  			} else {
  1340  				f = i
  1341  			}
  1342  		}
  1343  		printf("\t0x%04X,\n", f)
  1344  	}
  1345  	printf("}\n\n")
  1346  }
  1347  
  1348  func printCaseOrbit() {
  1349  	if *test {
  1350  		for j := range chars {
  1351  			i := rune(j)
  1352  			c := &chars[i]
  1353  			f := c.caseOrbit
  1354  			if f == 0 {
  1355  				if c.lowerCase != i && c.lowerCase != 0 {
  1356  					f = c.lowerCase
  1357  				} else if c.upperCase != i && c.upperCase != 0 {
  1358  					f = c.upperCase
  1359  				} else {
  1360  					f = i
  1361  				}
  1362  			}
  1363  			if g := unicode.SimpleFold(i); g != f {
  1364  				fmt.Fprintf(os.Stderr, "unicode.SimpleFold(%#U) = %#U, want %#U\n", i, g, f)
  1365  			}
  1366  		}
  1367  		return
  1368  	}
  1369  
  1370  	printf("var caseOrbit = []foldPair{\n")
  1371  	for i := range chars {
  1372  		c := &chars[i]
  1373  		if c.caseOrbit != 0 {
  1374  			printf("\t{0x%04X, 0x%04X},\n", i, c.caseOrbit)
  1375  			foldPairCount++
  1376  		}
  1377  	}
  1378  	printf("}\n\n")
  1379  }
  1380  
  1381  func printCatFold(name string, m map[string]map[rune]bool) {
  1382  	if *test {
  1383  		var pkgMap map[string]*unicode.RangeTable
  1384  		if name == "FoldCategory" {
  1385  			pkgMap = unicode.FoldCategory
  1386  		} else {
  1387  			pkgMap = unicode.FoldScript
  1388  		}
  1389  		if len(pkgMap) != len(m) {
  1390  			fmt.Fprintf(os.Stderr, "unicode.%s has %d elements, want %d\n", name, len(pkgMap), len(m))
  1391  			return
  1392  		}
  1393  		for k, v := range m {
  1394  			t, ok := pkgMap[k]
  1395  			if !ok {
  1396  				fmt.Fprintf(os.Stderr, "unicode.%s[%q] missing\n", name, k)
  1397  				continue
  1398  			}
  1399  			n := 0
  1400  			for _, r := range t.R16 {
  1401  				for c := rune(r.Lo); c <= rune(r.Hi); c += rune(r.Stride) {
  1402  					if !v[c] {
  1403  						fmt.Fprintf(os.Stderr, "unicode.%s[%q] contains %#U, should not\n", name, k, c)
  1404  					}
  1405  					n++
  1406  				}
  1407  			}
  1408  			for _, r := range t.R32 {
  1409  				for c := rune(r.Lo); c <= rune(r.Hi); c += rune(r.Stride) {
  1410  					if !v[c] {
  1411  						fmt.Fprintf(os.Stderr, "unicode.%s[%q] contains %#U, should not\n", name, k, c)
  1412  					}
  1413  					n++
  1414  				}
  1415  			}
  1416  			if n != len(v) {
  1417  				fmt.Fprintf(os.Stderr, "unicode.%s[%q] has %d code points, want %d\n", name, k, n, len(v))
  1418  			}
  1419  		}
  1420  		return
  1421  	}
  1422  
  1423  	print(comment[name])
  1424  	printf("var %s = map[string]*RangeTable{\n", name)
  1425  	for _, name := range allCatFold(m) {
  1426  		printf("\t%q: fold%s,\n", name, name)
  1427  	}
  1428  	printf("}\n\n")
  1429  	for _, name := range allCatFold(m) {
  1430  		class := m[name]
  1431  		dumpRange(
  1432  			fmt.Sprintf("var fold%s = &RangeTable{\n", name),
  1433  			func(code rune) bool { return class[code] })
  1434  	}
  1435  }
  1436  
  1437  var range16Count = 0  // Number of entries in the 16-bit range tables.
  1438  var range32Count = 0  // Number of entries in the 32-bit range tables.
  1439  var foldPairCount = 0 // Number of fold pairs in the exception tables.
  1440  
  1441  func printSizes() {
  1442  	if *test {
  1443  		return
  1444  	}
  1445  	println()
  1446  	printf("// Range entries: %d 16-bit, %d 32-bit, %d total.\n", range16Count, range32Count, range16Count+range32Count)
  1447  	range16Bytes := range16Count * 3 * 2
  1448  	range32Bytes := range32Count * 3 * 4
  1449  	printf("// Range bytes: %d 16-bit, %d 32-bit, %d total.\n", range16Bytes, range32Bytes, range16Bytes+range32Bytes)
  1450  	println()
  1451  	printf("// Fold orbit bytes: %d pairs, %d bytes\n", foldPairCount, foldPairCount*2*2)
  1452  }
  1453  

View as plain text