mirror of
https://github.com/golang/go.git
synced 2025-12-08 06:10:04 +00:00
1) Change default gofmt default settings for
parsing and printing to new syntax. Use -oldparser to parse the old syntax, use -oldprinter to print the old syntax. 2) Change default gofmt formatting settings to use tabs for indentation only and to use spaces for alignment. This will make the code alignment insensitive to an editor's tabwidth. Use -spaces=false to use tabs for alignment. 3) Manually changed src/exp/parser/parser_test.go so that it doesn't try to parse the parser's source files using the old syntax (they have new syntax now). 4) gofmt -w src misc test/bench 5th and last set of files. R=rsc CC=golang-dev https://golang.org/cl/180050
This commit is contained in:
Robert Griesemer
parent
d65a5cce89
commit
45ca9f7a9e
59 changed files with 5907 additions and 5907 deletions
|
|
@ -8,26 +8,26 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bufio";
|
||||
"flag";
|
||||
"fmt";
|
||||
"http";
|
||||
"log";
|
||||
"os";
|
||||
"sort";
|
||||
"strconv";
|
||||
"strings";
|
||||
"regexp";
|
||||
"unicode";
|
||||
"bufio"
|
||||
"flag"
|
||||
"fmt"
|
||||
"http"
|
||||
"log"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"regexp"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
func main() {
|
||||
flag.Parse();
|
||||
loadChars(); // always needed
|
||||
printCategories();
|
||||
printScriptOrProperty(false);
|
||||
printScriptOrProperty(true);
|
||||
printCases();
|
||||
flag.Parse()
|
||||
loadChars() // always needed
|
||||
printCategories()
|
||||
printScriptOrProperty(false)
|
||||
printScriptOrProperty(true)
|
||||
printCases()
|
||||
}
|
||||
|
||||
var dataURL = flag.String("data", "", "full URL for UnicodeData.txt; defaults to --url/UnicodeData.txt")
|
||||
|
|
@ -53,7 +53,7 @@ var test = flag.Bool("test",
|
|||
var scriptRe = regexp.MustCompile(`([0-9A-F]+)(\.\.[0-9A-F]+)? *; ([A-Za-z_]+)`)
|
||||
var die = log.New(os.Stderr, nil, "", log.Lexit|log.Lshortfile)
|
||||
|
||||
var category = map[string]bool{"letter": true} // Nd Lu etc. letter is a special case
|
||||
var category = map[string]bool{"letter": true} // Nd Lu etc. letter is a special case
|
||||
|
||||
// UnicodeData.txt has form:
|
||||
// 0037;DIGIT SEVEN;Nd;0;EN;;7;7;7;N;;;;;
|
||||
|
|
@ -61,24 +61,24 @@ var category = map[string]bool{"letter": true} // Nd Lu etc. letter is a special
|
|||
// See http://www.unicode.org/Public/5.1.0/ucd/UCD.html for full explanation
|
||||
// The fields:
|
||||
const (
|
||||
FCodePoint = iota;
|
||||
FName;
|
||||
FGeneralCategory;
|
||||
FCanonicalCombiningClass;
|
||||
FBidiClass;
|
||||
FDecompositionType;
|
||||
FDecompositionMapping;
|
||||
FNumericType;
|
||||
FNumericValue;
|
||||
FBidiMirrored;
|
||||
FUnicode1Name;
|
||||
FISOComment;
|
||||
FSimpleUppercaseMapping;
|
||||
FSimpleLowercaseMapping;
|
||||
FSimpleTitlecaseMapping;
|
||||
NumField;
|
||||
FCodePoint = iota
|
||||
FName
|
||||
FGeneralCategory
|
||||
FCanonicalCombiningClass
|
||||
FBidiClass
|
||||
FDecompositionType
|
||||
FDecompositionMapping
|
||||
FNumericType
|
||||
FNumericValue
|
||||
FBidiMirrored
|
||||
FUnicode1Name
|
||||
FISOComment
|
||||
FSimpleUppercaseMapping
|
||||
FSimpleLowercaseMapping
|
||||
FSimpleTitlecaseMapping
|
||||
NumField
|
||||
|
||||
MaxChar = 0x10FFFF; // anything above this shouldn't exist
|
||||
MaxChar = 0x10FFFF // anything above this shouldn't exist
|
||||
)
|
||||
|
||||
var fieldName = []string{
|
||||
|
|
@ -101,12 +101,12 @@ var fieldName = []string{
|
|||
|
||||
// This contains only the properties we're interested in.
|
||||
type Char struct {
|
||||
field []string; // debugging only; could be deleted if we take out char.dump()
|
||||
codePoint uint32; // if zero, this index is not a valid code point.
|
||||
category string;
|
||||
upperCase int;
|
||||
lowerCase int;
|
||||
titleCase int;
|
||||
field []string // debugging only; could be deleted if we take out char.dump()
|
||||
codePoint uint32 // if zero, this index is not a valid code point.
|
||||
category string
|
||||
upperCase int
|
||||
lowerCase int
|
||||
titleCase int
|
||||
}
|
||||
|
||||
// Scripts.txt has form:
|
||||
|
|
@ -115,13 +115,13 @@ type Char struct {
|
|||
// See http://www.unicode.org/Public/5.1.0/ucd/UCD.html for full explanation
|
||||
|
||||
type Script struct {
|
||||
lo, hi uint32; // range of code points
|
||||
script string;
|
||||
lo, hi uint32 // range of code points
|
||||
script string
|
||||
}
|
||||
|
||||
var chars = make([]Char, MaxChar+1)
|
||||
var scripts = make(map[string][]Script)
|
||||
var props = make(map[string][]Script) // a property looks like a script; can share the format
|
||||
var props = make(map[string][]Script) // a property looks like a script; can share the format
|
||||
|
||||
var lastChar uint32 = 0
|
||||
|
||||
|
|
@ -132,40 +132,40 @@ var lastChar uint32 = 0
|
|||
type State int
|
||||
|
||||
const (
|
||||
SNormal State = iota; // known to be zero for the type
|
||||
SFirst;
|
||||
SLast;
|
||||
SMissing;
|
||||
SNormal State = iota // known to be zero for the type
|
||||
SFirst
|
||||
SLast
|
||||
SMissing
|
||||
)
|
||||
|
||||
func parseCategory(line string) (state State) {
|
||||
field := strings.Split(line, ";", -1);
|
||||
field := strings.Split(line, ";", -1)
|
||||
if len(field) != NumField {
|
||||
die.Logf("%5s: %d fields (expected %d)\n", line, len(field), NumField)
|
||||
}
|
||||
point, err := strconv.Btoui64(field[FCodePoint], 16);
|
||||
point, err := strconv.Btoui64(field[FCodePoint], 16)
|
||||
if err != nil {
|
||||
die.Log("%.5s...:", err)
|
||||
}
|
||||
lastChar = uint32(point);
|
||||
lastChar = uint32(point)
|
||||
if point == 0 {
|
||||
return // not interesting and we use 0 as unset
|
||||
return // not interesting and we use 0 as unset
|
||||
}
|
||||
if point > MaxChar {
|
||||
return
|
||||
}
|
||||
char := &chars[point];
|
||||
char.field = field;
|
||||
char := &chars[point]
|
||||
char.field = field
|
||||
if char.codePoint != 0 {
|
||||
die.Logf("point U+%04x reused\n")
|
||||
}
|
||||
char.codePoint = lastChar;
|
||||
char.category = field[FGeneralCategory];
|
||||
category[char.category] = true;
|
||||
char.codePoint = lastChar
|
||||
char.category = field[FGeneralCategory]
|
||||
category[char.category] = true
|
||||
switch char.category {
|
||||
case "Nd":
|
||||
// Decimal digit
|
||||
_, err := strconv.Atoi(field[FNumericValue]);
|
||||
_, err := strconv.Atoi(field[FNumericValue])
|
||||
if err != nil {
|
||||
die.Log("U+%04x: bad numeric field: %s", point, err)
|
||||
}
|
||||
|
|
@ -184,66 +184,66 @@ func parseCategory(line string) (state State) {
|
|||
case strings.Index(field[FName], ", Last>") > 0:
|
||||
state = SLast
|
||||
}
|
||||
return;
|
||||
return
|
||||
}
|
||||
|
||||
func (char *Char) dump(s string) {
|
||||
fmt.Print(s, " ");
|
||||
fmt.Print(s, " ")
|
||||
for i := 0; i < len(char.field); i++ {
|
||||
fmt.Printf("%s:%q ", fieldName[i], char.field[i])
|
||||
}
|
||||
fmt.Print("\n");
|
||||
fmt.Print("\n")
|
||||
}
|
||||
|
||||
func (char *Char) letter(u, l, t string) {
|
||||
char.upperCase = char.letterValue(u, "U");
|
||||
char.lowerCase = char.letterValue(l, "L");
|
||||
char.titleCase = char.letterValue(t, "T");
|
||||
char.upperCase = char.letterValue(u, "U")
|
||||
char.lowerCase = char.letterValue(l, "L")
|
||||
char.titleCase = char.letterValue(t, "T")
|
||||
}
|
||||
|
||||
func (char *Char) letterValue(s string, cas string) int {
|
||||
if s == "" {
|
||||
return 0
|
||||
}
|
||||
v, err := strconv.Btoui64(s, 16);
|
||||
v, err := strconv.Btoui64(s, 16)
|
||||
if err != nil {
|
||||
char.dump(cas);
|
||||
die.Logf("U+%04x: bad letter(%s): %s", char.codePoint, s, err);
|
||||
char.dump(cas)
|
||||
die.Logf("U+%04x: bad letter(%s): %s", char.codePoint, s, err)
|
||||
}
|
||||
return int(v);
|
||||
return int(v)
|
||||
}
|
||||
|
||||
func allCategories() []string {
|
||||
a := make([]string, len(category));
|
||||
i := 0;
|
||||
a := make([]string, len(category))
|
||||
i := 0
|
||||
for k := range category {
|
||||
a[i] = k;
|
||||
i++;
|
||||
a[i] = k
|
||||
i++
|
||||
}
|
||||
return a;
|
||||
return a
|
||||
}
|
||||
|
||||
func all(scripts map[string][]Script) []string {
|
||||
a := make([]string, len(scripts));
|
||||
i := 0;
|
||||
a := make([]string, len(scripts))
|
||||
i := 0
|
||||
for k := range scripts {
|
||||
a[i] = k;
|
||||
i++;
|
||||
a[i] = k
|
||||
i++
|
||||
}
|
||||
return a;
|
||||
return a
|
||||
}
|
||||
|
||||
// Extract the version number from the URL
|
||||
func version() string {
|
||||
// Break on slashes and look for the first numeric field
|
||||
fields := strings.Split(*url, "/", 0);
|
||||
fields := strings.Split(*url, "/", 0)
|
||||
for _, f := range fields {
|
||||
if len(f) > 0 && '0' <= f[0] && f[0] <= '9' {
|
||||
return f
|
||||
}
|
||||
}
|
||||
die.Log("unknown version");
|
||||
return "Unknown";
|
||||
die.Log("unknown version")
|
||||
return "Unknown"
|
||||
}
|
||||
|
||||
func letterOp(code int) bool {
|
||||
|
|
@ -251,29 +251,29 @@ func letterOp(code int) bool {
|
|||
case "Lu", "Ll", "Lt", "Lm", "Lo":
|
||||
return true
|
||||
}
|
||||
return false;
|
||||
return false
|
||||
}
|
||||
|
||||
func loadChars() {
|
||||
if *dataURL == "" {
|
||||
flag.Set("data", *url+"UnicodeData.txt")
|
||||
}
|
||||
resp, _, err := http.Get(*dataURL);
|
||||
resp, _, err := http.Get(*dataURL)
|
||||
if err != nil {
|
||||
die.Log(err)
|
||||
}
|
||||
if resp.StatusCode != 200 {
|
||||
die.Log("bad GET status for UnicodeData.txt", resp.Status)
|
||||
}
|
||||
input := bufio.NewReader(resp.Body);
|
||||
var first uint32 = 0;
|
||||
input := bufio.NewReader(resp.Body)
|
||||
var first uint32 = 0
|
||||
for {
|
||||
line, err := input.ReadString('\n');
|
||||
line, err := input.ReadString('\n')
|
||||
if err != nil {
|
||||
if err == os.EOF {
|
||||
break
|
||||
}
|
||||
die.Log(err);
|
||||
die.Log(err)
|
||||
}
|
||||
switch parseCategory(line[0 : len(line)-1]) {
|
||||
case SNormal:
|
||||
|
|
@ -284,19 +284,19 @@ func loadChars() {
|
|||
if first != 0 {
|
||||
die.Logf("bad state first at U+%04X", lastChar)
|
||||
}
|
||||
first = lastChar;
|
||||
first = lastChar
|
||||
case SLast:
|
||||
if first == 0 {
|
||||
die.Logf("bad state last at U+%04X", lastChar)
|
||||
}
|
||||
for i := first + 1; i <= lastChar; i++ {
|
||||
chars[i] = chars[first];
|
||||
chars[i].codePoint = i;
|
||||
chars[i] = chars[first]
|
||||
chars[i].codePoint = i
|
||||
}
|
||||
first = 0;
|
||||
first = 0
|
||||
}
|
||||
}
|
||||
resp.Body.Close();
|
||||
resp.Body.Close()
|
||||
}
|
||||
|
||||
func printCategories() {
|
||||
|
|
@ -304,13 +304,13 @@ func printCategories() {
|
|||
return
|
||||
}
|
||||
// Find out which categories to dump
|
||||
list := strings.Split(*tablelist, ",", 0);
|
||||
list := strings.Split(*tablelist, ",", 0)
|
||||
if *tablelist == "all" {
|
||||
list = allCategories()
|
||||
}
|
||||
if *test {
|
||||
fullCategoryTest(list);
|
||||
return;
|
||||
fullCategoryTest(list)
|
||||
return
|
||||
}
|
||||
fmt.Printf(
|
||||
"// Generated by running\n"+
|
||||
|
|
@ -318,22 +318,22 @@ func printCategories() {
|
|||
"// DO NOT EDIT\n\n"+
|
||||
"package unicode\n\n",
|
||||
*tablelist,
|
||||
*dataURL);
|
||||
*dataURL)
|
||||
|
||||
fmt.Println("// Version is the Unicode edition from which the tables are derived.");
|
||||
fmt.Printf("const Version = %q\n\n", version());
|
||||
fmt.Println("// Version is the Unicode edition from which the tables are derived.")
|
||||
fmt.Printf("const Version = %q\n\n", version())
|
||||
|
||||
if *tablelist == "all" {
|
||||
fmt.Println("// Categories is the set of Unicode data tables.");
|
||||
fmt.Println("var Categories = map[string] []Range {");
|
||||
fmt.Println("// Categories is the set of Unicode data tables.")
|
||||
fmt.Println("var Categories = map[string] []Range {")
|
||||
for k, _ := range category {
|
||||
fmt.Printf("\t%q: %s,\n", k, k)
|
||||
}
|
||||
fmt.Printf("}\n\n");
|
||||
fmt.Printf("}\n\n")
|
||||
}
|
||||
|
||||
decl := make(sort.StringArray, len(list));
|
||||
ndecl := 0;
|
||||
decl := make(sort.StringArray, len(list))
|
||||
ndecl := 0
|
||||
for _, name := range list {
|
||||
if _, ok := category[name]; !ok {
|
||||
die.Log("unknown category", name)
|
||||
|
|
@ -342,7 +342,7 @@ func printCategories() {
|
|||
// name to store the data. This stops godoc dumping all the tables but keeps them
|
||||
// available to clients.
|
||||
// Cases deserving special comments
|
||||
varDecl := "";
|
||||
varDecl := ""
|
||||
switch name {
|
||||
case "letter":
|
||||
varDecl = "\tLetter = letter; // Letter is the set of Unicode letters.\n"
|
||||
|
|
@ -360,24 +360,24 @@ func printCategories() {
|
|||
"\t%s = _%s; // %s is the set of Unicode characters in category %s.\n",
|
||||
name, name, name, name)
|
||||
}
|
||||
decl[ndecl] = varDecl;
|
||||
ndecl++;
|
||||
if name == "letter" { // special case
|
||||
decl[ndecl] = varDecl
|
||||
ndecl++
|
||||
if name == "letter" { // special case
|
||||
dumpRange(
|
||||
"var letter = []Range {\n",
|
||||
letterOp);
|
||||
continue;
|
||||
letterOp)
|
||||
continue
|
||||
}
|
||||
dumpRange(
|
||||
fmt.Sprintf("var _%s = []Range {\n", name),
|
||||
func(code int) bool { return chars[code].category == name });
|
||||
func(code int) bool { return chars[code].category == name })
|
||||
}
|
||||
decl.Sort();
|
||||
fmt.Println("var (");
|
||||
decl.Sort()
|
||||
fmt.Println("var (")
|
||||
for _, d := range decl {
|
||||
fmt.Print(d)
|
||||
}
|
||||
fmt.Println(")\n");
|
||||
fmt.Println(")\n")
|
||||
}
|
||||
|
||||
type Op func(code int) bool
|
||||
|
|
@ -385,8 +385,8 @@ type Op func(code int) bool
|
|||
const format = "\tRange{0x%04x, 0x%04x, %d},\n"
|
||||
|
||||
func dumpRange(header string, inCategory Op) {
|
||||
fmt.Print(header);
|
||||
next := 0;
|
||||
fmt.Print(header)
|
||||
next := 0
|
||||
// one Range for each iteration
|
||||
for {
|
||||
// look for start of range
|
||||
|
|
@ -399,22 +399,22 @@ func dumpRange(header string, inCategory Op) {
|
|||
}
|
||||
|
||||
// start of range
|
||||
lo := next;
|
||||
hi := next;
|
||||
stride := 1;
|
||||
lo := next
|
||||
hi := next
|
||||
stride := 1
|
||||
// accept lo
|
||||
next++;
|
||||
next++
|
||||
// look for another character to set the stride
|
||||
for next < len(chars) && !inCategory(next) {
|
||||
next++
|
||||
}
|
||||
if next >= len(chars) {
|
||||
// no more characters
|
||||
fmt.Printf(format, lo, hi, stride);
|
||||
break;
|
||||
fmt.Printf(format, lo, hi, stride)
|
||||
break
|
||||
}
|
||||
// set stride
|
||||
stride = next - lo;
|
||||
stride = next - lo
|
||||
// check for length of run. next points to first jump in stride
|
||||
for i := next; i < len(chars); i++ {
|
||||
if inCategory(i) == (((i - lo) % stride) == 0) {
|
||||
|
|
@ -427,11 +427,11 @@ func dumpRange(header string, inCategory Op) {
|
|||
break
|
||||
}
|
||||
}
|
||||
fmt.Printf(format, lo, hi, stride);
|
||||
fmt.Printf(format, lo, hi, stride)
|
||||
// next range: start looking where this range ends
|
||||
next = hi + 1;
|
||||
next = hi + 1
|
||||
}
|
||||
fmt.Print("}\n\n");
|
||||
fmt.Print("}\n\n")
|
||||
}
|
||||
|
||||
func fullCategoryTest(list []string) {
|
||||
|
|
@ -439,7 +439,7 @@ func fullCategoryTest(list []string) {
|
|||
if _, ok := category[name]; !ok {
|
||||
die.Log("unknown category", name)
|
||||
}
|
||||
r, ok := unicode.Categories[name];
|
||||
r, ok := unicode.Categories[name]
|
||||
if !ok {
|
||||
die.Log("unknown table", name)
|
||||
}
|
||||
|
|
@ -456,8 +456,8 @@ func fullCategoryTest(list []string) {
|
|||
|
||||
func verifyRange(name string, inCategory Op, table []unicode.Range) {
|
||||
for i := range chars {
|
||||
web := inCategory(i);
|
||||
pkg := unicode.Is(table, i);
|
||||
web := inCategory(i)
|
||||
pkg := unicode.Is(table, i)
|
||||
if web != pkg {
|
||||
fmt.Fprintf(os.Stderr, "%s: U+%04X: web=%t pkg=%t\n", name, i, web, pkg)
|
||||
}
|
||||
|
|
@ -465,61 +465,61 @@ func verifyRange(name string, inCategory Op, table []unicode.Range) {
|
|||
}
|
||||
|
||||
func parseScript(line string, scripts map[string][]Script) {
|
||||
comment := strings.Index(line, "#");
|
||||
comment := strings.Index(line, "#")
|
||||
if comment >= 0 {
|
||||
line = line[0:comment]
|
||||
}
|
||||
line = strings.TrimSpace(line);
|
||||
line = strings.TrimSpace(line)
|
||||
if len(line) == 0 {
|
||||
return
|
||||
}
|
||||
field := strings.Split(line, ";", -1);
|
||||
field := strings.Split(line, ";", -1)
|
||||
if len(field) != 2 {
|
||||
die.Logf("%s: %d fields (expected 2)\n", line, len(field))
|
||||
}
|
||||
matches := scriptRe.MatchStrings(line);
|
||||
matches := scriptRe.MatchStrings(line)
|
||||
if len(matches) != 4 {
|
||||
die.Logf("%s: %d matches (expected 3)\n", line, len(matches))
|
||||
}
|
||||
lo, err := strconv.Btoui64(matches[1], 16);
|
||||
lo, err := strconv.Btoui64(matches[1], 16)
|
||||
if err != nil {
|
||||
die.Log("%.5s...:", err)
|
||||
}
|
||||
hi := lo;
|
||||
if len(matches[2]) > 2 { // ignore leading ..
|
||||
hi, err = strconv.Btoui64(matches[2][2:], 16);
|
||||
hi := lo
|
||||
if len(matches[2]) > 2 { // ignore leading ..
|
||||
hi, err = strconv.Btoui64(matches[2][2:], 16)
|
||||
if err != nil {
|
||||
die.Log("%.5s...:", err)
|
||||
}
|
||||
}
|
||||
name := matches[3];
|
||||
s, ok := scripts[name];
|
||||
name := matches[3]
|
||||
s, ok := scripts[name]
|
||||
if !ok || len(s) == cap(s) {
|
||||
ns := make([]Script, len(s), len(s)+100);
|
||||
ns := make([]Script, len(s), len(s)+100)
|
||||
for i, sc := range s {
|
||||
ns[i] = sc
|
||||
}
|
||||
s = ns;
|
||||
s = ns
|
||||
}
|
||||
s = s[0 : len(s)+1];
|
||||
s[len(s)-1] = Script{uint32(lo), uint32(hi), name};
|
||||
scripts[name] = s;
|
||||
s = s[0 : len(s)+1]
|
||||
s[len(s)-1] = Script{uint32(lo), uint32(hi), name}
|
||||
scripts[name] = s
|
||||
}
|
||||
|
||||
// The script tables have a lot of adjacent elements. Fold them together.
|
||||
func foldAdjacent(r []Script) []unicode.Range {
|
||||
s := make([]unicode.Range, 0, len(r));
|
||||
j := 0;
|
||||
s := make([]unicode.Range, 0, len(r))
|
||||
j := 0
|
||||
for i := 0; i < len(r); i++ {
|
||||
if j > 0 && int(r[i].lo) == s[j-1].Hi+1 {
|
||||
s[j-1].Hi = int(r[i].hi)
|
||||
} else {
|
||||
s = s[0 : j+1];
|
||||
s[j] = unicode.Range{int(r[i].lo), int(r[i].hi), 1};
|
||||
j++;
|
||||
s = s[0 : j+1]
|
||||
s[j] = unicode.Range{int(r[i].lo), int(r[i].hi), 1}
|
||||
j++
|
||||
}
|
||||
}
|
||||
return s;
|
||||
return s
|
||||
}
|
||||
|
||||
func fullScriptTest(list []string, installed map[string][]unicode.Range, scripts map[string][]Script) {
|
||||
|
|
@ -527,7 +527,7 @@ func fullScriptTest(list []string, installed map[string][]unicode.Range, scripts
|
|||
if _, ok := scripts[name]; !ok {
|
||||
die.Log("unknown script", name)
|
||||
}
|
||||
_, ok := installed[name];
|
||||
_, ok := installed[name]
|
||||
if !ok {
|
||||
die.Log("unknown table", name)
|
||||
}
|
||||
|
|
@ -543,50 +543,50 @@ func fullScriptTest(list []string, installed map[string][]unicode.Range, scripts
|
|||
|
||||
// PropList.txt has the same format as Scripts.txt so we can share its parser.
|
||||
func printScriptOrProperty(doProps bool) {
|
||||
flag := "scripts";
|
||||
flaglist := *scriptlist;
|
||||
file := "Scripts.txt";
|
||||
table := scripts;
|
||||
installed := unicode.Scripts;
|
||||
flag := "scripts"
|
||||
flaglist := *scriptlist
|
||||
file := "Scripts.txt"
|
||||
table := scripts
|
||||
installed := unicode.Scripts
|
||||
if doProps {
|
||||
flag = "props";
|
||||
flaglist = *proplist;
|
||||
file = "PropList.txt";
|
||||
table = props;
|
||||
installed = unicode.Properties;
|
||||
flag = "props"
|
||||
flaglist = *proplist
|
||||
file = "PropList.txt"
|
||||
table = props
|
||||
installed = unicode.Properties
|
||||
}
|
||||
if flaglist == "" {
|
||||
return
|
||||
}
|
||||
var err os.Error;
|
||||
resp, _, err := http.Get(*url + file);
|
||||
var err os.Error
|
||||
resp, _, err := http.Get(*url + file)
|
||||
if err != nil {
|
||||
die.Log(err)
|
||||
}
|
||||
if resp.StatusCode != 200 {
|
||||
die.Log("bad GET status for ", file, ":", resp.Status)
|
||||
}
|
||||
input := bufio.NewReader(resp.Body);
|
||||
input := bufio.NewReader(resp.Body)
|
||||
for {
|
||||
line, err := input.ReadString('\n');
|
||||
line, err := input.ReadString('\n')
|
||||
if err != nil {
|
||||
if err == os.EOF {
|
||||
break
|
||||
}
|
||||
die.Log(err);
|
||||
die.Log(err)
|
||||
}
|
||||
parseScript(line[0:len(line)-1], table);
|
||||
parseScript(line[0:len(line)-1], table)
|
||||
}
|
||||
resp.Body.Close();
|
||||
resp.Body.Close()
|
||||
|
||||
// Find out which scripts to dump
|
||||
list := strings.Split(flaglist, ",", 0);
|
||||
list := strings.Split(flaglist, ",", 0)
|
||||
if flaglist == "all" {
|
||||
list = all(table)
|
||||
}
|
||||
if *test {
|
||||
fullScriptTest(list, installed, table);
|
||||
return;
|
||||
fullScriptTest(list, installed, table)
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Printf(
|
||||
|
|
@ -595,23 +595,23 @@ func printScriptOrProperty(doProps bool) {
|
|||
"// DO NOT EDIT\n\n",
|
||||
flag,
|
||||
flaglist,
|
||||
*url);
|
||||
*url)
|
||||
if flaglist == "all" {
|
||||
if doProps {
|
||||
fmt.Println("// Properties is the set of Unicode property tables.");
|
||||
fmt.Println("var Properties = map[string] []Range {");
|
||||
fmt.Println("// Properties is the set of Unicode property tables.")
|
||||
fmt.Println("var Properties = map[string] []Range {")
|
||||
} else {
|
||||
fmt.Println("// Scripts is the set of Unicode script tables.");
|
||||
fmt.Println("var Scripts = map[string] []Range {");
|
||||
fmt.Println("// Scripts is the set of Unicode script tables.")
|
||||
fmt.Println("var Scripts = map[string] []Range {")
|
||||
}
|
||||
for k, _ := range table {
|
||||
fmt.Printf("\t%q: %s,\n", k, k)
|
||||
}
|
||||
fmt.Printf("}\n\n");
|
||||
fmt.Printf("}\n\n")
|
||||
}
|
||||
|
||||
decl := make(sort.StringArray, len(list));
|
||||
ndecl := 0;
|
||||
decl := make(sort.StringArray, len(list))
|
||||
ndecl := 0
|
||||
for _, name := range list {
|
||||
if doProps {
|
||||
decl[ndecl] = fmt.Sprintf(
|
||||
|
|
@ -622,36 +622,36 @@ func printScriptOrProperty(doProps bool) {
|
|||
"\t%s = _%s;\t// %s is the set of Unicode characters in script %s.\n",
|
||||
name, name, name, name)
|
||||
}
|
||||
ndecl++;
|
||||
fmt.Printf("var _%s = []Range {\n", name);
|
||||
ranges := foldAdjacent(table[name]);
|
||||
ndecl++
|
||||
fmt.Printf("var _%s = []Range {\n", name)
|
||||
ranges := foldAdjacent(table[name])
|
||||
for _, s := range ranges {
|
||||
fmt.Printf(format, s.Lo, s.Hi, s.Stride)
|
||||
}
|
||||
fmt.Printf("}\n\n");
|
||||
fmt.Printf("}\n\n")
|
||||
}
|
||||
decl.Sort();
|
||||
fmt.Println("var (");
|
||||
decl.Sort()
|
||||
fmt.Println("var (")
|
||||
for _, d := range decl {
|
||||
fmt.Print(d)
|
||||
}
|
||||
fmt.Println(")\n");
|
||||
fmt.Println(")\n")
|
||||
}
|
||||
|
||||
const (
|
||||
CaseUpper = 1 << iota;
|
||||
CaseLower;
|
||||
CaseTitle;
|
||||
CaseNone = 0; // must be zero
|
||||
CaseMissing = -1; // character not present; not a valid case state
|
||||
CaseUpper = 1 << iota
|
||||
CaseLower
|
||||
CaseTitle
|
||||
CaseNone = 0 // must be zero
|
||||
CaseMissing = -1 // character not present; not a valid case state
|
||||
)
|
||||
|
||||
type caseState struct {
|
||||
point int;
|
||||
_case int;
|
||||
deltaToUpper int;
|
||||
deltaToLower int;
|
||||
deltaToTitle int;
|
||||
point int
|
||||
_case int
|
||||
deltaToUpper int
|
||||
deltaToLower int
|
||||
deltaToTitle int
|
||||
}
|
||||
|
||||
// Is d a continuation of the state of c?
|
||||
|
|
@ -660,9 +660,9 @@ func (c *caseState) adjacent(d *caseState) bool {
|
|||
c, d = d, c
|
||||
}
|
||||
switch {
|
||||
case d.point != c.point+1: // code points not adjacent (shouldn't happen)
|
||||
case d.point != c.point+1: // code points not adjacent (shouldn't happen)
|
||||
return false
|
||||
case d._case != c._case: // different cases
|
||||
case d._case != c._case: // different cases
|
||||
return c.upperLowerAdjacent(d)
|
||||
case c._case == CaseNone:
|
||||
return false
|
||||
|
|
@ -675,7 +675,7 @@ func (c *caseState) adjacent(d *caseState) bool {
|
|||
case d.deltaToTitle != c.deltaToTitle:
|
||||
return false
|
||||
}
|
||||
return true;
|
||||
return true
|
||||
}
|
||||
|
||||
// Is d the same as c, but opposite in upper/lower case? this would make it
|
||||
|
|
@ -709,7 +709,7 @@ func (c *caseState) upperLowerAdjacent(d *caseState) bool {
|
|||
case d.deltaToTitle != -1:
|
||||
return false
|
||||
}
|
||||
return true;
|
||||
return true
|
||||
}
|
||||
|
||||
// Does this character start an UpperLower sequence?
|
||||
|
|
@ -724,7 +724,7 @@ func (c *caseState) isUpperLower() bool {
|
|||
case c.deltaToTitle != 0:
|
||||
return false
|
||||
}
|
||||
return true;
|
||||
return true
|
||||
}
|
||||
|
||||
// Does this character start a LowerUpper sequence?
|
||||
|
|
@ -739,16 +739,16 @@ func (c *caseState) isLowerUpper() bool {
|
|||
case c.deltaToTitle != -1:
|
||||
return false
|
||||
}
|
||||
return true;
|
||||
return true
|
||||
}
|
||||
|
||||
func getCaseState(i int) (c *caseState) {
|
||||
c = &caseState{point: i, _case: CaseNone};
|
||||
ch := &chars[i];
|
||||
c = &caseState{point: i, _case: CaseNone}
|
||||
ch := &chars[i]
|
||||
switch int(ch.codePoint) {
|
||||
case 0:
|
||||
c._case = CaseMissing; // Will get NUL wrong but that doesn't matter
|
||||
return;
|
||||
c._case = CaseMissing // Will get NUL wrong but that doesn't matter
|
||||
return
|
||||
case ch.upperCase:
|
||||
c._case = CaseUpper
|
||||
case ch.lowerCase:
|
||||
|
|
@ -765,7 +765,7 @@ func getCaseState(i int) (c *caseState) {
|
|||
if ch.titleCase != 0 {
|
||||
c.deltaToTitle = ch.titleCase - i
|
||||
}
|
||||
return;
|
||||
return
|
||||
}
|
||||
|
||||
func printCases() {
|
||||
|
|
@ -773,8 +773,8 @@ func printCases() {
|
|||
return
|
||||
}
|
||||
if *test {
|
||||
fullCaseTest();
|
||||
return;
|
||||
fullCaseTest()
|
||||
return
|
||||
}
|
||||
fmt.Printf(
|
||||
"// Generated by running\n"+
|
||||
|
|
@ -784,25 +784,25 @@ func printCases() {
|
|||
"// non-self mappings.\n"+
|
||||
"var CaseRanges = _CaseRanges\n"+
|
||||
"var _CaseRanges = []CaseRange {\n",
|
||||
*dataURL);
|
||||
*dataURL)
|
||||
|
||||
var startState *caseState; // the start of a run; nil for not active
|
||||
var prevState = &caseState{}; // the state of the previous character
|
||||
var startState *caseState // the start of a run; nil for not active
|
||||
var prevState = &caseState{} // the state of the previous character
|
||||
for i := range chars {
|
||||
state := getCaseState(i);
|
||||
state := getCaseState(i)
|
||||
if state.adjacent(prevState) {
|
||||
prevState = state;
|
||||
continue;
|
||||
prevState = state
|
||||
continue
|
||||
}
|
||||
// end of run (possibly)
|
||||
printCaseRange(startState, prevState);
|
||||
startState = nil;
|
||||
printCaseRange(startState, prevState)
|
||||
startState = nil
|
||||
if state._case != CaseMissing && state._case != CaseNone {
|
||||
startState = state
|
||||
}
|
||||
prevState = state;
|
||||
prevState = state
|
||||
}
|
||||
fmt.Printf("}\n");
|
||||
fmt.Printf("}\n")
|
||||
}
|
||||
|
||||
func printCaseRange(lo, hi *caseState) {
|
||||
|
|
@ -818,9 +818,9 @@ func printCaseRange(lo, hi *caseState) {
|
|||
fmt.Printf("\tCaseRange{0x%04X, 0x%04X, d{UpperLower, UpperLower, UpperLower}},\n",
|
||||
lo.point, hi.point)
|
||||
case hi.point > lo.point && lo.isLowerUpper():
|
||||
die.Log("LowerUpper sequence: should not happen: U+%04X. If it's real, need to fix To()", lo.point);
|
||||
die.Log("LowerUpper sequence: should not happen: U+%04X. If it's real, need to fix To()", lo.point)
|
||||
fmt.Printf("\tCaseRange{0x%04X, 0x%04X, d{LowerUpper, LowerUpper, LowerUpper}},\n",
|
||||
lo.point, hi.point);
|
||||
lo.point, hi.point)
|
||||
default:
|
||||
fmt.Printf("\tCaseRange{0x%04X, 0x%04X, d{%d, %d, %d}},\n",
|
||||
lo.point, hi.point,
|
||||
|
|
@ -833,23 +833,23 @@ func caseIt(rune, cased int) int {
|
|||
if cased == 0 {
|
||||
return rune
|
||||
}
|
||||
return cased;
|
||||
return cased
|
||||
}
|
||||
|
||||
func fullCaseTest() {
|
||||
for i, c := range chars {
|
||||
lower := unicode.ToLower(i);
|
||||
want := caseIt(i, c.lowerCase);
|
||||
lower := unicode.ToLower(i)
|
||||
want := caseIt(i, c.lowerCase)
|
||||
if lower != want {
|
||||
fmt.Fprintf(os.Stderr, "lower U+%04X should be U+%04X is U+%04X\n", i, want, lower)
|
||||
}
|
||||
upper := unicode.ToUpper(i);
|
||||
want = caseIt(i, c.upperCase);
|
||||
upper := unicode.ToUpper(i)
|
||||
want = caseIt(i, c.upperCase)
|
||||
if upper != want {
|
||||
fmt.Fprintf(os.Stderr, "upper U+%04X should be U+%04X is U+%04X\n", i, want, upper)
|
||||
}
|
||||
title := unicode.ToTitle(i);
|
||||
want = caseIt(i, c.titleCase);
|
||||
title := unicode.ToTitle(i)
|
||||
want = caseIt(i, c.titleCase)
|
||||
if title != want {
|
||||
fmt.Fprintf(os.Stderr, "title U+%04X should be U+%04X is U+%04X\n", i, want, title)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue