mirror of
https://github.com/golang/go.git
synced 2025-12-08 06:10:04 +00:00
unicode tables for all categories
maketables now has a --test option to verify the data. R=rsc DELTA=3087 (1982 added, 1001 deleted, 104 changed) OCL=33947 CL=33950
This commit is contained in:
Rob Pike
parent
5cbc96d958
commit
94e691552f
6 changed files with 2086 additions and 1105 deletions
|
|
@ -16,13 +16,23 @@ import (
|
|||
"os";
|
||||
"strconv";
|
||||
"strings";
|
||||
"unicode";
|
||||
)
|
||||
|
||||
var url = flag.String("url", "http://www.unicode.org/Public/5.1.0/ucd/UnicodeData.txt", "URL of Unicode database")
|
||||
var digits = flag.Bool("digits", false, "whether to generate digit tables; default is letter tables");
|
||||
var url = flag.String("url",
|
||||
"http://www.unicode.org/Public/5.1.0/ucd/UnicodeData.txt",
|
||||
"URL of Unicode database")
|
||||
var tables = flag.String("tables",
|
||||
"all",
|
||||
"comma-separated list of which tables to generate; default is all; can be letter");
|
||||
var test = flag.Bool("test",
|
||||
false,
|
||||
"test existing tables; can be used to compare web data with package data");
|
||||
|
||||
var die = log.New(os.Stderr, nil, "", log.Lexit|log.Lshortfile);
|
||||
|
||||
var category = map[string] bool{ "letter":true } // Nd Lu etc. letter is a special case
|
||||
|
||||
// Data has form:
|
||||
// 0037;DIGIT SEVEN;Nd;0;EN;;7;7;7;N;;;;;
|
||||
// 007A;LATIN SMALL LETTER Z;Ll;0;L;;;;;N;;;005A;;005A
|
||||
|
|
@ -46,7 +56,7 @@ const (
|
|||
FSimpleTitlecaseMapping;
|
||||
NumField;
|
||||
|
||||
MaxChar = 0xF0000; // anything above this doesn't have useful properties
|
||||
MaxChar = 0x10FFFF; // anything above this shouldn't exist
|
||||
)
|
||||
|
||||
var fieldName = []string{
|
||||
|
|
@ -72,7 +82,6 @@ type Char struct {
|
|||
field []string; // debugging only; could be deleted if we take out char.dump()
|
||||
codePoint uint32; // redundant (it's the index in the chars table) but useful
|
||||
category string;
|
||||
numValue int;
|
||||
upperCase uint32;
|
||||
lowerCase uint32;
|
||||
titleCase uint32;
|
||||
|
|
@ -85,7 +94,7 @@ var lastChar uint32 = 0;
|
|||
func parse(line string) {
|
||||
field := strings.Split(line, ";", -1);
|
||||
if len(field) != NumField {
|
||||
die.Logf("%.5s...: %d fields (expected %d)\n", line, len(field), NumField);
|
||||
die.Logf("%5s: %d fields (expected %d)\n", line, len(field), NumField);
|
||||
}
|
||||
point, err := strconv.Btoui64(field[FCodePoint], 16);
|
||||
if err != nil {
|
||||
|
|
@ -96,7 +105,6 @@ func parse(line string) {
|
|||
return // not interesting and we use 0 as unset
|
||||
}
|
||||
if point >= MaxChar {
|
||||
fmt.Fprintf(os.Stderr, "ignoring char U+%04x\n", point);
|
||||
return;
|
||||
}
|
||||
char := &chars[point];
|
||||
|
|
@ -106,6 +114,7 @@ func parse(line string) {
|
|||
}
|
||||
char.codePoint = lastChar;
|
||||
char.category = field[FGeneralCategory];
|
||||
category[char.category] = true;
|
||||
switch char.category {
|
||||
case "Nd":
|
||||
// Decimal digit
|
||||
|
|
@ -113,7 +122,6 @@ func parse(line string) {
|
|||
if err != nil {
|
||||
die.Log("U+%04x: bad numeric field: %s", point, err);
|
||||
}
|
||||
char.numValue = v;
|
||||
case "Lu":
|
||||
char.letter(field[FCodePoint], field[FSimpleLowercaseMapping], field[FSimpleTitlecaseMapping]);
|
||||
case "Ll":
|
||||
|
|
@ -151,6 +159,37 @@ func (char *Char) letterValue(s string, cas string) uint32 {
|
|||
return uint32(v)
|
||||
}
|
||||
|
||||
func allCategories() []string {
|
||||
a := make([]string, len(category));
|
||||
i := 0;
|
||||
for k := range category {
|
||||
a[i] = k;
|
||||
i++;
|
||||
}
|
||||
return a;
|
||||
}
|
||||
|
||||
// Extract the version number from the URL
|
||||
func version() string {
|
||||
// Break on slashes and look for the first numeric field
|
||||
fields := strings.Split(*url, "/", 0);
|
||||
for _, f := range fields {
|
||||
if len(f) > 0 && '0' <= f[0] && f[0] <= '9' {
|
||||
return f
|
||||
}
|
||||
}
|
||||
die.Log("unknown version");
|
||||
return "Unknown";
|
||||
}
|
||||
|
||||
func letterOp(code int) bool {
|
||||
switch chars[code].category {
|
||||
case "Lu", "Ll", "Lt", "Lm", "Lo":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func main() {
|
||||
flag.Parse();
|
||||
|
||||
|
|
@ -158,6 +197,9 @@ func main() {
|
|||
if err != nil {
|
||||
die.Log(err);
|
||||
}
|
||||
if resp.StatusCode != 200 {
|
||||
die.Log("bad GET status", resp.StatusCode);
|
||||
}
|
||||
input := bufio.NewReader(resp.Body);
|
||||
for {
|
||||
line, err := input.ReadLineString('\n', false);
|
||||
|
|
@ -170,59 +212,85 @@ func main() {
|
|||
parse(line);
|
||||
}
|
||||
resp.Body.Close();
|
||||
// Find out which categories to dump
|
||||
list := strings.Split(*tables, ",", 0);
|
||||
if *tables == "all" {
|
||||
list = allCategories();
|
||||
}
|
||||
if *test {
|
||||
fullTest(list);
|
||||
return
|
||||
}
|
||||
fmt.Printf(
|
||||
"// Generated by running\n"
|
||||
"// maketables --digits=%t --url=%s\n"
|
||||
"// maketables --tables=%t --url=%s\n"
|
||||
"// DO NOT EDIT\n\n"
|
||||
"package unicode\n",
|
||||
*digits,
|
||||
"package unicode\n\n",
|
||||
*tables,
|
||||
*url
|
||||
);
|
||||
// We generate an UpperCase name to serve as concise documentation and a lowerCase
|
||||
// name to store the data. This stops godoc dumping all the tables but keeps them
|
||||
// available to clients.
|
||||
if *digits {
|
||||
|
||||
fmt.Println("// Version is the Unicode edition from which the tables are derived.");
|
||||
fmt.Printf("var Version = %q\n\n", version());
|
||||
|
||||
if *tables == "all" {
|
||||
fmt.Println("// Tables is the set of Unicode data tables.");
|
||||
fmt.Println("var Tables = map[string] []Range {");
|
||||
for k, _ := range category {
|
||||
fmt.Printf("\t%q: %s,\n", k, k);
|
||||
}
|
||||
fmt.Printf("}\n\n");
|
||||
}
|
||||
|
||||
for _, name := range list {
|
||||
if _, ok := category[name]; !ok {
|
||||
die.Log("unknown category", name);
|
||||
}
|
||||
// We generate an UpperCase name to serve as concise documentation and an _UnderScored
|
||||
// name to store the data. This stops godoc dumping all the tables but keeps them
|
||||
// available to clients.
|
||||
if name == "letter" { // special case
|
||||
dumpRange(
|
||||
"\n// Letter is the set of Unicode letters.\n"
|
||||
"var Letter = letter\n"
|
||||
"var letter = []Range {\n",
|
||||
letterOp,
|
||||
"}\n"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
// Cases deserving special comments
|
||||
switch name {
|
||||
case "Nd":
|
||||
fmt.Printf(
|
||||
"\n// Digit is the set of Unicode characters with the \"decimal digit\" property.\n"
|
||||
"var Digit = Nd\n\n"
|
||||
)
|
||||
case "Lu":
|
||||
fmt.Printf(
|
||||
"\n// Upper is the set of Unicode upper case letters.\n"
|
||||
"var Upper = Lu\n\n"
|
||||
)
|
||||
case "Ll":
|
||||
fmt.Printf(
|
||||
"\n// Lower is the set of Unicode lower case letters.\n"
|
||||
"var Lower = Ll\n\n"
|
||||
)
|
||||
case "Lt":
|
||||
fmt.Printf(
|
||||
"\n// Title is the set of Unicode title case letters.\n"
|
||||
"var Title = Lt\n\n"
|
||||
)
|
||||
}
|
||||
dumpRange(
|
||||
"\n// Digit is the set of Unicode characters with the \"decimal digit\" (Nd) property.\n"
|
||||
"var Digit = digit\n"
|
||||
"var digit = []Range {\n",
|
||||
func(code int) bool { return chars[code].category == "Nd" },
|
||||
"}\n"
|
||||
);
|
||||
} else {
|
||||
dumpRange(
|
||||
"\n// Letter is the set of Unicode letters.\n"
|
||||
"var Letter = letter\n"
|
||||
"var letter = []Range {\n",
|
||||
func(code int) bool {
|
||||
switch chars[code].category {
|
||||
case "Lu", "Ll", "Lt", "Lm", "Lo":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
},
|
||||
"}\n"
|
||||
);
|
||||
dumpRange(
|
||||
"\n// Upper is the set of Unicode upper case letters.\n"
|
||||
"var Upper = upper\n"
|
||||
"var upper = []Range {\n",
|
||||
func(code int) bool { return chars[code].category == "Lu" },
|
||||
"}\n"
|
||||
);
|
||||
dumpRange(
|
||||
"\n// Lower is the set of Unicode lower case letters.\n"
|
||||
"var Lower = lower\n"
|
||||
"var lower = []Range {\n",
|
||||
func(code int) bool { return chars[code].category == "Ll" },
|
||||
"}\n"
|
||||
);
|
||||
dumpRange(
|
||||
"\n// Title is the set of Unicode title case letters.\n"
|
||||
"var Title = title\n"
|
||||
"var title = []Range {\n",
|
||||
func(code int) bool { return chars[code].category == "Lt" },
|
||||
"}\n"
|
||||
fmt.Sprintf(
|
||||
"// %s is the set of Unicode characters in category %s\n"
|
||||
"var %s = _%s\n"
|
||||
"var _%s = []Range {\n",
|
||||
name, name, name, name, name
|
||||
),
|
||||
func(code int) bool { return chars[code].category == name },
|
||||
"}\n\n"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -279,3 +347,34 @@ func dumpRange(header string, inCategory Op, trailer string) {
|
|||
}
|
||||
fmt.Print(trailer);
|
||||
}
|
||||
|
||||
func fullTest(list []string) {
|
||||
for _, name := range list {
|
||||
if _, ok := category[name]; !ok {
|
||||
die.Log("unknown category", name);
|
||||
}
|
||||
r, ok := unicode.Tables[name];
|
||||
if !ok {
|
||||
die.Log("unknown table", name);
|
||||
}
|
||||
if name == "letter" {
|
||||
verifyRange(name, letterOp, r);
|
||||
} else {
|
||||
verifyRange(
|
||||
name,
|
||||
func(code int) bool { return chars[code].category == name },
|
||||
r
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func verifyRange(name string, inCategory Op, table []unicode.Range) {
|
||||
for i, c := range chars {
|
||||
web := inCategory(i);
|
||||
pkg := unicode.Is(table, i);
|
||||
if web != pkg {
|
||||
fmt.Fprintf(os.Stderr, "%s: U+%04X: web=%t pkg=%t\n", name, i, web, pkg);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue