summaryrefslogtreecommitdiff
path: root/vendor/golang.org/x/text/language/display/maketables.go
diff options
context:
space:
mode:
Diffstat (limited to 'vendor/golang.org/x/text/language/display/maketables.go')
-rw-r--r--vendor/golang.org/x/text/language/display/maketables.go602
1 files changed, 602 insertions, 0 deletions
diff --git a/vendor/golang.org/x/text/language/display/maketables.go b/vendor/golang.org/x/text/language/display/maketables.go
new file mode 100644
index 0000000..8f2fd07
--- /dev/null
+++ b/vendor/golang.org/x/text/language/display/maketables.go
@@ -0,0 +1,602 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+// Generator for display name tables.
+
+package main
+
+import (
+ "bytes"
+ "flag"
+ "fmt"
+ "log"
+ "reflect"
+ "sort"
+ "strings"
+
+ "golang.org/x/text/internal/gen"
+ "golang.org/x/text/language"
+ "golang.org/x/text/unicode/cldr"
+)
+
+var (
+ test = flag.Bool("test", false,
+ "test existing tables; can be used to compare web data with package data.")
+ outputFile = flag.String("output", "tables.go", "output file")
+
+ stats = flag.Bool("stats", false, "prints statistics to stderr")
+
+ short = flag.Bool("short", false, `Use "short" alternatives, when available.`)
+ draft = flag.String("draft",
+ "contributed",
+ `Minimal draft requirements (approved, contributed, provisional, unconfirmed).`)
+ pkg = flag.String("package",
+ "display",
+ "the name of the package in which the generated file is to be included")
+
+ tags = newTagSet("tags",
+ []language.Tag{},
+ "space-separated list of tags to include or empty for all")
+ dict = newTagSet("dict",
+ dictTags(),
+ "space-separated list or tags for which to include a Dictionary. "+
+ `"" means the common list from go.text/language.`)
+)
+
+func dictTags() (tag []language.Tag) {
+ // TODO: replace with language.Common.Tags() once supported.
+ const str = "af am ar ar-001 az bg bn ca cs da de el en en-US en-GB " +
+ "es es-ES es-419 et fa fi fil fr fr-CA gu he hi hr hu hy id is it ja " +
+ "ka kk km kn ko ky lo lt lv mk ml mn mr ms my ne nl no pa pl pt pt-BR " +
+ "pt-PT ro ru si sk sl sq sr sr-Latn sv sw ta te th tr uk ur uz vi " +
+ "zh zh-Hans zh-Hant zu"
+
+ for _, s := range strings.Split(str, " ") {
+ tag = append(tag, language.MustParse(s))
+ }
+ return tag
+}
+
+func main() {
+ gen.Init()
+
+ // Read the CLDR zip file.
+ r := gen.OpenCLDRCoreZip()
+ defer r.Close()
+
+ d := &cldr.Decoder{}
+ d.SetDirFilter("main", "supplemental")
+ d.SetSectionFilter("localeDisplayNames")
+ data, err := d.DecodeZip(r)
+ if err != nil {
+ log.Fatalf("DecodeZip: %v", err)
+ }
+
+ w := gen.NewCodeWriter()
+ defer w.WriteGoFile(*outputFile, "display")
+
+ gen.WriteCLDRVersion(w)
+
+ b := builder{
+ w: w,
+ data: data,
+ group: make(map[string]*group),
+ }
+ b.generate()
+}
+
+const tagForm = language.All
+
+// tagSet is used to parse command line flags of tags. It implements the
+// flag.Value interface.
+type tagSet map[language.Tag]bool
+
+func newTagSet(name string, tags []language.Tag, usage string) tagSet {
+ f := tagSet(make(map[language.Tag]bool))
+ for _, t := range tags {
+ f[t] = true
+ }
+ flag.Var(f, name, usage)
+ return f
+}
+
+// String implements the String method of the flag.Value interface.
+func (f tagSet) String() string {
+ tags := []string{}
+ for t := range f {
+ tags = append(tags, t.String())
+ }
+ sort.Strings(tags)
+ return strings.Join(tags, " ")
+}
+
+// Set implements Set from the flag.Value interface.
+func (f tagSet) Set(s string) error {
+ if s != "" {
+ for _, s := range strings.Split(s, " ") {
+ if s != "" {
+ tag, err := tagForm.Parse(s)
+ if err != nil {
+ return err
+ }
+ f[tag] = true
+ }
+ }
+ }
+ return nil
+}
+
+func (f tagSet) contains(t language.Tag) bool {
+ if len(f) == 0 {
+ return true
+ }
+ return f[t]
+}
+
+// builder is used to create all tables with display name information.
+type builder struct {
+ w *gen.CodeWriter
+
+ data *cldr.CLDR
+
+ fromLocs []string
+
+ // destination tags for the current locale.
+ toTags []string
+ toTagIndex map[string]int
+
+ // list of supported tags
+ supported []language.Tag
+
+ // key-value pairs per group
+ group map[string]*group
+
+ // statistics
+ sizeIndex int // total size of all indexes of headers
+ sizeData int // total size of all data of headers
+ totalSize int
+}
+
+type group struct {
+ // Maps from a given language to the Namer data for this language.
+ lang map[language.Tag]keyValues
+ headers []header
+
+ toTags []string
+ threeStart int
+ fourPlusStart int
+}
+
+// set sets the typ to the name for locale loc.
+func (g *group) set(t language.Tag, typ, name string) {
+ kv := g.lang[t]
+ if kv == nil {
+ kv = make(keyValues)
+ g.lang[t] = kv
+ }
+ if kv[typ] == "" {
+ kv[typ] = name
+ }
+}
+
+type keyValues map[string]string
+
+type header struct {
+ tag language.Tag
+ data string
+ index []uint16
+}
+
+var versionInfo = `// Version is deprecated. Use CLDRVersion.
+const Version = %#v
+
+`
+
+var self = language.MustParse("mul")
+
+// generate builds and writes all tables.
+func (b *builder) generate() {
+ fmt.Fprintf(b.w, versionInfo, cldr.Version)
+
+ b.filter()
+ b.setData("lang", func(g *group, loc language.Tag, ldn *cldr.LocaleDisplayNames) {
+ if ldn.Languages != nil {
+ for _, v := range ldn.Languages.Language {
+ lang := v.Type
+ if lang == "root" {
+ // We prefer the data from "und"
+ // TODO: allow both the data for root and und somehow.
+ continue
+ }
+ tag := tagForm.MustParse(lang)
+ if tags.contains(tag) {
+ g.set(loc, tag.String(), v.Data())
+ }
+ }
+ }
+ })
+ b.setData("script", func(g *group, loc language.Tag, ldn *cldr.LocaleDisplayNames) {
+ if ldn.Scripts != nil {
+ for _, v := range ldn.Scripts.Script {
+ code := language.MustParseScript(v.Type)
+ if code.IsPrivateUse() { // Qaaa..Qabx
+ // TODO: data currently appears to be very meager.
+ // Reconsider if we have data for English.
+ if loc == language.English {
+ log.Fatal("Consider including data for private use scripts.")
+ }
+ continue
+ }
+ g.set(loc, code.String(), v.Data())
+ }
+ }
+ })
+ b.setData("region", func(g *group, loc language.Tag, ldn *cldr.LocaleDisplayNames) {
+ if ldn.Territories != nil {
+ for _, v := range ldn.Territories.Territory {
+ g.set(loc, language.MustParseRegion(v.Type).String(), v.Data())
+ }
+ }
+ })
+
+ b.makeSupported()
+
+ b.writeParents()
+
+ b.writeGroup("lang")
+ b.writeGroup("script")
+ b.writeGroup("region")
+
+ b.w.WriteConst("numSupported", len(b.supported))
+ buf := bytes.Buffer{}
+ for _, tag := range b.supported {
+ fmt.Fprint(&buf, tag.String(), "|")
+ }
+ b.w.WriteConst("supported", buf.String())
+
+ b.writeDictionaries()
+
+ b.supported = []language.Tag{self}
+
+ // Compute the names of locales in their own language. Some of these names
+ // may be specified in their parent locales. We iterate the maximum depth
+ // of the parent three times to match successive parents of tags until a
+ // possible match is found.
+ for i := 0; i < 4; i++ {
+ b.setData("self", func(g *group, tag language.Tag, ldn *cldr.LocaleDisplayNames) {
+ parent := tag
+ if b, s, r := tag.Raw(); i > 0 && (s != language.Script{} && r == language.Region{}) {
+ parent, _ = language.Raw.Compose(b)
+ }
+ if ldn.Languages != nil {
+ for _, v := range ldn.Languages.Language {
+ key := tagForm.MustParse(v.Type)
+ saved := key
+ if key == parent {
+ g.set(self, tag.String(), v.Data())
+ }
+ for k := 0; k < i; k++ {
+ key = key.Parent()
+ }
+ if key == tag {
+ g.set(self, saved.String(), v.Data()) // set does not overwrite a value.
+ }
+ }
+ }
+ })
+ }
+
+ b.writeGroup("self")
+}
+
+func (b *builder) setData(name string, f func(*group, language.Tag, *cldr.LocaleDisplayNames)) {
+ b.sizeIndex = 0
+ b.sizeData = 0
+ b.toTags = nil
+ b.fromLocs = nil
+ b.toTagIndex = make(map[string]int)
+
+ g := b.group[name]
+ if g == nil {
+ g = &group{lang: make(map[language.Tag]keyValues)}
+ b.group[name] = g
+ }
+ for _, loc := range b.data.Locales() {
+ // We use RawLDML instead of LDML as we are managing our own inheritance
+ // in this implementation.
+ ldml := b.data.RawLDML(loc)
+
+ // We do not support the POSIX variant (it is not a supported BCP 47
+ // variant). This locale also doesn't happen to contain any data, so
+ // we'll skip it by checking for this.
+ tag, err := tagForm.Parse(loc)
+ if err != nil {
+ if ldml.LocaleDisplayNames != nil {
+ log.Fatalf("setData: %v", err)
+ }
+ continue
+ }
+ if ldml.LocaleDisplayNames != nil && tags.contains(tag) {
+ f(g, tag, ldml.LocaleDisplayNames)
+ }
+ }
+}
+
+func (b *builder) filter() {
+ filter := func(s *cldr.Slice) {
+ if *short {
+ s.SelectOnePerGroup("alt", []string{"short", ""})
+ } else {
+ s.SelectOnePerGroup("alt", []string{"stand-alone", ""})
+ }
+ d, err := cldr.ParseDraft(*draft)
+ if err != nil {
+ log.Fatalf("filter: %v", err)
+ }
+ s.SelectDraft(d)
+ }
+ for _, loc := range b.data.Locales() {
+ if ldn := b.data.RawLDML(loc).LocaleDisplayNames; ldn != nil {
+ if ldn.Languages != nil {
+ s := cldr.MakeSlice(&ldn.Languages.Language)
+ if filter(&s); len(ldn.Languages.Language) == 0 {
+ ldn.Languages = nil
+ }
+ }
+ if ldn.Scripts != nil {
+ s := cldr.MakeSlice(&ldn.Scripts.Script)
+ if filter(&s); len(ldn.Scripts.Script) == 0 {
+ ldn.Scripts = nil
+ }
+ }
+ if ldn.Territories != nil {
+ s := cldr.MakeSlice(&ldn.Territories.Territory)
+ if filter(&s); len(ldn.Territories.Territory) == 0 {
+ ldn.Territories = nil
+ }
+ }
+ }
+ }
+}
+
+// makeSupported creates a list of all supported locales.
+func (b *builder) makeSupported() {
+ // tags across groups
+ for _, g := range b.group {
+ for t, _ := range g.lang {
+ b.supported = append(b.supported, t)
+ }
+ }
+ b.supported = b.supported[:unique(tagsSorter(b.supported))]
+
+}
+
+type tagsSorter []language.Tag
+
+func (a tagsSorter) Len() int { return len(a) }
+func (a tagsSorter) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a tagsSorter) Less(i, j int) bool { return a[i].String() < a[j].String() }
+
+func (b *builder) writeGroup(name string) {
+ g := b.group[name]
+
+ for _, kv := range g.lang {
+ for t, _ := range kv {
+ g.toTags = append(g.toTags, t)
+ }
+ }
+ g.toTags = g.toTags[:unique(tagsBySize(g.toTags))]
+
+ // Allocate header per supported value.
+ g.headers = make([]header, len(b.supported))
+ for i, sup := range b.supported {
+ kv, ok := g.lang[sup]
+ if !ok {
+ g.headers[i].tag = sup
+ continue
+ }
+ data := []byte{}
+ index := make([]uint16, len(g.toTags), len(g.toTags)+1)
+ for j, t := range g.toTags {
+ index[j] = uint16(len(data))
+ data = append(data, kv[t]...)
+ }
+ index = append(index, uint16(len(data)))
+
+ // Trim the tail of the index.
+ // TODO: indexes can be reduced in size quite a bit more.
+ n := len(index)
+ for ; n >= 2 && index[n-2] == index[n-1]; n-- {
+ }
+ index = index[:n]
+
+ // Workaround for a bug in CLDR 26.
+ // See http://unicode.org/cldr/trac/ticket/8042.
+ if cldr.Version == "26" && sup.String() == "hsb" {
+ data = bytes.Replace(data, []byte{'"'}, nil, 1)
+ }
+ g.headers[i] = header{sup, string(data), index}
+ }
+ g.writeTable(b.w, name)
+}
+
+type tagsBySize []string
+
+func (l tagsBySize) Len() int { return len(l) }
+func (l tagsBySize) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
+func (l tagsBySize) Less(i, j int) bool {
+ a, b := l[i], l[j]
+ // Sort single-tag entries based on size first. Otherwise alphabetic.
+ if len(a) != len(b) && (len(a) <= 4 || len(b) <= 4) {
+ return len(a) < len(b)
+ }
+ return a < b
+}
+
+// parentIndices returns slice a of len(tags) where tags[a[i]] is the parent
+// of tags[i].
+func parentIndices(tags []language.Tag) []int16 {
+ index := make(map[language.Tag]int16)
+ for i, t := range tags {
+ index[t] = int16(i)
+ }
+
+ // Construct default parents.
+ parents := make([]int16, len(tags))
+ for i, t := range tags {
+ parents[i] = -1
+ for t = t.Parent(); t != language.Und; t = t.Parent() {
+ if j, ok := index[t]; ok {
+ parents[i] = j
+ break
+ }
+ }
+ }
+ return parents
+}
+
+func (b *builder) writeParents() {
+ parents := parentIndices(b.supported)
+ fmt.Fprintf(b.w, "var parents = ")
+ b.w.WriteArray(parents)
+}
+
+// writeKeys writes keys to a special index used by the display package.
+// tags are assumed to be sorted by length.
+func writeKeys(w *gen.CodeWriter, name string, keys []string) {
+ w.Size += int(3 * reflect.TypeOf("").Size())
+ w.WriteComment("Number of keys: %d", len(keys))
+ fmt.Fprintf(w, "var (\n\t%sIndex = tagIndex{\n", name)
+ for i := 2; i <= 4; i++ {
+ sub := []string{}
+ for _, t := range keys {
+ if len(t) != i {
+ break
+ }
+ sub = append(sub, t)
+ }
+ s := strings.Join(sub, "")
+ w.WriteString(s)
+ fmt.Fprintf(w, ",\n")
+ keys = keys[len(sub):]
+ }
+ fmt.Fprintln(w, "\t}")
+ if len(keys) > 0 {
+ w.Size += int(reflect.TypeOf([]string{}).Size())
+ fmt.Fprintf(w, "\t%sTagsLong = ", name)
+ w.WriteSlice(keys)
+ }
+ fmt.Fprintln(w, ")\n")
+}
+
+// identifier creates an identifier from the given tag.
+func identifier(t language.Tag) string {
+ return strings.Replace(t.String(), "-", "", -1)
+}
+
+func (h *header) writeEntry(w *gen.CodeWriter, name string) {
+ if len(dict) > 0 && dict.contains(h.tag) {
+ fmt.Fprintf(w, "\t{ // %s\n", h.tag)
+ fmt.Fprintf(w, "\t\t%[1]s%[2]sStr,\n\t\t%[1]s%[2]sIdx,\n", identifier(h.tag), name)
+ fmt.Fprintln(w, "\t},")
+ } else if len(h.data) == 0 {
+ fmt.Fprintln(w, "\t\t{}, //", h.tag)
+ } else {
+ fmt.Fprintf(w, "\t{ // %s\n", h.tag)
+ w.WriteString(h.data)
+ fmt.Fprintln(w, ",")
+ w.WriteSlice(h.index)
+ fmt.Fprintln(w, ",\n\t},")
+ }
+}
+
+// write the data for the given header as single entries. The size for this data
+// was already accounted for in writeEntry.
+func (h *header) writeSingle(w *gen.CodeWriter, name string) {
+ if len(dict) > 0 && dict.contains(h.tag) {
+ tag := identifier(h.tag)
+ w.WriteConst(tag+name+"Str", h.data)
+
+ // Note that we create a slice instead of an array. If we use an array
+ // we need to refer to it as a[:] in other tables, which will cause the
+ // array to always be included by the linker. See Issue 7651.
+ w.WriteVar(tag+name+"Idx", h.index)
+ }
+}
+
+// WriteTable writes an entry for a single Namer.
+func (g *group) writeTable(w *gen.CodeWriter, name string) {
+ start := w.Size
+ writeKeys(w, name, g.toTags)
+ w.Size += len(g.headers) * int(reflect.ValueOf(g.headers[0]).Type().Size())
+
+ fmt.Fprintf(w, "var %sHeaders = [%d]header{\n", name, len(g.headers))
+
+ title := strings.Title(name)
+ for _, h := range g.headers {
+ h.writeEntry(w, title)
+ }
+ fmt.Fprintln(w, "}\n")
+
+ for _, h := range g.headers {
+ h.writeSingle(w, title)
+ }
+ n := w.Size - start
+ fmt.Fprintf(w, "// Total size for %s: %d bytes (%d KB)\n\n", name, n, n/1000)
+}
+
+func (b *builder) writeDictionaries() {
+ fmt.Fprintln(b.w, "// Dictionary entries of frequent languages")
+ fmt.Fprintln(b.w, "var (")
+ parents := parentIndices(b.supported)
+
+ for i, t := range b.supported {
+ if dict.contains(t) {
+ ident := identifier(t)
+ fmt.Fprintf(b.w, "\t%s = Dictionary{ // %s\n", ident, t)
+ if p := parents[i]; p == -1 {
+ fmt.Fprintln(b.w, "\t\tnil,")
+ } else {
+ fmt.Fprintf(b.w, "\t\t&%s,\n", identifier(b.supported[p]))
+ }
+ fmt.Fprintf(b.w, "\t\theader{%[1]sLangStr, %[1]sLangIdx},\n", ident)
+ fmt.Fprintf(b.w, "\t\theader{%[1]sScriptStr, %[1]sScriptIdx},\n", ident)
+ fmt.Fprintf(b.w, "\t\theader{%[1]sRegionStr, %[1]sRegionIdx},\n", ident)
+ fmt.Fprintln(b.w, "\t}")
+ }
+ }
+ fmt.Fprintln(b.w, ")")
+
+ var s string
+ var a []uint16
+ sz := reflect.TypeOf(s).Size()
+ sz += reflect.TypeOf(a).Size()
+ sz *= 3
+ sz += reflect.TypeOf(&a).Size()
+ n := int(sz) * len(dict)
+ fmt.Fprintf(b.w, "// Total size for %d entries: %d bytes (%d KB)\n\n", len(dict), n, n/1000)
+
+ b.w.Size += n
+}
+
+// unique sorts the given lists and removes duplicate entries by swapping them
+// past position k, where k is the number of unique values. It returns k.
+func unique(a sort.Interface) int {
+ if a.Len() == 0 {
+ return 0
+ }
+ sort.Sort(a)
+ k := 1
+ for i := 1; i < a.Len(); i++ {
+ if a.Less(k-1, i) {
+ if k != i {
+ a.Swap(k, i)
+ }
+ k++
+ }
+ }
+ return k
+}