From 32dd62da990484cf15db8eb7a14bd646bc0fda8a Mon Sep 17 00:00:00 2001 From: Dimitri Sokolyuk Date: Sat, 6 Jul 2019 18:56:43 +0200 Subject: switch to go.mod --- vendor/golang.org/x/text/language/language.go | 720 ++++++++------------------ 1 file changed, 207 insertions(+), 513 deletions(-) (limited to 'vendor/golang.org/x/text/language/language.go') diff --git a/vendor/golang.org/x/text/language/language.go b/vendor/golang.org/x/text/language/language.go index b65e213..abfa17f 100644 --- a/vendor/golang.org/x/text/language/language.go +++ b/vendor/golang.org/x/text/language/language.go @@ -2,8 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:generate go run gen.go gen_common.go -output tables.go -//go:generate go run gen_index.go +//go:generate go run gen.go -output tables.go package language @@ -11,47 +10,34 @@ package language // - verifying that tables are dropped correctly (most notably matcher tables). import ( - "errors" - "fmt" "strings" -) - -const ( - // maxCoreSize is the maximum size of a BCP 47 tag without variants and - // extensions. Equals max lang (3) + script (4) + max reg (3) + 2 dashes. - maxCoreSize = 12 - // max99thPercentileSize is a somewhat arbitrary buffer size that presumably - // is large enough to hold at least 99% of the BCP 47 tags. - max99thPercentileSize = 32 - - // maxSimpleUExtensionSize is the maximum size of a -u extension with one - // key-type pair. Equals len("-u-") + key (2) + dash + max value (8). - maxSimpleUExtensionSize = 14 + "golang.org/x/text/internal/language" + "golang.org/x/text/internal/language/compact" ) // Tag represents a BCP 47 language tag. It is used to specify an instance of a // specific language or locale. All language tag values are guaranteed to be // well-formed. -type Tag struct { - lang langID - region regionID - // TODO: we will soon run out of positions for script. Idea: instead of - // storing lang, region, and script codes, store only the compact index and - // have a lookup table from this code to its expansion. This greatly speeds - // up table lookup, speed up common variant cases. - // This will also immediately free up 3 extra bytes. Also, the pVariant - // field can now be moved to the lookup table, as the compact index uniquely - // determines the offset of a possible variant. - script scriptID - pVariant byte // offset in str, includes preceding '-' - pExt uint16 // offset of first extension, includes preceding '-' - - // str is the string representation of the Tag. It will only be used if the - // tag has variants or extensions. - str string +type Tag compact.Tag + +func makeTag(t language.Tag) (tag Tag) { + return Tag(compact.Make(t)) +} + +func (t *Tag) tag() language.Tag { + return (*compact.Tag)(t).Tag() +} + +func (t *Tag) isCompact() bool { + return (*compact.Tag)(t).IsCompact() } +// TODO: improve performance. +func (t *Tag) lang() language.Language { return t.tag().LangID } +func (t *Tag) region() language.Region { return t.tag().RegionID } +func (t *Tag) script() language.Script { return t.tag().ScriptID } + // Make is a convenience wrapper for Parse that omits the error. // In case of an error, a sensible default is returned. func Make(s string) Tag { @@ -68,25 +54,13 @@ func (c CanonType) Make(s string) Tag { // Raw returns the raw base language, script and region, without making an // attempt to infer their values. func (t Tag) Raw() (b Base, s Script, r Region) { - return Base{t.lang}, Script{t.script}, Region{t.region} -} - -// equalTags compares language, script and region subtags only. -func (t Tag) equalTags(a Tag) bool { - return t.lang == a.lang && t.script == a.script && t.region == a.region + tt := t.tag() + return Base{tt.LangID}, Script{tt.ScriptID}, Region{tt.RegionID} } // IsRoot returns true if t is equal to language "und". func (t Tag) IsRoot() bool { - if int(t.pVariant) < len(t.str) { - return false - } - return t.equalTags(und) -} - -// private reports whether the Tag consists solely of a private use tag. -func (t Tag) private() bool { - return t.str != "" && t.pVariant == 0 + return compact.Tag(t).IsRoot() } // CanonType can be used to enable or disable various types of canonicalization. @@ -138,73 +112,73 @@ const ( // canonicalize returns the canonicalized equivalent of the tag and // whether there was any change. -func (t Tag) canonicalize(c CanonType) (Tag, bool) { +func canonicalize(c CanonType, t language.Tag) (language.Tag, bool) { if c == Raw { return t, false } changed := false if c&SuppressScript != 0 { - if t.lang < langNoIndexOffset && uint8(t.script) == suppressScript[t.lang] { - t.script = 0 + if t.LangID.SuppressScript() == t.ScriptID { + t.ScriptID = 0 changed = true } } if c&canonLang != 0 { for { - if l, aliasType := normLang(t.lang); l != t.lang { + if l, aliasType := t.LangID.Canonicalize(); l != t.LangID { switch aliasType { - case langLegacy: + case language.Legacy: if c&Legacy != 0 { - if t.lang == _sh && t.script == 0 { - t.script = _Latn + if t.LangID == _sh && t.ScriptID == 0 { + t.ScriptID = _Latn } - t.lang = l + t.LangID = l changed = true } - case langMacro: + case language.Macro: if c&Macro != 0 { // We deviate here from CLDR. The mapping "nb" -> "no" // qualifies as a typical Macro language mapping. However, // for legacy reasons, CLDR maps "no", the macro language // code for Norwegian, to the dominant variant "nb". This // change is currently under consideration for CLDR as well. - // See http://unicode.org/cldr/trac/ticket/2698 and also - // http://unicode.org/cldr/trac/ticket/1790 for some of the + // See https://unicode.org/cldr/trac/ticket/2698 and also + // https://unicode.org/cldr/trac/ticket/1790 for some of the // practical implications. TODO: this check could be removed // if CLDR adopts this change. - if c&CLDR == 0 || t.lang != _nb { + if c&CLDR == 0 || t.LangID != _nb { changed = true - t.lang = l + t.LangID = l } } - case langDeprecated: + case language.Deprecated: if c&DeprecatedBase != 0 { - if t.lang == _mo && t.region == 0 { - t.region = _MD + if t.LangID == _mo && t.RegionID == 0 { + t.RegionID = _MD } - t.lang = l + t.LangID = l changed = true // Other canonicalization types may still apply. continue } } - } else if c&Legacy != 0 && t.lang == _no && c&CLDR != 0 { - t.lang = _nb + } else if c&Legacy != 0 && t.LangID == _no && c&CLDR != 0 { + t.LangID = _nb changed = true } break } } if c&DeprecatedScript != 0 { - if t.script == _Qaai { + if t.ScriptID == _Qaai { changed = true - t.script = _Zinh + t.ScriptID = _Zinh } } if c&DeprecatedRegion != 0 { - if r := normRegion(t.region); r != 0 { + if r := t.RegionID.Canonicalize(); r != t.RegionID { changed = true - t.region = r + t.RegionID = r } } return t, changed @@ -212,11 +186,20 @@ func (t Tag) canonicalize(c CanonType) (Tag, bool) { // Canonicalize returns the canonicalized equivalent of the tag. func (c CanonType) Canonicalize(t Tag) (Tag, error) { - t, changed := t.canonicalize(c) - if changed { - t.remakeString() + // First try fast path. + if t.isCompact() { + if _, changed := canonicalize(c, compact.Tag(t).Tag()); !changed { + return t, nil + } + } + // It is unlikely that one will canonicalize a tag after matching. So do + // a slow but simple approach here. + if tag, changed := canonicalize(c, t.tag()); changed { + tag.RemakeString() + return makeTag(tag), nil } return t, nil + } // Confidence indicates the level of certainty for a given return value. @@ -239,83 +222,21 @@ func (c Confidence) String() string { return confName[c] } -// remakeString is used to update t.str in case lang, script or region changed. -// It is assumed that pExt and pVariant still point to the start of the -// respective parts. -func (t *Tag) remakeString() { - if t.str == "" { - return - } - extra := t.str[t.pVariant:] - if t.pVariant > 0 { - extra = extra[1:] - } - if t.equalTags(und) && strings.HasPrefix(extra, "x-") { - t.str = extra - t.pVariant = 0 - t.pExt = 0 - return - } - var buf [max99thPercentileSize]byte // avoid extra memory allocation in most cases. - b := buf[:t.genCoreBytes(buf[:])] - if extra != "" { - diff := len(b) - int(t.pVariant) - b = append(b, '-') - b = append(b, extra...) - t.pVariant = uint8(int(t.pVariant) + diff) - t.pExt = uint16(int(t.pExt) + diff) - } else { - t.pVariant = uint8(len(b)) - t.pExt = uint16(len(b)) - } - t.str = string(b) -} - -// genCoreBytes writes a string for the base languages, script and region tags -// to the given buffer and returns the number of bytes written. It will never -// write more than maxCoreSize bytes. -func (t *Tag) genCoreBytes(buf []byte) int { - n := t.lang.stringToBuf(buf[:]) - if t.script != 0 { - n += copy(buf[n:], "-") - n += copy(buf[n:], t.script.String()) - } - if t.region != 0 { - n += copy(buf[n:], "-") - n += copy(buf[n:], t.region.String()) - } - return n -} - // String returns the canonical string representation of the language tag. func (t Tag) String() string { - if t.str != "" { - return t.str - } - if t.script == 0 && t.region == 0 { - return t.lang.String() - } - buf := [maxCoreSize]byte{} - return string(buf[:t.genCoreBytes(buf[:])]) + return t.tag().String() } // MarshalText implements encoding.TextMarshaler. func (t Tag) MarshalText() (text []byte, err error) { - if t.str != "" { - text = append(text, t.str...) - } else if t.script == 0 && t.region == 0 { - text = append(text, t.lang.String()...) - } else { - buf := [maxCoreSize]byte{} - text = buf[:t.genCoreBytes(buf[:])] - } - return text, nil + return t.tag().MarshalText() } // UnmarshalText implements encoding.TextUnmarshaler. func (t *Tag) UnmarshalText(text []byte) error { - tag, err := Raw.Parse(string(text)) - *t = tag + var tag language.Tag + err := tag.UnmarshalText(text) + *t = makeTag(tag) return err } @@ -323,15 +244,16 @@ func (t *Tag) UnmarshalText(text []byte) error { // unspecified, an attempt will be made to infer it from the context. // It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change. func (t Tag) Base() (Base, Confidence) { - if t.lang != 0 { - return Base{t.lang}, Exact + if b := t.lang(); b != 0 { + return Base{b}, Exact } + tt := t.tag() c := High - if t.script == 0 && !(Region{t.region}).IsCountry() { + if tt.ScriptID == 0 && !tt.RegionID.IsCountry() { c = Low } - if tag, err := addTags(t); err == nil && tag.lang != 0 { - return Base{tag.lang}, c + if tag, err := tt.Maximize(); err == nil && tag.LangID != 0 { + return Base{tag.LangID}, c } return Base{0}, No } @@ -344,35 +266,34 @@ func (t Tag) Base() (Base, Confidence) { // If a script cannot be inferred (Zzzz, No) is returned. We do not use Zyyy (undetermined) // as one would suspect from the IANA registry for BCP 47. In a Unicode context Zyyy marks // common characters (like 1, 2, 3, '.', etc.) and is therefore more like multiple scripts. -// See http://www.unicode.org/reports/tr24/#Values for more details. Zzzz is also used for +// See https://www.unicode.org/reports/tr24/#Values for more details. Zzzz is also used for // unknown value in CLDR. (Zzzz, Exact) is returned if Zzzz was explicitly specified. // Note that an inferred script is never guaranteed to be the correct one. Latin is // almost exclusively used for Afrikaans, but Arabic has been used for some texts // in the past. Also, the script that is commonly used may change over time. // It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change. func (t Tag) Script() (Script, Confidence) { - if t.script != 0 { - return Script{t.script}, Exact - } - sc, c := scriptID(_Zzzz), No - if t.lang < langNoIndexOffset { - if scr := scriptID(suppressScript[t.lang]); scr != 0 { - // Note: it is not always the case that a language with a suppress - // script value is only written in one script (e.g. kk, ms, pa). - if t.region == 0 { - return Script{scriptID(scr)}, High - } - sc, c = scr, High + if scr := t.script(); scr != 0 { + return Script{scr}, Exact + } + tt := t.tag() + sc, c := language.Script(_Zzzz), No + if scr := tt.LangID.SuppressScript(); scr != 0 { + // Note: it is not always the case that a language with a suppress + // script value is only written in one script (e.g. kk, ms, pa). + if tt.RegionID == 0 { + return Script{scr}, High } + sc, c = scr, High } - if tag, err := addTags(t); err == nil { - if tag.script != sc { - sc, c = tag.script, Low + if tag, err := tt.Maximize(); err == nil { + if tag.ScriptID != sc { + sc, c = tag.ScriptID, Low } } else { - t, _ = (Deprecated | Macro).Canonicalize(t) - if tag, err := addTags(t); err == nil && tag.script != sc { - sc, c = tag.script, Low + tt, _ = canonicalize(Deprecated|Macro, tt) + if tag, err := tt.Maximize(); err == nil && tag.ScriptID != sc { + sc, c = tag.ScriptID, Low } } return Script{sc}, c @@ -382,28 +303,31 @@ func (t Tag) Script() (Script, Confidence) { // infer a most likely candidate from the context. // It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change. func (t Tag) Region() (Region, Confidence) { - if t.region != 0 { - return Region{t.region}, Exact + if r := t.region(); r != 0 { + return Region{r}, Exact } - if t, err := addTags(t); err == nil { - return Region{t.region}, Low // TODO: differentiate between high and low. + tt := t.tag() + if tt, err := tt.Maximize(); err == nil { + return Region{tt.RegionID}, Low // TODO: differentiate between high and low. } - t, _ = (Deprecated | Macro).Canonicalize(t) - if tag, err := addTags(t); err == nil { - return Region{tag.region}, Low + tt, _ = canonicalize(Deprecated|Macro, tt) + if tag, err := tt.Maximize(); err == nil { + return Region{tag.RegionID}, Low } return Region{_ZZ}, No // TODO: return world instead of undetermined? } -// Variant returns the variants specified explicitly for this language tag. +// Variants returns the variants specified explicitly for this language tag. // or nil if no variant was specified. func (t Tag) Variants() []Variant { + if !compact.Tag(t).MayHaveVariants() { + return nil + } v := []Variant{} - if int(t.pVariant) < int(t.pExt) { - for x, str := "", t.str[t.pVariant:t.pExt]; str != ""; { - x, str = nextToken(str) - v = append(v, Variant{x}) - } + x, str := "", t.tag().Variants() + for str != "" { + x, str = nextToken(str) + v = append(v, Variant{x}) } return v } @@ -411,57 +335,13 @@ func (t Tag) Variants() []Variant { // Parent returns the CLDR parent of t. In CLDR, missing fields in data for a // specific language are substituted with fields from the parent language. // The parent for a language may change for newer versions of CLDR. +// +// Parent returns a tag for a less specific language that is mutually +// intelligible or Und if there is no such language. This may not be the same as +// simply stripping the last BCP 47 subtag. For instance, the parent of "zh-TW" +// is "zh-Hant", and the parent of "zh-Hant" is "und". func (t Tag) Parent() Tag { - if t.str != "" { - // Strip the variants and extensions. - t, _ = Raw.Compose(t.Raw()) - if t.region == 0 && t.script != 0 && t.lang != 0 { - base, _ := addTags(Tag{lang: t.lang}) - if base.script == t.script { - return Tag{lang: t.lang} - } - } - return t - } - if t.lang != 0 { - if t.region != 0 { - maxScript := t.script - if maxScript == 0 { - max, _ := addTags(t) - maxScript = max.script - } - - for i := range parents { - if langID(parents[i].lang) == t.lang && scriptID(parents[i].maxScript) == maxScript { - for _, r := range parents[i].fromRegion { - if regionID(r) == t.region { - return Tag{ - lang: t.lang, - script: scriptID(parents[i].script), - region: regionID(parents[i].toRegion), - } - } - } - } - } - - // Strip the script if it is the default one. - base, _ := addTags(Tag{lang: t.lang}) - if base.script != maxScript { - return Tag{lang: t.lang, script: maxScript} - } - return Tag{lang: t.lang} - } else if t.script != 0 { - // The parent for an base-script pair with a non-default script is - // "und" instead of the base language. - base, _ := addTags(Tag{lang: t.lang}) - if base.script != t.script { - return und - } - return Tag{lang: t.lang} - } - } - return und + return Tag(compact.Tag(t).Parent()) } // returns token t and the rest of the string. @@ -487,17 +367,8 @@ func (e Extension) String() string { // ParseExtension parses s as an extension and returns it on success. func ParseExtension(s string) (e Extension, err error) { - scan := makeScannerString(s) - var end int - if n := len(scan.token); n != 1 { - return Extension{}, errSyntax - } - scan.toLower(0, len(scan.b)) - end = parseExtension(&scan) - if end != len(s) { - return Extension{}, errSyntax - } - return Extension{string(scan.b)}, nil + ext, err := language.ParseExtension(s) + return Extension{ext}, err } // Type returns the one-byte extension type of e. It returns 0 for the zero @@ -518,22 +389,20 @@ func (e Extension) Tokens() []string { // false for ok if t does not have the requested extension. The returned // extension will be invalid in this case. func (t Tag) Extension(x byte) (ext Extension, ok bool) { - for i := int(t.pExt); i < len(t.str)-1; { - var ext string - i, ext = getExtension(t.str, i) - if ext[0] == x { - return Extension{ext}, true - } + if !compact.Tag(t).MayHaveExtensions() { + return Extension{}, false } - return Extension{}, false + e, ok := t.tag().Extension(x) + return Extension{e}, ok } // Extensions returns all extensions of t. func (t Tag) Extensions() []Extension { + if !compact.Tag(t).MayHaveExtensions() { + return nil + } e := []Extension{} - for i := int(t.pExt); i < len(t.str)-1; { - var ext string - i, ext = getExtension(t.str, i) + for _, ext := range t.tag().Extensions() { e = append(e, Extension{ext}) } return e @@ -541,259 +410,105 @@ func (t Tag) Extensions() []Extension { // TypeForKey returns the type associated with the given key, where key and type // are of the allowed values defined for the Unicode locale extension ('u') in -// http://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers. +// https://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers. // TypeForKey will traverse the inheritance chain to get the correct value. func (t Tag) TypeForKey(key string) string { - if start, end, _ := t.findTypeForKey(key); end != start { - return t.str[start:end] + if !compact.Tag(t).MayHaveExtensions() { + if key != "rg" && key != "va" { + return "" + } } - return "" + return t.tag().TypeForKey(key) } -var ( - errPrivateUse = errors.New("cannot set a key on a private use tag") - errInvalidArguments = errors.New("invalid key or type") -) - // SetTypeForKey returns a new Tag with the key set to type, where key and type // are of the allowed values defined for the Unicode locale extension ('u') in -// http://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers. +// https://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers. // An empty value removes an existing pair with the same key. func (t Tag) SetTypeForKey(key, value string) (Tag, error) { - if t.private() { - return t, errPrivateUse - } - if len(key) != 2 { - return t, errInvalidArguments - } - - // Remove the setting if value is "". - if value == "" { - start, end, _ := t.findTypeForKey(key) - if start != end { - // Remove key tag and leading '-'. - start -= 4 - - // Remove a possible empty extension. - if (end == len(t.str) || t.str[end+2] == '-') && t.str[start-2] == '-' { - start -= 2 - } - if start == int(t.pVariant) && end == len(t.str) { - t.str = "" - t.pVariant, t.pExt = 0, 0 - } else { - t.str = fmt.Sprintf("%s%s", t.str[:start], t.str[end:]) - } - } - return t, nil - } - - if len(value) < 3 || len(value) > 8 { - return t, errInvalidArguments - } - - var ( - buf [maxCoreSize + maxSimpleUExtensionSize]byte - uStart int // start of the -u extension. - ) - - // Generate the tag string if needed. - if t.str == "" { - uStart = t.genCoreBytes(buf[:]) - buf[uStart] = '-' - uStart++ - } - - // Create new key-type pair and parse it to verify. - b := buf[uStart:] - copy(b, "u-") - copy(b[2:], key) - b[4] = '-' - b = b[:5+copy(b[5:], value)] - scan := makeScanner(b) - if parseExtensions(&scan); scan.err != nil { - return t, scan.err - } - - // Assemble the replacement string. - if t.str == "" { - t.pVariant, t.pExt = byte(uStart-1), uint16(uStart-1) - t.str = string(buf[:uStart+len(b)]) - } else { - s := t.str - start, end, hasExt := t.findTypeForKey(key) - if start == end { - if hasExt { - b = b[2:] - } - t.str = fmt.Sprintf("%s-%s%s", s[:start], b, s[end:]) - } else { - t.str = fmt.Sprintf("%s%s%s", s[:start], value, s[end:]) - } - } - return t, nil + tt, err := t.tag().SetTypeForKey(key, value) + return makeTag(tt), err } -// findKeyAndType returns the start and end position for the type corresponding -// to key or the point at which to insert the key-value pair if the type -// wasn't found. The hasExt return value reports whether an -u extension was present. -// Note: the extensions are typically very small and are likely to contain -// only one key-type pair. -func (t Tag) findTypeForKey(key string) (start, end int, hasExt bool) { - p := int(t.pExt) - if len(key) != 2 || p == len(t.str) || p == 0 { - return p, p, false - } - s := t.str - - // Find the correct extension. - for p++; s[p] != 'u'; p++ { - if s[p] > 'u' { - p-- - return p, p, false - } - if p = nextExtension(s, p); p == len(s) { - return len(s), len(s), false - } - } - // Proceed to the hyphen following the extension name. - p++ - - // curKey is the key currently being processed. - curKey := "" - - // Iterate over keys until we get the end of a section. - for { - // p points to the hyphen preceding the current token. - if p3 := p + 3; s[p3] == '-' { - // Found a key. - // Check whether we just processed the key that was requested. - if curKey == key { - return start, p, true - } - // Set to the next key and continue scanning type tokens. - curKey = s[p+1 : p3] - if curKey > key { - return p, p, true - } - // Start of the type token sequence. - start = p + 4 - // A type is at least 3 characters long. - p += 7 // 4 + 3 - } else { - // Attribute or type, which is at least 3 characters long. - p += 4 - } - // p points past the third character of a type or attribute. - max := p + 5 // maximum length of token plus hyphen. - if len(s) < max { - max = len(s) - } - for ; p < max && s[p] != '-'; p++ { - } - // Bail if we have exhausted all tokens or if the next token starts - // a new extension. - if p == len(s) || s[p+2] == '-' { - if curKey == key { - return start, p, true - } - return p, p, true - } - } -} +// NumCompactTags is the number of compact tags. The maximum tag is +// NumCompactTags-1. +const NumCompactTags = compact.NumCompactTags // CompactIndex returns an index, where 0 <= index < NumCompactTags, for tags -// for which data exists in the text repository. The index will change over time -// and should not be stored in persistent storage. Extensions, except for the -// 'va' type of the 'u' extension, are ignored. It will return 0, false if no -// compact tag exists, where 0 is the index for the root language (Und). -func CompactIndex(t Tag) (index int, ok bool) { - // TODO: perhaps give more frequent tags a lower index. - // TODO: we could make the indexes stable. This will excluded some - // possibilities for optimization, so don't do this quite yet. - b, s, r := t.Raw() - if len(t.str) > 0 { - if strings.HasPrefix(t.str, "x-") { - // We have no entries for user-defined tags. - return 0, false - } - if uint16(t.pVariant) != t.pExt { - // There are no tags with variants and an u-va type. - if t.TypeForKey("va") != "" { - return 0, false - } - t, _ = Raw.Compose(b, s, r, t.Variants()) - } else if _, ok := t.Extension('u'); ok { - // Strip all but the 'va' entry. - variant := t.TypeForKey("va") - t, _ = Raw.Compose(b, s, r) - t, _ = t.SetTypeForKey("va", variant) - } - if len(t.str) > 0 { - // We have some variants. - for i, s := range specialTags { - if s == t { - return i + 1, true - } - } - return 0, false - } - } - // No variants specified: just compare core components. - // The key has the form lllssrrr, where l, s, and r are nibbles for - // respectively the langID, scriptID, and regionID. - key := uint32(b.langID) << (8 + 12) - key |= uint32(s.scriptID) << 12 - key |= uint32(r.regionID) - x, ok := coreTags[key] - return int(x), ok +// for which data exists in the text repository.The index will change over time +// and should not be stored in persistent storage. If t does not match a compact +// index, exact will be false and the compact index will be returned for the +// first match after repeatedly taking the Parent of t. +func CompactIndex(t Tag) (index int, exact bool) { + id, exact := compact.LanguageID(compact.Tag(t)) + return int(id), exact } +var root = language.Tag{} + // Base is an ISO 639 language code, used for encoding the base language // of a language tag. type Base struct { - langID + langID language.Language } // ParseBase parses a 2- or 3-letter ISO 639 code. // It returns a ValueError if s is a well-formed but unknown language identifier // or another error if another error occurred. func ParseBase(s string) (Base, error) { - if n := len(s); n < 2 || 3 < n { - return Base{}, errSyntax - } - var buf [3]byte - l, err := getLangID(buf[:copy(buf[:], s)]) + l, err := language.ParseBase(s) return Base{l}, err } +// String returns the BCP 47 representation of the base language. +func (b Base) String() string { + return b.langID.String() +} + +// ISO3 returns the ISO 639-3 language code. +func (b Base) ISO3() string { + return b.langID.ISO3() +} + +// IsPrivateUse reports whether this language code is reserved for private use. +func (b Base) IsPrivateUse() bool { + return b.langID.IsPrivateUse() +} + // Script is a 4-letter ISO 15924 code for representing scripts. // It is idiomatically represented in title case. type Script struct { - scriptID + scriptID language.Script } // ParseScript parses a 4-letter ISO 15924 code. // It returns a ValueError if s is a well-formed but unknown script identifier // or another error if another error occurred. func ParseScript(s string) (Script, error) { - if len(s) != 4 { - return Script{}, errSyntax - } - var buf [4]byte - sc, err := getScriptID(script, buf[:copy(buf[:], s)]) + sc, err := language.ParseScript(s) return Script{sc}, err } +// String returns the script code in title case. +// It returns "Zzzz" for an unspecified script. +func (s Script) String() string { + return s.scriptID.String() +} + +// IsPrivateUse reports whether this script code is reserved for private use. +func (s Script) IsPrivateUse() bool { + return s.scriptID.IsPrivateUse() +} + // Region is an ISO 3166-1 or UN M.49 code for representing countries and regions. type Region struct { - regionID + regionID language.Region } // EncodeM49 returns the Region for the given UN M.49 code. // It returns an error if r is not a valid code. func EncodeM49(r int) (Region, error) { - rid, err := getRegionM49(r) + rid, err := language.EncodeM49(r) return Region{rid}, err } @@ -801,62 +516,54 @@ func EncodeM49(r int) (Region, error) { // It returns a ValueError if s is a well-formed but unknown region identifier // or another error if another error occurred. func ParseRegion(s string) (Region, error) { - if n := len(s); n < 2 || 3 < n { - return Region{}, errSyntax - } - var buf [3]byte - r, err := getRegionID(buf[:copy(buf[:], s)]) + r, err := language.ParseRegion(s) return Region{r}, err } +// String returns the BCP 47 representation for the region. +// It returns "ZZ" for an unspecified region. +func (r Region) String() string { + return r.regionID.String() +} + +// ISO3 returns the 3-letter ISO code of r. +// Note that not all regions have a 3-letter ISO code. +// In such cases this method returns "ZZZ". +func (r Region) ISO3() string { + return r.regionID.ISO3() +} + +// M49 returns the UN M.49 encoding of r, or 0 if this encoding +// is not defined for r. +func (r Region) M49() int { + return r.regionID.M49() +} + +// IsPrivateUse reports whether r has the ISO 3166 User-assigned status. This +// may include private-use tags that are assigned by CLDR and used in this +// implementation. So IsPrivateUse and IsCountry can be simultaneously true. +func (r Region) IsPrivateUse() bool { + return r.regionID.IsPrivateUse() +} + // IsCountry returns whether this region is a country or autonomous area. This // includes non-standard definitions from CLDR. func (r Region) IsCountry() bool { - if r.regionID == 0 || r.IsGroup() || r.IsPrivateUse() && r.regionID != _XK { - return false - } - return true + return r.regionID.IsCountry() } // IsGroup returns whether this region defines a collection of regions. This // includes non-standard definitions from CLDR. func (r Region) IsGroup() bool { - if r.regionID == 0 { - return false - } - return int(regionInclusion[r.regionID]) < len(regionContainment) + return r.regionID.IsGroup() } // Contains returns whether Region c is contained by Region r. It returns true // if c == r. func (r Region) Contains(c Region) bool { - return r.regionID.contains(c.regionID) + return r.regionID.Contains(c.regionID) } -func (r regionID) contains(c regionID) bool { - if r == c { - return true - } - g := regionInclusion[r] - if g >= nRegionGroups { - return false - } - m := regionContainment[g] - - d := regionInclusion[c] - b := regionInclusionBits[d] - - // A contained country may belong to multiple disjoint groups. Matching any - // of these indicates containment. If the contained region is a group, it - // must strictly be a subset. - if d >= nRegionGroups { - return b&m != 0 - } - return b&^m == 0 -} - -var errNoTLD = errors.New("language: region is not a valid ccTLD") - // TLD returns the country code top-level domain (ccTLD). UK is returned for GB. // In all other cases it returns either the region itself or an error. // @@ -865,25 +572,15 @@ var errNoTLD = errors.New("language: region is not a valid ccTLD") // region will already be canonicalized it was obtained from a Tag that was // obtained using any of the default methods. func (r Region) TLD() (Region, error) { - // See http://en.wikipedia.org/wiki/Country_code_top-level_domain for the - // difference between ISO 3166-1 and IANA ccTLD. - if r.regionID == _GB { - r = Region{_UK} - } - if (r.typ() & ccTLD) == 0 { - return Region{}, errNoTLD - } - return r, nil + tld, err := r.regionID.TLD() + return Region{tld}, err } // Canonicalize returns the region or a possible replacement if the region is // deprecated. It will not return a replacement for deprecated regions that // are split into multiple regions. func (r Region) Canonicalize() Region { - if cr := normRegion(r.regionID); cr != 0 { - return Region{cr} - } - return r + return Region{r.regionID.Canonicalize()} } // Variant represents a registered variant of a language as defined by BCP 47. @@ -894,11 +591,8 @@ type Variant struct { // ParseVariant parses and returns a Variant. An error is returned if s is not // a valid variant. func ParseVariant(s string) (Variant, error) { - s = strings.ToLower(s) - if _, ok := variantIndex[s]; ok { - return Variant{s}, nil - } - return Variant{}, mkErrInvalid([]byte(s)) + v, err := language.ParseVariant(s) + return Variant{v.String()}, err } // String returns the string representation of the variant. -- cgit v1.2.3