From 354da79bb2edaa1af7d909d2774e7d67eb4e198c Mon Sep 17 00:00:00 2001 From: Dimitri Sokolyuk Date: Tue, 23 Jan 2018 18:17:51 +0100 Subject: Add vendor --- vendor/golang.org/x/text/language/parse_test.go | 517 ++++++++++++++++++++++++ 1 file changed, 517 insertions(+) create mode 100644 vendor/golang.org/x/text/language/parse_test.go (limited to 'vendor/golang.org/x/text/language/parse_test.go') diff --git a/vendor/golang.org/x/text/language/parse_test.go b/vendor/golang.org/x/text/language/parse_test.go new file mode 100644 index 0000000..9b40eb4 --- /dev/null +++ b/vendor/golang.org/x/text/language/parse_test.go @@ -0,0 +1,517 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package language + +import ( + "bytes" + "strings" + "testing" + + "golang.org/x/text/internal/tag" +) + +type scanTest struct { + ok bool // true if scanning does not result in an error + in string + tok []string // the expected tokens +} + +var tests = []scanTest{ + {true, "", []string{}}, + {true, "1", []string{"1"}}, + {true, "en", []string{"en"}}, + {true, "root", []string{"root"}}, + {true, "maxchars", []string{"maxchars"}}, + {false, "bad/", []string{}}, + {false, "morethan8", []string{}}, + {false, "-", []string{}}, + {false, "----", []string{}}, + {false, "_", []string{}}, + {true, "en-US", []string{"en", "US"}}, + {true, "en_US", []string{"en", "US"}}, + {false, "en-US-", []string{"en", "US"}}, + {false, "en-US--", []string{"en", "US"}}, + {false, "en-US---", []string{"en", "US"}}, + {false, "en--US", []string{"en", "US"}}, + {false, "-en-US", []string{"en", "US"}}, + {false, "-en--US-", []string{"en", "US"}}, + {false, "-en--US-", []string{"en", "US"}}, + {false, "en-.-US", []string{"en", "US"}}, + {false, ".-en--US-.", []string{"en", "US"}}, + {false, "en-u.-US", []string{"en", "US"}}, + {true, "en-u1-US", []string{"en", "u1", "US"}}, + {true, "maxchar1_maxchar2-maxchar3", []string{"maxchar1", "maxchar2", "maxchar3"}}, + {false, "moreThan8-moreThan8-e", []string{"e"}}, +} + +func TestScan(t *testing.T) { + for i, tt := range tests { + scan := makeScannerString(tt.in) + for j := 0; !scan.done; j++ { + if j >= len(tt.tok) { + t.Errorf("%d: extra token %q", i, scan.token) + } else if tag.Compare(tt.tok[j], scan.token) != 0 { + t.Errorf("%d: token %d: found %q; want %q", i, j, scan.token, tt.tok[j]) + break + } + scan.scan() + } + if s := strings.Join(tt.tok, "-"); tag.Compare(s, bytes.Replace(scan.b, b("_"), b("-"), -1)) != 0 { + t.Errorf("%d: input: found %q; want %q", i, scan.b, s) + } + if (scan.err == nil) != tt.ok { + t.Errorf("%d: ok: found %v; want %v", i, scan.err == nil, tt.ok) + } + } +} + +func TestAcceptMinSize(t *testing.T) { + for i, tt := range tests { + // count number of successive tokens with a minimum size. + for sz := 1; sz <= 8; sz++ { + scan := makeScannerString(tt.in) + scan.end, scan.next = 0, 0 + end := scan.acceptMinSize(sz) + n := 0 + for i := 0; i < len(tt.tok) && len(tt.tok[i]) >= sz; i++ { + n += len(tt.tok[i]) + if i > 0 { + n++ + } + } + if end != n { + t.Errorf("%d:%d: found len %d; want %d", i, sz, end, n) + } + } + } +} + +type parseTest struct { + i int // the index of this test + in string + lang, script, region string + variants, ext string + extList []string // only used when more than one extension is present + invalid bool + rewrite bool // special rewrite not handled by parseTag + changed bool // string needed to be reformatted +} + +func parseTests() []parseTest { + tests := []parseTest{ + {in: "root", lang: "und"}, + {in: "und", lang: "und"}, + {in: "en", lang: "en"}, + {in: "xy", lang: "und", invalid: true}, + {in: "en-ZY", lang: "en", invalid: true}, + {in: "gsw", lang: "gsw"}, + {in: "sr_Latn", lang: "sr", script: "Latn"}, + {in: "af-Arab", lang: "af", script: "Arab"}, + {in: "nl-BE", lang: "nl", region: "BE"}, + {in: "es-419", lang: "es", region: "419"}, + {in: "und-001", lang: "und", region: "001"}, + {in: "de-latn-be", lang: "de", script: "Latn", region: "BE"}, + // Variants + {in: "de-1901", lang: "de", variants: "1901"}, + // Accept with unsuppressed script. + {in: "de-Latn-1901", lang: "de", script: "Latn", variants: "1901"}, + // Specialized. + {in: "sl-rozaj", lang: "sl", variants: "rozaj"}, + {in: "sl-rozaj-lipaw", lang: "sl", variants: "rozaj-lipaw"}, + {in: "sl-rozaj-biske", lang: "sl", variants: "rozaj-biske"}, + {in: "sl-rozaj-biske-1994", lang: "sl", variants: "rozaj-biske-1994"}, + {in: "sl-rozaj-1994", lang: "sl", variants: "rozaj-1994"}, + // Maximum number of variants while adhering to prefix rules. + {in: "sl-rozaj-biske-1994-alalc97-fonipa-fonupa-fonxsamp", lang: "sl", variants: "rozaj-biske-1994-alalc97-fonipa-fonupa-fonxsamp"}, + + // Sorting. + {in: "sl-1994-biske-rozaj", lang: "sl", variants: "rozaj-biske-1994", changed: true}, + {in: "sl-rozaj-biske-1994-alalc97-fonupa-fonipa-fonxsamp", lang: "sl", variants: "rozaj-biske-1994-alalc97-fonipa-fonupa-fonxsamp", changed: true}, + {in: "nl-fonxsamp-alalc97-fonipa-fonupa", lang: "nl", variants: "alalc97-fonipa-fonupa-fonxsamp", changed: true}, + + // Duplicates variants are removed, but not an error. + {in: "nl-fonupa-fonupa", lang: "nl", variants: "fonupa"}, + + // Variants that do not have correct prefixes. We still accept these. + {in: "de-Cyrl-1901", lang: "de", script: "Cyrl", variants: "1901"}, + {in: "sl-rozaj-lipaw-1994", lang: "sl", variants: "rozaj-lipaw-1994"}, + {in: "sl-1994-biske-rozaj-1994-biske-rozaj", lang: "sl", variants: "rozaj-biske-1994", changed: true}, + {in: "de-Cyrl-1901", lang: "de", script: "Cyrl", variants: "1901"}, + + // Invalid variant. + {in: "de-1902", lang: "de", variants: "", invalid: true}, + + {in: "EN_CYRL", lang: "en", script: "Cyrl"}, + // private use and extensions + {in: "x-a-b-c-d", ext: "x-a-b-c-d"}, + {in: "x_A.-B-C_D", ext: "x-b-c-d", invalid: true, changed: true}, + {in: "x-aa-bbbb-cccccccc-d", ext: "x-aa-bbbb-cccccccc-d"}, + {in: "en-c_cc-b-bbb-a-aaa", lang: "en", changed: true, extList: []string{"a-aaa", "b-bbb", "c-cc"}}, + {in: "en-x_cc-b-bbb-a-aaa", lang: "en", ext: "x-cc-b-bbb-a-aaa", changed: true}, + {in: "en-c_cc-b-bbb-a-aaa-x-x", lang: "en", changed: true, extList: []string{"a-aaa", "b-bbb", "c-cc", "x-x"}}, + {in: "en-v-c", lang: "en", ext: "", invalid: true}, + {in: "en-v-abcdefghi", lang: "en", ext: "", invalid: true}, + {in: "en-v-abc-x", lang: "en", ext: "v-abc", invalid: true}, + {in: "en-v-abc-x-", lang: "en", ext: "v-abc", invalid: true}, + {in: "en-v-abc-w-x-xx", lang: "en", extList: []string{"v-abc", "x-xx"}, invalid: true, changed: true}, + {in: "en-v-abc-w-y-yx", lang: "en", extList: []string{"v-abc", "y-yx"}, invalid: true, changed: true}, + {in: "en-v-c-abc", lang: "en", ext: "c-abc", invalid: true, changed: true}, + {in: "en-v-w-abc", lang: "en", ext: "w-abc", invalid: true, changed: true}, + {in: "en-v-x-abc", lang: "en", ext: "x-abc", invalid: true, changed: true}, + {in: "en-v-x-a", lang: "en", ext: "x-a", invalid: true, changed: true}, + {in: "en-9-aa-0-aa-z-bb-x-a", lang: "en", extList: []string{"0-aa", "9-aa", "z-bb", "x-a"}, changed: true}, + {in: "en-u-c", lang: "en", ext: "", invalid: true}, + {in: "en-u-co-phonebk", lang: "en", ext: "u-co-phonebk"}, + {in: "en-u-co-phonebk-ca", lang: "en", ext: "u-co-phonebk", invalid: true}, + {in: "en-u-nu-arabic-co-phonebk-ca", lang: "en", ext: "u-co-phonebk-nu-arabic", invalid: true, changed: true}, + {in: "en-u-nu-arabic-co-phonebk-ca-x", lang: "en", ext: "u-co-phonebk-nu-arabic", invalid: true, changed: true}, + {in: "en-u-nu-arabic-co-phonebk-ca-s", lang: "en", ext: "u-co-phonebk-nu-arabic", invalid: true, changed: true}, + {in: "en-u-nu-arabic-co-phonebk-ca-a12345678", lang: "en", ext: "u-co-phonebk-nu-arabic", invalid: true, changed: true}, + {in: "en-u-co-phonebook", lang: "en", ext: "", invalid: true}, + {in: "en-u-co-phonebook-cu-xau", lang: "en", ext: "u-cu-xau", invalid: true, changed: true}, + {in: "en-Cyrl-u-co-phonebk", lang: "en", script: "Cyrl", ext: "u-co-phonebk"}, + {in: "en-US-u-co-phonebk", lang: "en", region: "US", ext: "u-co-phonebk"}, + {in: "en-US-u-co-phonebk-cu-xau", lang: "en", region: "US", ext: "u-co-phonebk-cu-xau"}, + {in: "en-scotland-u-co-phonebk", lang: "en", variants: "scotland", ext: "u-co-phonebk"}, + {in: "en-u-cu-xua-co-phonebk", lang: "en", ext: "u-co-phonebk-cu-xua", changed: true}, + {in: "en-u-def-abc-cu-xua-co-phonebk", lang: "en", ext: "u-abc-def-co-phonebk-cu-xua", changed: true}, + {in: "en-u-def-abc", lang: "en", ext: "u-abc-def", changed: true}, + {in: "en-u-cu-xua-co-phonebk-a-cd", lang: "en", extList: []string{"a-cd", "u-co-phonebk-cu-xua"}, changed: true}, + // Invalid "u" extension. Drop invalid parts. + {in: "en-u-cu-co-phonebk", lang: "en", extList: []string{"u-co-phonebk"}, invalid: true, changed: true}, + {in: "en-u-cu-xau-co", lang: "en", extList: []string{"u-cu-xau"}, invalid: true}, + // We allow duplicate keys as the LDML spec does not explicitly prohibit it. + // TODO: Consider eliminating duplicates and returning an error. + {in: "en-u-cu-xau-co-phonebk-cu-xau", lang: "en", ext: "u-co-phonebk-cu-xau-cu-xau", changed: true}, + {in: "en-t-en-Cyrl-NL-fonipa", lang: "en", ext: "t-en-cyrl-nl-fonipa", changed: true}, + {in: "en-t-en-Cyrl-NL-fonipa-t0-abc-def", lang: "en", ext: "t-en-cyrl-nl-fonipa-t0-abc-def", changed: true}, + {in: "en-t-t0-abcd", lang: "en", ext: "t-t0-abcd"}, + // Not necessary to have changed here. + {in: "en-t-nl-abcd", lang: "en", ext: "t-nl", invalid: true}, + {in: "en-t-nl-latn", lang: "en", ext: "t-nl-latn"}, + {in: "en-t-t0-abcd-x-a", lang: "en", extList: []string{"t-t0-abcd", "x-a"}}, + // invalid + {in: "", lang: "und", invalid: true}, + {in: "-", lang: "und", invalid: true}, + {in: "x", lang: "und", invalid: true}, + {in: "x-", lang: "und", invalid: true}, + {in: "x--", lang: "und", invalid: true}, + {in: "a-a-b-c-d", lang: "und", invalid: true}, + {in: "en-", lang: "en", invalid: true}, + {in: "enne-", lang: "und", invalid: true}, + {in: "en.", lang: "und", invalid: true}, + {in: "en.-latn", lang: "und", invalid: true}, + {in: "en.-en", lang: "en", invalid: true}, + {in: "x-a-tooManyChars-c-d", ext: "x-a-c-d", invalid: true, changed: true}, + {in: "a-tooManyChars-c-d", lang: "und", invalid: true}, + // TODO: check key-value validity + // { in: "en-u-cu-xd", lang: "en", ext: "u-cu-xd", invalid: true }, + {in: "en-t-abcd", lang: "en", invalid: true}, + {in: "en-Latn-US-en", lang: "en", script: "Latn", region: "US", invalid: true}, + // rewrites (more tests in TestGrandfathered) + {in: "zh-min-nan", lang: "nan"}, + {in: "zh-yue", lang: "yue"}, + {in: "zh-xiang", lang: "hsn", rewrite: true}, + {in: "zh-guoyu", lang: "cmn", rewrite: true}, + {in: "iw", lang: "iw"}, + {in: "sgn-BE-FR", lang: "sfb", rewrite: true}, + {in: "i-klingon", lang: "tlh", rewrite: true}, + } + for i, tt := range tests { + tests[i].i = i + if tt.extList != nil { + tests[i].ext = strings.Join(tt.extList, "-") + } + if tt.ext != "" && tt.extList == nil { + tests[i].extList = []string{tt.ext} + } + } + return tests +} + +func TestParseExtensions(t *testing.T) { + for i, tt := range parseTests() { + if tt.ext == "" || tt.rewrite { + continue + } + scan := makeScannerString(tt.in) + if len(scan.b) > 1 && scan.b[1] != '-' { + scan.end = nextExtension(string(scan.b), 0) + scan.next = scan.end + 1 + scan.scan() + } + start := scan.start + scan.toLower(start, len(scan.b)) + parseExtensions(&scan) + ext := string(scan.b[start:]) + if ext != tt.ext { + t.Errorf("%d(%s): ext was %v; want %v", i, tt.in, ext, tt.ext) + } + if changed := !strings.HasPrefix(tt.in[start:], ext); changed != tt.changed { + t.Errorf("%d(%s): changed was %v; want %v", i, tt.in, changed, tt.changed) + } + } +} + +// partChecks runs checks for each part by calling the function returned by f. +func partChecks(t *testing.T, f func(*parseTest) (Tag, bool)) { + for i, tt := range parseTests() { + tag, skip := f(&tt) + if skip { + continue + } + if l, _ := getLangID(b(tt.lang)); l != tag.lang { + t.Errorf("%d: lang was %q; want %q", i, tag.lang, l) + } + if sc, _ := getScriptID(script, b(tt.script)); sc != tag.script { + t.Errorf("%d: script was %q; want %q", i, tag.script, sc) + } + if r, _ := getRegionID(b(tt.region)); r != tag.region { + t.Errorf("%d: region was %q; want %q", i, tag.region, r) + } + if tag.str == "" { + continue + } + p := int(tag.pVariant) + if p < int(tag.pExt) { + p++ + } + if s, g := tag.str[p:tag.pExt], tt.variants; s != g { + t.Errorf("%d: variants was %q; want %q", i, s, g) + } + p = int(tag.pExt) + if p > 0 && p < len(tag.str) { + p++ + } + if s, g := (tag.str)[p:], tt.ext; s != g { + t.Errorf("%d: extensions were %q; want %q", i, s, g) + } + } +} + +func TestParseTag(t *testing.T) { + partChecks(t, func(tt *parseTest) (id Tag, skip bool) { + if strings.HasPrefix(tt.in, "x-") || tt.rewrite { + return Tag{}, true + } + scan := makeScannerString(tt.in) + id, end := parseTag(&scan) + id.str = string(scan.b[:end]) + tt.ext = "" + tt.extList = []string{} + return id, false + }) +} + +func TestParse(t *testing.T) { + partChecks(t, func(tt *parseTest) (id Tag, skip bool) { + id, err := Raw.Parse(tt.in) + ext := "" + if id.str != "" { + if strings.HasPrefix(id.str, "x-") { + ext = id.str + } else if int(id.pExt) < len(id.str) && id.pExt > 0 { + ext = id.str[id.pExt+1:] + } + } + if tag, _ := Raw.Parse(id.String()); tag.String() != id.String() { + t.Errorf("%d:%s: reparse was %q; want %q", tt.i, tt.in, id.String(), tag.String()) + } + if ext != tt.ext { + t.Errorf("%d:%s: ext was %q; want %q", tt.i, tt.in, ext, tt.ext) + } + changed := id.str != "" && !strings.HasPrefix(tt.in, id.str) + if changed != tt.changed { + t.Errorf("%d:%s: changed was %v; want %v", tt.i, tt.in, changed, tt.changed) + } + if (err != nil) != tt.invalid { + t.Errorf("%d:%s: invalid was %v; want %v. Error: %v", tt.i, tt.in, err != nil, tt.invalid, err) + } + return id, false + }) +} + +func TestErrors(t *testing.T) { + mkInvalid := func(s string) error { + return mkErrInvalid([]byte(s)) + } + tests := []struct { + in string + out error + }{ + // invalid subtags. + {"ac", mkInvalid("ac")}, + {"AC", mkInvalid("ac")}, + {"aa-Uuuu", mkInvalid("Uuuu")}, + {"aa-AB", mkInvalid("AB")}, + // ill-formed wins over invalid. + {"ac-u", errSyntax}, + {"ac-u-ca", errSyntax}, + {"ac-u-ca-co-pinyin", errSyntax}, + {"noob", errSyntax}, + } + for _, tt := range tests { + _, err := Parse(tt.in) + if err != tt.out { + t.Errorf("%s: was %q; want %q", tt.in, err, tt.out) + } + } +} + +func TestCompose1(t *testing.T) { + partChecks(t, func(tt *parseTest) (id Tag, skip bool) { + l, _ := ParseBase(tt.lang) + s, _ := ParseScript(tt.script) + r, _ := ParseRegion(tt.region) + v := []Variant{} + for _, x := range strings.Split(tt.variants, "-") { + p, _ := ParseVariant(x) + v = append(v, p) + } + e := []Extension{} + for _, x := range tt.extList { + p, _ := ParseExtension(x) + e = append(e, p) + } + id, _ = Raw.Compose(l, s, r, v, e) + return id, false + }) +} + +func TestCompose2(t *testing.T) { + partChecks(t, func(tt *parseTest) (id Tag, skip bool) { + l, _ := ParseBase(tt.lang) + s, _ := ParseScript(tt.script) + r, _ := ParseRegion(tt.region) + p := []interface{}{l, s, r, s, r, l} + for _, x := range strings.Split(tt.variants, "-") { + v, _ := ParseVariant(x) + p = append(p, v) + } + for _, x := range tt.extList { + e, _ := ParseExtension(x) + p = append(p, e) + } + id, _ = Raw.Compose(p...) + return id, false + }) +} + +func TestCompose3(t *testing.T) { + partChecks(t, func(tt *parseTest) (id Tag, skip bool) { + id, _ = Raw.Parse(tt.in) + id, _ = Raw.Compose(id) + return id, false + }) +} + +func mk(s string) Tag { + return Raw.Make(s) +} + +func TestParseAcceptLanguage(t *testing.T) { + type res struct { + t Tag + q float32 + } + en := []res{{mk("en"), 1.0}} + tests := []struct { + out []res + in string + ok bool + }{ + {en, "en", true}, + {en, " en", true}, + {en, "en ", true}, + {en, " en ", true}, + {en, "en,", true}, + {en, ",en", true}, + {en, ",,,en,,,", true}, + {en, ",en;q=1", true}, + + // We allow an empty input, contrary to spec. + {nil, "", true}, + {[]res{{mk("aa"), 1}}, "aa;", true}, // allow unspecified weight + + // errors + {nil, ";", false}, + {nil, "$", false}, + {nil, "e;", false}, + {nil, "x;", false}, + {nil, "x", false}, + {nil, "ac", false}, // non-existing language + {nil, "aa;q", false}, + {nil, "aa;q=", false}, + {nil, "aa;q=.", false}, + + // odd fallbacks + { + []res{{mk("en"), 0.1}}, + " english ;q=.1", + true, + }, + { + []res{{mk("it"), 1.0}, {mk("de"), 1.0}, {mk("fr"), 1.0}}, + " italian, deutsch, french", + true, + }, + + // lists + { + []res{{mk("en"), 0.1}}, + "en;q=.1", + true, + }, + { + []res{{mk("mul"), 1.0}}, + "*", + true, + }, + { + []res{{mk("en"), 1.0}, {mk("de"), 1.0}}, + "en,de", + true, + }, + { + []res{{mk("en"), 1.0}, {mk("de"), .5}}, + "en,de;q=0.5", + true, + }, + { + []res{{mk("de"), 0.8}, {mk("en"), 0.5}}, + " en ; q = 0.5 , , de;q=0.8", + true, + }, + { + []res{{mk("en"), 1.0}, {mk("de"), 1.0}, {mk("fr"), 1.0}, {mk("tlh"), 1.0}}, + "en,de,fr,i-klingon", + true, + }, + // sorting + { + []res{{mk("tlh"), 0.4}, {mk("de"), 0.2}, {mk("fr"), 0.2}, {mk("en"), 0.1}}, + "en;q=0.1,de;q=0.2,fr;q=0.2,i-klingon;q=0.4", + true, + }, + // dropping + { + []res{{mk("fr"), 0.2}, {mk("en"), 0.1}}, + "en;q=0.1,de;q=0,fr;q=0.2,i-klingon;q=0.0", + true, + }, + } + for i, tt := range tests { + tags, qs, e := ParseAcceptLanguage(tt.in) + if e == nil != tt.ok { + t.Errorf("%d:%s:err: was %v; want %v", i, tt.in, e == nil, tt.ok) + } + for j, tag := range tags { + if out := tt.out[j]; !tag.equalTags(out.t) || qs[j] != out.q { + t.Errorf("%d:%s: was %s, %1f; want %s, %1f", i, tt.in, tag, qs[j], out.t, out.q) + break + } + } + } +} -- cgit v1.2.3