summaryrefslogtreecommitdiff
path: root/vendor/golang.org/x/net/html/charset
diff options
context:
space:
mode:
Diffstat (limited to 'vendor/golang.org/x/net/html/charset')
-rw-r--r--vendor/golang.org/x/net/html/charset/charset.go257
-rw-r--r--vendor/golang.org/x/net/html/charset/charset_test.go237
2 files changed, 494 insertions, 0 deletions
diff --git a/vendor/golang.org/x/net/html/charset/charset.go b/vendor/golang.org/x/net/html/charset/charset.go
new file mode 100644
index 0000000..13bed15
--- /dev/null
+++ b/vendor/golang.org/x/net/html/charset/charset.go
@@ -0,0 +1,257 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package charset provides common text encodings for HTML documents.
+//
+// The mapping from encoding labels to encodings is defined at
+// https://encoding.spec.whatwg.org/.
+package charset // import "golang.org/x/net/html/charset"
+
+import (
+ "bytes"
+ "fmt"
+ "io"
+ "mime"
+ "strings"
+ "unicode/utf8"
+
+ "golang.org/x/net/html"
+ "golang.org/x/text/encoding"
+ "golang.org/x/text/encoding/charmap"
+ "golang.org/x/text/encoding/htmlindex"
+ "golang.org/x/text/transform"
+)
+
+// Lookup returns the encoding with the specified label, and its canonical
+// name. It returns nil and the empty string if label is not one of the
+// standard encodings for HTML. Matching is case-insensitive and ignores
+// leading and trailing whitespace. Encoders will use HTML escape sequences for
+// runes that are not supported by the character set.
+func Lookup(label string) (e encoding.Encoding, name string) {
+ e, err := htmlindex.Get(label)
+ if err != nil {
+ return nil, ""
+ }
+ name, _ = htmlindex.Name(e)
+ return &htmlEncoding{e}, name
+}
+
+type htmlEncoding struct{ encoding.Encoding }
+
+func (h *htmlEncoding) NewEncoder() *encoding.Encoder {
+ // HTML requires a non-terminating legacy encoder. We use HTML escapes to
+ // substitute unsupported code points.
+ return encoding.HTMLEscapeUnsupported(h.Encoding.NewEncoder())
+}
+
+// DetermineEncoding determines the encoding of an HTML document by examining
+// up to the first 1024 bytes of content and the declared Content-Type.
+//
+// See http://www.whatwg.org/specs/web-apps/current-work/multipage/parsing.html#determining-the-character-encoding
+func DetermineEncoding(content []byte, contentType string) (e encoding.Encoding, name string, certain bool) {
+ if len(content) > 1024 {
+ content = content[:1024]
+ }
+
+ for _, b := range boms {
+ if bytes.HasPrefix(content, b.bom) {
+ e, name = Lookup(b.enc)
+ return e, name, true
+ }
+ }
+
+ if _, params, err := mime.ParseMediaType(contentType); err == nil {
+ if cs, ok := params["charset"]; ok {
+ if e, name = Lookup(cs); e != nil {
+ return e, name, true
+ }
+ }
+ }
+
+ if len(content) > 0 {
+ e, name = prescan(content)
+ if e != nil {
+ return e, name, false
+ }
+ }
+
+ // Try to detect UTF-8.
+ // First eliminate any partial rune at the end.
+ for i := len(content) - 1; i >= 0 && i > len(content)-4; i-- {
+ b := content[i]
+ if b < 0x80 {
+ break
+ }
+ if utf8.RuneStart(b) {
+ content = content[:i]
+ break
+ }
+ }
+ hasHighBit := false
+ for _, c := range content {
+ if c >= 0x80 {
+ hasHighBit = true
+ break
+ }
+ }
+ if hasHighBit && utf8.Valid(content) {
+ return encoding.Nop, "utf-8", false
+ }
+
+ // TODO: change default depending on user's locale?
+ return charmap.Windows1252, "windows-1252", false
+}
+
+// NewReader returns an io.Reader that converts the content of r to UTF-8.
+// It calls DetermineEncoding to find out what r's encoding is.
+func NewReader(r io.Reader, contentType string) (io.Reader, error) {
+ preview := make([]byte, 1024)
+ n, err := io.ReadFull(r, preview)
+ switch {
+ case err == io.ErrUnexpectedEOF:
+ preview = preview[:n]
+ r = bytes.NewReader(preview)
+ case err != nil:
+ return nil, err
+ default:
+ r = io.MultiReader(bytes.NewReader(preview), r)
+ }
+
+ if e, _, _ := DetermineEncoding(preview, contentType); e != encoding.Nop {
+ r = transform.NewReader(r, e.NewDecoder())
+ }
+ return r, nil
+}
+
+// NewReaderLabel returns a reader that converts from the specified charset to
+// UTF-8. It uses Lookup to find the encoding that corresponds to label, and
+// returns an error if Lookup returns nil. It is suitable for use as
+// encoding/xml.Decoder's CharsetReader function.
+func NewReaderLabel(label string, input io.Reader) (io.Reader, error) {
+ e, _ := Lookup(label)
+ if e == nil {
+ return nil, fmt.Errorf("unsupported charset: %q", label)
+ }
+ return transform.NewReader(input, e.NewDecoder()), nil
+}
+
+func prescan(content []byte) (e encoding.Encoding, name string) {
+ z := html.NewTokenizer(bytes.NewReader(content))
+ for {
+ switch z.Next() {
+ case html.ErrorToken:
+ return nil, ""
+
+ case html.StartTagToken, html.SelfClosingTagToken:
+ tagName, hasAttr := z.TagName()
+ if !bytes.Equal(tagName, []byte("meta")) {
+ continue
+ }
+ attrList := make(map[string]bool)
+ gotPragma := false
+
+ const (
+ dontKnow = iota
+ doNeedPragma
+ doNotNeedPragma
+ )
+ needPragma := dontKnow
+
+ name = ""
+ e = nil
+ for hasAttr {
+ var key, val []byte
+ key, val, hasAttr = z.TagAttr()
+ ks := string(key)
+ if attrList[ks] {
+ continue
+ }
+ attrList[ks] = true
+ for i, c := range val {
+ if 'A' <= c && c <= 'Z' {
+ val[i] = c + 0x20
+ }
+ }
+
+ switch ks {
+ case "http-equiv":
+ if bytes.Equal(val, []byte("content-type")) {
+ gotPragma = true
+ }
+
+ case "content":
+ if e == nil {
+ name = fromMetaElement(string(val))
+ if name != "" {
+ e, name = Lookup(name)
+ if e != nil {
+ needPragma = doNeedPragma
+ }
+ }
+ }
+
+ case "charset":
+ e, name = Lookup(string(val))
+ needPragma = doNotNeedPragma
+ }
+ }
+
+ if needPragma == dontKnow || needPragma == doNeedPragma && !gotPragma {
+ continue
+ }
+
+ if strings.HasPrefix(name, "utf-16") {
+ name = "utf-8"
+ e = encoding.Nop
+ }
+
+ if e != nil {
+ return e, name
+ }
+ }
+ }
+}
+
+func fromMetaElement(s string) string {
+ for s != "" {
+ csLoc := strings.Index(s, "charset")
+ if csLoc == -1 {
+ return ""
+ }
+ s = s[csLoc+len("charset"):]
+ s = strings.TrimLeft(s, " \t\n\f\r")
+ if !strings.HasPrefix(s, "=") {
+ continue
+ }
+ s = s[1:]
+ s = strings.TrimLeft(s, " \t\n\f\r")
+ if s == "" {
+ return ""
+ }
+ if q := s[0]; q == '"' || q == '\'' {
+ s = s[1:]
+ closeQuote := strings.IndexRune(s, rune(q))
+ if closeQuote == -1 {
+ return ""
+ }
+ return s[:closeQuote]
+ }
+
+ end := strings.IndexAny(s, "; \t\n\f\r")
+ if end == -1 {
+ end = len(s)
+ }
+ return s[:end]
+ }
+ return ""
+}
+
+var boms = []struct {
+ bom []byte
+ enc string
+}{
+ {[]byte{0xfe, 0xff}, "utf-16be"},
+ {[]byte{0xff, 0xfe}, "utf-16le"},
+ {[]byte{0xef, 0xbb, 0xbf}, "utf-8"},
+}
diff --git a/vendor/golang.org/x/net/html/charset/charset_test.go b/vendor/golang.org/x/net/html/charset/charset_test.go
new file mode 100644
index 0000000..e4e7d86
--- /dev/null
+++ b/vendor/golang.org/x/net/html/charset/charset_test.go
@@ -0,0 +1,237 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package charset
+
+import (
+ "bytes"
+ "encoding/xml"
+ "io/ioutil"
+ "runtime"
+ "strings"
+ "testing"
+
+ "golang.org/x/text/transform"
+)
+
+func transformString(t transform.Transformer, s string) (string, error) {
+ r := transform.NewReader(strings.NewReader(s), t)
+ b, err := ioutil.ReadAll(r)
+ return string(b), err
+}
+
+type testCase struct {
+ utf8, other, otherEncoding string
+}
+
+// testCases for encoding and decoding.
+var testCases = []testCase{
+ {"Résumé", "Résumé", "utf8"},
+ {"Résumé", "R\xe9sum\xe9", "latin1"},
+ {"これは漢字です。", "S0\x8c0o0\"oW[g0Y0\x020", "UTF-16LE"},
+ {"これは漢字です。", "0S0\x8c0oo\"[W0g0Y0\x02", "UTF-16BE"},
+ {"Hello, world", "Hello, world", "ASCII"},
+ {"Gdańsk", "Gda\xf1sk", "ISO-8859-2"},
+ {"Ââ Čč Đđ Ŋŋ Õõ Šš Žž Åå Ää", "\xc2\xe2 \xc8\xe8 \xa9\xb9 \xaf\xbf \xd5\xf5 \xaa\xba \xac\xbc \xc5\xe5 \xc4\xe4", "ISO-8859-10"},
+ {"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "ISO-8859-11"},
+ {"latviešu", "latvie\xf0u", "ISO-8859-13"},
+ {"Seònaid", "Se\xf2naid", "ISO-8859-14"},
+ {"€1 is cheap", "\xa41 is cheap", "ISO-8859-15"},
+ {"românește", "rom\xe2ne\xbate", "ISO-8859-16"},
+ {"nutraĵo", "nutra\xbco", "ISO-8859-3"},
+ {"Kalâdlit", "Kal\xe2dlit", "ISO-8859-4"},
+ {"русский", "\xe0\xe3\xe1\xe1\xda\xd8\xd9", "ISO-8859-5"},
+ {"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "ISO-8859-7"},
+ {"Kağan", "Ka\xf0an", "ISO-8859-9"},
+ {"Résumé", "R\x8esum\x8e", "macintosh"},
+ {"Gdańsk", "Gda\xf1sk", "windows-1250"},
+ {"русский", "\xf0\xf3\xf1\xf1\xea\xe8\xe9", "windows-1251"},
+ {"Résumé", "R\xe9sum\xe9", "windows-1252"},
+ {"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "windows-1253"},
+ {"Kağan", "Ka\xf0an", "windows-1254"},
+ {"עִבְרִית", "\xf2\xc4\xe1\xc0\xf8\xc4\xe9\xfa", "windows-1255"},
+ {"العربية", "\xc7\xe1\xda\xd1\xc8\xed\xc9", "windows-1256"},
+ {"latviešu", "latvie\xf0u", "windows-1257"},
+ {"Việt", "Vi\xea\xf2t", "windows-1258"},
+ {"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "windows-874"},
+ {"русский", "\xd2\xd5\xd3\xd3\xcb\xc9\xca", "KOI8-R"},
+ {"українська", "\xd5\xcb\xd2\xc1\xa7\xce\xd3\xd8\xcb\xc1", "KOI8-U"},
+ {"Hello 常用國字標準字體表", "Hello \xb1`\xa5\u03b0\xea\xa6r\xbc\u0437\u01e6r\xc5\xe9\xaa\xed", "big5"},
+ {"Hello 常用國字標準字體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gbk"},
+ {"Hello 常用國字標準字體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gb18030"},
+ {"עִבְרִית", "\x81\x30\xfb\x30\x81\x30\xf6\x34\x81\x30\xf9\x33\x81\x30\xf6\x30\x81\x30\xfb\x36\x81\x30\xf6\x34\x81\x30\xfa\x31\x81\x30\xfb\x38", "gb18030"},
+ {"㧯", "\x82\x31\x89\x38", "gb18030"},
+ {"これは漢字です。", "\x82\xb1\x82\xea\x82\xcd\x8a\xbf\x8e\x9a\x82\xc5\x82\xb7\x81B", "SJIS"},
+ {"Hello, 世界!", "Hello, \x90\xa2\x8aE!", "SJIS"},
+ {"イウエオカ", "\xb2\xb3\xb4\xb5\xb6", "SJIS"},
+ {"これは漢字です。", "\xa4\xb3\xa4\xec\xa4\u03f4\xc1\xbb\xfa\xa4\u01e4\xb9\xa1\xa3", "EUC-JP"},
+ {"Hello, 世界!", "Hello, \x1b$B@$3&\x1b(B!", "ISO-2022-JP"},
+ {"다음과 같은 조건을 따라야 합니다: 저작자표시", "\xb4\xd9\xc0\xbd\xb0\xfa \xb0\xb0\xc0\xba \xc1\xb6\xb0\xc7\xc0\xbb \xb5\xfb\xb6\xf3\xbe\xdf \xc7մϴ\xd9: \xc0\xfa\xc0\xdb\xc0\xdaǥ\xbd\xc3", "EUC-KR"},
+}
+
+func TestDecode(t *testing.T) {
+ testCases := append(testCases, []testCase{
+ // Replace multi-byte maximum subpart of ill-formed subsequence with
+ // single replacement character (WhatWG requirement).
+ {"Rés\ufffdumé", "Rés\xe1\x80umé", "utf8"},
+ }...)
+ for _, tc := range testCases {
+ e, _ := Lookup(tc.otherEncoding)
+ if e == nil {
+ t.Errorf("%s: not found", tc.otherEncoding)
+ continue
+ }
+ s, err := transformString(e.NewDecoder(), tc.other)
+ if err != nil {
+ t.Errorf("%s: decode %q: %v", tc.otherEncoding, tc.other, err)
+ continue
+ }
+ if s != tc.utf8 {
+ t.Errorf("%s: got %q, want %q", tc.otherEncoding, s, tc.utf8)
+ }
+ }
+}
+
+func TestEncode(t *testing.T) {
+ testCases := append(testCases, []testCase{
+ // Use Go-style replacement.
+ {"Rés\xe1\x80umé", "Rés\ufffd\ufffdumé", "utf8"},
+ // U+0144 LATIN SMALL LETTER N WITH ACUTE not supported by encoding.
+ {"Gdańsk", "Gda&#324;sk", "ISO-8859-11"},
+ {"\ufffd", "&#65533;", "ISO-8859-11"},
+ {"a\xe1\x80b", "a&#65533;&#65533;b", "ISO-8859-11"},
+ }...)
+ for _, tc := range testCases {
+ e, _ := Lookup(tc.otherEncoding)
+ if e == nil {
+ t.Errorf("%s: not found", tc.otherEncoding)
+ continue
+ }
+ s, err := transformString(e.NewEncoder(), tc.utf8)
+ if err != nil {
+ t.Errorf("%s: encode %q: %s", tc.otherEncoding, tc.utf8, err)
+ continue
+ }
+ if s != tc.other {
+ t.Errorf("%s: got %q, want %q", tc.otherEncoding, s, tc.other)
+ }
+ }
+}
+
+var sniffTestCases = []struct {
+ filename, declared, want string
+}{
+ {"HTTP-charset.html", "text/html; charset=iso-8859-15", "iso-8859-15"},
+ {"UTF-16LE-BOM.html", "", "utf-16le"},
+ {"UTF-16BE-BOM.html", "", "utf-16be"},
+ {"meta-content-attribute.html", "text/html", "iso-8859-15"},
+ {"meta-charset-attribute.html", "text/html", "iso-8859-15"},
+ {"No-encoding-declaration.html", "text/html", "utf-8"},
+ {"HTTP-vs-UTF-8-BOM.html", "text/html; charset=iso-8859-15", "utf-8"},
+ {"HTTP-vs-meta-content.html", "text/html; charset=iso-8859-15", "iso-8859-15"},
+ {"HTTP-vs-meta-charset.html", "text/html; charset=iso-8859-15", "iso-8859-15"},
+ {"UTF-8-BOM-vs-meta-content.html", "text/html", "utf-8"},
+ {"UTF-8-BOM-vs-meta-charset.html", "text/html", "utf-8"},
+}
+
+func TestSniff(t *testing.T) {
+ switch runtime.GOOS {
+ case "nacl": // platforms that don't permit direct file system access
+ t.Skipf("not supported on %q", runtime.GOOS)
+ }
+
+ for _, tc := range sniffTestCases {
+ content, err := ioutil.ReadFile("testdata/" + tc.filename)
+ if err != nil {
+ t.Errorf("%s: error reading file: %v", tc.filename, err)
+ continue
+ }
+
+ _, name, _ := DetermineEncoding(content, tc.declared)
+ if name != tc.want {
+ t.Errorf("%s: got %q, want %q", tc.filename, name, tc.want)
+ continue
+ }
+ }
+}
+
+func TestReader(t *testing.T) {
+ switch runtime.GOOS {
+ case "nacl": // platforms that don't permit direct file system access
+ t.Skipf("not supported on %q", runtime.GOOS)
+ }
+
+ for _, tc := range sniffTestCases {
+ content, err := ioutil.ReadFile("testdata/" + tc.filename)
+ if err != nil {
+ t.Errorf("%s: error reading file: %v", tc.filename, err)
+ continue
+ }
+
+ r, err := NewReader(bytes.NewReader(content), tc.declared)
+ if err != nil {
+ t.Errorf("%s: error creating reader: %v", tc.filename, err)
+ continue
+ }
+
+ got, err := ioutil.ReadAll(r)
+ if err != nil {
+ t.Errorf("%s: error reading from charset.NewReader: %v", tc.filename, err)
+ continue
+ }
+
+ e, _ := Lookup(tc.want)
+ want, err := ioutil.ReadAll(transform.NewReader(bytes.NewReader(content), e.NewDecoder()))
+ if err != nil {
+ t.Errorf("%s: error decoding with hard-coded charset name: %v", tc.filename, err)
+ continue
+ }
+
+ if !bytes.Equal(got, want) {
+ t.Errorf("%s: got %q, want %q", tc.filename, got, want)
+ continue
+ }
+ }
+}
+
+var metaTestCases = []struct {
+ meta, want string
+}{
+ {"", ""},
+ {"text/html", ""},
+ {"text/html; charset utf-8", ""},
+ {"text/html; charset=latin-2", "latin-2"},
+ {"text/html; charset; charset = utf-8", "utf-8"},
+ {`charset="big5"`, "big5"},
+ {"charset='shift_jis'", "shift_jis"},
+}
+
+func TestFromMeta(t *testing.T) {
+ for _, tc := range metaTestCases {
+ got := fromMetaElement(tc.meta)
+ if got != tc.want {
+ t.Errorf("%q: got %q, want %q", tc.meta, got, tc.want)
+ }
+ }
+}
+
+func TestXML(t *testing.T) {
+ const s = "<?xml version=\"1.0\" encoding=\"windows-1252\"?><a><Word>r\xe9sum\xe9</Word></a>"
+
+ d := xml.NewDecoder(strings.NewReader(s))
+ d.CharsetReader = NewReaderLabel
+
+ var a struct {
+ Word string
+ }
+ err := d.Decode(&a)
+ if err != nil {
+ t.Fatalf("Decode: %v", err)
+ }
+
+ want := "résumé"
+ if a.Word != want {
+ t.Errorf("got %q, want %q", a.Word, want)
+ }
+}