// Copyright 2013 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. //go:generate go run maketables.go // Package charmap provides simple character encodings such as IBM Code Page 437 // and Windows 1252. package charmap // import "golang.org/x/text/encoding/charmap" import ( "unicode/utf8" "golang.org/x/text/encoding" "golang.org/x/text/encoding/internal" "golang.org/x/text/encoding/internal/identifier" "golang.org/x/text/transform" ) // These encodings vary only in the way clients should interpret them. Their // coded character set is identical and a single implementation can be shared. var ( // ISO8859_6E is the ISO 8859-6E encoding. ISO8859_6E encoding.Encoding = &iso8859_6E // ISO8859_6I is the ISO 8859-6I encoding. ISO8859_6I encoding.Encoding = &iso8859_6I // ISO8859_8E is the ISO 8859-8E encoding. ISO8859_8E encoding.Encoding = &iso8859_8E // ISO8859_8I is the ISO 8859-8I encoding. ISO8859_8I encoding.Encoding = &iso8859_8I iso8859_6E = internal.Encoding{ Encoding: ISO8859_6, Name: "ISO-8859-6E", MIB: identifier.ISO88596E, } iso8859_6I = internal.Encoding{ Encoding: ISO8859_6, Name: "ISO-8859-6I", MIB: identifier.ISO88596I, } iso8859_8E = internal.Encoding{ Encoding: ISO8859_8, Name: "ISO-8859-8E", MIB: identifier.ISO88598E, } iso8859_8I = internal.Encoding{ Encoding: ISO8859_8, Name: "ISO-8859-8I", MIB: identifier.ISO88598I, } ) // All is a list of all defined encodings in this package. var All []encoding.Encoding = listAll // TODO: implement these encodings, in order of importance. // ASCII, ISO8859_1: Rather common. Close to Windows 1252. // ISO8859_9: Close to Windows 1254. // utf8Enc holds a rune's UTF-8 encoding in data[:len]. type utf8Enc struct { len uint8 data [3]byte } // Charmap is an 8-bit character set encoding. type Charmap struct { // name is the encoding's name. name string // mib is the encoding type of this encoder. mib identifier.MIB // asciiSuperset states whether the encoding is a superset of ASCII. asciiSuperset bool // low is the lower bound of the encoded byte for a non-ASCII rune. If // Charmap.asciiSuperset is true then this will be 0x80, otherwise 0x00. low uint8 // replacement is the encoded replacement character. replacement byte // decode is the map from encoded byte to UTF-8. decode [256]utf8Enc // encoding is the map from runes to encoded bytes. Each entry is a // uint32: the high 8 bits are the encoded byte and the low 24 bits are // the rune. The table entries are sorted by ascending rune. encode [256]uint32 } // NewDecoder implements the encoding.Encoding interface. func (m *Charmap) NewDecoder() *encoding.Decoder { return &encoding.Decoder{Transformer: charmapDecoder{charmap: m}} } // NewEncoder implements the encoding.Encoding interface. func (m *Charmap) NewEncoder() *encoding.Encoder { return &encoding.Encoder{Transformer: charmapEncoder{charmap: m}} } // String returns the Charmap's name. func (m *Charmap) String() string { return m.name } // ID implements an internal interface. func (m *Charmap) ID() (mib identifier.MIB, other string) { return m.mib, "" } // charmapDecoder implements transform.Transformer by decoding to UTF-8. type charmapDecoder struct { transform.NopResetter charmap *Charmap } func (m charmapDecoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { for i, c := range src { if m.charmap.asciiSuperset && c < utf8.RuneSelf { if nDst >= len(dst) { err = transform.ErrShortDst break } dst[nDst] = c nDst++ nSrc = i + 1 continue } decode := &m.charmap.decode[c] n := int(decode.len) if nDst+n > len(dst) { err = transform.ErrShortDst break } // It's 15% faster to avoid calling copy for these tiny slices. for j := 0; j < n; j++ { dst[nDst] = decode.data[j] nDst++ } nSrc = i + 1 } return nDst, nSrc, err } // DecodeByte returns the Charmap's rune decoding of the byte b. func (m *Charmap) DecodeByte(b byte) rune { switch x := &m.decode[b]; x.len { case 1: return rune(x.data[0]) case 2: return rune(x.data[0]&0x1f)<<6 | rune(x.data[1]&0x3f) default: return rune(x.data[0]&0x0f)<<12 | rune(x.data[1]&0x3f)<<6 | rune(x.data[2]&0x3f) } } // charmapEncoder implements transform.Transformer by encoding from UTF-8. type charmapEncoder struct { transform.NopResetter charmap *Charmap } func (m charmapEncoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { r, size := rune(0), 0 loop: for nSrc < len(src) { if nDst >= len(dst) { err = transform.ErrShortDst break } r = rune(src[nSrc]) // Decode a 1-byte rune. if r < utf8.RuneSelf { if m.charmap.asciiSuperset { nSrc++ dst[nDst] = uint8(r) nDst++ continue } size = 1 } else { // Decode a multi-byte rune. r, size = utf8.DecodeRune(src[nSrc:]) if size == 1 { // All valid runes of size 1 (those below utf8.RuneSelf) were // handled above. We have invalid UTF-8 or we haven't seen the // full character yet. if !atEOF && !utf8.FullRune(src[nSrc:]) { err = transform.ErrShortSrc } else { err = internal.RepertoireError(m.charmap.replacement) } break } } // Binary search in [low, high) for that rune in the m.charmap.encode table. for low, high := int(m.charmap.low), 0x100; ; { if low >= high { err = internal.RepertoireError(m.charmap.replacement) break loop } mid := (low + high) / 2 got := m.charmap.encode[mid] gotRune := rune(got & (1<<24 - 1)) if gotRune < r { low = mid + 1 } else if gotRune > r { high = mid } else { dst[nDst] = byte(got >> 24) nDst++ break } } nSrc += size } return nDst, nSrc, err } // EncodeRune returns the Charmap's byte encoding of the rune r. ok is whether // r is in the Charmap's repertoire. If not, b is set to the Charmap's // replacement byte. This is often the ASCII substitute character '\x1a'. func (m *Charmap) EncodeRune(r rune) (b byte, ok bool) { if r < utf8.RuneSelf && m.asciiSuperset { return byte(r), true } for low, high := int(m.low), 0x100; ; { if low >= high { return m.replacement, false } mid := (low + high) / 2 got := m.encode[mid] gotRune := rune(got & (1<<24 - 1)) if gotRune < r { low = mid + 1 } else if gotRune > r { high = mid } else { return byte(got >> 24), true } } }