diff options
Diffstat (limited to 'Godeps/_workspace/src/golang.org/x/net/html/charset/charset.go')
-rw-r--r-- | Godeps/_workspace/src/golang.org/x/net/html/charset/charset.go | 257 |
1 files changed, 0 insertions, 257 deletions
diff --git a/Godeps/_workspace/src/golang.org/x/net/html/charset/charset.go b/Godeps/_workspace/src/golang.org/x/net/html/charset/charset.go deleted file mode 100644 index 449607253..000000000 --- a/Godeps/_workspace/src/golang.org/x/net/html/charset/charset.go +++ /dev/null @@ -1,257 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package charset provides common text encodings for HTML documents. -// -// The mapping from encoding labels to encodings is defined at -// https://encoding.spec.whatwg.org/. -package charset - -import ( - "bytes" - "fmt" - "io" - "mime" - "strings" - "unicode/utf8" - - "golang.org/x/net/html" - "golang.org/x/text/encoding" - "golang.org/x/text/encoding/charmap" - "golang.org/x/text/encoding/htmlindex" - "golang.org/x/text/transform" -) - -// Lookup returns the encoding with the specified label, and its canonical -// name. It returns nil and the empty string if label is not one of the -// standard encodings for HTML. Matching is case-insensitive and ignores -// leading and trailing whitespace. Encoders will use HTML escape sequences for -// runes that are not supported by the character set. -func Lookup(label string) (e encoding.Encoding, name string) { - e, err := htmlindex.Get(label) - if err != nil { - return nil, "" - } - name, _ = htmlindex.Name(e) - return &htmlEncoding{e}, name -} - -type htmlEncoding struct{ encoding.Encoding } - -func (h *htmlEncoding) NewEncoder() *encoding.Encoder { - // HTML requires a non-terminating legacy encoder. We use HTML escapes to - // substitute unsupported code points. - return encoding.HTMLEscapeUnsupported(h.Encoding.NewEncoder()) -} - -// DetermineEncoding determines the encoding of an HTML document by examining -// up to the first 1024 bytes of content and the declared Content-Type. -// -// See http://www.whatwg.org/specs/web-apps/current-work/multipage/parsing.html#determining-the-character-encoding -func DetermineEncoding(content []byte, contentType string) (e encoding.Encoding, name string, certain bool) { - if len(content) > 1024 { - content = content[:1024] - } - - for _, b := range boms { - if bytes.HasPrefix(content, b.bom) { - e, name = Lookup(b.enc) - return e, name, true - } - } - - if _, params, err := mime.ParseMediaType(contentType); err == nil { - if cs, ok := params["charset"]; ok { - if e, name = Lookup(cs); e != nil { - return e, name, true - } - } - } - - if len(content) > 0 { - e, name = prescan(content) - if e != nil { - return e, name, false - } - } - - // Try to detect UTF-8. - // First eliminate any partial rune at the end. - for i := len(content) - 1; i >= 0 && i > len(content)-4; i-- { - b := content[i] - if b < 0x80 { - break - } - if utf8.RuneStart(b) { - content = content[:i] - break - } - } - hasHighBit := false - for _, c := range content { - if c >= 0x80 { - hasHighBit = true - break - } - } - if hasHighBit && utf8.Valid(content) { - return encoding.Nop, "utf-8", false - } - - // TODO: change default depending on user's locale? - return charmap.Windows1252, "windows-1252", false -} - -// NewReader returns an io.Reader that converts the content of r to UTF-8. -// It calls DetermineEncoding to find out what r's encoding is. -func NewReader(r io.Reader, contentType string) (io.Reader, error) { - preview := make([]byte, 1024) - n, err := io.ReadFull(r, preview) - switch { - case err == io.ErrUnexpectedEOF: - preview = preview[:n] - r = bytes.NewReader(preview) - case err != nil: - return nil, err - default: - r = io.MultiReader(bytes.NewReader(preview), r) - } - - if e, _, _ := DetermineEncoding(preview, contentType); e != encoding.Nop { - r = transform.NewReader(r, e.NewDecoder()) - } - return r, nil -} - -// NewReaderLabel returns a reader that converts from the specified charset to -// UTF-8. It uses Lookup to find the encoding that corresponds to label, and -// returns an error if Lookup returns nil. It is suitable for use as -// encoding/xml.Decoder's CharsetReader function. -func NewReaderLabel(label string, input io.Reader) (io.Reader, error) { - e, _ := Lookup(label) - if e == nil { - return nil, fmt.Errorf("unsupported charset: %q", label) - } - return transform.NewReader(input, e.NewDecoder()), nil -} - -func prescan(content []byte) (e encoding.Encoding, name string) { - z := html.NewTokenizer(bytes.NewReader(content)) - for { - switch z.Next() { - case html.ErrorToken: - return nil, "" - - case html.StartTagToken, html.SelfClosingTagToken: - tagName, hasAttr := z.TagName() - if !bytes.Equal(tagName, []byte("meta")) { - continue - } - attrList := make(map[string]bool) - gotPragma := false - - const ( - dontKnow = iota - doNeedPragma - doNotNeedPragma - ) - needPragma := dontKnow - - name = "" - e = nil - for hasAttr { - var key, val []byte - key, val, hasAttr = z.TagAttr() - ks := string(key) - if attrList[ks] { - continue - } - attrList[ks] = true - for i, c := range val { - if 'A' <= c && c <= 'Z' { - val[i] = c + 0x20 - } - } - - switch ks { - case "http-equiv": - if bytes.Equal(val, []byte("content-type")) { - gotPragma = true - } - - case "content": - if e == nil { - name = fromMetaElement(string(val)) - if name != "" { - e, name = Lookup(name) - if e != nil { - needPragma = doNeedPragma - } - } - } - - case "charset": - e, name = Lookup(string(val)) - needPragma = doNotNeedPragma - } - } - - if needPragma == dontKnow || needPragma == doNeedPragma && !gotPragma { - continue - } - - if strings.HasPrefix(name, "utf-16") { - name = "utf-8" - e = encoding.Nop - } - - if e != nil { - return e, name - } - } - } -} - -func fromMetaElement(s string) string { - for s != "" { - csLoc := strings.Index(s, "charset") - if csLoc == -1 { - return "" - } - s = s[csLoc+len("charset"):] - s = strings.TrimLeft(s, " \t\n\f\r") - if !strings.HasPrefix(s, "=") { - continue - } - s = s[1:] - s = strings.TrimLeft(s, " \t\n\f\r") - if s == "" { - return "" - } - if q := s[0]; q == '"' || q == '\'' { - s = s[1:] - closeQuote := strings.IndexRune(s, rune(q)) - if closeQuote == -1 { - return "" - } - return s[:closeQuote] - } - - end := strings.IndexAny(s, "; \t\n\f\r") - if end == -1 { - end = len(s) - } - return s[:end] - } - return "" -} - -var boms = []struct { - bom []byte - enc string -}{ - {[]byte{0xfe, 0xff}, "utf-16be"}, - {[]byte{0xff, 0xfe}, "utf-16le"}, - {[]byte{0xef, 0xbb, 0xbf}, "utf-8"}, -} |