This commit is contained in:
Unknwon 2014-12-09 02:18:25 -05:00
parent 6f71632e3e
commit 9a1fe801e5
12 changed files with 84 additions and 48 deletions

View file

@ -47,18 +47,23 @@ func ShortSha(sha1 string) string {
return sha1
}
func ToUtf8WithErr(content []byte) (error, string) {
func DetectEncoding(content []byte) (string, error) {
detector := chardet.NewTextDetector()
result, err := detector.DetectBest(content)
return result.Charset, err
}
func ToUtf8WithErr(content []byte) (error, string) {
charset, err := DetectEncoding(content)
if err != nil {
return err, ""
}
if result.Charset == "utf8" {
if charset == "utf8" {
return nil, string(content)
}
decoder := mahonia.NewDecoder(result.Charset)
decoder := mahonia.NewDecoder(charset)
if decoder != nil {
return nil, decoder.ConvertString(string(content))
}