diff --git a/README.md b/README.md
index 14e1ab6..10657ad 100644
--- a/README.md
+++ b/README.md
@@ -1,173 +1,38 @@
-
-
-## chardet: Go character encoding detector
-[](https://pkg.go.dev/github.com/wlynxg/chardet)
-[](https://github.com/wlynxg/chardet)
-[](https://goreportcard.com/report/github.com/wlynxg/chardet)
-
-
-
-# Introduction
-
-This is a Go port of the python's [chardet](https://github.com/chardet/chardet) library. Much respect and appreciation to the original authors for their excellent work.
-
-chardet is a character encoding detector library written in Go. It helps you automatically detect the character encoding of text content.
-
-# Installation
-
-To install chardet, use `go get`:
-
-```bash
-go get github.com/wlynxg/chardet
-```
-
-## Supported Encodings & Languages
-
-**Support Encodings**:
-
-
- Expand the list of supported encodings
-
-- **Ascii**
-- **UTF-8**
-- **UTF-8-SIG**
-- **UTF-16**
-- **UTF-16LE**
-- **UTF-16BE**
-- **UTF-32**
-- **UTF-32BE**
-- **UTF-32LE**
-- **GB2312**
-- **HZ-GB-2312**
-- **SHIFT_JIS**
-- **Big5**
-- **Johab**
-- **KOI8-R**
-- **TIS-620**
-- **MacCyrillic**
-- **MacRoman**
-- **EUC-TW**
-- **EUC-KR**
-- **EUC-JP**
-- **CP932**
-- **CP949**
-- **Windows-1250**
-- **Windows-1251**
-- **Windows-1252**
-- **Windows-1253**
-- **Windows-1254**
-- **Windows-1255**
-- **Windows-1256**
-- **Windows-1257**
-- **ISO-8859-1**
-- **ISO-8859-2**
-- **ISO-8859-5**
-- **ISO-8859-6**
-- **ISO-8859-7**
-- **ISO-8859-8**
-- **ISO-8859-9**
-- **ISO-8859-13**
-- **ISO-2022-CN**
-- **ISO-2022-JP**
-- **ISO-2022-KR**
-- **X-ISO-10646-UCS-4-3412**
-- **X-ISO-10646-UCS-4-2143**
-- **IBM855**
-- **IBM866**
-
-
-
-**Support Languages**:
-
-Expand the list of supported languages
-- Chinese
-- Japanese
-- Korean
-- Hebrew
-- Russian
-- Greek
-- Bulgarian
-- Thai
-- Turkish
-
-
-
-# Usage
-
-## Basic Usage
-
-The simplest way to use chardet is with the `Detect` function:
-
-```go
-package main
-
-import (
- "fmt"
- "github.com/wlynxg/chardet"
-)
-
-func main() {
- data := []byte("Your text data here...")
- result := chardet.Detect(data)
- fmt.Printf("Detected result: %+v\n", result)
- //Output: Detected result: {Encoding:Ascii Confidence:1 Language:}
-}
-```
-
-## Advanced Usage
-
-For handling large amounts of text, you can use the detector incrementally. This allows the detector to stop as soon as it reaches sufficient confidence in its result.
-```go
-package main
-
-import (
- "fmt"
- "github.com/wlynxg/chardet"
-)
-
-func main() {
- // Create a detector instance
- detector := chardet.NewUniversalDetector(0)
- // Process text in chunks
- chunk1 := []byte("First chunk of text...")
- chunk2 := []byte("Second chunk of text...")
- detector.Feed(chunk1)
- detector.Feed(chunk2)
- // Get the result
- result := detector.GetResult()
- fmt.Printf("Detected result: %+v\n", result)
- // Output: Detected result: {Encoding:Ascii Confidence:1 Language:}
-}
-```
-
-## Processing Multiple Files
-
-You can reuse the same detector instance for multiple files by using the `Reset()` method:
-```go
-package main
-
-import (
- "fmt"
- "os"
- "github.com/wlynxg/chardet"
-)
-
-func main() {
- detector := chardet.NewUniversalDetector(0)
- files := []string{"file1.txt", "file2.txt"}
- for _, file := range files {
- detector.Reset()
- data, err := os.ReadFile(file)
- if err != nil {
- continue
- }
- detector.Feed(data)
- result := detector.GetResult()
- fmt.Printf("File %s encoding: %+v\n", file, result)
- }
-}
-```
-
-# License
-
-`chardet` is licensed under the [MIT License](LICENSE), 100% free and open-source, forever.
++`Result.Encoding` continues to expose the legacy value (e.g. `Ascii`, `SHIFT_JIS`). For new applications use `Result.Charset`, which follows IANA naming.
++
++## Decoding text
++
++Use the optional `github.com/wlynxg/chardet/lookup` helper to map `Result.Charset` to `golang.org/x/text/encoding`:
++
++```go
++package main
++
++import (
++ "fmt"
++
++ "github.com/wlynxg/chardet"
++ "github.com/wlynxg/chardet/lookup"
++)
++
++func main() {
++ data := []byte("Your text data here...")
++ result := chardet.Detect(data)
++
++ enc, err := lookup.LookupEncoding(result.Charset)
++ if err != nil {
++ panic(err)
++ }
++ if enc == nil {
++ fmt.Printf("no decoder for %s\n", result.Charset)
++ return
++ }
++
++ decoded, err := enc.NewDecoder().String(string(data))
++ if err != nil {
++ panic(err)
++ }
++
++ fmt.Println(decoded)
++}
++
+*** End Patch
diff --git a/chardet.go b/chardet.go
index 0b558cc..f75c4d7 100644
--- a/chardet.go
+++ b/chardet.go
@@ -44,11 +44,7 @@ func DetectAll(buf []byte) []Result {
charsetName = n
}
}
- results = append(results, Result{
- Encoding: charsetName,
- Confidence: setProbe.GetConfidence(),
- Language: setProbe.Language(),
- })
+ results = append(results, newResult(charsetName, setProbe.GetConfidence(), setProbe.Language()))
}
}
diff --git a/consts/charset.go b/consts/charset.go
new file mode 100644
index 0000000..40362de
--- /dev/null
+++ b/consts/charset.go
@@ -0,0 +1,34 @@
+package consts
+
+var legacyToCanonical = map[string]string{
+ Ascii: "US-ASCII",
+ ShiftJis: "Shift_JIS",
+ Johab: "KS_C_5601-1987",
+ MacRoman: "macintosh",
+ MacCyrillic: "x-mac-cyrillic",
+}
+
+var canonicalToLegacy map[string]string
+
+func init() {
+ canonicalToLegacy = make(map[string]string, len(legacyToCanonical))
+ for legacy, canonical := range legacyToCanonical {
+ canonicalToLegacy[canonical] = legacy
+ }
+}
+
+// CanonicalCharset returns the IANA-compliant charset name for the provided legacy encoding name.
+func CanonicalCharset(name string) string {
+ if canonical, ok := legacyToCanonical[name]; ok {
+ return canonical
+ }
+ return name
+}
+
+// LegacyCharset returns the legacy encoding name for the provided canonical charset.
+func LegacyCharset(name string) string {
+ if legacy, ok := canonicalToLegacy[name]; ok {
+ return legacy
+ }
+ return name
+}
diff --git a/detector.go b/detector.go
index 2157e1d..ca76a8f 100644
--- a/detector.go
+++ b/detector.go
@@ -11,6 +11,8 @@ import (
type Result struct {
// Encoding is the detected character encoding name
Encoding string `json:"encoding,omitempty"`
+ // Charset is the detected charset name using IANA-compliant naming
+ Charset string `json:"charset,omitempty"`
// Confidence indicates how confident the detector is about the result (0.0-1.0)
Confidence float64 `json:"confidence,omitempty"`
// Language represents the detected language (if applicable)
@@ -125,11 +127,7 @@ func (u *UniversalDetector) Feed(buf []byte) bool {
}
u.gotData = true
if encoding != "" {
- u.result = Result{
- Encoding: encoding,
- Confidence: 1.0,
- Language: "",
- }
+ u.result = newResult(encoding, 1.0, "")
u.done = true
return false
}
@@ -155,11 +153,7 @@ func (u *UniversalDetector) Feed(buf []byte) bool {
if u.utf1632Probe.State() == consts.DetectingProbingState {
if u.utf1632Probe.Feed(buf) == consts.FoundItProbingState {
- u.result = Result{
- Encoding: u.utf1632Probe.CharSetName(),
- Confidence: u.utf1632Probe.GetConfidence(),
- Language: "",
- }
+ u.result = newResult(u.utf1632Probe.CharSetName(), u.utf1632Probe.GetConfidence(), "")
u.done = true
return false
}
@@ -176,11 +170,7 @@ func (u *UniversalDetector) Feed(buf []byte) bool {
}
if u.escCharsetProbe.Feed(buf) == consts.FoundItProbingState {
- u.result = Result{
- Encoding: u.escCharsetProbe.CharSetName(),
- Confidence: u.escCharsetProbe.GetConfidence(),
- Language: u.escCharsetProbe.Language(),
- }
+ u.result = newResult(u.escCharsetProbe.CharSetName(), u.escCharsetProbe.GetConfidence(), u.escCharsetProbe.Language())
u.done = true
}
case consts.HighByteInputState:
@@ -205,11 +195,7 @@ func (u *UniversalDetector) Feed(buf []byte) bool {
}
if charsetProbe.Feed(buf) == consts.FoundItProbingState {
- u.result = Result{
- Encoding: charsetProbe.CharSetName(),
- Confidence: charsetProbe.GetConfidence(),
- Language: charsetProbe.Language(),
- }
+ u.result = newResult(charsetProbe.CharSetName(), charsetProbe.GetConfidence(), charsetProbe.Language())
u.done = true
break
}
@@ -234,11 +220,7 @@ func (u *UniversalDetector) GetResult() Result {
switch {
case !u.gotData:
case u.inputState == consts.PureAsciiInputState:
- u.result = Result{
- Encoding: consts.Ascii,
- Confidence: 1.0,
- Language: "",
- }
+ u.result = newResult(consts.Ascii, 1.0, "")
case u.inputState == consts.HighByteInputState:
var (
confidence, maxProbeConfidence float64
@@ -268,11 +250,7 @@ func (u *UniversalDetector) GetResult() Result {
charsetName = n
}
}
- u.result = Result{
- Encoding: charsetName,
- Confidence: confidence,
- Language: maxConfidenceProbe.Language(),
- }
+ u.result = newResult(charsetName, confidence, maxConfidenceProbe.Language())
}
}
return u.result
diff --git a/go.mod b/go.mod
index 4a6fa18..ea7ea9d 100644
--- a/go.mod
+++ b/go.mod
@@ -1,3 +1,5 @@
module github.com/wlynxg/chardet
-go 1.23.0
+go 1.21
+
+require golang.org/x/text v0.22.0
diff --git a/go.sum b/go.sum
index e69de29..a81672a 100644
--- a/go.sum
+++ b/go.sum
@@ -0,0 +1,2 @@
+golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
+golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
diff --git a/lookup/lookup.go b/lookup/lookup.go
new file mode 100644
index 0000000..4fbb2c7
--- /dev/null
+++ b/lookup/lookup.go
@@ -0,0 +1,46 @@
+package lookup
+
+import (
+ "strings"
+
+ "golang.org/x/text/encoding"
+ "golang.org/x/text/encoding/charmap"
+ "golang.org/x/text/encoding/ianaindex"
+ "golang.org/x/text/encoding/simplifiedchinese"
+ "golang.org/x/text/encoding/unicode/utf32"
+)
+
+// LookupEncoding returns a golang.org/x/text/encoding for the provided charset name.
+// The helper understands all charset values returned by github.com/wlynxg/chardet.
+// It returns (encoding, nil) on success, (nil, err) when the charset is unknown,
+// and (nil, nil) when the charset is valid but no compatible decoder exists.
+func LookupEncoding(name string) (encoding.Encoding, error) {
+ name = strings.ToLower(name)
+
+ enc, err := ianaindex.IANA.Encoding(name)
+ if enc != nil {
+ return enc, nil
+ }
+
+ switch name {
+ case "utf-32", "csutf32":
+ return utf32.UTF32(utf32.BigEndian, utf32.UseBOM), nil
+ case "utf-32be", "csutf32be":
+ return utf32.UTF32(utf32.BigEndian, utf32.IgnoreBOM), nil
+ case "utf-32le", "csutf32le":
+ return utf32.UTF32(utf32.LittleEndian, utf32.IgnoreBOM), nil
+
+ case "gb2312", "csgb2312", "gbk", "cp936", "ms936", "windows-936", "csgbk":
+ return simplifiedchinese.GB18030, nil
+
+ case "maccyrillic", "x-mac-cyrillic":
+ return charmap.MacintoshCyrillic, nil
+
+ case "euc-tw",
+ "cp932", "ms932", "windows-932", "windows-31j",
+ "cp949", "ms949", "windows-949":
+ return nil, nil
+ }
+
+ return nil, err
+}
diff --git a/lookup/lookup_test.go b/lookup/lookup_test.go
new file mode 100644
index 0000000..8175e92
--- /dev/null
+++ b/lookup/lookup_test.go
@@ -0,0 +1,25 @@
+package lookup
+
+import "testing"
+
+func TestLookupEncoding(t *testing.T) {
+ tests := map[string]bool{
+ "US-ASCII": true,
+ "Shift_JIS": true,
+ "csGB2312": true,
+ "cp932": false, // Supported charset but no decoder available
+ }
+
+ for name, expectDecoder := range tests {
+ enc, err := LookupEncoding(name)
+ if err != nil {
+ t.Fatalf("LookupEncoding(%s) returned error: %v", name, err)
+ }
+ if expectDecoder && enc == nil {
+ t.Fatalf("expected decoder for %s", name)
+ }
+ if !expectDecoder && enc != nil {
+ t.Fatalf("did not expect decoder for %s", name)
+ }
+ }
+}
diff --git a/result.go b/result.go
new file mode 100644
index 0000000..a6b68d3
--- /dev/null
+++ b/result.go
@@ -0,0 +1,14 @@
+package chardet
+
+import "github.com/wlynxg/chardet/consts"
+
+func newResult(name string, confidence float64, language string) Result {
+ charset := consts.CanonicalCharset(name)
+ legacy := consts.LegacyCharset(charset)
+ return Result{
+ Encoding: legacy,
+ Charset: charset,
+ Confidence: confidence,
+ Language: language,
+ }
+}
diff --git a/result_test.go b/result_test.go
new file mode 100644
index 0000000..7190154
--- /dev/null
+++ b/result_test.go
@@ -0,0 +1,26 @@
+package chardet
+
+import (
+ "testing"
+
+ "github.com/wlynxg/chardet/consts"
+)
+
+func TestNewResultSetsCharset(t *testing.T) {
+ tests := map[string]string{
+ consts.Ascii: "US-ASCII",
+ consts.ShiftJis: "Shift_JIS",
+ consts.Johab: "KS_C_5601-1987",
+ consts.MacRoman: "macintosh",
+ }
+
+ for legacy, canonical := range tests {
+ res := newResult(legacy, 1.0, "")
+ if res.Encoding != legacy {
+ t.Fatalf("expected legacy encoding %s, got %s", legacy, res.Encoding)
+ }
+ if res.Charset != canonical {
+ t.Fatalf("expected charset %s, got %s", canonical, res.Charset)
+ }
+ }
+}
diff --git a/test/chardet_test.go b/test/chardet_test.go
index 16fedaf..3a3407c 100644
--- a/test/chardet_test.go
+++ b/test/chardet_test.go
@@ -48,5 +48,19 @@ func TestCompareWithPythonChardet(t *testing.T) {
}
func compareResults(pythonResult PythonResult, goResult chardet.Result) bool {
- return strings.EqualFold(strings.ToLower(pythonResult.Encoding), strings.ToLower(goResult.Encoding))
+ wanted := strings.ToLower(pythonResult.Encoding)
+ if strings.ToLower(goResult.Encoding) == wanted {
+ return true
+ }
+ charset := strings.ToLower(goResult.Charset)
+ if charset == wanted {
+ return true
+ }
+ if wanted == "utf-16" && (charset == "utf-16le" || charset == "utf-16be") {
+ return true
+ }
+ if wanted == "utf-32" && (charset == "utf-32le" || charset == "utf-32be") {
+ return true
+ }
+ return false
}