feat: utf-16 initial support

greut · Jun 12, 2023 · 6c5ad63 · 6c5ad63
1 parent ab8e593
commit 6c5ad63
Show file tree

Hide file tree

Showing 5 changed files with 22 additions and 4 deletions.
diff --git a/README.md b/README.md
@@ -62,7 +62,7 @@ $ eclint -exclude "testdata/**/*"
 
 ## Missing features
 
-- `max_line_length` counting UTF-16 and UTF-32 characters
+- `max_line_length` counting UTF-32 characters
 - more tests
 - ability to fix: `insert_final_newline`, etc
 - etc.

diff --git a/go.mod b/go.mod
@@ -11,6 +11,7 @@ require (
 	github.com/logrusorgru/aurora v2.0.3+incompatible
 	github.com/mattn/go-colorable v0.1.13
 	golang.org/x/term v0.8.0
+	golang.org/x/text v0.9.0
 	k8s.io/klog/v2 v2.100.1
 )
 

diff --git a/go.sum b/go.sum
@@ -31,6 +31,8 @@ golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
 gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=

diff --git a/lint.go b/lint.go
@@ -11,6 +11,9 @@ import (
 
 	"github.com/editorconfig/editorconfig-core-go/v2"
 	"github.com/go-logr/logr"
+	"golang.org/x/text/encoding"
+	"golang.org/x/text/encoding/unicode"
+	"golang.org/x/text/transform"
 )
 
 // DefaultTabWidth sets the width of a tab used when counting the line length.
@@ -92,9 +95,21 @@ func LintWithDefinition(ctx context.Context, d *editorconfig.Definition, filenam
 		return nil
 	}
 
-	log.V(2).Info("charset probed", "charset", charset)
+	log.V(2).Info("charset probed", "filename", filename, "charset", charset)
 
-	errs := validate(ctx, r, fileSize, charset, def)
+	var decoder *encoding.Decoder
+
+	switch charset {
+	case "utf-16be":
+		decoder = unicode.UTF16(unicode.BigEndian, unicode.ExpectBOM).NewDecoder()
+	case "utf-16le":
+		decoder = unicode.UTF16(unicode.LittleEndian, unicode.ExpectBOM).NewDecoder()
+	default:
+		decoder = unicode.UTF8.NewDecoder()
+	}
+
+	t := transform.NewReader(r, unicode.BOMOverride(decoder))
+	errs := validate(ctx, t, fileSize, charset, def)
 
 	// Enrich the errors with the filename
 	for i, err := range errs {

diff --git a/validators.go b/validators.go
@@ -53,7 +53,7 @@ func endOfLine(eol string, data []byte) error {
 			}
 		}
 	case "crlf":
-		if !bytes.HasSuffix(data, []byte{cr, lf}) {
+		if !bytes.HasSuffix(data, []byte{cr, lf}) && !bytes.HasSuffix(data, []byte{0x00, cr, 0x00, lf}) {
 			return ValidationError{
 				Message:  "line does not end with crlf (`\\r\\n`)",
 				Position: len(data),