feat: fix unicode var bug by handling multi-byte characters before single byte characters in input (#17)

Author: pboyd04

evaluation_test.go

@@ -1453,6 +1453,12 @@ func TestParameterizedEvaluation(test *testing.T) {
 			Parameters: []EvaluationParameter{fooParameter},
 			Expected:   false,
 		},
+		{
+			Name:       "Test with cyrilic parameter",
+			Input:      "переменная * 5",
+			Parameters: []EvaluationParameter{{Name: "переменная", Value: 2}},
+			Expected:   float64(10),
+		},
 	}
 
 	runEvaluationTests(evaluationTests, test)

lexerStream.go

@@ -1,10 +1,14 @@
 package govaluate
 
-import "sync"
+import (
+	"sync"
+	"unicode/utf8"
+)
 
 type lexerStream struct {
 	sourceString string
 	source       []rune
+	strPosition  int
 	position     int
 	length       int
 }
@@ -25,18 +29,34 @@ func newLexerStream(source string) *lexerStream {
 	}
 	ret.sourceString = source
 	ret.position = 0
+	ret.strPosition = 0
 	ret.length = len(ret.source)
 	return ret
 }
 
 func (this *lexerStream) readCharacter() rune {
 	character := this.source[this.position]
 	this.position += 1
+	this.strPosition += utf8.RuneLen(character)
 	return character
 }
 
 func (this *lexerStream) rewind(amount int) {
-	this.position -= amount
+	if amount < 0 {
+		this.position -= amount
+		this.strPosition -= amount
+	}
+	strAmount := 0
+	for i := 0; i < amount; i++ {
+		if this.position >= this.length {
+			strAmount += 1
+			this.position -= 1
+			continue
+		}
+		strAmount += utf8.RuneLen(this.source[this.position])
+		this.position -= 1
+	}
+	this.strPosition -= strAmount
 }
 
 func (this lexerStream) canRead() bool {

parsing.go

@@ -308,9 +308,10 @@ Returns false if the stream ended before whitespace was broken or condition was
 func readUntilFalse(stream *lexerStream, includeWhitespace bool, breakWhitespace bool, allowEscaping bool, condition func(rune) bool) (string, bool) {
 
 	tokenBuffer := tokenBufferPool.Get().(*bytes.Buffer)
+	tokenBuffer.Reset()
 	var character rune
 
-	startPosition := stream.position
+	startPosition := stream.strPosition
 	reuseString := true
 	trimString := false
 	conditioned := false
@@ -356,7 +357,7 @@ func readUntilFalse(stream *lexerStream, includeWhitespace bool, breakWhitespace
 	if reuseString {
 		tokenBuffer.Reset()
 		tokenBufferPool.Put(tokenBuffer)
-		ret := stream.sourceString[startPosition:stream.position]
+		ret := stream.sourceString[startPosition:stream.strPosition]
 		if trimString {
 			ret = ret[:len(ret)-1]
 		}

parsing_test.go

@@ -1141,6 +1141,24 @@ func TestModifierParsing(test *testing.T) {
 				},
 			},
 		},
+		{
+			Name:  "Single cyrilic parameter with op",
+			Input: "переменная * 1",
+			Expected: []ExpressionToken{
+				{
+					Kind:  VARIABLE,
+					Value: "переменная",
+				},
+				{
+					Kind:  MODIFIER,
+					Value: "*",
+				},
+				{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+			},
+		},
 	}
 
 	tokenParsingTests = combineWhitespaceExpressions(tokenParsingTests)