perf: more interning (#7636)

This PR adds interning of strings representing common integer values,
which greatly speeds up "to string" operations on numbers, and updates
some built-ins commonly used for this to make use of interned values
where possible.

This is "light" version of a previous PR that did this more aggressively,
but also came with more caveats. Importantly, interning of new strings is
now never done at "runtime", but only allowed at init time. The API for
interning is marked experimental and should not relied upon by anyone
who expects a stable API.

Signed-off-by: Anders Eknert <[email protected]>

Author: anderseknert

internal/edittree/edittree.go

@@ -723,15 +723,17 @@ func (e *EditTree) Unfold(path ast.Ref) (*EditTree, error) {
 			return child.Unfold(path[1:])
 		}
 
+		idxt := ast.InternedIntNumberTerm(idx)
+
 		// Fall back to looking up the key in e.value.
 		// Extend the tree if key is present. Error otherwise.
-		if v, err := x.Find(ast.Ref{ast.InternedIntNumberTerm(idx)}); err == nil {
+		if v, err := x.Find(ast.Ref{idxt}); err == nil {
 			// TODO: Consider a more efficient "Replace" function that special-cases this for arrays instead?
-			_, err := e.Delete(ast.InternedIntNumberTerm(idx))
+			_, err := e.Delete(idxt)
 			if err != nil {
 				return nil, err
 			}
-			child, err := e.Insert(ast.IntNumberTerm(idx), ast.NewTerm(v))
+			child, err := e.Insert(idxt, ast.NewTerm(v))
 			if err != nil {
 				return nil, err
 			}

internal/future/filter_imports.go

@@ -19,30 +19,28 @@ func FilterFutureImports(imps []*ast.Import) []*ast.Import {
 	return ret
 }
 
-var keywordsTerm = ast.StringTerm("keywords")
-
 // IsAllFutureKeywords returns true if the passed *ast.Import is `future.keywords`
 func IsAllFutureKeywords(imp *ast.Import) bool {
 	path := imp.Path.Value.(ast.Ref)
 	return len(path) == 2 &&
 		ast.FutureRootDocument.Equal(path[0]) &&
-		path[1].Equal(keywordsTerm)
+		path[1].Equal(ast.InternedStringTerm("keywords"))
 }
 
 // IsFutureKeyword returns true if the passed *ast.Import is `future.keywords.{kw}`
 func IsFutureKeyword(imp *ast.Import, kw string) bool {
 	path := imp.Path.Value.(ast.Ref)
 	return len(path) == 3 &&
 		ast.FutureRootDocument.Equal(path[0]) &&
-		path[1].Equal(keywordsTerm) &&
+		path[1].Equal(ast.InternedStringTerm("keywords")) &&
 		path[2].Equal(ast.StringTerm(kw))
 }
 
 func WhichFutureKeyword(imp *ast.Import) (string, bool) {
 	path := imp.Path.Value.(ast.Ref)
 	if len(path) == 3 &&
 		ast.FutureRootDocument.Equal(path[0]) &&
-		path[1].Equal(keywordsTerm) {
+		path[1].Equal(ast.InternedStringTerm("keywords")) {
 		if str, ok := path[2].Value.(ast.String); ok {
 			return string(str), true
 		}

internal/runtime/runtime.go

@@ -21,15 +21,6 @@ type Params struct {
 	SkipKnownSchemaCheck   bool
 }
 
-var (
-	configKey               = ast.StringTerm("config")
-	envKey                  = ast.StringTerm("env")
-	versionKey              = ast.StringTerm("version")
-	commitKey               = ast.StringTerm("commit")
-	authorizationEnabledKey = ast.StringTerm("authorization_enabled")
-	skipKnownSchemaCheckKey = ast.StringTerm("skip_known_schema_check")
-)
-
 // Term returns the runtime information as an ast.Term object.
 func Term(params Params) (*ast.Term, error) {
 
@@ -47,7 +38,7 @@ func Term(params Params) (*ast.Term, error) {
 			return nil, err
 		}
 
-		obj.Insert(configKey, ast.NewTerm(v))
+		obj.Insert(ast.InternedStringTerm("config"), ast.NewTerm(v))
 	}
 
 	env := ast.NewObject()
@@ -61,11 +52,11 @@ func Term(params Params) (*ast.Term, error) {
 		}
 	}
 
-	obj.Insert(envKey, ast.NewTerm(env))
-	obj.Insert(versionKey, ast.StringTerm(version.Version))
-	obj.Insert(commitKey, ast.StringTerm(version.Vcs))
-	obj.Insert(authorizationEnabledKey, ast.InternedBooleanTerm(params.IsAuthorizationEnabled))
-	obj.Insert(skipKnownSchemaCheckKey, ast.InternedBooleanTerm(params.SkipKnownSchemaCheck))
+	obj.Insert(ast.InternedStringTerm("env"), ast.NewTerm(env))
+	obj.Insert(ast.InternedStringTerm("version"), ast.StringTerm(version.Version))
+	obj.Insert(ast.InternedStringTerm("commit"), ast.StringTerm(version.Vcs))
+	obj.Insert(ast.InternedStringTerm("authorization_enabled"), ast.InternedBooleanTerm(params.IsAuthorizationEnabled))
+	obj.Insert(ast.InternedStringTerm("skip_known_schema_check"), ast.InternedBooleanTerm(params.SkipKnownSchemaCheck))
 
 	return ast.NewTerm(obj), nil
 }

v1/ast/annotations.go

@@ -23,27 +23,6 @@ const (
 	annotationScopeSubpackages = "subpackages"
 )
 
-var (
-	scopeTerm            = StringTerm("scope")
-	titleTerm            = StringTerm("title")
-	entrypointTerm       = StringTerm("entrypoint")
-	descriptionTerm      = StringTerm("description")
-	organizationsTerm    = StringTerm("organizations")
-	authorsTerm          = StringTerm("authors")
-	relatedResourcesTerm = StringTerm("related_resources")
-	schemasTerm          = StringTerm("schemas")
-	customTerm           = StringTerm("custom")
-	refTerm              = StringTerm("ref")
-	nameTerm             = StringTerm("name")
-	emailTerm            = StringTerm("email")
-	schemaTerm           = StringTerm("schema")
-	definitionTerm       = StringTerm("definition")
-	documentTerm         = StringTerm(annotationScopeDocument)
-	packageTerm          = StringTerm(annotationScopePackage)
-	ruleTerm             = StringTerm(annotationScopeRule)
-	subpackagesTerm      = StringTerm(annotationScopeSubpackages)
-)
-
 type (
 	// Annotations represents metadata attached to other AST nodes such as rules.
 	Annotations struct {
@@ -444,93 +423,93 @@ func (a *Annotations) toObject() (*Object, *Error) {
 	if len(a.Scope) > 0 {
 		switch a.Scope {
 		case annotationScopeDocument:
-			obj.Insert(scopeTerm, documentTerm)
+			obj.Insert(InternedStringTerm("scope"), InternedStringTerm("document"))
 		case annotationScopePackage:
-			obj.Insert(scopeTerm, packageTerm)
+			obj.Insert(InternedStringTerm("scope"), InternedStringTerm("package"))
 		case annotationScopeRule:
-			obj.Insert(scopeTerm, ruleTerm)
+			obj.Insert(InternedStringTerm("scope"), InternedStringTerm("rule"))
 		case annotationScopeSubpackages:
-			obj.Insert(scopeTerm, subpackagesTerm)
+			obj.Insert(InternedStringTerm("scope"), InternedStringTerm("subpackages"))
 		default:
-			obj.Insert(scopeTerm, StringTerm(a.Scope))
+			obj.Insert(InternedStringTerm("scope"), StringTerm(a.Scope))
 		}
 	}
 
 	if len(a.Title) > 0 {
-		obj.Insert(titleTerm, StringTerm(a.Title))
+		obj.Insert(InternedStringTerm("title"), StringTerm(a.Title))
 	}
 
 	if a.Entrypoint {
-		obj.Insert(entrypointTerm, InternedBooleanTerm(true))
+		obj.Insert(InternedStringTerm("entrypoint"), InternedBooleanTerm(true))
 	}
 
 	if len(a.Description) > 0 {
-		obj.Insert(descriptionTerm, StringTerm(a.Description))
+		obj.Insert(InternedStringTerm("description"), StringTerm(a.Description))
 	}
 
 	if len(a.Organizations) > 0 {
 		orgs := make([]*Term, 0, len(a.Organizations))
 		for _, org := range a.Organizations {
 			orgs = append(orgs, StringTerm(org))
 		}
-		obj.Insert(organizationsTerm, ArrayTerm(orgs...))
+		obj.Insert(InternedStringTerm("organizations"), ArrayTerm(orgs...))
 	}
 
 	if len(a.RelatedResources) > 0 {
 		rrs := make([]*Term, 0, len(a.RelatedResources))
 		for _, rr := range a.RelatedResources {
-			rrObj := NewObject(Item(refTerm, StringTerm(rr.Ref.String())))
+			rrObj := NewObject(Item(InternedStringTerm("ref"), StringTerm(rr.Ref.String())))
 			if len(rr.Description) > 0 {
-				rrObj.Insert(descriptionTerm, StringTerm(rr.Description))
+				rrObj.Insert(InternedStringTerm("description"), StringTerm(rr.Description))
 			}
 			rrs = append(rrs, NewTerm(rrObj))
 		}
-		obj.Insert(relatedResourcesTerm, ArrayTerm(rrs...))
+		obj.Insert(InternedStringTerm("related_resources"), ArrayTerm(rrs...))
 	}
 
 	if len(a.Authors) > 0 {
 		as := make([]*Term, 0, len(a.Authors))
 		for _, author := range a.Authors {
 			aObj := NewObject()
 			if len(author.Name) > 0 {
-				aObj.Insert(nameTerm, StringTerm(author.Name))
+				aObj.Insert(InternedStringTerm("name"), StringTerm(author.Name))
 			}
 			if len(author.Email) > 0 {
-				aObj.Insert(emailTerm, StringTerm(author.Email))
+				aObj.Insert(InternedStringTerm("email"), StringTerm(author.Email))
 			}
 			as = append(as, NewTerm(aObj))
 		}
-		obj.Insert(authorsTerm, ArrayTerm(as...))
+		obj.Insert(InternedStringTerm("authors"), ArrayTerm(as...))
 	}
 
 	if len(a.Schemas) > 0 {
 		ss := make([]*Term, 0, len(a.Schemas))
 		for _, s := range a.Schemas {
 			sObj := NewObject()
 			if len(s.Path) > 0 {
-				sObj.Insert(pathTerm, NewTerm(s.Path.toArray()))
+				sObj.Insert(InternedStringTerm("path"), NewTerm(s.Path.toArray()))
 			}
 			if len(s.Schema) > 0 {
-				sObj.Insert(schemaTerm, NewTerm(s.Schema.toArray()))
+				sObj.Insert(InternedStringTerm("schema"), NewTerm(s.Schema.toArray()))
 			}
 			if s.Definition != nil {
 				def, err := InterfaceToValue(s.Definition)
 				if err != nil {
 					return nil, NewError(CompileErr, a.Location, "invalid definition in schema annotation: %s", err.Error())
 				}
-				sObj.Insert(definitionTerm, NewTerm(def))
+				sObj.Insert(InternedStringTerm("definition"), NewTerm(def))
 			}
 			ss = append(ss, NewTerm(sObj))
 		}
-		obj.Insert(schemasTerm, ArrayTerm(ss...))
+		obj.Insert(InternedStringTerm("schemas"), ArrayTerm(ss...))
 	}
 
 	if len(a.Custom) > 0 {
 		c, err := InterfaceToValue(a.Custom)
 		if err != nil {
 			return nil, NewError(CompileErr, a.Location, "invalid custom annotation %s", err.Error())
 		}
-		obj.Insert(customTerm, NewTerm(c))
+		obj.Insert(InternedStringTerm("custom"), NewTerm(c))
 	}
 
 	return &obj, nil

v1/ast/builtins.go

@@ -20,6 +20,13 @@ func RegisterBuiltin(b *Builtin) {
 	BuiltinMap[b.Name] = b
 	if len(b.Infix) > 0 {
 		BuiltinMap[b.Infix] = b
+
+		InternStringTerm(b.Infix)
+	}
+
+	InternStringTerm(b.Name)
+	if strings.Contains(b.Name, ".") {
+		InternStringTerm(strings.Split(b.Name, ".")...)
 	}
 }
 
@@ -3388,7 +3395,7 @@ func (b *Builtin) Ref() Ref {
 	ref := make(Ref, len(parts))
 	ref[0] = VarTerm(parts[0])
 	for i := 1; i < len(parts); i++ {
-		ref[i] = StringTerm(parts[i])
+		ref[i] = InternedStringTerm(parts[i])
 	}
 	return ref
 }

v1/ast/compile.go

@@ -964,12 +964,7 @@ func (c *Compiler) buildComprehensionIndices() {
 	}
 }
 
-var (
-	keywordsTerm         = StringTerm("keywords")
-	pathTerm             = StringTerm("path")
-	annotationsTerm      = StringTerm("annotations")
-	futureKeywordsPrefix = Ref{FutureRootDocument, keywordsTerm}
-)
+var futureKeywordsPrefix = Ref{FutureRootDocument, InternedStringTerm("keywords")}
 
 // buildRequiredCapabilities updates the required capabilities on the compiler
 // to include any keyword and feature dependencies present in the modules. The
@@ -2534,17 +2529,15 @@ func createMetadataChain(chain []*AnnotationsRef) (*Term, *Error) {
 
 	metaArray := NewArray()
 	for _, link := range chain {
-		p := link.Path.toArray().
-			Slice(1, -1) // Dropping leading 'data' element of path
-		obj := NewObject(
-			Item(pathTerm, NewTerm(p)),
-		)
+		// Dropping leading 'data' element of path
+		p := link.Path[1:].toArray()
+		obj := NewObject(Item(InternedStringTerm("path"), NewTerm(p)))
 		if link.Annotations != nil {
 			annotObj, err := link.Annotations.toObject()
 			if err != nil {
 				return nil, err
 			}
-			obj.Insert(annotationsTerm, NewTerm(*annotObj))
+			obj.Insert(InternedStringTerm("annotations"), NewTerm(*annotObj))
 		}
 		metaArray = metaArray.Append(NewTerm(obj))
 	}
@@ -4642,6 +4635,8 @@ func rewriteComprehensionTerms(f *equalityFactory, node any) (any, error) {
 	})
 }
 
+var doubleEq = Equal.Ref()
+
 // rewriteEquals will rewrite exprs under x as unification calls instead of ==
 // calls. For example:
 //
@@ -4656,7 +4651,6 @@ func rewriteComprehensionTerms(f *equalityFactory, node any) (any, error) {
 // partial evaluation cases we do want to rewrite == to = to simplify the
 // result.
 func rewriteEquals(x any) (modified bool) {
-	doubleEq := Equal.Ref()
 	unifyOp := Equality.Ref()
 	t := NewGenericTransformer(func(x any) (any, error) {
 		if x, ok := x.(*Expr); ok && x.IsCall() {