implement shared tags functionality to avoid duplicates in exports

Author: teresaromero

internal/export/dashboards.go

@@ -7,11 +7,13 @@ package export
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
 
 	"github.com/Masterminds/semver/v3"
+	"github.com/stretchr/testify/assert/yaml"
 
 	"github.com/elastic/elastic-package/internal/common"
 	"github.com/elastic/elastic-package/internal/kibana"
@@ -46,8 +48,14 @@ func Dashboards(ctx context.Context, kibanaClient *kibana.Client, dashboardsIDs
 		return fmt.Errorf("exporting dashboards using Kibana client failed: %w", err)
 	}
 
+	sharedTags, err := readSharedTagsFile(packageRoot)
+	if err != nil {
+		return fmt.Errorf("reading shared tags file failed: %w", err)
+	}
+
 	transformContext := &transformationContext{
 		packageName: m.Name,
+		sharedTags:  sharedTags,
 	}
 
 	objects, err = applyTransformations(transformContext, objects)
@@ -87,6 +95,7 @@ func applyTransformations(ctx *transformationContext, objects []common.MapStr) (
 			stripObjectProperties,
 			standardizeObjectProperties,
 			removeFleetManagedTags,
+			removeDuplicateSharedTags,
 			standardizeObjectID).
 		transform(objects)
 }
@@ -120,3 +129,26 @@ func saveObjectsToFiles(packageRoot string, objects []common.MapStr) error {
 	}
 	return nil
 }
+
+func readSharedTagsFile(packageRoot string) ([]string, error) {
+	b, err := os.ReadFile(filepath.Join(packageRoot, "kibana", "tags.yml"))
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			// No shared tags file, return empty list
+			return nil, nil
+		}
+		return nil, fmt.Errorf("reading shared tags file failed: %w", err)
+	}
+	var sharedTags []struct {
+		Text string `yaml:"text"`
+	}
+	err = yaml.Unmarshal(b, &sharedTags)
+	if err != nil {
+		return nil, fmt.Errorf("unmarshalling shared tags file failed: %w", err)
+	}
+	tags := make([]string, 0, len(sharedTags))
+	for _, tag := range sharedTags {
+		tags = append(tags, tag.Text)
+	}
+	return tags, nil
+}

internal/export/dashboards_test.go

@@ -7,6 +7,7 @@ package export
 import (
 	"encoding/json"
 	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -38,3 +39,45 @@ func TestTransform(t *testing.T) {
 
 	require.Equal(t, string(expected), string(result))
 }
+
+func TestReadSharedTagsFile(t *testing.T) {
+
+	t.Run("file exists", func(t *testing.T) {
+		tmpDir := t.TempDir()
+
+		err := os.MkdirAll(filepath.Join(tmpDir, "kibana"), 0755)
+		require.NoError(t, err)
+		err = os.WriteFile(filepath.Join(tmpDir, "kibana", "tags.yml"), []byte(`- text: tag1
+- text: tag2
+- text: tag3
+`), 0644)
+		require.NoError(t, err)
+
+		tags, err := readSharedTagsFile(tmpDir)
+		require.NoError(t, err)
+		require.Equal(t, []string{"tag1", "tag2", "tag3"}, tags)
+	})
+
+	t.Run("file does not exist", func(t *testing.T) {
+		tmpDir := t.TempDir()
+
+		tags, err := readSharedTagsFile(tmpDir)
+		require.NoError(t, err)
+		require.Empty(t, tags)
+	})
+
+	t.Run("invalid YAML", func(t *testing.T) {
+		tmpDir := t.TempDir()
+
+		err := os.MkdirAll(filepath.Join(tmpDir, "kibana"), 0755)
+		require.NoError(t, err)
+		err = os.WriteFile(filepath.Join(tmpDir, "kibana", "tags.yml"), []byte(`- text: tag1
+- text
+- text: tag3
+`), 0644)
+		require.NoError(t, err)
+
+		_, err = readSharedTagsFile(tmpDir)
+		require.Error(t, err)
+	})
+}

internal/export/object_transformer.go

@@ -17,6 +17,7 @@ type objectTransformer struct {
 
 type transformationContext struct {
 	packageName string
+	sharedTags  []string
 }
 
 func newObjectTransformer() *objectTransformer {

internal/export/transform_remove_fleet_managed_tags.go

@@ -6,6 +6,7 @@ package export
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/elastic/elastic-package/internal/common"
 )
@@ -38,6 +39,11 @@ func removeTagReferences(ctx *transformationContext, object common.MapStr) (comm
 		return nil, err
 	}
 
+	newReferences, err = filterOutSharedTags(ctx, newReferences)
+	if err != nil {
+		return nil, err
+	}
+
 	_, err = object.Put("references", newReferences)
 	if err != nil {
 		return nil, fmt.Errorf("can't update references: %w", err)
@@ -80,6 +86,42 @@ func isTagFleetManaged(aId, packageName string) bool {
 	return ok
 }
 
+func isSharedTag(aId string, sharedTags []string) bool {
+	for _, tag := range sharedTags {
+		id := fmt.Sprintf("tag-ref-%s-default", strings.ReplaceAll(strings.ToLower(tag), " ", "-"))
+		if aId == id {
+			return true
+		}
+	}
+	return false
+}
+
+func filterOutSharedTags(ctx *transformationContext, references []interface{}) ([]interface{}, error) {
+	newReferences := make([]interface{}, 0)
+	for _, r := range references {
+		reference := r.(map[string]interface{})
+
+		aType, ok := reference["type"]
+		if !ok {
+			continue
+		}
+		if aType != "tag" {
+			newReferences = append(newReferences, r)
+			continue
+		}
+
+		aNameString, ok := reference["name"].(string)
+		if !ok {
+			return nil, fmt.Errorf("failed to assert name as a string: %v", reference["name"])
+		}
+		if isSharedTag(aNameString, ctx.sharedTags) {
+			continue
+		}
+		newReferences = append(newReferences, r)
+	}
+	return newReferences, nil
+}
+
 func filterOutFleetManagedTags(ctx *transformationContext, references []interface{}) ([]interface{}, error) {
 	newReferences := make([]interface{}, 0)
 	for _, r := range references {

internal/export/transform_remove_shared_tags.go

@@ -0,0 +1,37 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package export
+
+import (
+	"fmt"
+	"slices"
+
+	"github.com/elastic/elastic-package/internal/common"
+)
+
+func removeDuplicateSharedTags(ctx *transformationContext, object common.MapStr) (common.MapStr, error) {
+	aType, err := object.GetValue("type")
+	if err != nil {
+		return nil, fmt.Errorf("failed to read type field: %w", err)
+	}
+
+	if aType != "tag" {
+		return object, nil
+	}
+	tagName, err := object.GetValue("attributes.name")
+	if err != nil {
+		return nil, fmt.Errorf("failed to read attributes.name field: %w", err)
+	}
+
+	tagNameStr, ok := tagName.(string)
+	if !ok {
+		return nil, fmt.Errorf("failed to convert tag name to string")
+	}
+	if slices.Contains(ctx.sharedTags, tagNameStr) {
+		// Tag is already in the shared tags list, remove it
+		return nil, nil
+	}
+	return object, nil
+}

internal/export/transform_remove_shared_tags_test.go

@@ -0,0 +1,69 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package export
+
+import (
+	"testing"
+
+	"github.com/elastic/elastic-package/internal/common"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRemoveDuplicateSharedTags(t *testing.T) {
+	tests := []struct {
+		name           string
+		sharedTags     []string
+		inputObject    common.MapStr
+		expectedObject common.MapStr
+	}{
+		{
+			name:       "Tag already in shared tags",
+			sharedTags: []string{"shared-tag-1", "shared-tag-2"},
+			inputObject: map[string]interface{}{
+				"type":       "tag",
+				"attributes": map[string]interface{}{"name": "shared-tag-1"},
+			},
+			expectedObject: nil,
+		},
+		{
+			name:       "Tag not in shared tags",
+			sharedTags: []string{"shared-tag-1", "shared-tag-2"},
+			inputObject: map[string]interface{}{
+				"type":       "tag",
+				"attributes": map[string]interface{}{"name": "unique-tag"},
+			},
+			expectedObject: map[string]interface{}{
+				"type":       "tag",
+				"attributes": map[string]interface{}{"name": "unique-tag"},
+			},
+		},
+		{
+			name:       "Non-tag object",
+			sharedTags: []string{"shared-tag-1", "shared-tag-2"},
+			inputObject: map[string]interface{}{
+				"type":       "dashboard",
+				"attributes": map[string]interface{}{"title": "My Dashboard"},
+			},
+			expectedObject: map[string]interface{}{
+				"type":       "dashboard",
+				"attributes": map[string]interface{}{"title": "My Dashboard"},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ctx := &transformationContext{
+				sharedTags: tt.sharedTags,
+			}
+			inputMapStr := tt.inputObject
+			result, err := removeDuplicateSharedTags(ctx, inputMapStr)
+			require.NoError(t, err)
+
+			assert.Equal(t, tt.expectedObject, result)
+		})
+	}
+}