From 444f32f9e415bdd2ff4c167199d3b37ac5049d05 Mon Sep 17 00:00:00 2001 From: Suhas Karanth Date: Mon, 5 Dec 2022 14:01:58 +0530 Subject: [PATCH] feat: populate new fields of Entity in caramlstore extractor - Populate fields 'Description' and 'Type' for FeatureTable_Entity while extracting assets from CaraML Store. - Bump proton commit to include changes "feat: add attributes field for v1beta2 asset types (#233)". --- Makefile | 2 +- plugins/extractors/application_yaml/README.md | 4 +-- plugins/extractors/caramlstore/README.md | 4 +-- .../extractors/caramlstore/asset_builder.go | 14 +++------- .../caramlstore/testdata/expected-assets.json | 24 ++++++----------- plugins/extractors/merlin/README.md | 6 ++--- plugins/processors/script/README.md | 26 +++++++++---------- 7 files changed, 33 insertions(+), 47 deletions(-) diff --git a/Makefile b/Makefile index d546bbc96..6e84f6bf7 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ NAME="github.com/odpf/meteor" VERSION=$(shell git describe --always --tags 2>/dev/null) COVERFILE="/tmp/app.coverprofile" -PROTON_COMMIT := "5267e1fdf3abc8d9a06938290e202efdd060f665" +PROTON_COMMIT := "fabbde821d6f74cf5aec8437a1d13dcb7ad13e8c" .PHONY: all build clean test all: build diff --git a/plugins/extractors/application_yaml/README.md b/plugins/extractors/application_yaml/README.md index fcadc057d..0da81b6db 100644 --- a/plugins/extractors/application_yaml/README.md +++ b/plugins/extractors/application_yaml/README.md @@ -91,6 +91,6 @@ Refer to the [contribution guidelines](../../../docs/docs/contribute/guide.md#adding-a-new-extractor) for information on contributing to this module. -[proton-asset]: https://github.com/odpf/proton/blob/5267e1f/odpf/assets/v1beta2/asset.proto#L14 +[proton-asset]: https://github.com/odpf/proton/blob/fabbde8/odpf/assets/v1beta2/asset.proto#L14 -[proton-application]: https://github.com/odpf/proton/blob/5267e1f/odpf/assets/v1beta2/application.proto#L11 +[proton-application]: https://github.com/odpf/proton/blob/fabbde8/odpf/assets/v1beta2/application.proto#L11 diff --git a/plugins/extractors/caramlstore/README.md b/plugins/extractors/caramlstore/README.md index a752fb0c3..9e6834dd6 100644 --- a/plugins/extractors/caramlstore/README.md +++ b/plugins/extractors/caramlstore/README.md @@ -64,5 +64,5 @@ for information on contributing to this module. [caraml-store]: https://github.com/caraml-dev/caraml-store [coreservice.proto]: https://github.com/caraml-dev/caraml-store/blob/v0.1.1/caraml-store-protobuf/src/main/proto/feast/core/CoreService.proto#L12 -[proton-asset]: https://github.com/odpf/proton/blob/5267e1f/odpf/assets/v1beta2/asset.proto#L14 -[proton-featuretable]: https://github.com/odpf/proton/blob/5267e1f/odpf/assets/v1beta2/feature_table.proto#L32 +[proton-asset]: https://github.com/odpf/proton/blob/fabbde8/odpf/assets/v1beta2/asset.proto#L14 +[proton-featuretable]: https://github.com/odpf/proton/blob/fabbde8/odpf/assets/v1beta2/feature_table.proto#L32 diff --git a/plugins/extractors/caramlstore/asset_builder.go b/plugins/extractors/caramlstore/asset_builder.go index aa51f68e2..99dbbfe02 100644 --- a/plugins/extractors/caramlstore/asset_builder.go +++ b/plugins/extractors/caramlstore/asset_builder.go @@ -83,17 +83,11 @@ func (b featureTableBuilder) buildEntities(ft *core.FeatureTable) ([]*v1beta2.Fe return nil, fmt.Errorf("entity '%s' not found in project '%s", e, b.project) } - labels := map[string]string{ - "value_type": entity.Spec.ValueType.String(), - "description": entity.Spec.Description, - } - for k, v := range entity.Spec.Labels { - labels[k] = v - } - entities = append(entities, &v1beta2.FeatureTable_Entity{ - Name: entity.Spec.Name, - Labels: labels, + Name: entity.Spec.Name, + Description: entity.Spec.Description, + Type: entity.Spec.ValueType.String(), + Labels: entity.Spec.Labels, }) } diff --git a/plugins/extractors/caramlstore/testdata/expected-assets.json b/plugins/extractors/caramlstore/testdata/expected-assets.json index 0d8c21376..8433e1960 100644 --- a/plugins/extractors/caramlstore/testdata/expected-assets.json +++ b/plugins/extractors/caramlstore/testdata/expected-assets.json @@ -10,10 +10,8 @@ "entities": [ { "name": "merchant_uuid", - "labels": { - "description": "merchant uuid", - "value_type": "STRING" - } + "description": "merchant uuid", + "type": "STRING" } ], "features": [ @@ -98,10 +96,8 @@ "entities": [ { "name": "merchant_uuid", - "labels": { - "description": "merchant uuid", - "value_type": "STRING" - } + "description": "merchant uuid", + "type": "STRING" } ], "features": [ @@ -146,10 +142,8 @@ "entities": [ { "name": "merchant_uuid", - "labels": { - "description": "merchant uuid", - "value_type": "STRING" - } + "description": "merchant uuid", + "type": "STRING" } ], "features": [ @@ -234,10 +228,8 @@ "entities": [ { "name": "merchant_uuid", - "labels": { - "description": "merchant uuid", - "value_type": "STRING" - } + "description": "merchant uuid", + "type": "STRING" } ], "features": [ diff --git a/plugins/extractors/merlin/README.md b/plugins/extractors/merlin/README.md index e8f7890ff..e35642938 100644 --- a/plugins/extractors/merlin/README.md +++ b/plugins/extractors/merlin/README.md @@ -116,10 +116,10 @@ for information on contributing to this module. [google-default-auth]: https://cloud.google.com/docs/authentication/production#automatically -[proton-asset]: https://github.com/odpf/proton/blob/5267e1f/odpf/assets/v1beta2/asset.proto#L14 +[proton-asset]: https://github.com/odpf/proton/blob/fabbde8/odpf/assets/v1beta2/asset.proto#L14 -[proton-model]: https://github.com/odpf/proton/blob/5267e1f/odpf/assets/v1beta2/model.proto#L73 +[proton-model]: https://github.com/odpf/proton/blob/fabbde8/odpf/assets/v1beta2/model.proto#L73 -[proton-modelversion]: https://github.com/odpf/proton/blob/5267e1f/odpf/assets/v1beta2/model.proto#L31 +[proton-modelversion]: https://github.com/odpf/proton/blob/fabbde8/odpf/assets/v1beta2/model.proto#L31 [caraml-store]: https://github.com/caraml-dev/caraml-store diff --git a/plugins/processors/script/README.md b/plugins/processors/script/README.md index 5da759d3b..dd9ff8481 100644 --- a/plugins/processors/script/README.md +++ b/plugins/processors/script/README.md @@ -224,29 +224,29 @@ for information on contributing to this module. [tengo-stdlib]: https://github.com/d5/tengo/blob/v2.13.0/docs/stdlib.md -[proton-asset]: https://github.com/odpf/proton/blob/5267e1f/odpf/assets/v1beta2/asset.proto#L14 +[proton-asset]: https://github.com/odpf/proton/blob/fabbde8/odpf/assets/v1beta2/asset.proto#L14 -[proton-bucket]: https://github.com/odpf/proton/blob/5267e1f/odpf/assets/v1beta2/bucket.proto#L13 +[proton-bucket]: https://github.com/odpf/proton/blob/fabbde8/odpf/assets/v1beta2/bucket.proto#L13 -[proton-dashboard]: https://github.com/odpf/proton/blob/5267e1f/odpf/assets/v1beta2/dashboard.proto#L14 +[proton-dashboard]: https://github.com/odpf/proton/blob/fabbde8/odpf/assets/v1beta2/dashboard.proto#L14 -[proton-experiment]: https://github.com/odpf/proton/blob/5267e1f/odpf/assets/v1beta2/experiment.proto#L15 +[proton-experiment]: https://github.com/odpf/proton/blob/fabbde8/odpf/assets/v1beta2/experiment.proto#L15 -[proton-featuretable]: https://github.com/odpf/proton/blob/5267e1f/odpf/assets/v1beta2/feature_table.proto#L32 +[proton-featuretable]: https://github.com/odpf/proton/blob/fabbde8/odpf/assets/v1beta2/feature_table.proto#L32 -[proton-group]: https://github.com/odpf/proton/blob/5267e1f/odpf/assets/v1beta2/group.proto#L12 +[proton-group]: https://github.com/odpf/proton/blob/fabbde8/odpf/assets/v1beta2/group.proto#L12 -[proton-job]: https://github.com/odpf/proton/blob/5267e1f/odpf/assets/v1beta2/job.proto#L13 +[proton-job]: https://github.com/odpf/proton/blob/fabbde8/odpf/assets/v1beta2/job.proto#L13 -[proton-metric]: https://github.com/odpf/proton/blob/5267e1f/odpf/assets/v1beta2/metric.proto#L13 +[proton-metric]: https://github.com/odpf/proton/blob/fabbde8/odpf/assets/v1beta2/metric.proto#L13 -[proton-model]: https://github.com/odpf/proton/blob/5267e1f/odpf/assets/v1beta2/model.proto#L73 +[proton-model]: https://github.com/odpf/proton/blob/fabbde8/odpf/assets/v1beta2/model.proto#L73 -[proton-service]: https://github.com/odpf/proton/blob/5267e1f/odpf/assets/v1beta2/service.proto#L11 +[proton-service]: https://github.com/odpf/proton/blob/fabbde8/odpf/assets/v1beta2/service.proto#L11 -[proton-table]: https://github.com/odpf/proton/blob/5267e1f/odpf/assets/v1beta2/table.proto#L14 +[proton-table]: https://github.com/odpf/proton/blob/fabbde8/odpf/assets/v1beta2/table.proto#L14 -[proton-topic]: https://github.com/odpf/proton/blob/5267e1f/odpf/assets/v1beta2/topic.proto#L14 +[proton-topic]: https://github.com/odpf/proton/blob/fabbde8/odpf/assets/v1beta2/topic.proto#L14 -[proton-user]: https://github.com/odpf/proton/blob/5267e1f/odpf/assets/v1beta2/user.proto#L15 +[proton-user]: https://github.com/odpf/proton/blob/fabbde8/odpf/assets/v1beta2/user.proto#L15