feat: add metadata models to enable E2E lineage

odpf/assets/v1beta2/common.proto

@@ -25,7 +25,7 @@ message Event {
   string description = 3;
 }
 
-// Linage reprsents the relationship of resource to other resources.
+// Linage represents the relationship of resource to other resources.
 // Relation is way of describing the relationship between two resources.
 message Lineage {
   // The resource that is the source of the relationship.

odpf/assets/v1beta2/experiment.proto

@@ -0,0 +1,67 @@
+syntax = "proto3";
+
+package odpf.assets.v1beta2;
+
+import "google/protobuf/struct.proto";
+import "google/protobuf/timestamp.proto";
+
+option go_package = "github.com/odpf/proton/assets/v1beta2;assetsv1beta2";
+option java_outer_classname = "ExperimentProto";
+option java_package = "io.odpf.assets";
+
+// An experiment is the set of configurations and filters that allow for
+// systematically varying some independent variables to impact some other
+// dependent variables.
+message Experiment {
+  // Instance of configurations to be compared in the experiment.
+  message Variant {
+    // Name of the experiment variant.
+    string name = 1;
+
+    // Traffic percent enabled for the variant.
+    float traffic_percent = 2;
+
+    // Indicated whether the variant is the control for the experiment.
+    bool is_control = 3;
+
+    // List of properties the entity has.
+    google.protobuf.Struct attributes = 4;
+
+    // Whether the variant has been promoted to all users.
+    bool is_promoted = 5;
+  }
+
+  // Optional: Type of the entity being experimented over. ex: customer, session,
+  // device, driver etc.
+  string entity = 1;
+
+  // Optional: Percentage of the traffic that the experiment is enabled for.
+  float traffic_percent = 2;
+
+  // The variants of the experiment possibly including the control group.
+  repeated Variant variants = 3;
+
+  // Optional: List of attributes the experiment has. This could include the
+  // following:
+  // - client_id[string]: The ID if the client running the experiment.
+  // - client_name[string]: The name of the client running the experiment.
+  // - primary_metric[string]: Used to determine a statistically significant
+  //   winning or losing variant.
+  // - guardrail_metric[string]: Business metric designed to indirectly measure
+  //   business value and track any potentially misleading or erroneous results
+  //   and analysis.
+  // - variant_sample_size[double]: Sample size per variant.
+  // - filter_rules[repeated string]: Textual representation of rules required
+  //   to be satisfied for experiment to be shown to the user.
+  // - start_time[RFC 3339 string]: The timestamp at which the
+  //   experiment would start.
+  // - end_time[RFC 3339 string]: The timestamp at which the
+  //   experiment would end.
+  google.protobuf.Struct attributes = 5;
+
+  // The timestamp of the experiment's creation.
+  google.protobuf.Timestamp create_time = 101;
+
+  // The timestamp when the experiment was last modified.
+  google.protobuf.Timestamp update_time = 102;
+}

odpf/assets/v1beta2/metric.proto

@@ -0,0 +1,42 @@
+syntax = "proto3";
+
+package odpf.assets.v1beta2;
+
+import "google/protobuf/struct.proto";
+import "google/protobuf/timestamp.proto";
+
+option go_package = "github.com/odpf/proton/assets/v1beta2;assetsv1beta2";
+option java_outer_classname = "MetricProto";
+option java_package = "io.odpf.assets";
+
+// A metric is a timeseries aggregation over a table that supports zero or more dimensions.
+message Metric {
+  // Namespace of the metric. Something like `{project}-{model}` for dbt and
+  // `schema` for Cube.
+  string namespace = 1;
+
+  // The field being used to calculate a metric.
+  string field_name = 2;
+
+  // Type of the evaluated metric. ex: count_distinct, average etc.
+  string measure_type = 3;
+
+  // Optional: The query, possibly in SQL representation, with filters and aggregations.
+  string query = 4;
+
+  // Optional: List of attributes the metric has. This could include the
+  // following:
+  // - time_grains[repeated string]: One or more "grains" at which the metric
+  //   can be evaluated. Ex: [day, week, month].
+  // - dimensions[repeated string]: A list of dimensions to group or filter the
+  //   metric by. Ex: [plan, country].
+  // - filters[repeated map<string, string>]: Predicates for the metric. Ex:
+  //   [{"field": "is_paying", "operator": "is", "value": "true"}].
+  google.protobuf.Struct attributes = 5;
+
+  // The timestamp of the metric's creation.
+  google.protobuf.Timestamp create_time = 101;
+
+  // The timestamp when the metric was last modified.
+  google.protobuf.Timestamp update_time = 102;
+}

odpf/assets/v1beta2/ml_feature.proto

@@ -0,0 +1,64 @@
+syntax = "proto3";
+
+package odpf.assets.v1beta2;
+
+import "google/protobuf/timestamp.proto";
+
+option go_package = "github.com/odpf/proton/assets/v1beta2;assetsv1beta2";
+option java_outer_classname = "MLFeatureProto";
+option java_package = "io.odpf.assets";
+
+// MLFeature is a Machine Learning(ML) feature. In machine learning, a feature
+// is an individual measurable property, typically represented by a column,
+// that serves as an input for Machine Learning (ML) algorithms.
+message MLFeature {
+  // The name of the field.
+  string name = 1;
+
+  // The data type associated with an individual ML Feature.
+  string data_type = 2;
+
+  // Optional: Name of the algorithm used to compute the feature, e.g., PCA,
+  // bucketing etc.
+  string algorithm = 3;
+
+  // Optional: Name of the entity instance.
+  string entity_name = 4;
+}
+
+// MLFeatureTable is a Machine Learning(ML) feature table or view that
+// represents a logical group of time-series feature data as it is found in a
+// data source.
+message MLFeatureTable {
+  // An entity is a collection of semantically related features. Users define
+  // entities to map to the domain of their use case. For example, a
+  // ride-hailing service could have customers and drivers as their entities,
+  // which group related features that correspond to these customers and drivers.
+  message Entity {
+    // The unique name of the entity.
+    string name = 1;
+
+    // A property that uniquely identifies different entities within the
+    // collection. The join_key property is typically used for joining entities
+    // with their associated features.
+    repeated string join_keys = 2;
+
+    // Optional: Arbitrary metadata.
+    map<string, string> labels = 3;
+  }
+
+  // Optional: Feature store's namespace or project.
+  string namespace = 1;
+
+  // Optional: The list of entities that this feature view is associated with.
+  repeated Entity entities = 2;
+
+  // Features that are part of the table, akin to columns in a table.
+  repeated MLFeature features = 3;
+
+  // The timestamp when the feature table was created.
+  google.protobuf.Timestamp create_time = 101;
+
+  // The timestamp when the feature table was last modified.
+  google.protobuf.Timestamp update_time = 102;
+}

odpf/assets/v1beta2/ml_model.proto

@@ -0,0 +1,72 @@
+syntax = "proto3";
+
+package odpf.assets.v1beta2;
+
+import "google/protobuf/struct.proto";
+import "google/protobuf/timestamp.proto";
+
+option go_package = "github.com/odpf/proton/assets/v1beta2;assetsv1beta2";
+option java_outer_classname = "MLModelProto";
+option java_package = "io.odpf.assets";
+
+// MLModel represents a Machine Learning(ML) Model. Models are algorithms trained
+// on data to find patterns or make predictions. Models typically consume
+// ML features to generate a meaningful output. The inputs can also include
+// contextual information that is made available in realtime as part of the
+// request to the model server.
+message MLModel {
+  // Schema of the model's inputs and outputs. Strongly inspired by
+  // https://mlflow.org/docs/latest/python_api/mlflow.models.html#mlflow.models.ModelSignature.
+  message Signature {
+    // Specification of name and type of a single column in a dataset.
+    message Parameter {
+      // Optional: Name of the input or output parameter.
+      string name = 1;
+
+      // Data type of the parameter. Ex: boolean, double, numpy's dtypes etc.
+      string data_type = 2;
+
+      // Optional: The tensor shape.
+      repeated int64 shape = 3;
+    }
+
+    repeated Parameter inputs = 1;
+    repeated Parameter outputs = 2;
+  }
+
+  // Optional: Model's namespace or project.
+  string namespace = 1;
+
+  // Flavor of the ML Model. ex: pytorch, tensorflow etc.
+  string flavor = 2;
+
+  // Optional: Algorithm used to train the ML Model.
+  string algorithm = 3;
+
+  // The schema of a model’s inputs and outputs.
+  Signature signature = 4;
+
+  // Status of the model. ex: pending/ready/serving/terminated etc.
+  string state = 5;
+
+  // Version of the model
+  string version = 6;
+
+  // List of attributes the model has. This could include the following:
+  // - endpoint_url[string]: Endpoint that the model is serving requests on.
+  //   Ex: http://<model_name>-<version>.<project_name>.<merlin_base_url>.
+  // - version_endpoint_url[string]: Endpoint that the model is serving
+  //   requests on for the specific version. Ex:
+  //   http://<model_name>-<version>.<project_name>.<merlin_base_url>.
+  // - traffic[double]: Percentage of traffic being served by this version of
+  //   the model.
+  // - params[map<string, string>]: Parameters for the Model's run.
+  // - metrics[map<string, double>]: Metrics for the model's run.
+  google.protobuf.Struct attributes = 7;
+
+  // The timestamp of the model's creation.
+  google.protobuf.Timestamp create_time = 101;
+
+  // The timestamp when the model was last modified.
+  google.protobuf.Timestamp update_time = 102;
+}

odpf/assets/v1beta2/service.proto

@@ -0,0 +1,23 @@
+syntax = "proto3";
+
+package odpf.assets.v1beta2;
+
+import "google/protobuf/timestamp.proto";
+
+option go_package = "github.com/odpf/proton/assets/v1beta2;assetsv1beta2";
+option java_outer_classname = "ServiceProto";
+option java_package = "io.odpf.assets";
+
+message Service {
+  // The service/application's ID
+  string application_id = 1;
+
+  // Optional: The version of the service.
+  string version = 2;
+
+  // The timestamp of the service's creation.
+  google.protobuf.Timestamp create_time = 101;
+
+  // The timestamp when the service was last modified.
+  google.protobuf.Timestamp update_time = 102;
+}