diff --git a/plugins/extractors/application_yaml/README.md b/plugins/extractors/application_yaml/README.md index 77d5bf31e..f967ad94f 100644 --- a/plugins/extractors/application_yaml/README.md +++ b/plugins/extractors/application_yaml/README.md @@ -33,21 +33,12 @@ description: "string" url: "string" version: "string" inputs: # OPTIONAL - - type: "batch" # ONE OF {batch, kafka} - platform: "bigquery" # If type is batch then bigquery - options: - table_ref: "{project_id}:{dataset_id}.{table_id}" # REQUIRED - - type: "stream" - platform: "kafka" # If type is stream then kafka - options: - topic: "estimate-log" # REQUIRED - bootstrap_servers: "host:port" # REQUIRED + # Format: "urn:{service}:{scope}:{type}:{name}" + - urn:bigquery:bq-raw-internal:table:bq-raw-internal:dagstream.production_feast09_s2id13_30min_demand + - urn:kafka:int-dagstream-kafka.yonkou.io:topic:staging_feast09_s2id13_30min_demand outputs: # OPTIONAL - - type: "stream" - platform: "kafka" - options: - topic: "string" - bootstrap_servers: "host:port" + # Format: "urn:{service}:{scope}:{type}:{name}" + - urn:kafka:1-my-kafka.com:topic:staging_feast09_mixed_granularity_demand_forecast_3es create_time: "2006-01-02T15:04:05Z" update_time: "2006-01-02T15:04:05Z" labels: @@ -90,12 +81,8 @@ proto definitions for more information. | `ownership.owners[0].urn` | `{application.team.id}` | `9ebcc2f8-5894-47c6-83a9-160b7eaa3f6b` | | `ownership.owners[0].name` | `{application.team.name}` | `Search` | | `ownership.owners[0].email` | `{application.team.email}` | `search@mycompany.com` | -| `lineage.upstreams[].urn` | `urn:{service}:{scope}:{type}:{name}` | `urn:kafka:int-kafka.yonkou.io:topic:staging_30min_demand` | -| `lineage.upstreams[].type` | `{type}` | `topic` | -| `lineage.upstreams[].service` | `{service}` | `kafka` | -| `lineage.downstreams[].urn` | `urn:{service}:{scope}:{type}:{name}` | `urn:bigquery:bq-internal:table:bq-internal:dagstream.production_30min_demand` | -| `lineage.downstreams[].type` | `{type}` | `table` | -| `lineage.downstreams[].service` | `{service}` | `bigquery` | +| `lineage.upstreams[].urn` | `{application.inputs[]}` | `urn:kafka:int-kafka.yonkou.io:topic:staging_30min_demand` | +| `lineage.downstreams[].urn` | `{application.outputs[]}` | `urn:bigquery:bq-internal:table:bq-internal:dagstream.production_30min_demand` | | `resource.labels` | `map[string]string` | `{"team": "Booking Experience"}` | ## Contributing diff --git a/plugins/extractors/bigquery/README.md b/plugins/extractors/bigquery/README.md index 19df5b5df..25f88c875 100644 --- a/plugins/extractors/bigquery/README.md +++ b/plugins/extractors/bigquery/README.md @@ -47,8 +47,8 @@ source: ### *Notes* -- Leaving `credentials_json` blank will default - to [Google's default authentication](https://cloud.google.com/docs/authentication/production#automatically). It is +- Leaving `service_account_json` and `service_account_base64` blank will default + to [Google's default authentication][google-default-auth]. It is recommended if Meteor instance runs inside the same Google Cloud environment as the BigQuery project. - Service account needs to have `bigquery.privateLogsViewer` role to be able to collect bigquery audit logs @@ -89,3 +89,5 @@ source: Refer to the [contribution guidelines](../../../docs/docs/contribute/guide.md#adding-a-new-extractor) for information on contributing to this module. + +[google-default-auth]: https://cloud.google.com/docs/authentication/production#automatically diff --git a/plugins/extractors/caramlstore/README.md b/plugins/extractors/caramlstore/README.md index 3fda3d109..076e77bc7 100644 --- a/plugins/extractors/caramlstore/README.md +++ b/plugins/extractors/caramlstore/README.md @@ -31,17 +31,17 @@ The feature tables are mapped to an [`Asset`][proton-asset] with model specific metadata stored using [`FeatureTable`][proton-featuretable]. Please refer the proto definitions for more information. -| Field | Value | Sample Value | -|:-------------------|:----------------------------------------------------------------|:---------------------------------------------------------------------------| -| `resource.urn` | `urn:caramlstore:{scope}:ml-feature-table:{feature_table_name}` | `urn:caramlstore:caramlstore-stg:feature_table:merchant_uuid_t2_discovery` | -| `resource.name` | `{feature_table.name}` | `merchant_uuid_t2_discovery` | -| `resource.service` | `caramlstore` | `caramlstore` | -| `resource.type` | `feature_table` | `feature_table` | -| `namespace` | `{feature_table.project}` | `my_project` | -| `entities` | [`[]Entity`](#entity) | | -| `features` | [`[]Feature`](#feature) | | -| `create_time` | `{feature_table.created_timestamp}` | `2022-08-08T03:17:54Z` | -| `update_time` | `{feature_table.updated_timestamp}` | `2022-08-08T03:57:54Z` | +| Field | Value | Sample Value | +|:-------------------|:-------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------| +| `resource.urn` | `urn:caramlstore:{scope}:feature_table:{feature_table.project}.{feature_table.name}` | `urn:caramlstore:caramlstore-stg:feature_table:my_project.merchant_uuid_t2_discovery` | +| `resource.name` | `{feature_table.name}` | `merchant_uuid_t2_discovery` | +| `resource.service` | `caramlstore` | `caramlstore` | +| `resource.type` | `feature_table` | `feature_table` | +| `namespace` | `{feature_table.project}` | `my_project` | +| `entities` | [`[]Entity`](#entity) | | +| `features` | [`[]Feature`](#feature) | | +| `create_time` | `{feature_table.created_timestamp}` | `2022-08-08T03:17:54Z` | +| `update_time` | `{feature_table.updated_timestamp}` | `2022-08-08T03:57:54Z` | ### Entity