diff --git a/SandBox_Jaggi/Unique_Customer_ID_Test_SQL_SCRIPT.sql b/SandBox_Jaggi/Unique_Customer_ID_Test_SQL_SCRIPT.sql new file mode 100644 index 0000000..3ff808d --- /dev/null +++ b/SandBox_Jaggi/Unique_Customer_ID_Test_SQL_SCRIPT.sql @@ -0,0 +1,24 @@ +select + count(*) as failures, + count(*) != 0 as should_warn, + count(*) != 0 as should_error + from ( + +select + order_id as unique_field, + count(*) as n_records + +from DBT_TRAINING.dbt_jkavuru.stg_jaffle_shop__orders +where order_id is not null +group by order_id +having count(*) > 1 + + ) dbt_internal_test + +/************************************************ */ + +SELECT order_id, COUNT(*) +FROM DBT_TRAINING.dbt_jkavuru.stg_jaffle_shop__orders +GROUP BY order_id +HAVING COUNT(*) > 1; + diff --git a/SandBox_Jaggi/create_customer_tbl.sql b/SandBox_Jaggi/create_customer_tbl.sql new file mode 100644 index 0000000..cda2403 --- /dev/null +++ b/SandBox_Jaggi/create_customer_tbl.sql @@ -0,0 +1,19 @@ + +CREATE TABLE DBT_TRAINING.DBT_JKAVURU.customers +( + id NUMBER(9) PRIMARY KEY, + first_name VARCHAR(10) NOT NULL, + last_name VARCHAR(10) NOT NULL + +); + + + + +/* + +select id as customer_id, first_name, last_name + +from DBT_TRAINING.DBT_JKAVURU.customers + +*/ diff --git a/SandBox_Jaggi/fct_orders_jk1.sql b/SandBox_Jaggi/fct_orders_jk1.sql new file mode 100644 index 0000000..e5d6b93 --- /dev/null +++ b/SandBox_Jaggi/fct_orders_jk1.sql @@ -0,0 +1,35 @@ +with orders as ( + select * from {{ ref ('stg_jaffle_shop__orders' )}} +), + +payments as ( + select * from {{ ref ('stg_stripe__payments') }} +), + + +order_payments as ( + select + order_id, + payments.status as payment_status, + sum (case when status = 'success' then amount * 100 end) as amount, + sum (case when status = 'fail' then amount * 100 end) as amount_failed, + sum (case when status is null then 0 else amount * 100 end) as amount_total + from payments + group by 1,2 +), + + final as ( + + select + orders.order_id, + orders.customer_id, + orders.order_date, + coalesce (order_payments.amount, 0) as amount, + order_payments.payment_status as payment_status1 + from orders + left join order_payments using (order_id) +) + +select * from final order by 1 asc + + diff --git a/SandBox_Jaggi/jaffle_shop__sources.yml_DONOT_USE b/SandBox_Jaggi/jaffle_shop__sources.yml_DONOT_USE new file mode 100644 index 0000000..c7fed1f --- /dev/null +++ b/SandBox_Jaggi/jaffle_shop__sources.yml_DONOT_USE @@ -0,0 +1,15 @@ +version: 2 + +sources: + - name: jaffle_shop + database: raw + schema: jaffle_shop + freshness: # make this a little more strict + warn_after: { count: 24, period: hour } + error_after: { count: 1000, period: day } #Jaggi > increased from 12 to 9200 # + loaded_at_field: _etl_loaded_at + + tables: + - name: customers + freshness: null #Jaggi, changed the freshness to null for customers model. + - name: orders diff --git a/dbt_project.yml b/dbt_project.yml index 26771cc..4f0eb31 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -2,7 +2,7 @@ # Name your project! Project names should contain only lowercase characters # and underscores. A good package name should reflect your organization's # name or the intended use of these models -name: 'my_new_project' +name: 'jaffle_shop' version: '1.0.0' config-version: 2 @@ -32,7 +32,13 @@ clean-targets: # directories to be removed by `dbt clean` # as tables. These settings can be overridden in the individual model files # using the `{{ config(...) }}` macro. models: - my_new_project: + jaffle_shop: # Applies to all files under models/example/ example: materialized: view + + staging: + materialized: view + + marts: + materialized: table \ No newline at end of file diff --git a/models/example/my_first_dbt_model.sql b/models/example/my_first_dbt_model.sql index f31a12d..9d9c40d 100644 --- a/models/example/my_first_dbt_model.sql +++ b/models/example/my_first_dbt_model.sql @@ -1,4 +1,3 @@ - /* Welcome to your first dbt model! Did you know that you can also configure models directly within SQL files? @@ -6,22 +5,27 @@ Try changing "table" to "view" below */ +{{ config(materialized="table") }} -{{ config(materialized='table') }} - -with source_data as ( +with + source_data as ( - select 1 as id - union all - select null as id + select 1 as id, 'Jaggi' as fname, 'Krishna' as lname + union all + select 2 as id, 'Todd' as fname, 'Graham' as lname + union all + select 3 as id, 'Penny' as fname, 'Davis' as lname + union all + select null as id, 'Kelly' as fname, 'Greer' as lname -) + ) select * -from source_data +from + source_data -/* + /* Uncomment the line below to remove records with null `id` values */ - --- where id is not null + -- where id is not null + diff --git a/models/example/my_second_dbt_model.sql b/models/example/my_second_dbt_model.sql index c91f879..311b6c2 100644 --- a/models/example/my_second_dbt_model.sql +++ b/models/example/my_second_dbt_model.sql @@ -1,6 +1,2 @@ - -- Use the `ref` function to select from other models - -select * -from {{ ref('my_first_dbt_model') }} -where id = 1 +select * from {{ ref("my_first_dbt_model") }} where id = 2 diff --git a/models/example/schema.yml b/models/example/schema.yml index 8dee309..8a8431c 100644 --- a/models/example/schema.yml +++ b/models/example/schema.yml @@ -1,21 +1,39 @@ - version: 2 models: - - name: my_first_dbt_model - description: "A starter dbt model" - columns: - - name: id - description: "The primary key for this table" - tests: - - unique - - not_null + - name: my_first_dbt_model + description: "A starter dbt model" + columns: + - name: id + description: "The primary key for this table" + tests: + - unique + - not_null + + - name: FName + description: "The First Name of the Student" + tests: + - not_null + + - name: LName + description: "The Last Name of the Student" + tests: + - not_null + + - name: my_second_dbt_model + description: "A starter dbt model" + columns: + - name: id + description: "The primary key for this table" + tests: + - unique + - not_null + - name: FName + description: "The First Name of the Student" + tests: + - not_null - - name: my_second_dbt_model - description: "A starter dbt model" - columns: - - name: id - description: "The primary key for this table" - tests: - - unique - - not_null + - name: LName + description: "The Last Name of the Student" + tests: + - not_null diff --git a/models/example/source_data_view.sql b/models/example/source_data_view.sql new file mode 100644 index 0000000..4444bf7 --- /dev/null +++ b/models/example/source_data_view.sql @@ -0,0 +1,24 @@ +{{ config(materialized="view") }} + +with + source_data_view as ( + + select 1 as id, 'Jaggi' as fname, 'Krishna' as lname + union all + select 2 as id, 'Todd' as fname, 'Graham' as lname + union all + select 3 as id, 'Penny' as fname, 'Davis' as lname + union all + select 4 as id, 'John' as fname, 'Kiwa' as lname + union all + select null as id, 'Kelly' as fname, 'Greer' as lname + + ) + +select * +from source_data_view + +/* + Uncomment the line below to remove records with null `id` values +*/ +where id is not null diff --git a/models/marts/finance/fct_orders.sql b/models/marts/finance/fct_orders.sql new file mode 100644 index 0000000..8d22ce3 --- /dev/null +++ b/models/marts/finance/fct_orders.sql @@ -0,0 +1,31 @@ +with orders as ( + select * from {{ ref ('stg_jaffle_shop__orders' )}} +), + +payments as ( + select * from {{ ref ('stg_stripe__payments') }} +), + +order_payments as ( + select + order_id, + sum (case when status = 'success' then amount end) as amount + + from payments + group by 1 +), + + + final as ( + + select + orders.order_id, + orders.customer_id, + orders.order_date, + coalesce (order_payments.amount, 0) as amount + + from orders + left join order_payments using (order_id) +) + +select * from final \ No newline at end of file diff --git a/models/marts/marketing/dim_customers.sql b/models/marts/marketing/dim_customers.sql new file mode 100644 index 0000000..9f15ea3 --- /dev/null +++ b/models/marts/marketing/dim_customers.sql @@ -0,0 +1,45 @@ +with customers as ( + + select * from {{ ref('stg_jaffle_shop__customers') }} + +), + +orders as ( + + select * from {{ ref('stg_jaffle_shop__orders') }} + +), + +customer_orders as ( + + select + customer_id, + + min(order_date) as first_order_date, + max(order_date) as most_recent_order_date, + count(order_id) as number_of_orders + + from orders + + group by 1 + +), + +final as ( + + select + customers.customer_id, + customers.first_name, + customers.last_name, + customer_orders.first_order_date, + customer_orders.most_recent_order_date, + coalesce (customer_orders.number_of_orders, 0) + as number_of_orders + + from customers + + left join customer_orders using (customer_id) + +) + +select * from final \ No newline at end of file diff --git a/models/staging/jaffle_shop/jaffle_shop.md b/models/staging/jaffle_shop/jaffle_shop.md new file mode 100644 index 0000000..9d51151 --- /dev/null +++ b/models/staging/jaffle_shop/jaffle_shop.md @@ -0,0 +1,13 @@ +{% docs order_status %} + +One of the following values: + +| status | definition | +|----------------|--------------------------------------------------| +| placed | Order placed, not yet shipped | +| shipped | Order has been shipped, not yet been delivered | +| completed | Order has been received by customers | +| return pending | Customer indicated they want to return this item | +| returned | Item has been returned | + +{% enddocs %} diff --git a/models/staging/jaffle_shop/src_jaffle_shop.yml b/models/staging/jaffle_shop/src_jaffle_shop.yml new file mode 100644 index 0000000..448e954 --- /dev/null +++ b/models/staging/jaffle_shop/src_jaffle_shop.yml @@ -0,0 +1,25 @@ +version: 2 + +sources: + - name: jaffle_shop + description: A clone of a Postgre application database + database: raw + schema: jaffle_shop + tables: + - name: customers + description: This is raw customer data. + columns: + - name: id + description: Customer ID is the primary key + tests: + - unique + - not_null + + - name: orders + description: This is raw orders data + columns: + - name: id + description: Order ID is the Primary key + tests: + - unique + - not_null diff --git a/models/staging/jaffle_shop/stg_jaffle_shop.yml b/models/staging/jaffle_shop/stg_jaffle_shop.yml new file mode 100644 index 0000000..cb1fbe7 --- /dev/null +++ b/models/staging/jaffle_shop/stg_jaffle_shop.yml @@ -0,0 +1,33 @@ +version: 2 + +models: + - name: stg_jaffle_shop__customers + description: One Unique Customer per row + columns: + - name: customer_id + description: Primary Key + tests: + - unique + - not_null + + - name: stg_jaffle_shop__orders + description: One unique Order per row + columns: + - name: order_id + description: Primary Key + tests: + - unique + - not_null + - name: status + description: "{{ doc('order_status') }}" + tests: + - accepted_values: + values: + ["shipped", "completed", "return_pending", "returned", "placed"] + + - name: customer_id + description: Foreign key to stg_customers.customer_id. + tests: + - relationships: + to: ref('stg_jaffle_shop__customers') + field: customer_id \ No newline at end of file diff --git a/models/staging/jaffle_shop/stg_jaffle_shop__customers.sql b/models/staging/jaffle_shop/stg_jaffle_shop__customers.sql new file mode 100644 index 0000000..b614f35 --- /dev/null +++ b/models/staging/jaffle_shop/stg_jaffle_shop__customers.sql @@ -0,0 +1,16 @@ +/* +select id as customer_id, first_name, last_name +from DBT_TRAINING.DBT_JKAVURU.customers + +*/ + + +select + id as customer_id, + first_name, + last_name +from {{ source('jaffle_shop', 'customers')}} + +/* +from raw.jaffle_shop.customers +*/ \ No newline at end of file diff --git a/models/staging/jaffle_shop/stg_jaffle_shop__orders.sql b/models/staging/jaffle_shop/stg_jaffle_shop__orders.sql new file mode 100644 index 0000000..c034c06 --- /dev/null +++ b/models/staging/jaffle_shop/stg_jaffle_shop__orders.sql @@ -0,0 +1,21 @@ +select + id as order_id, + user_id as customer_id, + order_date, + status + +from {{ source('jaffle_shop', 'orders') }} +order by 1 + +/* raw.jaffle_shop.orders */ + +/* +select + id as order_id, + user_id as customer_id, + order_date, + status + +from DBT_TRAINING.DBT_JKAVURU.orders + +*/ \ No newline at end of file diff --git a/models/staging/stripe/src_stripe.yml b/models/staging/stripe/src_stripe.yml new file mode 100644 index 0000000..b021a35 --- /dev/null +++ b/models/staging/stripe/src_stripe.yml @@ -0,0 +1,12 @@ +version: 2 + +sources: + - name: stripe + database: raw + schema: stripe + tables: + - name: payment + loaded_at_field: _batched_at + freshness: + warn_after: { count: 24, period: hour } + error_after: { count: 1024, period: day } diff --git a/models/staging/stripe/stg_stripe__payments.sql b/models/staging/stripe/stg_stripe__payments.sql new file mode 100644 index 0000000..2cb3d0e --- /dev/null +++ b/models/staging/stripe/stg_stripe__payments.sql @@ -0,0 +1,15 @@ +select + id as payment_id, + orderid as order_id, + paymentmethod as payment_method, + status, + + -- amount is stored in cents, convert it to dollars + amount / 100 as amount, + created as created_at +from {{ source('stripe', 'payment') }} + +/* +from raw.stripe.payment + +*/ diff --git a/tests/assert_stg_payments_amount_is_postive.sql b/tests/assert_stg_payments_amount_is_postive.sql new file mode 100644 index 0000000..d4bbd85 --- /dev/null +++ b/tests/assert_stg_payments_amount_is_postive.sql @@ -0,0 +1,9 @@ +with payments as ( + select * from {{ ref ('stg_stripe__payments') }} +) + +select order_id, +sum(amount) as total_amount +from payments +group by order_id +having total_amount < 0