Merge pull request #1690 from bruin-data/cursor/core-concepts-docs-structure-4c27

Core concepts docs structure

Author: arsalann

docs/.vitepress/config.mjs

@@ -83,21 +83,36 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
                         ],
                     },
                     {
-                        text: "Features",
-                        link: "/getting-started/features",
+                        text: "Core Concepts",
+                        link: "/core-concepts/overview",
                         items: [
-                            {text: "Pipeline", link: "/getting-started/pipeline"},
-                            {text: "Glossary", link: "/getting-started/glossary"},
-                            {text: "Policies", link: "/getting-started/policies"},
-                            {text: "Environments", link: "/getting-started/devenv"},
-                            {text: "Variables", link: "/getting-started/pipeline-variables"},
-                            {text: "Concurrency", link: "/getting-started/concurrency"},
-                            {text: "Bruin MCP", link: "/getting-started/bruin-mcp"},
-                            {text: "Lakehouse Support", link: "/getting-started/lakehouse"},
-                        ]
+                            {
+                                text: "Environments",
+                                link: "/core-concepts/environments",
+                                items: [
+                                    {text: "Connections", link: "/core-concepts/connections"},
+                                    {text: "Secrets", link: "/core-concepts/secrets"},
+                                ],
+                            },
+                            {text: "Pipeline", link: "/pipelines/definition"},
+                            {text: "Asset", link: "/assets/definition-schema"},
+                            {
+                                text: "Variables",
+                                link: "/core-concepts/variables",
+                                items: [
+                                    {text: "Built-in Variables", link: "/core-concepts/variables#built-in-variables"},
+                                    {text: "Custom Variables", link: "/core-concepts/variables#custom-variables"},
+                                ],
+                            },
+                            {text: "Commands", link: "/commands/overview"},
+                        ],
                     },
-                    {text: "Concepts", link: "/getting-started/concepts"},
                     {text: "Design Principles", link: "/getting-started/design-principles"},
+                    {text: "Glossary", link: "/getting-started/glossary"},
+                    {text: "Policies", link: "/getting-started/policies"},
+                    {text: "Concurrency", link: "/getting-started/concurrency"},
+                    {text: "Bruin MCP", link: "/getting-started/bruin-mcp"},
+                    {text: "Lakehouse Support", link: "/getting-started/lakehouse"},
 
                     {
                         text: "Templates",
@@ -180,6 +195,16 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
                     {text: "GCP Dataproc Serverless", link: "/platforms/dataproc_serverless"},
                 ],
             },
+            {
+                text: "Pipelines",
+                collapsed: false,
+                items: [
+                    {text: "Definition", link: "/pipelines/definition"},
+                    {text: "Scheduling", link: "/pipelines/definition#schedule"},
+                    {text: "Default Connections", link: "/pipelines/definition#default-connections"},
+                    {text: "Pipeline Defaults", link: "/pipelines/definition#default-pipeline-level-defaults"},
+                ],
+            },
             {
                 text: "Assets",
                 collapsed: false,
@@ -199,7 +224,6 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
                         ]
                     },
                     {text: "Columns", link: "/assets/columns"},
-                    {text: "Credentials", link: "/getting-started/credentials"},
                     {text: "Interval Modifiers", link: "/assets/interval-modifiers"},
                     {text: "Materialization", link: "/assets/materialization"},
                     {
@@ -363,9 +387,12 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
             },
             {
                 text: "Commands",
+                link: "/commands/overview",
                 collapsed: false,
                 items: [
-                    {text: "AI Enhance", link: "/commands/ai-enhance"},
+                    {text: "Overview", link: "/commands/overview"},
+                    {text: "Run", link: "/commands/run"},
+                    {text: "Validate", link: "/commands/validate"},
                     {text: "Init", link: "/commands/init"},
                     {text: "Clean", link: "/commands/clean"},
                     {text: "Connections", link: "/commands/connections"},
@@ -376,9 +403,8 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
                     {text: "Lineage", link: "/commands/lineage"},
                     {text: "Patch", link: "/commands/patch"},
                     {text: "Render", link: "/commands/render"},
-                    {text: "Run", link: "/commands/run"},
                     {text: "Query", link: "/commands/query"},
-                    {text: "Validate", link: "/commands/validate"},
+                    {text: "AI Enhance", link: "/commands/ai-enhance"},
                 ],
 
             },

docs/assets/definition-schema.md

@@ -179,7 +179,7 @@ hooks:
 
 Hooks are currently supported for SQL assets. Each hook entry supports a single `query` field and is executed in order. Queries may have a trailing `;` or not.
 
-Hooks can also be set as pipeline defaults (see [pipeline defaults](/getting-started/pipeline#default-pipeline-level-defaults)). Assets inherit default `pre` and `post` hooks independently - defining only `pre` hooks on an asset will still inherit default `post` hooks.
+Hooks can also be set as pipeline defaults (see [pipeline defaults](/pipelines/definition#default-pipeline-level-defaults)). Assets inherit default `pre` and `post` hooks independently - defining only `pre` hooks on an asset will still inherit default `post` hooks.
 
 - **Type:** `Object`
 

docs/assets/python.md

@@ -156,7 +156,7 @@ The following environment variables are available in every Python asset executio
 
 ### Pipeline
 
-Bruin supports user-defined variables at a pipeline level. These become available as a JSON document in your python asset as `BRUIN_VARS`. When no variables exist, `BRUIN_VARS` is set to `{}`. See [pipeline variables](/getting-started/pipeline-variables) for more information on how to define and override them, including the [full list of JSON Schema `type` options and complementary keywords](/getting-started/pipeline-variables#supported-json-schema-keywords).
+Bruin supports user-defined variables at a pipeline level. These become available as a JSON document in your python asset as `BRUIN_VARS`. When no variables exist, `BRUIN_VARS` is set to `{}`. See [Variables](/core-concepts/variables) for more information on how to define and override them, including the [full list of JSON Schema `type` options and complementary keywords](/core-concepts/variables#custom-variables).
 
 Here's a short example:
 ::: code-group

docs/assets/r.md

@@ -173,7 +173,7 @@ The following environment variables are available in every R asset execution:
 
 ### Pipeline
 
-Bruin supports user-defined variables at a pipeline level. These become available as a JSON document in your R asset as `BRUIN_VARS`. When no variables exist, `BRUIN_VARS` is set to `{}`. See [pipeline variables](/getting-started/pipeline-variables) for more information on how to define and override them.
+Bruin supports user-defined variables at a pipeline level. These become available as a JSON document in your R asset as `BRUIN_VARS`. When no variables exist, `BRUIN_VARS` is set to `{}`. See [Variables](/core-concepts/variables) for more information on how to define and override them.
 
 Here's an example:
 

docs/assets/templating/macros.md

@@ -465,7 +465,7 @@ AND region = '{{ region }}'
 :::
 
 > [!TIP]
-> The pipeline snippet above showcases array enums and numeric bounds. For additional JSON Schema keywords you can mix into macros-driven workflows—including nested objects and nullable values—refer to the [pipeline variables keyword reference](/getting-started/pipeline-variables#supported-json-schema-keywords).
+> The pipeline snippet above showcases array enums and numeric bounds. For additional JSON Schema keywords you can mix into macros-driven workflows—including nested objects and nullable values—refer to the [Variables keyword reference](/core-concepts/variables#custom-variables).
 
 ### Dynamic Column Generation
 
@@ -715,5 +715,5 @@ SELECT 1
 
 - [Templating](./templating.md) - Learn about Jinja templating basics
 - [Filters](./filters.md) - Use filters to transform variables
-- [Pipeline Variables](/getting-started/pipeline-variables) - Define custom variables
+- [Variables](/core-concepts/variables) - Define custom variables
 - [SQL Assets](../sql.md) - Learn about SQL assets in Bruin

docs/assets/templating/templating.md

@@ -40,7 +40,7 @@ GROUP BY 1,2
 :::
 
 > [!TIP]
-> Need enumerations, numeric bounds, or nested structures for your variables? Consult the [JSON Schema keyword reference](/getting-started/pipeline-variables#supported-json-schema-keywords) for the full list of `type` values and examples of arrays-of-objects and object-of-arrays patterns you can reuse in templated SQL.
+> Need enumerations, numeric bounds, or nested structures for your variables? Consult the [Variables documentation](/core-concepts/variables#custom-variables) for the full list of `type` values and examples of arrays-of-objects and object-of-arrays patterns you can reuse in templated SQL.
 
 This will render into the following SQL query:
 
@@ -76,7 +76,7 @@ Bruin injects various variables by default:
 | `execution_datetime` | The execution date and time in YYYY-MM-DDThh:mm:ss format | "2023-12-01T15:30:00" |
 | `execution_timestamp` | The execution timestamp in [RFC3339](https://datatracker.ietf.org/doc/html/rfc3339) format | "2023-12-01T15:30:00.000000Z07:00" |
 | `pipeline` | The name of the currently executing pipeline | `my_pipeline` |
-| `run_id` | The unique identifier for the current [pipeline run](../../getting-started/concepts.md#pipeline-run) | `run_1234567890` |
+| `run_id` | The unique identifier for the current pipeline run | `run_1234567890` |
 | `full_refresh` | Boolean indicating whether the `--full-refresh` flag was used | `True` or `False` |
 
 You can use these variables in your SQL queries by referencing them with the `{{ }}` syntax:
@@ -182,4 +182,4 @@ You can read more about [Jinja conditionals](https://jinja.palletsprojects.com/e
 
 ## Custom variables
 
-You can define your own variables and use them across your Assets. See [variables](/getting-started/pipeline-variables) for more information.
+You can define your own variables and use them across your Assets. See [Variables](/core-concepts/variables) for more information.

docs/commands/connections.md

@@ -47,7 +47,7 @@ bruin connections add --env someother --type generic --name MY_SECRET --credenti
 
 This will add the connection to the `.bruin.yml` file and the connection will be available in the given environment.
 
-The parameter after `--credentials` is the value of the connection in JSON format, as you would write it in the `.bruin.yml` file. For further reference, you can check the [Connections section](../getting-started/concepts.md#connection) of the documentation.
+The parameter after `--credentials` is the value of the connection in JSON format, as you would write it in the `.bruin.yml` file. For further reference, you can check the [Connections](/core-concepts/connections) documentation.
 
 ### Flags
 

docs/commands/overview.md

@@ -0,0 +1,155 @@
+# Commands Overview
+
+Bruin provides a comprehensive CLI for managing your data pipelines. Commands can be executed in multiple ways:
+
+- **Terminal**: Direct CLI usage via `bruin <command>`
+- **VS Code Extension**: Visual interface with integrated command execution
+- **AI Agents**: Programmatic access via [Bruin MCP](/getting-started/bruin-mcp)
+
+## Getting Help
+
+```bash
+# List all available commands
+bruin --help
+
+# Get help for a specific command
+bruin run --help
+bruin validate --help
+```
+
+## Command Reference
+
+### Pipeline Execution
+
+| Command | Description |
+|---------|-------------|
+| [`run`](/commands/run) | Execute pipelines or individual assets |
+| [`validate`](/commands/validate) | Check pipeline configuration and syntax without executing |
+
+### Project Management
+
+| Command | Description |
+|---------|-------------|
+| [`init`](/commands/init) | Create a new Bruin project from a template |
+| [`clean`](/commands/clean) | Remove temporary files and build artifacts |
+| [`format`](/commands/format) | Format asset files for consistency |
+
+### Connections & Environments
+
+| Command | Description |
+|---------|-------------|
+| [`connections`](/commands/connections) | List, add, delete, and test connections |
+| [`environments`](/commands/environments) | Manage deployment environments |
+
+### Development & Debugging
+
+| Command | Description |
+|---------|-------------|
+| [`render`](/commands/render) | Preview rendered Jinja templates |
+| [`lineage`](/commands/lineage) | Visualize asset dependencies |
+| [`query`](/commands/query) | Execute ad-hoc queries against connections |
+| [`data-diff`](/commands/data-diff) | Compare data between connections |
+
+### Asset Operations
+
+| Command | Description |
+|---------|-------------|
+| [`import`](/commands/import) | Import existing database tables as Bruin assets |
+| [`patch`](/commands/patch) | Apply patches to asset definitions |
+| [`ai-enhance`](/commands/ai-enhance) | Enhance asset metadata using AI |
+
+## Common Workflows
+
+### Running a Pipeline
+
+```bash
+# Run the pipeline in the current directory
+bruin run
+
+# Run a specific pipeline
+bruin run ./pipelines/analytics/
+
+# Run a specific asset
+bruin run ./pipelines/analytics/assets/daily_summary.sql
+
+# Run with a specific environment
+bruin run --environment production
+
+# Run for a specific date range
+bruin run --start-date 2024-01-01 --end-date 2024-01-31
+```
+
+### Validating Before Running
+
+```bash
+# Validate pipeline syntax and configuration
+bruin validate
+
+# Validate a specific pipeline
+bruin validate ./pipelines/analytics/
+```
+
+### Creating a New Project
+
+```bash
+# Initialize with the default template
+bruin init default my-project
+
+# Initialize with a specific template
+bruin init chess my-chess-project
+```
+
+### Testing Connections
+
+```bash
+# List all configured connections
+bruin connections list
+
+# Test a specific connection
+bruin connections ping my-postgres-connection
+```
+
+### Debugging Templates
+
+```bash
+# Render a SQL asset to see the final query
+bruin render ./assets/my_query.sql
+
+# Render with specific date parameters
+bruin render ./assets/my_query.sql --start-date 2024-01-01 --end-date 2024-01-02
+```
+
+## Global Flags
+
+These flags work with most commands:
+
+| Flag | Description |
+|------|-------------|
+| `--debug` | Enable debug logging |
+| `--environment` | Specify the environment to use |
+| `--config-file` | Path to a custom `.bruin.yml` file |
+| `--help` | Show help for the command |
+
+## VS Code Extension
+
+The [Bruin VS Code Extension](/vscode-extension/overview) provides a visual interface for many CLI commands:
+
+- Run pipelines and assets with a single click
+- View real-time execution logs
+- Explore asset lineage graphically
+- Preview rendered queries
+
+## Bruin MCP (AI Agent Integration)
+
+[Bruin MCP](/getting-started/bruin-mcp) enables AI agents and tools to interact with Bruin programmatically. This allows:
+
+- Natural language pipeline execution
+- Automated pipeline management
+- Integration with AI-powered development workflows
+
+## Related Topics
+
+- [Run Command](/commands/run) - Detailed execution options
+- [Validate Command](/commands/validate) - Pipeline validation
+- [VS Code Extension](/vscode-extension/overview) - Visual interface
+- [Bruin MCP](/getting-started/bruin-mcp) - AI agent integration

docs/commands/run.md

@@ -55,7 +55,7 @@ table td:first-child {
 | `--no-timestamp` | bool | `false` | Skip logging timestamps for this run. |
 | `--no-color` | bool | `false` | Plain log output for this run. |
 | `--minimal-logs` | bool | `false` | Skip initial pipeline analysis logs for this run. |
-| `--var` | []str | - | Override pipeline [variables](/getting-started/pipeline-variables.md) with custom values. |
+| `--var` | []str | - | Override pipeline [variables](/core-concepts/variables) with custom values. |
 | `--query-annotations` | str | - | Add annotations to SQL queries as comments. Use `default` to add asset name, pipeline name, and execution step, or provide custom JSON for additional fields. **BigQuery only.** |
 
 ### Continue from the last failed asset