Merge remote-tracking branch 'origin' into feat/new_faqs

Author: MMenchero

.github/workflows/ci.yaml

@@ -83,7 +83,7 @@ jobs:
           cache-environment: true
 
       - name: Install pip requirements
-        run: pip install ./ 
+        run: pip install -e ".[dev]"
 
       - name: Run tests 
         run: nbdev_test --skip_file_glob "*distributed*"

.github/workflows/lint.yaml

@@ -0,0 +1,25 @@
+name: Lint
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Clone repo
+        uses: actions/checkout@v2
+
+      - name: Set up python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: pip install black nbdev pre-commit
+
+      - name: Run pre-commit
+        run: pre-commit run

.pre-commit-config.yaml

@@ -0,0 +1,6 @@
+repos:
+  - repo: https://github.com/fastai/nbdev
+    rev: 2.2.10
+    hooks:
+      - id: nbdev_clean
+      - id: nbdev_export

CHANGELOG.md

@@ -1,5 +1,32 @@
 # Changelog
 
+## 0.1.21
+
+### 🚀 Feature Enhancements
+
+#### Introduction of Quantile Forecasts in `forecast` and `cross_validation` Methods 📈
+
+We're thrilled to announce the integration of the `quantiles` argument into TimeGP's `forecast` and `cross_validation` methods. This feature allows users to specify a list of quantiles, offering a comprehensive view of potential future values under uncertainty.
+
+- **Quantile Forecasting Capability:**
+  By providing a list of quantiles, users can now obtain forecasts at various percentiles of the forecast distribution. This is crucial for understanding the range of possible outcomes and assessing risks more effectively.
+
+``` python
+# Generate quantile forecasts
+quantiles = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
+timegpt_quantile_fcst_df = timegpt.forecast(df=df, h=12, quantiles=quantiles, ...)
+```
+
+- **Enhanced Cross-Validation with Quantiles:**
+  The `cross_validation` method has been updated to support quantile forecasting, enabling a more nuanced validation of model performance across different percentiles.
+
+``` python
+# Apply quantile forecasting in cross-validation
+timegpt_cv_quantile_fcst_df = timegpt.cross_validation(df=df, h=12, n_windows=5, quantiles=quantiles, ...)
+```
+
+*See full changelog [here](https://github.com/Nixtla/nixtla/releases/v0.1.21).*
+
 ## 0.1.20
 
 ### 🚀 Feature Enhancements

CONTRIBUTING.md

@@ -59,6 +59,7 @@ Please write to `[email protected]` if you're insterested in contributing to this pr
 Before doing any changes to the code, please install the git hooks that run automatic scripts during each commit and merge to strip the notebooks of superfluous metadata (and avoid merge conflicts).
 ```
 nbdev_install_hooks
+pre-commit install
 ```
 
 ### Preview Changes

action_files/models_performance/main.py

@@ -315,6 +315,7 @@ def summary_performance(
             results_comb = ["metric"] + models
             exp_config = [col for col in eval_df.columns if col not in results_comb]
             eval_df = eval_df.fillna("None")
+            f.write("<details><summary>Experiment Results</summary>\n\n")
             for exp_number, (exp_desc, eval_exp_df) in enumerate(
                 eval_df.groupby(exp_config), start=1
             ):
@@ -344,6 +345,7 @@ def summary_performance(
                 if os.getenv("GITHUB_ACTIONS"):
                     plot_path = f"{os.getenv('PLOTS_REPO_URL')}/{plot_path}?raw=true"
                 f.write(f"![]({plot_path})\n\n")
+            f.write("</details>\n")
 
 
 if __name__ == "__main__":

experiments/azure-automl-forecasting/.env.example

@@ -0,0 +1,5 @@
+AZURE_SUBSCRIPTION_ID=
+AZURE_RESOURCE_GROUP=
+AZURE_WORKSPACE_NAME=
+TIMEGPT_TOKEN=
+

experiments/azure-automl-forecasting/Makefile

@@ -0,0 +1,39 @@
+TS_FILES := Hourly_H.parquet Daily_D.parquet Weekly_W-MON.parquet Monthly_MS.parquet 
+FILTERED_TS_FILES := $(patsubst %,./data/filtered_datasets/%,$(TS_FILES))
+
+filter_data:
+	@for file in $(TS_FILES); do \
+		python -m src.utils.filter_data --dataset_path ./data/$$file; \
+	done
+
+run_timegpt: .require-dataset_path
+	@echo Running TimeGPT with dataset_path=$(dataset_path)
+	@python -m src.nixtla_timegpt --dataset_path $(dataset_path)
+
+run_sn: .require-dataset_path
+	@echo Running SN with dataset_path=$(dataset_path)
+	@python -m src.statsforecast_sn --dataset_path $(dataset_path)
+
+run_automl: .require-dataset_path
+	@echo Running AutoML with dataset_path=$(dataset_path)
+	@python -m src.azure_automl.forecasting --dataset_path $(dataset_path)
+
+run_methods:
+	@for file in $(TS_FILES); do \
+		echo "Running methods for $$file"; \
+		$(MAKE) run_timegpt dataset_path=./data/filtered_datasets/$$file; \
+		$(MAKE) run_sn dataset_path=./data/filtered_datasets/$$file; \
+		$(MAKE) run_automl dataset_path=./data/filtered_datasets/$$file; \
+	done
+
+download_automl_forecasts:
+	@python -m src.azure_automl.download_forecasts
+
+evaluate_experiments:
+	@python -m src.evaluation --datasets_paths "$(shell echo $(FILTERED_TS_FILES) | tr ' ' ',')"
+
+.require-dataset_path:
+ifndef dataset_path
+	$(error dataset_path is required)
+endif
+

experiments/azure-automl-forecasting/README.md

@@ -0,0 +1,75 @@
+# Nixtla TimeGPT vs. Azure AutoML: A Comprehensive Performance Analysis
+
+This experiment evaluates the performance of **Nixtla TimeGPT's zero-shot inference** against **Microsoft's Azure AutoML** in the domain of time series forecasting. Our analysis shows that TimeGPT **surpasses Azure AutoML by 12%, 12%, and 10% in MAE, RMSE, and MASE metrics** and has **300x improvement in computational efficiency**. This evaluation spanned over 3,000 distinct time series across various data frequencies, with considerations for Azure AutoML's cost constraints.
+
+# Introduction
+
+[Azure AutoML](https://learn.microsoft.com/en-us/azure/machine-learning/concept-automl-forecasting-methods?view=azureml-api-2), a product of Microsoft, offers a robust automated machine-learning solution that caters to a wide array of predictive tasks, including time series forecasting. TimeGPT is a foundational model for time series forecasting that can be accessed [through an API](https://docs.nixtla.io/). While Azure AutoML is known for its adaptability and ease of use, our findings reveal that TimeGPT offers superior accuracy and efficiency, especially in the context of time series data.
+
+## Empirical Evaluation
+
+Our study involved a detailed comparison of both models across various datasets, including Hourly, Daily, Weekly, and Monthly data frequencies. The datasets were chosen from the test set of the [TimeGPT-1 paper](https://arxiv.org/abs/2310.03589), ensuring a diverse set of time series for evaluation. The selection process was designed to manage computational complexity and adhere to Azure AutoML's dataset size requirements, with a cap of 3,000 observations to maintain cost-effectiveness.
+
+## Results
+
+The following table shows the main findings of our analysis, presenting a comparison of performance metrics (MASE, MAE, RMSE) and computational time (in seconds) across different datasets. The best results are highlighted in **bold** for clarity.
+
+<img width="632" alt="image" src="https://github.com/Nixtla/nixtla/assets/10517170/0cc4285e-2572-4f08-9846-94c68ad72e8b">
+
+
+## Reproducibility
+
+All experiments were conducted in controlled environments to uphold the integrity and reproducibility of our results. TimeGPT evaluations were performed using a 2020 MacBook Air with an M1 chip, ensuring accessibility and practicality. In contrast, Azure AutoML experiments were carried out on a cluster of 11 STANDARD_DS5_V2 virtual machines equipped with substantial computational resources to showcase its scalability and power.
+
+### Instructions
+
+1. Configure Azure AutoML according to the official Microsoft documentation.
+2. Set the environment variables in a `.env` file using `.env.example` as example.
+3. Set up a conda environment using:
+
+```bash
+mamba create -n azure-automl-fcst python=3.10
+conda activate azure-automl-fcst
+pip install uv
+uv pip install -r requirements.txt
+```
+
+4. Download the data using
+
+```python
+python -m src.utils.download_data
+```
+
+If you're interested in replicating the results, write us at `[email protected]` to give you access to the data.
+
+5. Filter the datasets to prevent AzureML from crashing
+
+```
+make filter_data
+```
+
+6. Run the forecasting tasks for TimeGPT, SeasonalNaive, and AzureAutoML using the following:
+
+```
+make run_methods
+```
+
+Notice that AzureAutoML will send the job to the predefined cluster. 
+
+7. Retrieve AzureAutoML forecasts once they are ready:
+
+```
+make download_automl_forecasts
+```
+
+8. Run evaluation
+
+```
+make evaluate_experiments
+```
+
+
+### References
+- [TimeGPT 1](https://arxiv.org/abs/2310.03589)
+- [StatsForecast](https://github.com/Nixtla/statsforecast/)
+- [Distributed AzureAutoML for forecasting](https://github.com/Azure/azureml-examples/blob/main/sdk/python/jobs/pipelines/1k_demand_forecasting_with_pipeline_components/automl-forecasting-demand-many-models-in-pipeline/automl-forecasting-demand-many-models-in-pipeline.ipynb)

experiments/azure-automl-forecasting/requirements.txt

@@ -0,0 +1,11 @@
+azure-ai-ml
+azure-identity
+azureml-core
+fire
+mltable
+nixtlats
+pandas
+python-dotenv
+rich
+statsforecast
+utilsforecast