From d0bdbc0a5943b5cfab547d7f8375274e92f5d340 Mon Sep 17 00:00:00 2001 From: Mark Jackson Date: Thu, 2 Mar 2023 17:46:54 -0700 Subject: [PATCH 1/5] adding adapter for spectral --- .gitignore | 1 + huma_signals/adapters/registry.py | 2 + huma_signals/adapters/spectral/README.md | 30 +++++++++ huma_signals/adapters/spectral/__init__.py | 0 huma_signals/adapters/spectral/adapter.py | 20 ++++++ .../adapters/spectral/spectral_client.py | 66 +++++++++++++++++++ huma_signals/settings.py | 3 + 7 files changed, 122 insertions(+) create mode 100644 huma_signals/adapters/spectral/README.md create mode 100644 huma_signals/adapters/spectral/__init__.py create mode 100644 huma_signals/adapters/spectral/adapter.py create mode 100644 huma_signals/adapters/spectral/spectral_client.py diff --git a/.gitignore b/.gitignore index 5a40c04..ca2ff34 100644 --- a/.gitignore +++ b/.gitignore @@ -131,3 +131,4 @@ dmypy.json # env files *.env *.envrc +.idea/ diff --git a/huma_signals/adapters/registry.py b/huma_signals/adapters/registry.py index e3d6754..4eb9662 100644 --- a/huma_signals/adapters/registry.py +++ b/huma_signals/adapters/registry.py @@ -6,6 +6,7 @@ from huma_signals.adapters.lending_pools import adapter as lending_pools_adapter from huma_signals.adapters.polygon_wallet import adapter as polygon_wallet_adapter from huma_signals.adapters.request_network import adapter as request_network_adapter +from huma_signals.adapters.spectral import adapter as spectral_adapter ADAPTER_REGISTRY: Dict[str, Type[models.SignalAdapterBase]] = { lending_pools_adapter.LendingPoolAdapter.name: lending_pools_adapter.LendingPoolAdapter, @@ -13,6 +14,7 @@ allowlist_adapter.AllowListAdapter.name: allowlist_adapter.AllowListAdapter, ethereum_wallet_adapter.EthereumWalletAdapter.name: ethereum_wallet_adapter.EthereumWalletAdapter, polygon_wallet_adapter.PolygonWalletAdapter.name: polygon_wallet_adapter.PolygonWalletAdapter, + spectral_adapter.SpectralWalletAdapter.name: spectral_adapter.SpectralWalletAdapter, } diff --git a/huma_signals/adapters/spectral/README.md b/huma_signals/adapters/spectral/README.md new file mode 100644 index 0000000..b727294 --- /dev/null +++ b/huma_signals/adapters/spectral/README.md @@ -0,0 +1,30 @@ +# Polygon Wallet Signal Adapter + +This is the repository for the Signal Adapter that fetch signal about Polygon addresses. + +## Type of signals + +- Address' tenure +- Address' number of transactions +- Address' current and historical balance +- ... + +## Local Development + +See [here](../../../docs/getting_started.md) for the development guide. + +## Required environment variable + +The following environment variable is required to run the adapter. +(Sign up at https://polygonscan.com/myapikey to get an API key) + +```bash +POLYGONSCAN_BASE_URL +POLYGONSCAN_API_KEY +``` + +## Tests + +```bash +make test +``` diff --git a/huma_signals/adapters/spectral/__init__.py b/huma_signals/adapters/spectral/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/huma_signals/adapters/spectral/adapter.py b/huma_signals/adapters/spectral/adapter.py new file mode 100644 index 0000000..50649aa --- /dev/null +++ b/huma_signals/adapters/spectral/adapter.py @@ -0,0 +1,20 @@ +import structlog +from typing import Any, ClassVar, List + +from huma_signals.adapters import models as adapter_models +from huma_signals.adapters.spectral.spectral_client import SpectralClient, SpectralWalletSignals + +logger = structlog.get_logger() + + +class SpectralWalletAdapter(adapter_models.SignalAdapterBase): + name: ClassVar[str] = "spectral_wallet" + required_inputs: ClassVar[List[str]] = ["borrower_wallet_address"] + signals: ClassVar[List[str]] = list(SpectralWalletSignals.__fields__.keys()) + + spectral_client: SpectralClient = SpectralClient() + + async def fetch( # pylint: disable=arguments-differ + self, borrower_wallet_address: str, *args: Any, **kwargs: Any + ) -> SpectralWalletSignals: + return await self.spectral_client.get_scores(borrower_wallet_address) diff --git a/huma_signals/adapters/spectral/spectral_client.py b/huma_signals/adapters/spectral/spectral_client.py new file mode 100644 index 0000000..7147182 --- /dev/null +++ b/huma_signals/adapters/spectral/spectral_client.py @@ -0,0 +1,66 @@ +from datetime import datetime + +from huma_signals import models +from huma_signals.settings import settings +import pydantic +import httpx +import structlog + +logger = structlog.get_logger() + + +class SpectralScoreIngredients(models.HumaBaseModel): + credit_mix: int + defi_actions: int + health_and_risk: int + liquidation: int + market: float + time: int + wallet: int + + +class SpectralWalletSignals(models.HumaBaseModel): + score: float + score_ingredients: SpectralScoreIngredients + score_timestamp: datetime + total_sent: int + total_received: int + wallet_tenure_in_days: int + probability_of_liquidation: float + risk_level: str + wallet_address: str + + +class SpectralClient(models.HumaBaseModel): + """Spectral Client""" + base_url: str = pydantic.Field(default='https://api.spectral.finance') + api_key: str = pydantic.Field( + default=settings.spectral_api_key, + description="Ethereum private key of the Spectral client" + ) + + @pydantic.validator("base_url") + def validate_pbase_url(cls, value: str) -> str: + if not value: + raise ValueError("spectral base_url is required") + return value + + @pydantic.validator("api_key") + def validate_api_key(cls, value: str) -> str: + if not value: + raise ValueError("spectral api_key is required") + return value + + async def get_scores(self, wallet_address: str) -> SpectralWalletSignals: + try: + async with httpx.AsyncClient(base_url=self.base_url) as client: + request = ( + f"/api/v1/addresses/{wallet_address}" + f"/calculate_score" + ) + resp = await client.get(request) + + resp.raise_for_status() + return SpectralWalletSignals.from_json(resp.json()) + except httpx.HTTPStatusError: + logger.error("Error fetching transactions", exc_info=True, request=request) diff --git a/huma_signals/settings.py b/huma_signals/settings.py index 7239a61..754650e 100644 --- a/huma_signals/settings.py +++ b/huma_signals/settings.py @@ -57,6 +57,9 @@ class Config: polygonscan_base_url: str = "https://api.polygonscan.com" polygonscan_api_key: str + # adapter: spectral + spectral_api_key: str + # adapter: request_network request_network_subgraph_endpoint_url: str request_network_invoice_api_url: str From c5f07a2fab851f9f89601728f2347af6adf5d375 Mon Sep 17 00:00:00 2001 From: Mark Jackson Date: Thu, 2 Mar 2023 22:23:59 -0700 Subject: [PATCH 2/5] have working spectral client --- .../adapters/spectral/spectral_client.py | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/huma_signals/adapters/spectral/spectral_client.py b/huma_signals/adapters/spectral/spectral_client.py index 7147182..063eafc 100644 --- a/huma_signals/adapters/spectral/spectral_client.py +++ b/huma_signals/adapters/spectral/spectral_client.py @@ -1,11 +1,12 @@ from datetime import datetime -from huma_signals import models -from huma_signals.settings import settings -import pydantic import httpx +import pydantic import structlog +from huma_signals import models +from huma_signals.settings import settings + logger = structlog.get_logger() @@ -23,9 +24,6 @@ class SpectralWalletSignals(models.HumaBaseModel): score: float score_ingredients: SpectralScoreIngredients score_timestamp: datetime - total_sent: int - total_received: int - wallet_tenure_in_days: int probability_of_liquidation: float risk_level: str wallet_address: str @@ -51,16 +49,27 @@ def validate_api_key(cls, value: str) -> str: raise ValueError("spectral api_key is required") return value - async def get_scores(self, wallet_address: str) -> SpectralWalletSignals: + async def _create_score(self, wallet_address: str) -> None: try: async with httpx.AsyncClient(base_url=self.base_url) as client: request = ( f"/api/v1/addresses/{wallet_address}" f"/calculate_score" ) - resp = await client.get(request) + headers = {"Authorization": f"Bearer {self.api_key}"} + resp = await client.post(request, headers=headers) + resp.raise_for_status() + except httpx.HTTPStatusError: + logger.error("Error fetching transactions", exc_info=True, request=request) + async def get_scores(self, wallet_address: str) -> SpectralWalletSignals: + await self._create_score(wallet_address) + try: + async with httpx.AsyncClient(base_url=self.base_url) as client: + request = f"/api/v1/addresses/{wallet_address}" + headers = {"Authorization": f"Bearer {self.api_key}"} + resp = await client.get(request, headers=headers) resp.raise_for_status() - return SpectralWalletSignals.from_json(resp.json()) + return SpectralWalletSignals(**resp.json()) except httpx.HTTPStatusError: logger.error("Error fetching transactions", exc_info=True, request=request) From 2a1f160ebc302f0867a2baa32ebeb31ba0caf266 Mon Sep 17 00:00:00 2001 From: Mark Jackson Date: Fri, 3 Mar 2023 07:45:37 -0700 Subject: [PATCH 3/5] fixing pre-commit issues --- .github/ISSUE_TEMPLATE.md | 2 +- .github/PULL_REQUEST_TEMPLATE.md | 2 +- .github/workflows/cd_develop.yaml | 4 ++-- .github/workflows/cd_new_repo.yaml | 2 +- .gitignore | 2 +- .pre-commit-config.yaml | 2 +- huma_signals/adapters/spectral/adapter.py | 21 ++++++++++++------- .../adapters/spectral/spectral_client.py | 17 +++++++-------- huma_signals/commons/web3_utils.py | 2 +- 9 files changed, 29 insertions(+), 25 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 002d1c4..cbd505f 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -25,4 +25,4 @@ https://plnkr.co or similar (you can use this template as a starting point: http -* **Other information** (e.g. detailed explanation, stacktraces, related issues, suggestions how to fix, links for us to have context, eg. stackoverflow, gitter, etc) \ No newline at end of file +* **Other information** (e.g. detailed explanation, stacktraces, related issues, suggestions how to fix, links for us to have context, eg. stackoverflow, gitter, etc) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index ada5359..4aa196e 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -20,4 +20,4 @@ -* **Other information**: \ No newline at end of file +* **Other information**: diff --git a/.github/workflows/cd_develop.yaml b/.github/workflows/cd_develop.yaml index bed50ab..a5ab412 100644 --- a/.github/workflows/cd_develop.yaml +++ b/.github/workflows/cd_develop.yaml @@ -3,7 +3,7 @@ name: Dev Huma Signals CD on: push: branches: [ "develop" ] - + workflow_dispatch: jobs: @@ -48,4 +48,4 @@ jobs: env: CLUSTER_NAME: dev-mainnet-huma-signals SERVICE_NAME: dev-matic-huma-signals - run: aws ecs update-service --cluster $CLUSTER_NAME --service $SERVICE_NAME --force-new-deployment \ No newline at end of file + run: aws ecs update-service --cluster $CLUSTER_NAME --service $SERVICE_NAME --force-new-deployment diff --git a/.github/workflows/cd_new_repo.yaml b/.github/workflows/cd_new_repo.yaml index c0fd1ac..521f830 100644 --- a/.github/workflows/cd_new_repo.yaml +++ b/.github/workflows/cd_new_repo.yaml @@ -3,7 +3,7 @@ name: Huma Decentralized Signal Portfolio CD on: push: branches: [ "main" ] - + workflow_dispatch: jobs: diff --git a/.gitignore b/.gitignore index ca2ff34..840162b 100644 --- a/.gitignore +++ b/.gitignore @@ -128,7 +128,7 @@ dmypy.json # Pyre type checker .pyre/ -# env files +# env files *.env *.envrc .idea/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a3b347b..a797fb3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,7 @@ repos: args: ['--target-version', 'py310'] types: [python] - repo: https://github.com/timothycrosley/isort - rev: 5.5.4 + rev: 5.12.0 hooks: - id: isort name: 'isort' diff --git a/huma_signals/adapters/spectral/adapter.py b/huma_signals/adapters/spectral/adapter.py index 50649aa..a057918 100644 --- a/huma_signals/adapters/spectral/adapter.py +++ b/huma_signals/adapters/spectral/adapter.py @@ -1,20 +1,25 @@ -import structlog from typing import Any, ClassVar, List -from huma_signals.adapters import models as adapter_models -from huma_signals.adapters.spectral.spectral_client import SpectralClient, SpectralWalletSignals +import structlog + +from huma_signals.adapters import models +from huma_signals.adapters.spectral import spectral_client logger = structlog.get_logger() -class SpectralWalletAdapter(adapter_models.SignalAdapterBase): +class SpectralWalletAdapter(models.SignalAdapterBase): name: ClassVar[str] = "spectral_wallet" required_inputs: ClassVar[List[str]] = ["borrower_wallet_address"] - signals: ClassVar[List[str]] = list(SpectralWalletSignals.__fields__.keys()) + signals: ClassVar[List[str]] = list( + spectral_client.SpectralWalletSignals.__fields__.keys() + ) - spectral_client: SpectralClient = SpectralClient() + spectral_api_client: spectral_client.SpectralClient = ( + spectral_client.SpectralClient() + ) async def fetch( # pylint: disable=arguments-differ self, borrower_wallet_address: str, *args: Any, **kwargs: Any - ) -> SpectralWalletSignals: - return await self.spectral_client.get_scores(borrower_wallet_address) + ) -> spectral_client.SpectralWalletSignals: + return await self.spectral_api_client.get_scores(borrower_wallet_address) diff --git a/huma_signals/adapters/spectral/spectral_client.py b/huma_signals/adapters/spectral/spectral_client.py index 063eafc..ce33bf6 100644 --- a/huma_signals/adapters/spectral/spectral_client.py +++ b/huma_signals/adapters/spectral/spectral_client.py @@ -1,4 +1,4 @@ -from datetime import datetime +import datetime import httpx import pydantic @@ -23,7 +23,7 @@ class SpectralScoreIngredients(models.HumaBaseModel): class SpectralWalletSignals(models.HumaBaseModel): score: float score_ingredients: SpectralScoreIngredients - score_timestamp: datetime + score_timestamp: datetime.datetime probability_of_liquidation: float risk_level: str wallet_address: str @@ -31,10 +31,11 @@ class SpectralWalletSignals(models.HumaBaseModel): class SpectralClient(models.HumaBaseModel): """Spectral Client""" - base_url: str = pydantic.Field(default='https://api.spectral.finance') + + base_url: str = pydantic.Field(default="https://api.spectral.finance") api_key: str = pydantic.Field( default=settings.spectral_api_key, - description="Ethereum private key of the Spectral client" + description="Ethereum private key of the Spectral client", ) @pydantic.validator("base_url") @@ -52,10 +53,7 @@ def validate_api_key(cls, value: str) -> str: async def _create_score(self, wallet_address: str) -> None: try: async with httpx.AsyncClient(base_url=self.base_url) as client: - request = ( - f"/api/v1/addresses/{wallet_address}" - f"/calculate_score" - ) + request = f"/api/v1/addresses/{wallet_address}" f"/calculate_score" headers = {"Authorization": f"Bearer {self.api_key}"} resp = await client.post(request, headers=headers) resp.raise_for_status() @@ -71,5 +69,6 @@ async def get_scores(self, wallet_address: str) -> SpectralWalletSignals: resp = await client.get(request, headers=headers) resp.raise_for_status() return SpectralWalletSignals(**resp.json()) - except httpx.HTTPStatusError: + except httpx.HTTPStatusError as e: logger.error("Error fetching transactions", exc_info=True, request=request) + raise e diff --git a/huma_signals/commons/web3_utils.py b/huma_signals/commons/web3_utils.py index 2dbada2..ca7fde1 100644 --- a/huma_signals/commons/web3_utils.py +++ b/huma_signals/commons/web3_utils.py @@ -29,7 +29,7 @@ async def get_w3(chain: chains.Chain, web3_provider_url: str) -> web3.Web3: w3 = web3.Web3( - provider=web3.Web3.AsyncHTTPProvider(web3_provider_url), modules=_MODULES + provider=web3.Web3.AsyncHTTPProvider(web3_provider_url), modules=_MODULES # type: ignore ) if await w3.net.version != _WEB3_CHAIN_NETWORK_ID.get(chain): # type: ignore raise ValueError(f"Web3 provider is not compatible with chain {chain.name}") From 17d1a0a81438c33e14b7259c542a3674c40a95f6 Mon Sep 17 00:00:00 2001 From: Mark Jackson Date: Fri, 3 Mar 2023 08:00:08 -0700 Subject: [PATCH 4/5] updating mypy errors and missing env --- docker-compose-ci.yml | 1 + huma_signals/commons/web3_utils.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-compose-ci.yml b/docker-compose-ci.yml index 06ff567..9757b58 100644 --- a/docker-compose-ci.yml +++ b/docker-compose-ci.yml @@ -30,3 +30,4 @@ services: - POLYGONSCAN_BASE_URL=https://api.polygonscan.com - REQUEST_NETWORK_SUBGRAPH_ENDPOINT_URL=https://api.thegraph.com/subgraphs/name/requestnetwork/request-payments-goerli - REQUEST_NETWORK_INVOICE_API_URL=https://goerli.api.huma.finance/invoice + - SPECTRAL_API_KEY diff --git a/huma_signals/commons/web3_utils.py b/huma_signals/commons/web3_utils.py index ca7fde1..2dbada2 100644 --- a/huma_signals/commons/web3_utils.py +++ b/huma_signals/commons/web3_utils.py @@ -29,7 +29,7 @@ async def get_w3(chain: chains.Chain, web3_provider_url: str) -> web3.Web3: w3 = web3.Web3( - provider=web3.Web3.AsyncHTTPProvider(web3_provider_url), modules=_MODULES # type: ignore + provider=web3.Web3.AsyncHTTPProvider(web3_provider_url), modules=_MODULES ) if await w3.net.version != _WEB3_CHAIN_NETWORK_ID.get(chain): # type: ignore raise ValueError(f"Web3 provider is not compatible with chain {chain.name}") From 278c8616a230fcc4401758068ccee966ca23eba9 Mon Sep 17 00:00:00 2001 From: Mark Jackson Date: Fri, 3 Mar 2023 08:10:47 -0700 Subject: [PATCH 5/5] trying to remove yaml file updates --- .github/ISSUE_TEMPLATE.md | 2 +- .github/PULL_REQUEST_TEMPLATE.md | 2 +- .github/workflows/cd_develop.yaml | 4 ++-- .github/workflows/cd_new_repo.yaml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index cbd505f..002d1c4 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -25,4 +25,4 @@ https://plnkr.co or similar (you can use this template as a starting point: http -* **Other information** (e.g. detailed explanation, stacktraces, related issues, suggestions how to fix, links for us to have context, eg. stackoverflow, gitter, etc) +* **Other information** (e.g. detailed explanation, stacktraces, related issues, suggestions how to fix, links for us to have context, eg. stackoverflow, gitter, etc) \ No newline at end of file diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 4aa196e..ada5359 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -20,4 +20,4 @@ -* **Other information**: +* **Other information**: \ No newline at end of file diff --git a/.github/workflows/cd_develop.yaml b/.github/workflows/cd_develop.yaml index a5ab412..bed50ab 100644 --- a/.github/workflows/cd_develop.yaml +++ b/.github/workflows/cd_develop.yaml @@ -3,7 +3,7 @@ name: Dev Huma Signals CD on: push: branches: [ "develop" ] - + workflow_dispatch: jobs: @@ -48,4 +48,4 @@ jobs: env: CLUSTER_NAME: dev-mainnet-huma-signals SERVICE_NAME: dev-matic-huma-signals - run: aws ecs update-service --cluster $CLUSTER_NAME --service $SERVICE_NAME --force-new-deployment + run: aws ecs update-service --cluster $CLUSTER_NAME --service $SERVICE_NAME --force-new-deployment \ No newline at end of file diff --git a/.github/workflows/cd_new_repo.yaml b/.github/workflows/cd_new_repo.yaml index 521f830..c0fd1ac 100644 --- a/.github/workflows/cd_new_repo.yaml +++ b/.github/workflows/cd_new_repo.yaml @@ -3,7 +3,7 @@ name: Huma Decentralized Signal Portfolio CD on: push: branches: [ "main" ] - + workflow_dispatch: jobs: