From e94e2d4bb31cc0cb884cc35b0aad1d2ba6aa3cb5 Mon Sep 17 00:00:00 2001 From: Luke McNinch Date: Tue, 23 Feb 2021 21:04:35 -0500 Subject: [PATCH 1/4] Add params keyword argument to _get method to enable use of URL parameters. --- Adafruit_IO/client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Adafruit_IO/client.py b/Adafruit_IO/client.py index 7ffe7d3..e51036c 100644 --- a/Adafruit_IO/client.py +++ b/Adafruit_IO/client.py @@ -111,10 +111,11 @@ def _handle_error(response): def _compose_url(self, path): return '{0}/api/{1}/{2}/{3}'.format(self.base_url, 'v2', self.username, path) - def _get(self, path): + def _get(self, path, params=None): response = requests.get(self._compose_url(path), headers=self._headers({'X-AIO-Key': self.key}), - proxies=self.proxies) + proxies=self.proxies, + params=params) self._handle_error(response) return response.json() From e9afd5b317efe3e890606d3d7ce2fc95d3f4866c Mon Sep 17 00:00:00 2001 From: Luke McNinch Date: Fri, 26 Feb 2021 09:51:14 -0500 Subject: [PATCH 2/4] Implement a "max_results" parameter for the data method, to allow for retrieval of more than 1000 data points. This implementation is subject to an existing bug in the pagination link header in the API that will break when and if that bug is fixed. --- Adafruit_IO/client.py | 58 ++++++++++++++++++++++++++++++++++++++----- docs/data.rst | 13 ++++++++++ tests/test_client.py | 8 +++--- 3 files changed, 69 insertions(+), 10 deletions(-) diff --git a/Adafruit_IO/client.py b/Adafruit_IO/client.py index e51036c..b7fe210 100644 --- a/Adafruit_IO/client.py +++ b/Adafruit_IO/client.py @@ -22,6 +22,9 @@ import json import platform import pkg_resources +import re +from urllib.parse import urlparse +from urllib.parse import parse_qs # import logging import requests @@ -29,6 +32,8 @@ from .errors import RequestError, ThrottlingError from .model import Data, Feed, Group +API_PAGE_LIMIT = 1000 + # set outgoing version, pulled from setup.py version = pkg_resources.require("Adafruit_IO")[0].version default_headers = { @@ -60,6 +65,9 @@ def __init__(self, username, key, proxies=None, base_url='https://io.adafruit.co # constructing the path. self.base_url = base_url.rstrip('/') + # Store the last response of a get or post + self._last_response = None + @staticmethod def to_red(data): """Hex color feed to red channel. @@ -116,6 +124,7 @@ def _get(self, path, params=None): headers=self._headers({'X-AIO-Key': self.key}), proxies=self.proxies, params=params) + self._last_response = response self._handle_error(response) return response.json() @@ -125,6 +134,7 @@ def _post(self, path, data): 'Content-Type': 'application/json'}), proxies=self.proxies, data=json.dumps(data)) + self._last_response = response self._handle_error(response) return response.json() @@ -133,6 +143,7 @@ def _delete(self, path): headers=self._headers({'X-AIO-Key': self.key, 'Content-Type': 'application/json'}), proxies=self.proxies) + self._last_response = response self._handle_error(response) # Data functionality. @@ -232,17 +243,52 @@ def receive_previous(self, feed): path = "feeds/{0}/data/previous".format(feed) return Data.from_dict(self._get(path)) - def data(self, feed, data_id=None): + def data(self, feed, data_id=None, max_results=API_PAGE_LIMIT): """Retrieve data from a feed. If data_id is not specified then all the data for the feed will be returned in an array. :param string feed: Name/Key/ID of Adafruit IO feed. :param string data_id: ID of the piece of data to delete. + :param int max_results: The maximum number of results to return. To + return all data, set to None. """ - if data_id is None: - path = "feeds/{0}/data".format(feed) - return list(map(Data.from_dict, self._get(path))) - path = "feeds/{0}/data/{1}".format(feed, data_id) - return Data.from_dict(self._get(path)) + if data_id: + path = "feeds/{0}/data/{1}".format(feed, data_id) + return Data.from_dict(self._get(path)) + + params = {'limit': max_results} if max_results else None + data = [] + path = "feeds/{0}/data".format(feed) + while True: + data.extend(list(map(Data.from_dict, self._get(path, + params=params)))) + nlink = self.get_next_link() + if not nlink: + break + # Parse the link for the query parameters + params = parse_qs(urlparse(nlink).query) + if max_results: + if len(data) >= max_results: + break + params['limit'] = max_results - len(data) + return data + + def get_next_link(self): + """Parse the `next` page URL in the pagination Link header. + + This is necessary because of a bug in the API's implementation of the + link header. If that bug is fixed, the link would be accesible by + response.links['next']['url'] and this method would be broken. + + :return: The url for the next page of data + :rtype: str + """ + if not self._last_response: + return + link_header = self._last_response.headers['link'] + res = re.search('rel="next", <(.+?)>', link_header) + if not res: + return + return res.groups()[0] def create_data(self, feed, data): """Create a new row of data in the specified feed. diff --git a/docs/data.rst b/docs/data.rst index ed76bf9..d162082 100644 --- a/docs/data.rst +++ b/docs/data.rst @@ -33,6 +33,19 @@ You can get all of the data for a feed by using the ``data(feed)`` method. The r for d in data: print('Data value: {0}'.format(d.value)) +By default, the maximum number of data points returned is 1000. This limit can be changed by using the max_results parameter. + +.. code-block:: python + + # Get less than the default number of data points + data = aio.data('Test', max_results=100) + + # Get more than the default number of data points + data = aio.data('Test', max_results=2000) + + # Get all of the points + data = aio.data('Test', max_results=None) + You can also get a specific value by ID by using the ``feeds(feed, data_id)`` method. This will return a single piece of feed data with the provided data ID if it exists in the feed. The returned object will be an instance of the Data class. diff --git a/tests/test_client.py b/tests/test_client.py index c037411..adfb646 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -22,8 +22,8 @@ class TestClient(base.IOTestCase): # If your IP isn't put on the list of non-throttled IPs, uncomment the # function below to waste time between tests to prevent throttling. - #def tearDown(self): - time.sleep(30.0) + def tearDown(self): + time.sleep(30.0) # Helper Methods def get_client(self): @@ -48,7 +48,7 @@ def ensure_group_deleted(self, client, group): def empty_feed(self, client, feed): # Remove all the data from a specified feed (but don't delete the feed). - data = client.data(feed) + data = client.data(feed, max_results=None) for d in data: client.delete(feed, d.id) @@ -138,7 +138,7 @@ def test_create_data(self): data = Data(value=42) result = aio.create_data('testfeed', data) self.assertEqual(int(result.value), 42) - + def test_location_data(self): """receive_location """ From 3d267e97df31a26642bb41e0bbefac3197061659 Mon Sep 17 00:00:00 2001 From: Luke McNinch Date: Thu, 15 Apr 2021 22:09:10 -0400 Subject: [PATCH 3/4] Eliminate while True: by querying the feed count if max_results is None (caller is requesting all data). --- Adafruit_IO/client.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Adafruit_IO/client.py b/Adafruit_IO/client.py index b7fe210..1b31720 100644 --- a/Adafruit_IO/client.py +++ b/Adafruit_IO/client.py @@ -251,6 +251,9 @@ def data(self, feed, data_id=None, max_results=API_PAGE_LIMIT): :param int max_results: The maximum number of results to return. To return all data, set to None. """ + if max_results is None: + res = self._get(f'feeds/{feed}/details') + max_results = res['details']['data']['count'] if data_id: path = "feeds/{0}/data/{1}".format(feed, data_id) return Data.from_dict(self._get(path)) @@ -258,7 +261,7 @@ def data(self, feed, data_id=None, max_results=API_PAGE_LIMIT): params = {'limit': max_results} if max_results else None data = [] path = "feeds/{0}/data".format(feed) - while True: + while len(data) < max_results: data.extend(list(map(Data.from_dict, self._get(path, params=params)))) nlink = self.get_next_link() @@ -267,8 +270,6 @@ def data(self, feed, data_id=None, max_results=API_PAGE_LIMIT): # Parse the link for the query parameters params = parse_qs(urlparse(nlink).query) if max_results: - if len(data) >= max_results: - break params['limit'] = max_results - len(data) return data From b53348282cde3951ea9dda5f69f01b0cde35fafb Mon Sep 17 00:00:00 2001 From: Luke McNinch Date: Mon, 17 Jan 2022 10:36:08 -0500 Subject: [PATCH 4/4] Lower default page limit from 1000 to 100. --- Adafruit_IO/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Adafruit_IO/client.py b/Adafruit_IO/client.py index cf46a03..08ffe6c 100644 --- a/Adafruit_IO/client.py +++ b/Adafruit_IO/client.py @@ -33,7 +33,7 @@ from .errors import RequestError, ThrottlingError from .model import Data, Feed, Group, Dashboard, Block, Layout -API_PAGE_LIMIT = 1000 +DEFAULT_PAGE_LIMIT = 100 # set outgoing version, pulled from setup.py version = pkg_resources.require("Adafruit_IO")[0].version @@ -254,7 +254,7 @@ def receive_previous(self, feed): path = "feeds/{0}/data/previous".format(feed) return Data.from_dict(self._get(path)) - def data(self, feed, data_id=None, max_results=API_PAGE_LIMIT): + def data(self, feed, data_id=None, max_results=DEFAULT_PAGE_LIMIT): """Retrieve data from a feed. If data_id is not specified then all the data for the feed will be returned in an array. :param string feed: Name/Key/ID of Adafruit IO feed.