From 5eb952089edb0b95d444612f2e456358217aaf9e Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 13 Oct 2015 11:46:28 -0400 Subject: [PATCH 1/4] Fix copy-pasta variable name. Also, add a missing assertion for the case that no arguments are passed to 'Client.list_indexes'. --- gcloud/search/test_client.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/gcloud/search/test_client.py b/gcloud/search/test_client.py index 1692be82b7b3..9ba7c65e4d52 100644 --- a/gcloud/search/test_client.py +++ b/gcloud/search/test_client.py @@ -54,10 +54,10 @@ def test_list_indexes_defaults(self): client = self._makeOne(self.PROJECT, creds) conn = client.connection = _Connection(DATA) - zones, token = client.list_indexes() + indexes, token = client.list_indexes() - self.assertEqual(len(zones), len(DATA['indexes'])) - for found, expected in zip(zones, DATA['indexes']): + self.assertEqual(len(indexes), len(DATA['indexes'])) + for found, expected in zip(indexes, DATA['indexes']): self.assertTrue(isinstance(found, Index)) self.assertEqual(found.name, expected['indexId']) self.assertEqual(found.text_fields, None) @@ -72,6 +72,7 @@ def test_list_indexes_defaults(self): req = conn._requested[0] self.assertEqual(req['method'], 'GET') self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['query_params'], {}) def test_list_indexes_explicit(self): from gcloud.search.index import Index @@ -93,11 +94,11 @@ def test_list_indexes_explicit(self): client = self._makeOne(self.PROJECT, creds) conn = client.connection = _Connection(DATA) - zones, token = client.list_indexes( + indexes, token = client.list_indexes( max_results=3, page_token=TOKEN, prefix='index', view='FULL') - self.assertEqual(len(zones), len(DATA['indexes'])) - for found, expected in zip(zones, DATA['indexes']): + self.assertEqual(len(indexes), len(DATA['indexes'])) + for found, expected in zip(indexes, DATA['indexes']): self.assertTrue(isinstance(found, Index)) self.assertEqual(found.name, expected['indexId']) field_info = expected['indexedField'] From 4567ccaa7ac06261fbe87c61ccd5bffaa58e03d9 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 13 Oct 2015 12:32:51 -0400 Subject: [PATCH 2/4] Add 'Index.list_documents' API wrapper. --- gcloud/search/index.py | 49 +++++++++++ gcloud/search/test_index.py | 159 ++++++++++++++++++++++++++++++++++++ 2 files changed, 208 insertions(+) diff --git a/gcloud/search/index.py b/gcloud/search/index.py index 31c2950b7105..5f418fa2b188 100644 --- a/gcloud/search/index.py +++ b/gcloud/search/index.py @@ -14,6 +14,8 @@ """Define API Indexes.""" +from gcloud.search.document import Document + class Index(object): """Indexes are containers for documents. @@ -148,3 +150,50 @@ def _set_properties(self, api_response): """ self._properties.clear() self._properties.update(api_response) + + def list_documents(self, max_results=None, page_token=None, + view=None): + """List documents created within this index. + + See: + https://cloud.google.com/search/reference/rest/v1/projects/indexes/documents/list + + :type max_results: int + :param max_results: maximum number of zones to return, If not + passed, defaults to a value set by the API. + + :type page_token: string + :param page_token: opaque marker for the next "page" of zones. If + not passed, the API will return the first page of + zones. + + :type view: string + :param view: One of 'ID_ONLY' (return only the document ID; the + default) or 'FULL' (return the full resource + representation for the document, including field + values) + + :rtype: tuple, (list, str) + :returns: list of :class:`gcloud.dns.document.Document`, plus a + "next page token" string: if the token is not None, + indicates that more zones can be retrieved with another + call (pass that value as ``page_token``). + """ + params = {} + + if max_results is not None: + params['pageSize'] = max_results + + if page_token is not None: + params['pageToken'] = page_token + + if view is not None: + params['view'] = view + + path = '%s/documents' % (self.path,) + connection = self._client.connection + resp = connection.api_request(method='GET', path=path, + query_params=params) + zones = [Document.from_api_repr(resource, self) + for resource in resp['documents']] + return zones, resp.get('nextPageToken') diff --git a/gcloud/search/test_index.py b/gcloud/search/test_index.py index 26f36879d710..c6b4d7ee2d5f 100644 --- a/gcloud/search/test_index.py +++ b/gcloud/search/test_index.py @@ -105,9 +105,168 @@ def test_from_api_repr_w_properties(self): self.assertTrue(index._client is client) self._verifyResourceProperties(index, RESOURCE) + def test_list_documents_defaults(self): + from gcloud.search.document import Document + from gcloud.search.document import StringValue + DOCID_1 = 'docid-one' + RANK_1 = 2345 + TITLE_1 = 'Title One' + DOCID_2 = 'docid-two' + RANK_2 = 1234 + TITLE_2 = 'Title Two' + PATH = 'projects/%s/indexes/%s/documents' % ( + self.PROJECT, self.INDEX_ID) + TOKEN = 'TOKEN' + DOC_1 = { + 'docId': DOCID_1, + 'rank': RANK_1, + 'fields': { + 'title': { + 'values': [{ + 'stringValue': TITLE_1, + 'stringFormat': 'text', + 'lang': 'en'}] + } + } + } + DOC_2 = { + 'docId': DOCID_2, + 'rank': RANK_2, + 'fields': { + 'title': { + 'values': [{ + 'stringValue': TITLE_2, + 'stringFormat': 'text', + 'lang': 'en'}], + } + } + } + DATA = { + 'nextPageToken': TOKEN, + 'documents': [DOC_1, DOC_2], + } + client = _Client(self.PROJECT) + conn = client.connection = _Connection(DATA) + index = self._makeOne(self.INDEX_ID, client) + + documents, token = index.list_documents() + + self.assertEqual(len(documents), len(DATA['documents'])) + for found, expected in zip(documents, DATA['documents']): + self.assertTrue(isinstance(found, Document)) + self.assertEqual(found.name, expected['docId']) + self.assertEqual(found.rank, expected['rank']) + self.assertEqual(sorted(found.fields), sorted(expected['fields'])) + for field, f_field in found.fields.items(): + e_field = expected['fields'][field] + for f_value, e_value in zip(f_field.values, e_field['values']): + self.assertTrue(isinstance(f_value, StringValue)) + self.assertEqual(f_value.string_value, + e_value['stringValue']) + self.assertEqual(f_value.string_format, + e_value['stringFormat']) + self.assertEqual(f_value.language, + e_value['lang']) + self.assertEqual(token, TOKEN) + + self.assertEqual(len(conn._requested), 1) + req = conn._requested[0] + self.assertEqual(req['method'], 'GET') + self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['query_params'], {}) + + def test_list_documents_explicit(self): + from gcloud.search.document import Document + from gcloud.search.document import StringValue + DOCID_1 = 'docid-one' + RANK_1 = 2345 + TITLE_1 = 'Title One' + DOCID_2 = 'docid-two' + RANK_2 = 1234 + TITLE_2 = 'Title Two' + PATH = 'projects/%s/indexes/%s/documents' % ( + self.PROJECT, self.INDEX_ID) + TOKEN = 'TOKEN' + DOC_1 = { + 'docId': DOCID_1, + 'rank': RANK_1, + 'fields': { + 'title': { + 'values': [{ + 'stringValue': TITLE_1, + 'stringFormat': 'text', + 'lang': 'en'}] + } + } + } + DOC_2 = { + 'docId': DOCID_2, + 'rank': RANK_2, + 'fields': { + 'title': { + 'values': [{ + 'stringValue': TITLE_2, + 'stringFormat': 'text', + 'lang': 'en'}], + } + } + } + DATA = {'documents': [DOC_1, DOC_2]} + client = _Client(self.PROJECT) + conn = client.connection = _Connection(DATA) + index = self._makeOne(self.INDEX_ID, client) + + documents, token = index.list_documents( + max_results=3, page_token=TOKEN, view='FULL') + + self.assertEqual(len(documents), len(DATA['documents'])) + for found, expected in zip(documents, DATA['documents']): + self.assertTrue(isinstance(found, Document)) + self.assertEqual(found.name, expected['docId']) + self.assertEqual(found.rank, expected['rank']) + self.assertEqual(sorted(found.fields), sorted(expected['fields'])) + for field, f_field in found.fields.items(): + e_field = expected['fields'][field] + for f_value, e_value in zip(f_field.values, e_field['values']): + self.assertTrue(isinstance(f_value, StringValue)) + self.assertEqual(f_value.string_value, + e_value['stringValue']) + self.assertEqual(f_value.string_format, + e_value['stringFormat']) + self.assertEqual(f_value.language, + e_value['lang']) + self.assertEqual(token, None) + + self.assertEqual(len(conn._requested), 1) + req = conn._requested[0] + self.assertEqual(req['method'], 'GET') + self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['query_params'], + {'pageSize': 3, + 'pageToken': TOKEN, + 'view': 'FULL'}) + class _Client(object): def __init__(self, project='project', connection=None): self.project = project self.connection = connection + + +class _Connection(object): + + def __init__(self, *responses): + self._responses = responses + self._requested = [] + + def api_request(self, **kw): + from gcloud.exceptions import NotFound + self._requested.append(kw) + + try: + response, self._responses = self._responses[0], self._responses[1:] + except: # pragma: NO COVER + raise NotFound('miss') + else: + return response From d8961e55d95b6348c2e908e4adb6a12a7ab1546a Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 14 Oct 2015 11:03:45 -0400 Subject: [PATCH 3/4] Factor out repeated document setup/verification. Addresses: https://github.com/GoogleCloudPlatform/gcloud-python/pull/1177#discussion_r41958488 --- gcloud/search/test_index.py | 124 ++++++++++++------------------------ 1 file changed, 40 insertions(+), 84 deletions(-) diff --git a/gcloud/search/test_index.py b/gcloud/search/test_index.py index c6b4d7ee2d5f..1ec33491411f 100644 --- a/gcloud/search/test_index.py +++ b/gcloud/search/test_index.py @@ -50,6 +50,20 @@ def _makeResource(self): } } + def _makeDocumentResource(self, doc_id, rank, title): + return { + 'docId': doc_id, + 'rank': rank, + 'fields': { + 'title': { + 'values': [{ + 'stringValue': title, + 'stringFormat': 'text', + 'lang': 'en'}] + } + } + } + def _verifyResourceProperties(self, index, resource): self.assertEqual(index.name, resource.get('indexId')) @@ -61,6 +75,26 @@ def _verifyResourceProperties(self, index, resource): self.assertEqual(index.number_fields, field_info.get('numberFields')) self.assertEqual(index.geo_fields, field_info.get('geoFields')) + def _verifyDocumentResource(self, documents, resource): + from gcloud.search.document import Document + from gcloud.search.document import StringValue + self.assertEqual(len(documents), len(resource['documents'])) + for found, expected in zip(documents, resource['documents']): + self.assertTrue(isinstance(found, Document)) + self.assertEqual(found.name, expected['docId']) + self.assertEqual(found.rank, expected['rank']) + self.assertEqual(sorted(found.fields), sorted(expected['fields'])) + for field, f_field in found.fields.items(): + e_field = expected['fields'][field] + for f_value, e_value in zip(f_field.values, e_field['values']): + self.assertTrue(isinstance(f_value, StringValue)) + self.assertEqual(f_value.string_value, + e_value['stringValue']) + self.assertEqual(f_value.string_format, + e_value['stringFormat']) + self.assertEqual(f_value.language, + e_value['lang']) + def test_ctor(self): client = _Client(self.PROJECT) index = self._makeOne(self.INDEX_ID, client) @@ -106,8 +140,6 @@ def test_from_api_repr_w_properties(self): self._verifyResourceProperties(index, RESOURCE) def test_list_documents_defaults(self): - from gcloud.search.document import Document - from gcloud.search.document import StringValue DOCID_1 = 'docid-one' RANK_1 = 2345 TITLE_1 = 'Title One' @@ -117,30 +149,8 @@ def test_list_documents_defaults(self): PATH = 'projects/%s/indexes/%s/documents' % ( self.PROJECT, self.INDEX_ID) TOKEN = 'TOKEN' - DOC_1 = { - 'docId': DOCID_1, - 'rank': RANK_1, - 'fields': { - 'title': { - 'values': [{ - 'stringValue': TITLE_1, - 'stringFormat': 'text', - 'lang': 'en'}] - } - } - } - DOC_2 = { - 'docId': DOCID_2, - 'rank': RANK_2, - 'fields': { - 'title': { - 'values': [{ - 'stringValue': TITLE_2, - 'stringFormat': 'text', - 'lang': 'en'}], - } - } - } + DOC_1 = self._makeDocumentResource(DOCID_1, RANK_1, TITLE_1) + DOC_2 = self._makeDocumentResource(DOCID_2, RANK_2, TITLE_2) DATA = { 'nextPageToken': TOKEN, 'documents': [DOC_1, DOC_2], @@ -151,22 +161,7 @@ def test_list_documents_defaults(self): documents, token = index.list_documents() - self.assertEqual(len(documents), len(DATA['documents'])) - for found, expected in zip(documents, DATA['documents']): - self.assertTrue(isinstance(found, Document)) - self.assertEqual(found.name, expected['docId']) - self.assertEqual(found.rank, expected['rank']) - self.assertEqual(sorted(found.fields), sorted(expected['fields'])) - for field, f_field in found.fields.items(): - e_field = expected['fields'][field] - for f_value, e_value in zip(f_field.values, e_field['values']): - self.assertTrue(isinstance(f_value, StringValue)) - self.assertEqual(f_value.string_value, - e_value['stringValue']) - self.assertEqual(f_value.string_format, - e_value['stringFormat']) - self.assertEqual(f_value.language, - e_value['lang']) + self._verifyDocumentResource(documents, DATA) self.assertEqual(token, TOKEN) self.assertEqual(len(conn._requested), 1) @@ -176,8 +171,6 @@ def test_list_documents_defaults(self): self.assertEqual(req['query_params'], {}) def test_list_documents_explicit(self): - from gcloud.search.document import Document - from gcloud.search.document import StringValue DOCID_1 = 'docid-one' RANK_1 = 2345 TITLE_1 = 'Title One' @@ -187,30 +180,8 @@ def test_list_documents_explicit(self): PATH = 'projects/%s/indexes/%s/documents' % ( self.PROJECT, self.INDEX_ID) TOKEN = 'TOKEN' - DOC_1 = { - 'docId': DOCID_1, - 'rank': RANK_1, - 'fields': { - 'title': { - 'values': [{ - 'stringValue': TITLE_1, - 'stringFormat': 'text', - 'lang': 'en'}] - } - } - } - DOC_2 = { - 'docId': DOCID_2, - 'rank': RANK_2, - 'fields': { - 'title': { - 'values': [{ - 'stringValue': TITLE_2, - 'stringFormat': 'text', - 'lang': 'en'}], - } - } - } + DOC_1 = self._makeDocumentResource(DOCID_1, RANK_1, TITLE_1) + DOC_2 = self._makeDocumentResource(DOCID_2, RANK_2, TITLE_2) DATA = {'documents': [DOC_1, DOC_2]} client = _Client(self.PROJECT) conn = client.connection = _Connection(DATA) @@ -219,22 +190,7 @@ def test_list_documents_explicit(self): documents, token = index.list_documents( max_results=3, page_token=TOKEN, view='FULL') - self.assertEqual(len(documents), len(DATA['documents'])) - for found, expected in zip(documents, DATA['documents']): - self.assertTrue(isinstance(found, Document)) - self.assertEqual(found.name, expected['docId']) - self.assertEqual(found.rank, expected['rank']) - self.assertEqual(sorted(found.fields), sorted(expected['fields'])) - for field, f_field in found.fields.items(): - e_field = expected['fields'][field] - for f_value, e_value in zip(f_field.values, e_field['values']): - self.assertTrue(isinstance(f_value, StringValue)) - self.assertEqual(f_value.string_value, - e_value['stringValue']) - self.assertEqual(f_value.string_format, - e_value['stringFormat']) - self.assertEqual(f_value.language, - e_value['lang']) + self._verifyDocumentResource(documents, DATA) self.assertEqual(token, None) self.assertEqual(len(conn._requested), 1) From a9728bac805f2b9037a3557b2b679630bb5235c8 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 14 Oct 2015 11:15:59 -0400 Subject: [PATCH 4/4] Accomodate case where 'list_documents' API returns only document IDs. This case is actually the default if no 'view' is passed. --- gcloud/search/test_index.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/gcloud/search/test_index.py b/gcloud/search/test_index.py index 1ec33491411f..54d31ac17646 100644 --- a/gcloud/search/test_index.py +++ b/gcloud/search/test_index.py @@ -50,11 +50,12 @@ def _makeResource(self): } } - def _makeDocumentResource(self, doc_id, rank, title): - return { - 'docId': doc_id, - 'rank': rank, - 'fields': { + def _makeDocumentResource(self, doc_id, rank=None, title=None): + resource = {'docId': doc_id} + if rank is not None: + resource['rank'] = rank + if title is not None: + resource['fields'] = { 'title': { 'values': [{ 'stringValue': title, @@ -62,7 +63,7 @@ def _makeDocumentResource(self, doc_id, rank, title): 'lang': 'en'}] } } - } + return resource def _verifyResourceProperties(self, index, resource): @@ -82,10 +83,11 @@ def _verifyDocumentResource(self, documents, resource): for found, expected in zip(documents, resource['documents']): self.assertTrue(isinstance(found, Document)) self.assertEqual(found.name, expected['docId']) - self.assertEqual(found.rank, expected['rank']) - self.assertEqual(sorted(found.fields), sorted(expected['fields'])) + self.assertEqual(found.rank, expected.get('rank')) + e_fields = expected.get('fields', ()) + self.assertEqual(sorted(found.fields), sorted(e_fields)) for field, f_field in found.fields.items(): - e_field = expected['fields'][field] + e_field = e_fields[field] for f_value, e_value in zip(f_field.values, e_field['values']): self.assertTrue(isinstance(f_value, StringValue)) self.assertEqual(f_value.string_value, @@ -141,16 +143,12 @@ def test_from_api_repr_w_properties(self): def test_list_documents_defaults(self): DOCID_1 = 'docid-one' - RANK_1 = 2345 - TITLE_1 = 'Title One' DOCID_2 = 'docid-two' - RANK_2 = 1234 - TITLE_2 = 'Title Two' PATH = 'projects/%s/indexes/%s/documents' % ( self.PROJECT, self.INDEX_ID) TOKEN = 'TOKEN' - DOC_1 = self._makeDocumentResource(DOCID_1, RANK_1, TITLE_1) - DOC_2 = self._makeDocumentResource(DOCID_2, RANK_2, TITLE_2) + DOC_1 = self._makeDocumentResource(DOCID_1) + DOC_2 = self._makeDocumentResource(DOCID_2) DATA = { 'nextPageToken': TOKEN, 'documents': [DOC_1, DOC_2],