From 92f196d43aefd40051932580f86889d064836d82 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Wed, 16 Dec 2015 17:13:58 -0800 Subject: [PATCH 1/2] Adding primitive Bigtable garbage collection rule classes. These are used as settings when creating new column families within a table. Also slipping in 3 small Bigtable docstring updates. --- gcloud/_helpers.py | 31 ++++++ gcloud/bigtable/cluster.py | 4 +- gcloud/bigtable/column_family.py | 109 +++++++++++++++++++++ gcloud/bigtable/table.py | 1 + gcloud/bigtable/test_column_family.py | 135 ++++++++++++++++++++++++++ gcloud/test__helpers.py | 28 ++++++ 6 files changed, 306 insertions(+), 2 deletions(-) diff --git a/gcloud/_helpers.py b/gcloud/_helpers.py index 2d44b99f379d..07fcc850e649 100644 --- a/gcloud/_helpers.py +++ b/gcloud/_helpers.py @@ -21,6 +21,7 @@ import os from threading import local as Local import socket +import sys import six from six.moves.http_client import HTTPConnection # pylint: disable=F0401 @@ -260,6 +261,36 @@ def _millis_from_datetime(value): return _millis(value) +def _total_seconds_backport(offset): + """Backport of timedelta.total_seconds() from python 2.7+. + + :type offset: :class:`datetime.timedelta` + :param offset: A timedelta object. + + :rtype: int + :returns: The total seconds (including microseconds) in the + duration. + """ + seconds = offset.days * 24 * 60 * 60 + offset.seconds + return seconds + offset.microseconds * 1e-6 + + +def _total_seconds(offset): + """Version independent total seconds for a time delta. + + :type offset: :class:`datetime.timedelta` + :param offset: A timedelta object. + + :rtype: int + :returns: The total seconds (including microseconds) in the + duration. + """ + if sys.version_info[:2] < (2, 7): # pragma: NO COVER + return _total_seconds_backport(offset) + else: + return offset.total_seconds() + + def _to_bytes(value, encoding='ascii'): """Converts a string value to bytes, if necessary. diff --git a/gcloud/bigtable/cluster.py b/gcloud/bigtable/cluster.py index 003c2760cc83..c015a9a4b485 100644 --- a/gcloud/bigtable/cluster.py +++ b/gcloud/bigtable/cluster.py @@ -213,7 +213,7 @@ def finished(self): operation_name = ('operations/' + self._cluster.name + '/operations/%d' % (self.op_id,)) request_pb = operations_pb2.GetOperationRequest(name=operation_name) - # We expact a `._generated.operations_pb2.Operation`. + # We expect a `._generated.operations_pb2.Operation`. operation_pb = self._cluster._client._operations_stub.GetOperation( request_pb, self._cluster._client.timeout_seconds) @@ -258,7 +258,7 @@ class Cluster(object): :type serve_nodes: int :param serve_nodes: (Optional) The number of nodes in the cluster. - Defaults to 3. + Defaults to 3 (``_DEFAULT_SERVE_NODES``). """ def __init__(self, zone, cluster_id, client, diff --git a/gcloud/bigtable/column_family.py b/gcloud/bigtable/column_family.py index 2b29b8f01666..cc0cf611675a 100644 --- a/gcloud/bigtable/column_family.py +++ b/gcloud/bigtable/column_family.py @@ -15,6 +15,115 @@ """User friendly container for Google Cloud Bigtable Column Family.""" +from gcloud._helpers import _total_seconds +from gcloud.bigtable._generated import bigtable_table_data_pb2 as data_pb2 +from gcloud.bigtable._generated import duration_pb2 + + +def _timedelta_to_duration_pb(timedelta_val): + """Convert a Python timedelta object to a duration protobuf. + + .. note:: + + The Python timedelta has a granularity of microseconds while + the protobuf duration type has a duration of nanoseconds. + + :type timedelta_val: :class:`datetime.timedelta` + :param timedelta_val: A timedelta object. + + :rtype: :class:`duration_pb2.Duration` + :returns: A duration object equivalent to the time delta. + """ + seconds_decimal = _total_seconds(timedelta_val) + # Truncate the parts other than the integer. + seconds = int(seconds_decimal) + if seconds_decimal < 0: + signed_micros = timedelta_val.microseconds - 10**6 + else: + signed_micros = timedelta_val.microseconds + # Convert nanoseconds to microseconds. + nanos = 1000 * signed_micros + return duration_pb2.Duration(seconds=seconds, nanos=nanos) + + +class GarbageCollectionRule(object): + """Garbage collection rule for column families within a table. + + Cells in the column family (within a table) fitting the rule will be + deleted during garbage collection. + + .. note:: + + This class is a do-nothing base class for all GC rules. + + .. note:: + + A string ``gc_expression`` can also be used with API requests, but + that value would be superceded by a ``gc_rule``. As a result, we + don't support that feature and instead support via native classes. + """ + + def to_pb(self): + """Converts the :class:`GarbageCollectionRule` to a protobuf. + + :raises: :class:`NotImplementedError ` + always since a virtual class. + """ + raise NotImplementedError + + def __ne__(self, other): + return not self.__eq__(other) + + +class MaxVersionsGCRule(GarbageCollectionRule): + """Garbage collection limiting the number of versions of a cell. + + :type max_num_versions: int + :param max_num_versions: The maximum number of versions + """ + + def __init__(self, max_num_versions): + self.max_num_versions = max_num_versions + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return other.max_num_versions == self.max_num_versions + + def to_pb(self): + """Converts the garbage collection rule to a protobuf. + + :rtype: :class:`.data_pb2.GcRule` + :returns: The converted current object. + """ + return data_pb2.GcRule(max_num_versions=self.max_num_versions) + + +class MaxAgeGCRule(GarbageCollectionRule): + """Garbage collection limiting the age of a cell. + + :type max_age: :class:`datetime.timedelta` + :param max_age: The maximum age allowed for a cell in the table. + """ + + def __init__(self, max_age): + self.max_age = max_age + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return other.max_age == self.max_age + + def to_pb(self): + """Converts the garbage collection rule to a protobuf. + + :rtype: :class:`.data_pb2.GcRule` + :returns: The converted current object. + """ + max_age = _timedelta_to_duration_pb(self.max_age) + return data_pb2.GcRule(max_age=max_age) + + class ColumnFamily(object): """Representation of a Google Cloud Bigtable Column Family. diff --git a/gcloud/bigtable/table.py b/gcloud/bigtable/table.py index f5365d017faa..53930649a5a3 100644 --- a/gcloud/bigtable/table.py +++ b/gcloud/bigtable/table.py @@ -41,6 +41,7 @@ class Table(object): We can use a :class:`Table` to: * :meth:`create` the table + * :meth:`rename` the table * :meth:`delete` the table :type table_id: str diff --git a/gcloud/bigtable/test_column_family.py b/gcloud/bigtable/test_column_family.py index b1b921d10acf..6182b8550136 100644 --- a/gcloud/bigtable/test_column_family.py +++ b/gcloud/bigtable/test_column_family.py @@ -16,6 +16,141 @@ import unittest2 +class Test__timedelta_to_duration_pb(unittest2.TestCase): + + def _callFUT(self, *args, **kwargs): + from gcloud.bigtable.column_family import _timedelta_to_duration_pb + return _timedelta_to_duration_pb(*args, **kwargs) + + def test_it(self): + import datetime + from gcloud.bigtable._generated import duration_pb2 + + seconds = microseconds = 1 + timedelta_val = datetime.timedelta(seconds=seconds, + microseconds=microseconds) + result = self._callFUT(timedelta_val) + self.assertTrue(isinstance(result, duration_pb2.Duration)) + self.assertEqual(result.seconds, seconds) + self.assertEqual(result.nanos, 1000 * microseconds) + + def test_with_negative_microseconds(self): + import datetime + from gcloud.bigtable._generated import duration_pb2 + + seconds = 1 + microseconds = -5 + timedelta_val = datetime.timedelta(seconds=seconds, + microseconds=microseconds) + result = self._callFUT(timedelta_val) + self.assertTrue(isinstance(result, duration_pb2.Duration)) + self.assertEqual(result.seconds, seconds - 1) + self.assertEqual(result.nanos, 10**9 + 1000 * microseconds) + + def test_with_negative_seconds(self): + import datetime + from gcloud.bigtable._generated import duration_pb2 + + seconds = -1 + microseconds = 5 + timedelta_val = datetime.timedelta(seconds=seconds, + microseconds=microseconds) + result = self._callFUT(timedelta_val) + self.assertTrue(isinstance(result, duration_pb2.Duration)) + self.assertEqual(result.seconds, seconds + 1) + self.assertEqual(result.nanos, -(10**9 - 1000 * microseconds)) + + +class TestGarbageCollectionRule(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.column_family import GarbageCollectionRule + return GarbageCollectionRule + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb_virtual(self): + gc_rule = self._makeOne() + self.assertRaises(NotImplementedError, gc_rule.to_pb) + + +class TestMaxVersionsGCRule(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.column_family import MaxVersionsGCRule + return MaxVersionsGCRule + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test___eq__max_num_versions(self): + gc_rule1 = self._makeOne(2) + gc_rule2 = self._makeOne(2) + self.assertEqual(gc_rule1, gc_rule2) + + def test___eq__type_differ(self): + gc_rule1 = self._makeOne(10) + gc_rule2 = object() + self.assertNotEqual(gc_rule1, gc_rule2) + + def test___ne__same_value(self): + gc_rule1 = self._makeOne(99) + gc_rule2 = self._makeOne(99) + comparison_val = (gc_rule1 != gc_rule2) + self.assertFalse(comparison_val) + + def test_to_pb(self): + from gcloud.bigtable._generated import ( + bigtable_table_data_pb2 as data_pb2) + max_num_versions = 1337 + gc_rule = self._makeOne(max_num_versions=max_num_versions) + pb_val = gc_rule.to_pb() + self.assertEqual(pb_val, + data_pb2.GcRule(max_num_versions=max_num_versions)) + + +class TestMaxAgeGCRule(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.column_family import MaxAgeGCRule + return MaxAgeGCRule + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test___eq__max_age(self): + max_age = object() + gc_rule1 = self._makeOne(max_age=max_age) + gc_rule2 = self._makeOne(max_age=max_age) + self.assertEqual(gc_rule1, gc_rule2) + + def test___eq__type_differ(self): + max_age = object() + gc_rule1 = self._makeOne(max_age=max_age) + gc_rule2 = object() + self.assertNotEqual(gc_rule1, gc_rule2) + + def test___ne__same_value(self): + max_age = object() + gc_rule1 = self._makeOne(max_age=max_age) + gc_rule2 = self._makeOne(max_age=max_age) + comparison_val = (gc_rule1 != gc_rule2) + self.assertFalse(comparison_val) + + def test_to_pb(self): + import datetime + from gcloud.bigtable._generated import ( + bigtable_table_data_pb2 as data_pb2) + from gcloud.bigtable._generated import duration_pb2 + + max_age = datetime.timedelta(seconds=1) + duration = duration_pb2.Duration(seconds=1) + gc_rule = self._makeOne(max_age=max_age) + pb_val = gc_rule.to_pb() + self.assertEqual(pb_val, data_pb2.GcRule(max_age=duration)) + + class TestColumnFamily(unittest2.TestCase): def _getTargetClass(self): diff --git a/gcloud/test__helpers.py b/gcloud/test__helpers.py index 684de4d907c8..03e1aede76f3 100644 --- a/gcloud/test__helpers.py +++ b/gcloud/test__helpers.py @@ -354,6 +354,34 @@ def test_it(self): self.assertEqual(self._callFUT(NOW_MICROS), NOW) +class Test__total_seconds_backport(unittest2.TestCase): + + def _callFUT(self, *args, **kwargs): + from gcloud._helpers import _total_seconds_backport + return _total_seconds_backport(*args, **kwargs) + + def test_it(self): + import datetime + offset = datetime.timedelta(seconds=3, + microseconds=140000) + result = self._callFUT(offset) + self.assertEqual(result, 3.14) + + +class Test__total_seconds(unittest2.TestCase): + + def _callFUT(self, *args, **kwargs): + from gcloud._helpers import _total_seconds + return _total_seconds(*args, **kwargs) + + def test_it(self): + import datetime + offset = datetime.timedelta(seconds=1, + microseconds=414000) + result = self._callFUT(offset) + self.assertEqual(result, 1.414) + + class Test__to_bytes(unittest2.TestCase): def _callFUT(self, *args, **kwargs): From d11980f86eeed1ba96c42ef5fdbd7fc8c37b1a6f Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Wed, 16 Dec 2015 17:31:53 -0800 Subject: [PATCH 2/2] Pinning on protobuf==3.0.0a3. Caused by a breakage in latest protobuf release (3.0.0b1.post1). --- setup.py | 2 +- tox.ini | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/setup.py b/setup.py index b657d199af3b..8be33426642a 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ REQUIREMENTS = [ 'httplib2 >= 0.9.1', 'oauth2client >= 1.4.6', - 'protobuf >= 3.0.0a3', + 'protobuf == 3.0.0a3', 'pycrypto', 'six', ] diff --git a/tox.ini b/tox.ini index 0b3b727ae735..4de28f423371 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,6 @@ commands = deps = nose unittest2 - protobuf>=3.0.0a3 setenv = PYTHONPATH = {toxinidir}/_testing covercmd = @@ -77,7 +76,6 @@ deps = pep8 pylint unittest2 - protobuf==3.0.0-alpha-1 passenv = {[testenv:system-tests]passenv} [testenv:system-tests]