Merge pull request #137 from octue/release/0.1.14

Release/0.1.14

Author: cortadocodes

.github/workflows/release.yml

@@ -1,4 +1,4 @@
-name: Publish on merge of release/x.y.z into main
+name: Release the package on merge of release/x.y.z into main
 
 # Only trigger when a pull request into main branch is closed.
 on:

docs/source/deploying_services.rst

@@ -15,7 +15,7 @@ Automated deployment with Octue means:
 
 All you need to enable automated deployments are the following files in your repository root:
 
-* A ``requirements.txt`` file that includes ``octue>=0.1.13`` and the rest of your service's dependencies
+* A ``requirements.txt`` file that includes ``octue>=0.1.14`` and the rest of your service's dependencies
 * A ``twine.json`` file
 * A ``deployment_configuration.json`` file (optional)
 

octue/deployment/google/Dockerfile

@@ -8,10 +8,6 @@ WORKDIR $PROJECT_ROOT
 
 RUN apt-get update -y && apt-get install -y --fix-missing build-essential && rm -rf /var/lib/apt/lists/*
 
-# This will cache bust if any of the requirements change.
-COPY requirements*.txt .
-COPY setup.* .
-
 COPY . .
 
 # Install requirements (supports requirements.txt, requirements-dev.txt, and setup.py; all will be run if all are present.)

octue/mixins/serialisable.py

@@ -7,29 +7,36 @@ class Serialisable:
     """Mixin class to make resources serialisable to JSON.
 
     Objects must have a `.logger` and a `.id` property
-
     """
 
     _SERIALISE_FIELDS = None
     _EXCLUDE_SERIALISE_FIELDS = ("logger",)
 
     def __init__(self, *args, **kwargs):
         """Constructor for serialisable mixin"""
-        # Ensure it passes construction arguments up the chain
         super().__init__(*args, **kwargs)
 
+    @classmethod
+    def deserialise(cls, serialised_object):
+        """Deserialise the given JSON-serialised object.
+
+        :param dict serialised_object:
+        :return any:
+        """
+        return cls(**serialised_object)
+
     def to_file(self, file_name, **kwargs):
-        """Write to a JSON file
+        """Write the object to a JSON file.
 
-        :parameter file_name:  file to write to, including relative or absolute path and .json extension
-        :type file_name: path-like
+        :parameter str file_name: file to write to, including relative or absolute path and .json extension
+        :return None:
         """
         self.logger.debug("Writing %s %s to file %s", self.__class__.__name__, self.id, file_name)
         with open(file_name, "w") as fp:
             fp.write(self.serialise(**kwargs, to_string=True))
 
     def serialise(self, to_string=False, **kwargs):
-        """Serialise into a primitive dict or JSON string
+        """Serialise into a primitive dict or JSON string.
 
         Serialises all non-private and non-protected attributes except for 'logger', unless the subclass has a
         `_serialise_fields` tuple of the attribute names to serialise. For example:

octue/resources/datafile.py

@@ -204,7 +204,7 @@ def to_cloud(self, project_name, bucket_name, path_in_bucket):
                 "hash_value": self.hash_value,
                 "cluster": self.cluster,
                 "sequence": self.sequence,
-                "tags": self.tags.serialise(to_string=False),
+                "tags": self.tags.serialise(),
             },
         )
 

octue/resources/dataset.py

@@ -10,6 +10,7 @@
 from octue.mixins import Hashable, Identifiable, Loggable, Pathable, Serialisable, Taggable
 from octue.resources.datafile import Datafile
 from octue.resources.filter_containers import FilterSet
+from octue.resources.tag import TagSet
 
 
 module_logger = logging.getLogger(__name__)
@@ -27,6 +28,7 @@ class Dataset(Taggable, Serialisable, Pathable, Loggable, Identifiable, Hashable
 
     _FILTERSET_ATTRIBUTE = "files"
     _ATTRIBUTES_TO_HASH = "files", "name", "tags"
+    _SERIALISE_FIELDS = "files", "name", "tags", "hash_value", "id", "path"
 
     def __init__(self, name=None, id=None, logger=None, path=None, path_from=None, tags=None, **kwargs):
         """Construct a Dataset"""
@@ -84,7 +86,7 @@ def from_cloud(cls, project_name, bucket_name, path_to_dataset_directory):
             name=serialised_dataset["name"],
             hash_value=serialised_dataset["hash_value"],
             path=storage.path.generate_gs_path(bucket_name, path_to_dataset_directory),
-            tags=json.loads(serialised_dataset["tags"]),
+            tags=TagSet(serialised_dataset["tags"]),
             files=datafiles,
         )
 
@@ -107,7 +109,6 @@ def to_cloud(self, project_name, bucket_name, output_directory):
 
         serialised_dataset = self.serialise()
         serialised_dataset["files"] = sorted(files)
-        del serialised_dataset["absolute_path"]
         del serialised_dataset["path"]
 
         GoogleCloudStorageClient(project_name=project_name).upload_from_string(

octue/resources/filter_containers.py

@@ -4,20 +4,9 @@
 def _filter(instance, filter_name=None, filter_value=None):
     """Returns a new instance containing only the Filterables to which the given filter criteria apply.
 
-    Say we want to filter by files whose extension equals "csv". We want to be able to do...
-
-    ds = DataSet(... initialise with csv files and other files)
-    Example of chaining:
-
-    ds.filter('files__extension_equals', 'csv')
-    is equvalent to
-    ds.files.filter(extension__equals', 'csv')
-    >>> FilterSet containing a set of datafiles
-
-
-    ds.filter('files__extension_equals', 'csv')
-    is equvalent to
-    ds.files.filter(extension__equals', 'csv')
+    :param str filter_name:
+    :param any filter_value:
+    :return octue.resources.filter_containers.FilterSet:
     """
     return instance.__class__((item for item in instance if item.satisfies(filter_name, filter_value)))
 

octue/resources/manifest.py

@@ -17,6 +17,7 @@ class Manifest(Pathable, Serialisable, Loggable, Identifiable, Hashable):
     (or leaving), a data service for an analysis at the configuration, input or output stage."""
 
     _ATTRIBUTES_TO_HASH = "datasets", "keys"
+    _SERIALISE_FIELDS = "datasets", "keys", "hash_value", "id", "name", "path"
 
     def __init__(self, id=None, logger=None, path=None, datasets=None, keys=None, **kwargs):
         super().__init__(id=id, logger=logger, path=path)
@@ -43,20 +44,6 @@ def __init__(self, id=None, logger=None, path=None, datasets=None, keys=None, **
         self._instantiate_datasets(datasets, key_list)
         vars(self).update(**kwargs)
 
-    @classmethod
-    def deserialise(cls, serialised_manifest, from_string=False):
-        """Deserialise a Manifest from a dictionary."""
-        if from_string:
-            serialised_manifest = json.loads(serialised_manifest)
-
-        return cls(
-            name=serialised_manifest["name"],
-            id=serialised_manifest["id"],
-            datasets=serialised_manifest["datasets"],
-            keys=serialised_manifest["keys"],
-            path=serialised_manifest["path"],
-        )
-
     @classmethod
     def from_cloud(cls, project_name, bucket_name, path_to_manifest_file):
         """Instantiate a Manifest from Google Cloud storage.
@@ -113,7 +100,6 @@ def to_cloud(self, project_name, bucket_name, path_to_manifest_file, store_datas
 
         serialised_manifest = self.serialise()
         serialised_manifest["datasets"] = sorted(datasets)
-        del serialised_manifest["absolute_path"]
         del serialised_manifest["path"]
 
         GoogleCloudStorageClient(project_name=project_name).upload_from_string(

octue/resources/tag.py

@@ -3,8 +3,9 @@
 from functools import lru_cache
 
 from octue.exceptions import InvalidTagException
-from octue.mixins import Filterable
-from octue.resources.filter_containers import FilterSet
+from octue.mixins import Filterable, Serialisable
+from octue.resources.filter_containers import FilterList, FilterSet
+from octue.utils.encoders import OctueJSONEncoder
 
 
 TAG_PATTERN = re.compile(r"^$|^[a-z0-9][a-z0-9:\-]*(?<![:-])$")
@@ -28,8 +29,11 @@ def name(self):
     @property
     @lru_cache(maxsize=1)
     def subtags(self):
-        """ Return the subtags of the tag as a new TagSet (e.g. TagSet({'a', 'b', 'c'}) for the Tag('a:b:c'). """
-        return TagSet({Tag(subtag_name) for subtag_name in (self.name.split(":"))})
+        """Return the subtags of the tag in order as a FilterList (e.g. FilterList(['a', 'b', 'c']) for Tag('a:b:c').
+
+        :return FilterList(Tag):
+        """
+        return FilterList(Tag(subtag_name) for subtag_name in self.name.split(":"))
 
     def __eq__(self, other):
         if isinstance(other, str):
@@ -85,7 +89,7 @@ def _clean(name):
         return cleaned_name
 
 
-class TagSet:
+class TagSet(Serialisable):
     """ Class to handle a set of tags as a string. """
 
     _FILTERSET_ATTRIBUTE = "tags"
@@ -114,10 +118,6 @@ def __init__(self, tags=None, *args, **kwargs):
                 "Tags must be expressed as a whitespace-delimited string or an iterable of strings or Tag instances."
             )
 
-    def __str__(self):
-        """ Serialise tags to a sorted list string. """
-        return self.serialise()
-
     def __eq__(self, other):
         """ Does this TagSet have the same tags as another TagSet? """
         if not isinstance(other, TagSet):
@@ -161,10 +161,35 @@ def any_tag_contains(self, value):
         """ Return True if any of the tags contains value. """
         return any(value in tag for tag in self)
 
-    def serialise(self, to_string=True):
-        """ Serialise tags to a sorted list string. """
-        serialised_tags = sorted(tag.name for tag in self)
+    def filter(self, filter_name=None, filter_value=None):
+        """Filter the tags with the given filter for the given value.
+
+        :param str filter_name:
+        :param any filter_value:
+        :return octue.resources.filter_containers.FilterSet:
+        """
+        return self.tags.filter(filter_name=filter_name, filter_value=filter_value)
+
+    def serialise(self, to_string=False, **kwargs):
+        """Serialise to a sorted list of tag names.
+
+        :param bool to_string:
+        :return list|str:
+        """
+        string = json.dumps(
+            sorted(tag.name for tag in self.tags), cls=OctueJSONEncoder, sort_keys=True, indent=4, **kwargs
+        )
 
         if to_string:
-            return str(serialised_tags)
-        return serialised_tags
+            return string
+
+        return json.loads(string)
+
+    @classmethod
+    def deserialise(cls, serialised_tagset):
+        """Deserialise from a sorted list of tag names.
+
+        :param list serialised_tagset:
+        :return TagSet:
+        """
+        return cls(tags=serialised_tagset)

octue/templates/template-child-services/elevation_service/requirements.txt

@@ -1 +1 @@
-octue==0.1.13
+octue==0.1.14