added support for generation-based query and delete of blobs.

gcloud/storage/_helpers.py

@@ -22,7 +22,7 @@
 
 
 class _PropertyMixin(object):
-    """Abstract mixin for cloud storage classes with associated propertties.
+    """Abstract mixin for cloud storage classes with associated properties.
 
     Non-abstract subclasses should implement:
       - client

gcloud/storage/blob.py

@@ -61,12 +61,13 @@ class Blob(_PropertyMixin):
     _CHUNK_SIZE_MULTIPLE = 256 * 1024
     """Number (256 KB, in bytes) that must divide the chunk size."""
 
-    def __init__(self, name, bucket, chunk_size=None):
+    def __init__(self, name, bucket, chunk_size=None, generation=None):
         super(Blob, self).__init__(name=name)
 
         self.chunk_size = chunk_size  # Check that setter accepts value.
         self.bucket = bucket
         self._acl = ObjectACL(self)
+        self.generation = generation
 
     @property
     def chunk_size(self):
@@ -132,6 +133,22 @@ def path(self):
 
         return self.path_helper(self.bucket.path, self.name)
 
+    @property
+    def path_with_params(self):
+        """Getter property for the URL path to this Blob, with version.
+
+        :rtype: tuple of ``path`` (a string) and ``params`` (a dictionary)
+        :returns: the URL path to this blob and a dictionary with the
+                 generation that can be used in query_params for
+                 connection.api_request
+        """
+
+        params = {}
+        if self.generation is not None:
+            params = {'generation': self.generation}
+
+        return (self.path, params)
+
     @property
     def client(self):
         """The client bound to this blob."""
@@ -241,6 +258,9 @@ def exists(self, client=None):
             # We only need the status code (200 or not) so we seek to
             # minimize the returned payload.
             query_params = {'fields': 'name'}
+            if self.generation is not None:
+                query_params['generation'] = self.generation
+
             # We intentionally pass `_target_object=None` since fields=name
             # would limit the local properties.
             client.connection.api_request(method='GET', path=self.path,
@@ -266,7 +286,8 @@ def delete(self, client=None):
                  (propagated from
                  :meth:`gcloud.storage.bucket.Bucket.delete_blob`).
         """
-        return self.bucket.delete_blob(self.name, client=client)
+        return self.bucket.delete_blob(self.name, client=client,
+                                       generation=self.generation)
 
     def download_to_file(self, file_obj, client=None):
         """Download the contents of this blob into a file-like object.
@@ -634,6 +655,19 @@ def generation(self):
         if generation is not None:
             return int(generation)
 
+    @generation.setter
+    def generation(self, value):
+        """Set the generation for this blob.
+
+        See: https://cloud.google.com/storage/docs/json_api/v1/objects
+
+        :type value: integer or ``NoneType``
+        :param value: the generation value for this blob.  Setting this
+                      value is useful when trying to retrieve specific
+                      versions of a blob.
+        """
+        self._patch_property('generation', value)
+
     @property
     def id(self):
         """Retrieve the ID for the object.

gcloud/storage/bucket.py

@@ -103,7 +103,7 @@ def client(self):
         """The client bound to this bucket."""
         return self._client
 
-    def blob(self, blob_name, chunk_size=None):
+    def blob(self, blob_name, chunk_size=None, generation=None):
         """Factory constructor for blob object.
 
         .. note::
@@ -118,10 +118,16 @@ def blob(self, blob_name, chunk_size=None):
                            (1 MB). This must be a multiple of 256 KB per the
                            API specification.
 
+        :type generation: integer
+        :param generation: The desired generation of the blob object.  This
+                           parameter affects generation based queries on
+                           buckets that support versioning.
+
         :rtype: :class:`gcloud.storage.blob.Blob`
         :returns: The blob object created.
         """
-        return Blob(name=blob_name, bucket=self, chunk_size=chunk_size)
+        return Blob(name=blob_name, bucket=self, chunk_size=chunk_size,
+                    generation=generation)
 
     def exists(self, client=None):
         """Determines whether or not this bucket exists.
@@ -202,7 +208,7 @@ def path(self):
 
         return self.path_helper(self.name)
 
-    def get_blob(self, blob_name, client=None):
+    def get_blob(self, blob_name, client=None, generation=None):
         """Get a blob object by name.
 
         This will return None if the blob doesn't exist::
@@ -214,6 +220,10 @@ def get_blob(self, blob_name, client=None):
           <Blob: my-bucket, /path/to/blob.txt>
           >>> print bucket.get_blob('/does-not-exist.txt')
           None
+          >>> print bucket.get_blob(
+          ... '/path/to/versioned_blob.txt',
+          ... generation=generation_id)
+          <Blob: my-bucket, /path/to/versioned_blob.txt>
 
         :type blob_name: string
         :param blob_name: The name of the blob to retrieve.
@@ -222,14 +232,20 @@ def get_blob(self, blob_name, client=None):
         :param client: Optional. The client to use.  If not passed, falls back
                        to the ``client`` stored on the current bucket.
 
+        :type generation: int
+        :param generation: Optional. The generation id to retrieve in a bucket
+                           that supports versioning.
+
         :rtype: :class:`gcloud.storage.blob.Blob` or None
         :returns: The blob object if it exists, otherwise None.
         """
         client = self._require_client(client)
-        blob = Blob(bucket=self, name=blob_name)
+        blob = Blob(bucket=self, name=blob_name, generation=generation)
+        blob_path, query_params = blob.path_with_params
         try:
             response = client.connection.api_request(
-                method='GET', path=blob.path, _target_object=blob)
+                method='GET', path=blob_path,
+                query_params=query_params, _target_object=blob)
             # NOTE: We assume response.get('name') matches `blob_name`.
             blob._set_properties(response)
             # NOTE: This will not fail immediately in a batch. However, when
@@ -357,7 +373,7 @@ def delete(self, force=False, client=None):
         client.connection.api_request(method='DELETE', path=self.path,
                                       _target_object=None)
 
-    def delete_blob(self, blob_name, client=None):
+    def delete_blob(self, blob_name, client=None, generation=None):
         """Deletes a blob from the current bucket.
 
         If the blob isn't found (backend 404), raises a
@@ -384,6 +400,10 @@ def delete_blob(self, blob_name, client=None):
         :param client: Optional. The client to use.  If not passed, falls back
                        to the ``client`` stored on the current bucket.
 
+        :type generation: int
+        :param generation: Optional. The generation of this object to delete.
+                           Only works on buckets with versioning enabled.
+
         :raises: :class:`gcloud.exceptions.NotFound` (to suppress
                  the exception, call ``delete_blobs``, passing a no-op
                  ``on_error`` callback, e.g.::
@@ -392,10 +412,15 @@ def delete_blob(self, blob_name, client=None):
         """
         client = self._require_client(client)
         blob_path = Blob.path_helper(self.path, blob_name)
+        query_params = {}
+        if generation is not None:
+            query_params = {'generation': generation}
+
         # We intentionally pass `_target_object=None` since a DELETE
         # request has no response value (whether in a standard request or
         # in a batch request).
         client.connection.api_request(method='DELETE', path=blob_path,
+                                      query_params=query_params,
                                       _target_object=None)
 
     def delete_blobs(self, blobs, on_error=None, client=None):
@@ -423,15 +448,21 @@ def delete_blobs(self, blobs, on_error=None, client=None):
                 blob_name = blob
                 if not isinstance(blob_name, six.string_types):
                     blob_name = blob.name
-                self.delete_blob(blob_name, client=client)
+
+                generation = None
+                if hasattr(blob, 'generation'):
+                    generation = blob.generation
+
+                self.delete_blob(blob_name, client=client,
+                                 generation=generation)
             except NotFound:
                 if on_error is not None:
                     on_error(blob)
                 else:
                     raise
 
     def copy_blob(self, blob, destination_bucket, new_name=None,
-                  client=None):
+                  client=None, versions=False):
         """Copy the given blob to the given bucket, optionally with a new name.
 
         :type blob: :class:`gcloud.storage.blob.Blob`
@@ -448,18 +479,38 @@ def copy_blob(self, blob, destination_bucket, new_name=None,
         :param client: Optional. The client to use.  If not passed, falls back
                        to the ``client`` stored on the current bucket.
 
+        :type versions: boolean
+        :param versions: Optional. Copy each version.
+
         :rtype: :class:`gcloud.storage.blob.Blob`
-        :returns: The new Blob.
+        :returns: The new Blob if versions is ``False``, or will return
+                  a list of new blob versions, and their old blob version
+                  counterparts.
         """
         client = self._require_client(client)
         if new_name is None:
             new_name = blob.name
-        new_blob = Blob(bucket=destination_bucket, name=new_name)
-        api_path = blob.path + '/copyTo' + new_blob.path
-        copy_result = client.connection.api_request(
-            method='POST', path=api_path, _target_object=new_blob)
-        new_blob._set_properties(copy_result)
-        return new_blob
+
+        tmp_blob = Blob(bucket=destination_bucket, name=new_name)
+        api_path = blob.path + '/copyTo' + tmp_blob.path
+        del tmp_blob
+
+        # TODO(tsinha): Support multi-page results from list_blobs
+        old_blobs = list(self.list_blobs(prefix=blob.name, versions=versions))
+        new_blobs = []
+        for old_blob in old_blobs:
+            new_blob = Blob(bucket=destination_bucket, name=new_name)
+            copy_result = client.connection.api_request(
+                method='POST', path=api_path,
+                query_params={'sourceGeneration': old_blob.generation},
+                _target_object=new_blob)
+            new_blob._set_properties(copy_result)
+            new_blobs.append(new_blob)
+
+        if versions:
+            return (new_blobs, old_blobs)
+        else:
+            return new_blobs[0]
 
     def rename_blob(self, blob, new_name, client=None):
         """Rename the given blob using copy and delete operations.
@@ -484,11 +535,19 @@ def rename_blob(self, blob, new_name, client=None):
                        to the ``client`` stored on the current bucket.
 
         :rtype: :class:`Blob`
-        :returns: The newly-renamed blob.
-        """
-        new_blob = self.copy_blob(blob, self, new_name, client=client)
-        blob.delete(client=client)
-        return new_blob
+        :returns: The newly-renamed blob if bucket versioning is off (or
+                  there is only one version), otherwise will return blobs
+                  for each newly-renamed version.
+        """
+        new_blobs, old_blobs = self.copy_blob(blob, self, new_name,
+                                              client=client, versions=True)
+        for old_blob in old_blobs:
+            old_blob.delete(client=client)
+
+        if len(new_blobs) == 1:
+            return new_blobs[0]
+        else:
+            return new_blobs
 
     @property
     def cors(self):

gcloud/storage/test_blob.py

@@ -241,6 +241,38 @@ def test_exists_hit(self):
         bucket._blobs[BLOB_NAME] = 1
         self.assertTrue(blob.exists())
 
+    def test_exists_hit_w_generation(self):
+        from six.moves.http_client import OK
+        BLOB_NAME = 'blob-name'
+        GENERATION = 999
+        found_response = {'status': OK}
+        connection = _Connection(found_response)
+        client = _Client(connection)
+        bucket = _Bucket(client)
+        blob = self._makeOne(BLOB_NAME, bucket=bucket,
+                             properties={'generation': GENERATION})
+        bucket._blobs[BLOB_NAME] = 1
+        self.assertTrue(blob.exists())
+        kw, = connection._requested
+        self.assertEqual(kw['method'], 'GET')
+        self.assertEqual(kw['query_params']['generation'], GENERATION)
+
+    def test_exists_hit_w_none_generation(self):
+        from six.moves.http_client import OK
+        BLOB_NAME = 'blob-name'
+        GENERATION = None
+        found_response = {'status': OK}
+        connection = _Connection(found_response)
+        client = _Client(connection)
+        bucket = _Bucket(client)
+        blob = self._makeOne(BLOB_NAME, bucket=bucket,
+                             properties={'generation': GENERATION})
+        bucket._blobs[BLOB_NAME] = 1
+        self.assertTrue(blob.exists())
+        kw, = connection._requested
+        self.assertEqual(kw['method'], 'GET')
+        self.assertNotIn('generation', kw['query_params'])
+
     def test_delete(self):
         from six.moves.http_client import NOT_FOUND
         BLOB_NAME = 'blob-name'
@@ -254,6 +286,42 @@ def test_delete(self):
         self.assertFalse(blob.exists())
         self.assertEqual(bucket._deleted, [(BLOB_NAME, None)])
 
+    def test_delete_w_generation(self):
+        from six.moves.http_client import NOT_FOUND
+        BLOB_NAME = 'blob-name'
+        GENERATION = 999
+        not_found_response = {'status': NOT_FOUND}
+        connection = _Connection(not_found_response)
+        client = _Client(connection)
+        bucket = _Bucket(client)
+        blob = self._makeOne(BLOB_NAME, bucket=bucket,
+                             properties={'generation': GENERATION})
+        bucket._blobs[BLOB_NAME] = 1
+        blob.delete()
+        self.assertFalse(blob.exists())
+        self.assertEqual(bucket._deleted, [(BLOB_NAME, None)])
+        kw, = connection._requested
+        self.assertEqual(kw['method'], 'GET')
+        self.assertEqual(kw['query_params']['generation'], GENERATION)
+
+    def test_delete_w_none_generation(self):
+        from six.moves.http_client import NOT_FOUND
+        BLOB_NAME = 'blob-name'
+        GENERATION = None
+        not_found_response = {'status': NOT_FOUND}
+        connection = _Connection(not_found_response)
+        client = _Client(connection)
+        bucket = _Bucket(client)
+        blob = self._makeOne(BLOB_NAME, bucket=bucket,
+                             properties={'generation': GENERATION})
+        bucket._blobs[BLOB_NAME] = 1
+        blob.delete()
+        self.assertFalse(blob.exists())
+        self.assertEqual(bucket._deleted, [(BLOB_NAME, None)])
+        kw, = connection._requested
+        self.assertEqual(kw['method'], 'GET')
+        self.assertNotIn('generation', kw['query_params'])
+
     def _download_to_file_helper(self, chunk_size=None):
         from six.moves.http_client import OK
         from six.moves.http_client import PARTIAL_CONTENT
@@ -1095,7 +1163,8 @@ def __init__(self, client=None):
         self._copied = []
         self._deleted = []
 
-    def delete_blob(self, blob_name, client=None):
+    def delete_blob(self, blob_name, client=None,
+                    generation=None):  # pylint: disable=W0613
         del self._blobs[blob_name]
         self._deleted.append((blob_name, client))