Blobstore API with Google Cloud Storage samples (#709)

Author: ryanmats

appengine/standard/blobstore/README.md renamed to appengine/standard/blobstore/api/README.md

@@ -0,0 +1,8 @@
+runtime: python27
+api_version: 1
+threadsafe: yes
+
+handlers:
+- url: .*
+  script: main.app
+  login: required

appengine/standard/blobstore/app.yaml renamed to appengine/standard/blobstore/api/app.yaml

@@ -0,0 +1,18 @@
+# Copyright 2016 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from google.appengine.ext import vendor
+
+# Add any libraries installed in the "lib" folder.
+vendor.add('lib')

appengine/standard/blobstore/main.py renamed to appengine/standard/blobstore/api/main.py

@@ -0,0 +1,69 @@
+"""A sample app that operates on GCS files with blobstore API's BlobReader."""
+
+import cloudstorage
+from google.appengine.api import app_identity
+from google.appengine.ext import blobstore
+import webapp2
+
+
+class BlobreaderHandler(webapp2.RequestHandler):
+    def get(self):
+        # Get the default Cloud Storage Bucket name and create a file name for
+        # the object in Cloud Storage.
+        bucket = app_identity.get_default_gcs_bucket_name()
+
+        # Cloud Storage file names are in the format /bucket/object.
+        filename = '/{}/blobreader_demo'.format(bucket)
+
+        # Create a file in Google Cloud Storage and write something to it.
+        with cloudstorage.open(filename, 'w') as filehandle:
+            filehandle.write('abcde\n')
+
+        # In order to read the contents of the file using the Blobstore API,
+        # you must create a blob_key from the Cloud Storage file name.
+        # Blobstore expects the filename to be in the format of:
+        # /gs/bucket/object
+        blobstore_filename = '/gs{}'.format(filename)
+        blob_key = blobstore.create_gs_key(blobstore_filename)
+
+        # [START blob_reader]
+        # Instantiate a BlobReader for a given Blobstore blob_key.
+        blob_reader = blobstore.BlobReader(blob_key)
+
+        # Instantiate a BlobReader for a given Blobstore blob_key, setting the
+        # buffer size to 1 MB.
+        blob_reader = blobstore.BlobReader(blob_key, buffer_size=1048576)
+
+        # Instantiate a BlobReader for a given Blobstore blob_key, setting the
+        # initial read position.
+        blob_reader = blobstore.BlobReader(blob_key, position=0)
+
+        # Read the entire value into memory. This may take a while depending
+        # on the size of the value and the size of the read buffer, and is not
+        # recommended for large values.
+        blob_reader_data = blob_reader.read()
+
+        # Write the contents to the response.
+        self.response.headers['Content-Type'] = 'text/plain'
+        self.response.write(blob_reader_data)
+
+        # Set the read position back to 0, then read and write 3 bytes.
+        blob_reader.seek(0)
+        blob_reader_data = blob_reader.read(3)
+        self.response.write(blob_reader_data)
+        self.response.write('\n')
+
+        # Set the read position back to 0, then read and write one line (up to
+        # and including a '\n' character) at a time.
+        blob_reader.seek(0)
+        for line in blob_reader:
+            self.response.write(line)
+        # [END blob_reader]
+
+        # Delete the file from Google Cloud Storage using the blob_key.
+        blobstore.delete(blob_key)
+
+
+app = webapp2.WSGIApplication([
+    ('/', BlobreaderHandler),
+    ('/blobreader', BlobreaderHandler)], debug=True)

appengine/standard/blobstore/main_test.py renamed to appengine/standard/blobstore/api/main_test.py

@@ -0,0 +1,25 @@
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import webtest
+
+import main
+
+
+def test_blobreader(testbed, login):
+    app = webtest.TestApp(main.app)
+
+    response = app.get('/blobreader')
+
+    assert 'abcde\nabc\nabcde\n' in response

appengine/standard/blobstore/blobreader/app.yaml

@@ -0,0 +1 @@
+GoogleAppEngineCloudStorageClient==1.9.22.1

appengine/standard/blobstore/blobreader/appengine_config.py

@@ -0,0 +1,8 @@
+runtime: python27
+api_version: 1
+threadsafe: yes
+
+handlers:
+- url: .*
+  script: main.app
+  login: required

appengine/standard/blobstore/blobreader/main.py

@@ -0,0 +1,18 @@
+# Copyright 2016 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from google.appengine.ext import vendor
+
+# Add any libraries installed in the "lib" folder.
+vendor.add('lib')

appengine/standard/blobstore/blobreader/main_test.py

@@ -0,0 +1,76 @@
+"""A sample app that operates on GCS files with blobstore API."""
+
+import cloudstorage
+from google.appengine.api import app_identity
+from google.appengine.ext import blobstore
+from google.appengine.ext.webapp import blobstore_handlers
+import webapp2
+
+
+# This handler creates a file in Cloud Storage using the cloudstorage
+# client library and then reads the data back using the Blobstore API.
+class CreateAndReadFileHandler(webapp2.RequestHandler):
+    def get(self):
+        # Get the default Cloud Storage Bucket name and create a file name for
+        # the object in Cloud Storage.
+        bucket = app_identity.get_default_gcs_bucket_name()
+
+        # Cloud Storage file names are in the format /bucket/object.
+        filename = '/{}/blobstore_demo'.format(bucket)
+
+        # Create a file in Google Cloud Storage and write something to it.
+        with cloudstorage.open(filename, 'w') as filehandle:
+            filehandle.write('abcde\n')
+
+        # In order to read the contents of the file using the Blobstore API,
+        # you must create a blob_key from the Cloud Storage file name.
+        # Blobstore expects the filename to be in the format of:
+        # /gs/bucket/object
+        blobstore_filename = '/gs{}'.format(filename)
+        blob_key = blobstore.create_gs_key(blobstore_filename)
+
+        # Read the file's contents using the Blobstore API.
+        # The last two parameters specify the start and end index of bytes we
+        # want to read.
+        data = blobstore.fetch_data(blob_key, 0, 6)
+
+        # Write the contents to the response.
+        self.response.headers['Content-Type'] = 'text/plain'
+        self.response.write(data)
+
+        # Delete the file from Google Cloud Storage using the blob_key.
+        blobstore.delete(blob_key)
+
+
+# This handler creates a file in Cloud Storage using the cloudstorage
+# client library and then serves the file back using the Blobstore API.
+class CreateAndServeFileHandler(blobstore_handlers.BlobstoreDownloadHandler):
+
+    def get(self):
+        # Get the default Cloud Storage Bucket name and create a file name for
+        # the object in Cloud Storage.
+        bucket = app_identity.get_default_gcs_bucket_name()
+
+        # Cloud Storage file names are in the format /bucket/object.
+        filename = '/{}/blobstore_serving_demo'.format(bucket)
+
+        # Create a file in Google Cloud Storage and write something to it.
+        with cloudstorage.open(filename, 'w') as filehandle:
+            filehandle.write('abcde\n')
+
+        # In order to read the contents of the file using the Blobstore API,
+        # you must create a blob_key from the Cloud Storage file name.
+        # Blobstore expects the filename to be in the format of:
+        # /gs/bucket/object
+        blobstore_filename = '/gs{}'.format(filename)
+        blob_key = blobstore.create_gs_key(blobstore_filename)
+
+        # BlobstoreDownloadHandler serves the file from Google Cloud Storage to
+        # your computer using blob_key.
+        self.send_blob(blob_key)
+
+
+app = webapp2.WSGIApplication([
+    ('/', CreateAndReadFileHandler),
+    ('/blobstore/read', CreateAndReadFileHandler),
+    ('/blobstore/serve', CreateAndServeFileHandler)], debug=True)

appengine/standard/blobstore/blobreader/requirements.txt

@@ -0,0 +1,34 @@
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import webtest
+
+import main
+
+
+def test_create_and_read(testbed, login):
+    app = webtest.TestApp(main.app)
+
+    response = app.get('/blobstore/read')
+
+    assert 'abcde' in response
+
+
+def test_create_and_serve(testbed, login):
+    app = webtest.TestApp(main.app)
+
+    response = app.get('/blobstore/serve')
+    served_file_header = response.headers['X-AppEngine-BlobKey']
+
+    assert 'encoded_gs_file' in served_file_header

appengine/standard/blobstore/gcs/app.yaml

@@ -0,0 +1 @@
+GoogleAppEngineCloudStorageClient==1.9.22.1