diff --git a/storage/cloud-client/customer_supplied_keys.py b/storage/cloud-client/customer_supplied_keys.py deleted file mode 100644 index 0ac70266fe9..00000000000 --- a/storage/cloud-client/customer_supplied_keys.py +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Command-line sample app demonstrating customer-supplied encryption keys. - -This sample demonstrates uploading an object while supplying an encryption key, -and retrieving that object's contents using gcloud API. The sample uses -the default credential and project. To review their values, run this command: - $ gcloud info - -This sample is used on this page: - https://cloud.google.com/storage/docs/encryption#customer-supplied - -For more information, see the README.md under /storage. -""" - -import argparse -import base64 -import filecmp -import os -import tempfile - -from gcloud import storage - -# An AES256 encryption key. It must be exactly 256 bits (32 bytes). You can -# (and should) generate your own encryption key. os.urandom(32) is a good way -# to accomplish this with Python. -# -# Although these keys are provided here for simplicity, please remember -# that it is a bad idea to store your encryption keys in your source code. -ENCRYPTION_KEY = os.urandom(32) - - -def upload_object(storage_client, bucket_name, filename, object_name, - encryption_key): - """Uploads an object, specifying a custom encryption key. - - Args: - storage_client: gcloud client to access cloud storage - bucket_name: name of the destination bucket - filename: name of file to be uploaded - object_name: name of resulting object - encryption_key: encryption key to encrypt the object, - either 32 raw bytes or a string of 32 bytes. - """ - bucket = storage_client.get_bucket(bucket_name) - blob = bucket.blob(object_name) - blob.upload_from_filename(filename, encryption_key=encryption_key) - - -def download_object(storage_client, bucket_name, object_name, filename, - encryption_key): - """Downloads an object protected by a custom encryption key. - - Args: - storage_client: gcloud client to access cloud storage - bucket_name: name of the source bucket - object_name: name of the object to be downloaded - filename: name of the resulting file - encryption_key: the encryption key that the object is encrypted by, - either 32 raw bytes or a string of 32 bytes. - """ - bucket = storage_client.get_bucket(bucket_name) - blob = bucket.blob(object_name) - blob.download_to_filename(filename, encryption_key=encryption_key) - - -def main(bucket, filename): - storage_client = storage.Client() - print('Uploading object gs://{}/{} using encryption key (base64 formatted)' - ' {}'.format(bucket, filename, base64.encodestring(ENCRYPTION_KEY))) - upload_object(storage_client, bucket, filename, filename, ENCRYPTION_KEY) - print('Downloading it back') - with tempfile.NamedTemporaryFile(mode='w+b') as tmpfile: - download_object( - storage_client, - bucket, - object_name=filename, - filename=tmpfile.name, - encryption_key=ENCRYPTION_KEY) - assert filecmp.cmp(filename, tmpfile.name), ( - 'Downloaded file has different content from the original file.') - print('Done') - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('bucket', help='Your Cloud Storage bucket.') - parser.add_argument('filename', help='A file to upload and download.') - - args = parser.parse_args() - - main(args.bucket, args.filename) diff --git a/storage/cloud-client/customer_supplied_keys_test.py b/storage/cloud-client/customer_supplied_keys_test.py deleted file mode 100644 index c5449a27ff5..00000000000 --- a/storage/cloud-client/customer_supplied_keys_test.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2016, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -from customer_supplied_keys import main - - -def test_main(cloud_config, capsys): - main(cloud_config.storage_bucket, __file__) - out, err = capsys.readouterr() - - assert not re.search(r'Downloaded file [!]=', out) - assert re.search(r'Uploading.*Downloading.*Done', out, re.DOTALL) diff --git a/storage/cloud-client/encryption.py b/storage/cloud-client/encryption.py new file mode 100644 index 00000000000..571b91b2120 --- /dev/null +++ b/storage/cloud-client/encryption.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python + +# Copyright 2016 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to upload and download encrypted blobs +(objects) in Google Cloud Storage. + +Use `generate-encryption-key` to generate an example key: + + python encryption.py generate-encryption-key + +Then use the key to upload and download files encrypted with a custom key. + +For more information, see the README.md under /storage and the documentation +at https://cloud.google.com/storage/docs/encryption. +""" + +import argparse +import base64 +import os + +from gcloud import storage + + +def generate_encryption_key(): + """Generates a 256 bit (32 byte) AES encryption key and prints the + base64 representation. + + This is included for demonstration purposes. You should generate your own + key. Please remember that encryption keys should be handled with a + comprehensive security policy. + """ + key = os.urandom(32) + encoded_key = base64.b64encode(key).decode('utf-8') + print('Base 64 encoded encryption key: {}'.format(encoded_key)) + + +def upload_encrypted_blob(bucket_name, source_file_name, + destination_blob_name, base64_encryption_key): + """Uploads a file to a Google Cloud Storage bucket using a custom + encryption key. + + The file will be encrypted by Google Cloud Storage and only + retrievable using the provided encryption key. + """ + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + blob = bucket.blob(destination_blob_name) + + # Encryption key must be an AES256 key represented as a bytestring with + # 32 bytes. Since it's passed in as a base64 encoded string, it needs + # to be decoded. + encryption_key = base64.b64decode(base64_encryption_key) + + blob.upload_from_filename( + source_file_name, encryption_key=encryption_key) + + print('File {} uploaded to {}.'.format( + source_file_name, + destination_blob_name)) + + +def download_encrypted_blob(bucket_name, source_blob_name, + destination_file_name, base64_encryption_key): + """Downloads a previously-encrypted blob from Google Cloud Storage. + + The encryption key provided must be the same key provided when uploading + the blob. + """ + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + blob = bucket.blob(source_blob_name) + + # Encryption key must be an AES256 key represented as a bytestring with + # 32 bytes. Since it's passed in as a base64 encoded string, it needs + # to be decoded. + encryption_key = base64.b64decode(base64_encryption_key) + + blob.download_to_filename( + destination_file_name, encryption_key=encryption_key) + + print('Blob {} downloaded to {}.'.format( + source_blob_name, + destination_file_name)) + + +def rotate_encryption_key(bucket_name, blob_name, base64_encryption_key, + base64_new_encryption_key): + """Performs a key rotation by re-writing an encrypted blob with a new + encryption key.""" + raise NotImplementedError( + 'This is currently not available using the Cloud Client Library.') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + subparsers = parser.add_subparsers(dest='command') + + subparsers.add_parser( + 'generate-encryption-key', help=generate_encryption_key.__doc__) + + upload_parser = subparsers.add_parser( + 'upload', help=upload_encrypted_blob.__doc__) + upload_parser.add_argument( + 'bucket_name', help='Your cloud storage bucket.') + upload_parser.add_argument('source_file_name') + upload_parser.add_argument('destination_blob_name') + upload_parser.add_argument('base64_encryption_key') + + download_parser = subparsers.add_parser( + 'download', help=download_encrypted_blob.__doc__) + download_parser.add_argument( + 'bucket_name', help='Your cloud storage bucket.') + download_parser.add_argument('source_blob_name') + download_parser.add_argument('destination_file_name') + download_parser.add_argument('base64_encryption_key') + + rotate_parser = subparsers.add_parser( + 'rotate', help=rotate_encryption_key.__doc__) + rotate_parser.add_argument( + 'bucket_name', help='Your cloud storage bucket.') + download_parser.add_argument('blob_name') + download_parser.add_argument('base64_encryption_key') + download_parser.add_argument('base64_new_encryption_key') + + args = parser.parse_args() + + if args.command == 'generate-encryption-key': + generate_encryption_key() + elif args.command == 'upload': + upload_encrypted_blob( + args.bucket_name, + args.source_file_name, + args.destination_blob_name, + args.base64_encryption_key) + elif args.command == 'download': + download_encrypted_blob( + args.bucket_name, + args.source_blob_name, + args.destination_file_name, + args.base64_encryption_key) + elif args.command == 'rotate': + rotate_encryption_key( + args.bucket_name, + args.blob_name, + args.base64_encryption_key, + args.base64_new_encryption_key) diff --git a/storage/cloud-client/encryption_test.py b/storage/cloud-client/encryption_test.py new file mode 100644 index 00000000000..4ebea22d1f5 --- /dev/null +++ b/storage/cloud-client/encryption_test.py @@ -0,0 +1,67 @@ +# Copyright 2016 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import tempfile + +import encryption +from gcloud import storage +import pytest + +TEST_ENCRYPTION_KEY = 'brtJUWneL92g5q0N2gyDSnlPSYAiIVZ/cWgjyZNeMy0=' +TEST_ENCRYPTION_KEY_DECODED = base64.b64decode(TEST_ENCRYPTION_KEY) + + +def test_generate_encryption_key(capsys): + encryption.generate_encryption_key() + out, _ = capsys.readouterr() + encoded_key = out.split(':', 1).pop().strip() + key = base64.b64decode(encoded_key) + assert len(key) == 32, 'Returned key should be 32 bytes' + + +def test_upload_encrypted_blob(cloud_config): + with tempfile.NamedTemporaryFile() as source_file: + source_file.write(b'test') + + encryption.upload_encrypted_blob( + cloud_config.storage_bucket, + source_file.name, + 'test_encrypted_upload_blob', + TEST_ENCRYPTION_KEY) + + +@pytest.fixture +def test_blob(cloud_config): + """Provides a pre-existing blob in the test bucket.""" + bucket = storage.Client().bucket(cloud_config.storage_bucket) + blob = bucket.blob('encrption_test_sigil') + content = 'Hello, is it me you\'re looking for?' + blob.upload_from_string( + content, + encryption_key=TEST_ENCRYPTION_KEY_DECODED) + return blob.name, content + + +def test_download_blob(test_blob, cloud_config): + test_blob_name, test_blob_content = test_blob + with tempfile.NamedTemporaryFile() as dest_file: + encryption.download_encrypted_blob( + cloud_config.storage_bucket, + test_blob_name, + dest_file.name, + TEST_ENCRYPTION_KEY) + + downloaded_content = dest_file.read().decode('utf-8') + assert downloaded_content == test_blob_content diff --git a/storage/cloud-client/manage_blobs.py b/storage/cloud-client/manage_blobs.py index 82dfc03837e..8116c8ea6ab 100644 --- a/storage/cloud-client/manage_blobs.py +++ b/storage/cloud-client/manage_blobs.py @@ -1,22 +1,24 @@ #!/usr/bin/env python -# Copyright (C) 2016 Google Inc. +# Copyright 2016 Google, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Command-line sample application for simple CRUD management of blobs in a -given bucket. -For more information, see the README.md under /storage. +"""This application demonstrates how to perform basic operations on blobs +(objects) in a Google Cloud Storage bucket. + +For more information, see the README.md under /storage and the documentation +at https://cloud.google.com/storage/docs. """ import argparse @@ -73,7 +75,9 @@ def delete_blob(bucket_name, blob_name): if __name__ == '__main__': - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('bucket_name', help='Your cloud storage bucket.') subparsers = parser.add_subparsers(dest='command') diff --git a/storage/cloud-client/manage_blobs_test.py b/storage/cloud-client/manage_blobs_test.py index 972d939aec8..8c78930a4e6 100644 --- a/storage/cloud-client/manage_blobs_test.py +++ b/storage/cloud-client/manage_blobs_test.py @@ -1,4 +1,5 @@ -# Copyright 2016, Google, Inc. +# Copyright 2016 Google, Inc. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -18,14 +19,6 @@ import pytest -@pytest.fixture -def test_blob(cloud_config): - bucket = storage.Client().bucket(cloud_config.storage_bucket) - blob = bucket.blob('manage_blobs_test_sigil') - blob.upload_from_string('Hello, is it me you\'re looking for?') - return blob.name - - def test_list_blobs(test_blob, cloud_config, capsys): manage_blobs.list_blobs(cloud_config.storage_bucket) out, _ = capsys.readouterr() @@ -38,7 +31,17 @@ def test_upload_blob(cloud_config): manage_blobs.upload_blob( cloud_config.storage_bucket, - source_file.name, 'test_upload_blob') + source_file.name, + 'test_upload_blob') + + +@pytest.fixture +def test_blob(cloud_config): + """Provides a pre-existing blob in the test bucket.""" + bucket = storage.Client().bucket(cloud_config.storage_bucket) + blob = bucket.blob('manage_blobs_test_sigil') + blob.upload_from_string('Hello, is it me you\'re looking for?') + return blob.name def test_download_blob(test_blob, cloud_config):