From b3c75343478f39b50b14d7fdcae4cd3a7c605ef6 Mon Sep 17 00:00:00 2001 From: ace-n Date: Thu, 6 Jun 2019 19:05:08 -0700 Subject: [PATCH 1/3] ImageMagick: use separate bucket for blurred images. --- functions/imagemagick/README.md | 14 +++++++++++--- functions/imagemagick/main.py | 12 +++++++----- functions/imagemagick/main_test.py | 9 ++++++--- functions/imagemagick/requirements-dev.txt | 4 ++++ 4 files changed, 28 insertions(+), 11 deletions(-) create mode 100644 functions/imagemagick/requirements-dev.txt diff --git a/functions/imagemagick/README.md b/functions/imagemagick/README.md index 549c2f32186..40ccafef3a9 100644 --- a/functions/imagemagick/README.md +++ b/functions/imagemagick/README.md @@ -21,15 +21,23 @@ Functions for your project. 1. Create a Cloud Storage Bucket: - gsutil mb gs://YOUR_BUCKET_NAME + gsutil mb gs://YOUR_INPUT_BUCKET_NAME This storage bucket is used to upload images for the function to check. +1. Create a second Cloud Storage Bucket: + + gsutil mb gs://YOUR_OUTPUT_BUCKET_NAME + + This second storage bucket is used to store blurred images. (Un-blurred images will not be saved to this bucket.) + + This is necessary because saving the blurred image to the input bucket would cause your function to be invoked a second time with the blurred image itself. + 1. Deploy the `blur_offensive_images` function with a Storage trigger: - gcloud functions deploy blur_offensive_images --trigger-bucket=YOUR_BUCKET_NAME --runtime python37 + gcloud functions deploy blur_offensive_images --trigger-bucket=YOUR_INPUT_BUCKET_NAME --set-env-vars BLURRED_BUCKET_NAME=YOUR_OUTPUT_BUCKET_NAME --runtime python37 - * Replace `YOUR_BUCKET_NAME` with the name of the Cloud Storage Bucket you created earlier. + * Replace `YOUR_INPUT_BUCKET_NAME` and `YOUR_OUTPUT_BUCKET_NAME` with the names of the respective Cloud Storage Buckets you created earlier. 1. Upload an offensive image to the Storage bucket, such as this image of a flesh-eating zombie: https://cdn.pixabay.com/photo/2015/09/21/14/24/zombie-949916_1280.jpg diff --git a/functions/imagemagick/main.py b/functions/imagemagick/main.py index 1fe57d09c1a..7f339e40b8a 100644 --- a/functions/imagemagick/main.py +++ b/functions/imagemagick/main.py @@ -73,12 +73,14 @@ def __blur_image(current_blob): print(f'Image {file_name} was blurred.') - # Send Blurred image back to the bucket (with a 'blurred-' prefix). - # The prefix is necessary to avoid re-invoking the function upon upload. - new_file_name = f'blurred-{file_name}' - new_blob = current_blob.bucket.blob(new_file_name) + # // Upload result to a different bucket, to avoid re-triggering this function. + # You can also re-upload it to the same bucket + tell your Cloud Function to + # ignore files marked as blurred (e.g. those with a "blurred" prefix) + blurred_bucket_name = os.getenv('BLURRED_BUCKET_NAME') + blurred_bucket = storage_client.bucket(blurred_bucket_name) + new_blob = blurred_bucket.blob(file_name) new_blob.upload_from_filename(temp_local_filename) - print(f'Blurred image was uploaded to {new_file_name}.') + print(f'Blurred image was uploaded to: gs://{blurred_bucket_name}/{file_name}') # Delete the temporary file. os.remove(temp_local_filename) diff --git a/functions/imagemagick/main_test.py b/functions/imagemagick/main_test.py index 1afc25337ce..180a04fa1aa 100644 --- a/functions/imagemagick/main_test.py +++ b/functions/imagemagick/main_test.py @@ -79,20 +79,23 @@ def test_process_safe_image( @patch('main.os') @patch('main.Image') -def test_blur_image(image_mock, os_mock, capsys): +@patch('main.storage_client') +def test_blur_image(storage_client, image_mock, os_mock, capsys): filename = str(uuid.uuid4()) + blurred_bucket_name = 'blurred-bucket-' + str(uuid.uuid4()) os_mock.remove = MagicMock() os_mock.path = MagicMock() os_mock.path.basename = MagicMock(side_effect=(lambda x: x)) + os_mock.getenv = MagicMock(return_value = blurred_bucket_name) + image_mock.return_value = image_mock image_mock.__enter__.return_value = image_mock blob = UserDict() blob.name = filename blob.bucket = UserDict() - blob.bucket.blob = MagicMock(return_value=blob) blob.download_to_filename = MagicMock() blob.upload_from_filename = MagicMock() @@ -102,6 +105,6 @@ def test_blur_image(image_mock, os_mock, capsys): assert f'Image {filename} was downloaded to' in out assert f'Image {filename} was blurred.' in out - assert f'Blurred image was uploaded to blurred-{filename}.' in out + assert f'Blurred image was uploaded to: gs://{blurred_bucket_name}/{filename}' in out assert os_mock.remove.called assert image_mock.resize.called diff --git a/functions/imagemagick/requirements-dev.txt b/functions/imagemagick/requirements-dev.txt new file mode 100644 index 00000000000..5d5417eb7c5 --- /dev/null +++ b/functions/imagemagick/requirements-dev.txt @@ -0,0 +1,4 @@ +mock==3.0.5 +six==1.12.0 +uuid==1.30 +pytest==4.6.2 From 06ae841136ea7b5e2654b011305f70d401bdf360 Mon Sep 17 00:00:00 2001 From: Ace Nassri Date: Fri, 7 Jun 2019 17:30:29 -0700 Subject: [PATCH 2/3] Address Charlie's comments --- functions/imagemagick/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/functions/imagemagick/main.py b/functions/imagemagick/main.py index 7f339e40b8a..351394c7213 100644 --- a/functions/imagemagick/main.py +++ b/functions/imagemagick/main.py @@ -74,8 +74,8 @@ def __blur_image(current_blob): print(f'Image {file_name} was blurred.') # // Upload result to a different bucket, to avoid re-triggering this function. - # You can also re-upload it to the same bucket + tell your Cloud Function to - # ignore files marked as blurred (e.g. those with a "blurred" prefix) + # You could instead re-upload it to the same bucket + tell your Cloud Function + # to ignore files marked as blurred (e.g. those with a "blurred" prefix) blurred_bucket_name = os.getenv('BLURRED_BUCKET_NAME') blurred_bucket = storage_client.bucket(blurred_bucket_name) new_blob = blurred_bucket.blob(file_name) From f402ed82049e5ce10eaf2d2f243b6b2c5c0b3235 Mon Sep 17 00:00:00 2001 From: ace-n Date: Fri, 7 Jun 2019 17:43:30 -0700 Subject: [PATCH 3/3] Fix lint --- functions/imagemagick/main.py | 12 ++++++------ functions/imagemagick/main_test.py | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/functions/imagemagick/main.py b/functions/imagemagick/main.py index 351394c7213..e0c8f2b9c6e 100644 --- a/functions/imagemagick/main.py +++ b/functions/imagemagick/main.py @@ -73,14 +73,14 @@ def __blur_image(current_blob): print(f'Image {file_name} was blurred.') - # // Upload result to a different bucket, to avoid re-triggering this function. - # You could instead re-upload it to the same bucket + tell your Cloud Function + # Upload result to a second bucket, to avoid re-triggering the function. + # You could instead re-upload it to the same bucket + tell your function # to ignore files marked as blurred (e.g. those with a "blurred" prefix) - blurred_bucket_name = os.getenv('BLURRED_BUCKET_NAME') - blurred_bucket = storage_client.bucket(blurred_bucket_name) - new_blob = blurred_bucket.blob(file_name) + blur_bucket_name = os.getenv('BLURRED_BUCKET_NAME') + blur_bucket = storage_client.bucket(blur_bucket_name) + new_blob = blur_bucket.blob(file_name) new_blob.upload_from_filename(temp_local_filename) - print(f'Blurred image was uploaded to: gs://{blurred_bucket_name}/{file_name}') + print(f'Blurred image uploaded to: gs://{blur_bucket_name}/{file_name}') # Delete the temporary file. os.remove(temp_local_filename) diff --git a/functions/imagemagick/main_test.py b/functions/imagemagick/main_test.py index 180a04fa1aa..5c02eade88f 100644 --- a/functions/imagemagick/main_test.py +++ b/functions/imagemagick/main_test.py @@ -82,13 +82,13 @@ def test_process_safe_image( @patch('main.storage_client') def test_blur_image(storage_client, image_mock, os_mock, capsys): filename = str(uuid.uuid4()) - blurred_bucket_name = 'blurred-bucket-' + str(uuid.uuid4()) + blur_bucket = 'blurred-bucket-' + str(uuid.uuid4()) os_mock.remove = MagicMock() os_mock.path = MagicMock() os_mock.path.basename = MagicMock(side_effect=(lambda x: x)) - os_mock.getenv = MagicMock(return_value = blurred_bucket_name) + os_mock.getenv = MagicMock(return_value=blur_bucket) image_mock.return_value = image_mock image_mock.__enter__.return_value = image_mock @@ -105,6 +105,6 @@ def test_blur_image(storage_client, image_mock, os_mock, capsys): assert f'Image {filename} was downloaded to' in out assert f'Image {filename} was blurred.' in out - assert f'Blurred image was uploaded to: gs://{blurred_bucket_name}/{filename}' in out + assert f'Blurred image uploaded to: gs://{blur_bucket}/{filename}' in out assert os_mock.remove.called assert image_mock.resize.called