Fix for downloading large files

Author: tino097

ckanext/s3filestore/controller.py

@@ -8,10 +8,11 @@
 import ckan.lib.base as base
 import ckan.model as model
 import ckan.lib.uploader as uploader
-from ckan.common import _, request, c, response
+from ckan.common import _, request, c, response, streaming_response
 from botocore.exceptions import ClientError
 
 from ckanext.s3filestore.uploader import S3Uploader
+import webob
 
 import logging
 log = logging.getLogger(__name__)
@@ -45,6 +46,7 @@ def resource_download(self, id, resource_id, filename=None):
             upload = uploader.get_resource_uploader(rsc)
             bucket_name = config.get('ckanext.s3filestore.aws_bucket_name')
             region = config.get('ckanext.s3filestore.region_name')
+            host_name = config.get('ckanext.s3filestore.host_name')
             bucket = upload.get_s3_bucket(bucket_name)
 
             if filename is None:
@@ -57,8 +59,15 @@ def resource_download(self, id, resource_id, filename=None):
                          .format(key_path, bucket_name))
 
             try:
-                obj = bucket.Object(key_path)
-                contents = str(obj.get()['Body'].read())
+                # Small workaround to manage downloading of large files
+                # We are using redirect to minio's resource public URL
+                s3 = upload.get_s3_session()
+                client = s3.client(service_name='s3', endpoint_url=host_name)
+                url = client.generate_presigned_url(ClientMethod='get_object',
+                                                    Params={'Bucket': bucket.name,
+                                                            'Key': key_path})
+                redirect(url)
+
             except ClientError as ex:
                 if ex.response['Error']['Code'] == 'NoSuchKey':
                     # attempt fallback
@@ -79,24 +88,6 @@ def resource_download(self, id, resource_id, filename=None):
                 else:
                     raise ex
 
-            dataapp = paste.fileapp.DataApp(contents)
-
-            try:
-                status, headers, app_iter = request.call_application(dataapp)
-            except OSError:
-                abort(404, _('Resource data not found'))
-
-            response.headers.update(dict(headers))
-            response.status = status
-            content_type, x = mimetypes.guess_type(rsc.get('url', ''))
-            if content_type:
-                response.headers['Content-Type'] = content_type
-            return app_iter
-
-        elif 'url' not in rsc:
-            abort(404, _('No download is available'))
-        redirect(str(rsc['url']))
-
     def filesystem_resource_download(self, id, resource_id, filename=None):
         """
         A fallback controller action to download resources from the

ckanext/s3filestore/uploader.py

@@ -39,23 +39,28 @@ def get_directory(self, id, storage_path):
         directory = os.path.join(storage_path, id)
         return directory
 
+    def get_s3_session(self):
+        return boto3.session.Session(aws_access_key_id=self.p_key,
+                                     aws_secret_access_key=self.s_key,
+                                     region_name=self.region)
+
     def get_s3_bucket(self, bucket_name):
         '''Return a boto bucket, creating it if it doesn't exist.'''
 
         # make s3 connection using boto3
-        session = boto3.session.Session(aws_access_key_id=self.p_key,
-                                        aws_secret_access_key=self.s_key,
-                                        region_name=self.region)
-        s3 = session.resource('s3', endpoint_url=self.host_name,
-                              config=botocore.client.Config(signature_version=self.signature))
+
+        s3 = self.get_s3_session().resource('s3', endpoint_url=self.host_name,
+                                            config=botocore.client.Config(
+                                             signature_version=self.signature))
         bucket = s3.Bucket(bucket_name)
         try:
             if s3.Bucket(bucket.name) in s3.buckets.all():
                 log.info('Bucket {0} found!'.format(bucket_name))
 
             else:
                 log.warning(
-                    'Bucket {0} could not be found, attempting to create it...'.format(bucket_name))
+                    'Bucket {0} could not be found,\
+                    attempting to create it...'.format(bucket_name))
                 try:
                     bucket = s3.create_bucket(Bucket=bucket_name, CreateBucketConfiguration={
                         'LocationConstraint': 'us-east-1'})