[WIP] Py3 #42

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open

smotornyuk wants to merge 52 commits into TkTech:master from DataShades:py3

.gitignore

-Original file line number
+Diff line change
@@ Expand Up / @@ -5,6 +5,7 @@ syntax: glob @@
     *.swo
     .DS_Store
     ckan.egg-info/*
+    ckanext_cloudstorage.egg-info/*
     sandbox/*
     dist
@@ Expand All / @@ -16,5 +17,6 @@ tmp/* @@
     solr_runtime/*
     fl_notes.txt
     *.ini
+    !alembic.ini
     .noseids
     *~

README.md

-Original file line number
+Diff line change
@@ Expand Up / @@ -37,7 +37,7 @@ below have been tested: @@
     | Provider | Uploads | Downloads | Secure URLs (private resources) |
     | --- | --- | --- | --- |
     | Azure    | YES | YES | YES (if `azure-storage` is installed) |
-    | AWS S3   | YES | YES | YES (if `boto` is installed) |
+    | AWS S3   | YES | YES | YES (if `boto` is installed and `host` key added to `driver_options`) |
     | Rackspace | YES | YES | No |
     # What are "Secure URLs"?
@@ Expand All / @@ -54,7 +54,12 @@ benefits of your CDN/blob storage. @@
     This option also enables multipart uploads, but you need to create database tables
     first. Run next command from extension folder:
-        `paster cloudstorage initdb -c /etc/ckan/default/production.ini `
+        paster cloudstorage initdb -c /etc/ckan/default/production.ini
+    For CKAN>=2.9 use the following command instead:
+        ckan -c /etc/ckan/default/production.ini db upgrade -p cloudstorage
     With that feature you can use `cloudstorage_clean_multipart` action, which is available
     only for sysadmins. After executing, all unfinished multipart uploads, older than 7 days,
@@ Expand Down @@

ckanext/cloudstorage/cli.py

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -1,170 +1,32 @@
  
    #!/usr/bin/env python

    # -*- coding: utf-8 -*-

    import os

    import os.path

    import cgi

    import tempfile

    from docopt import docopt

    from ckan.lib.cli import CkanCommand

    import click

    import ckanext.cloudstorage.utils as utils

    from ckanapi import LocalCKAN

    from ckanext.cloudstorage.storage import (

        CloudStorage,

        ResourceCloudStorage

    )

    from ckanext.cloudstorage.model import (

        create_tables,

        drop_tables

    )

    from ckan.logic import NotFound

    USAGE = """ckanext-cloudstorage

    @click.group()

    def cloudstorage():

        """CloudStorage management commands.

        """

        pass

    Commands:

        - fix-cors       Update CORS rules where possible.

        - migrate        Upload local storage to the remote.

        - initdb         Reinitalize database tables.

    Usage:

        cloudstorage fix-cors <domains>... [--c=<config>]

        cloudstorage migrate <path_to_storage> [<resource_id>] [--c=<config>]

        cloudstorage initdb [--c=<config>]

    @cloudstorage.command('fix-cors')

    @click.argument('domains', nargs=-1)

    def fix_cors(domains):

        """Update CORS rules where possible.

        """

        msg, ok = utils.fix_cors(domains)

        click.secho(msg, fg='green' if ok else 'red')

    Options:

        -c=<config>       The CKAN configuration file.

    """

    @cloudstorage.command()

    @click.argument('path')

    @click.argument('resource', required=False)

    def migrate(path, resource):

        """Upload local storage to the remote.

        """

        utils.migrate(path, resource)

    class FakeFileStorage(cgi.FieldStorage):

        def __init__(self, fp, filename):

            self.file = fp

            self.filename = filename

    class PasterCommand(CkanCommand):

        summary = 'ckanext-cloudstorage maintence utilities.'

        usage = USAGE

        def command(self):

            self._load_config()

            args = docopt(USAGE, argv=self.args)

            if args['fix-cors']:

                _fix_cors(args)

            elif args['migrate']:

                _migrate(args)

            elif args['initdb']:

                _initdb()

    def _migrate(args):

        path = args['<path_to_storage>']

        single_id = args['<resource_id>']

        if not os.path.isdir(path):

            print('The storage directory cannot be found.')

            return

        lc = LocalCKAN()

        resources = {}

        failed = []

        # The resource folder is stuctured like so on disk:

        # - storage/

        #   - ...

        # - resources/

        #   - <3 letter prefix>

        #     - <3 letter prefix>

        #       - <remaining resource_id as filename>

        #       ...

        #     ...

        #   ...

        for root, dirs, files in os.walk(path):

            # Only the bottom level of the tree actually contains any files. We

            # don't care at all about the overall structure.

            if not files:

                continue

            split_root = root.split('/')

            resource_id = split_root[-2] + split_root[-1]

            for file_ in files:

                ckan_res_id = resource_id + file_

                if single_id and ckan_res_id != single_id:

                    continue

                resources[ckan_res_id] = os.path.join(

                    root,

                    file_

                )

        for i, resource in enumerate(resources.iteritems(), 1):

            resource_id, file_path = resource

            print('[{i}/{count}] Working on {id}'.format(

                i=i,

                count=len(resources),

                id=resource_id

            ))

            try:

                resource = lc.action.resource_show(id=resource_id)

            except NotFound:

                print(u'\tResource not found')

                continue

            if resource['url_type'] != 'upload':

                print(u'\t`url_type` is not `upload`. Skip')

                continue

            with open(file_path, 'rb') as fin:

                resource['upload'] = FakeFileStorage(

                    fin,

                    resource['url'].split('/')[-1]

                )

                try:

                    uploader = ResourceCloudStorage(resource)

                    uploader.upload(resource['id'])

                except Exception as e:

                    failed.append(resource_id)

                    print(u'\tError of type {0} during upload: {1}'.format(type(e), e))

        if failed:

            log_file = tempfile.NamedTemporaryFile(delete=False)

            log_file.file.writelines(failed)

            print(u'ID of all failed uploads are saved to `{0}`'.format(log_file.name))

    def _fix_cors(args):

        cs = CloudStorage()

        if cs.can_use_advanced_azure:

            from azure.storage import blob as azure_blob

            from azure.storage import CorsRule

            blob_service = azure_blob.BlockBlobService(

                cs.driver_options['key'],

                cs.driver_options['secret']

            )

            blob_service.set_blob_service_properties(

                cors=[

                    CorsRule(

                        allowed_origins=args['<domains>'],

                        allowed_methods=['GET']

                    )

                ]

            )

            print('Done!')

        else:

            print(

                'The driver {driver_name} being used does not currently'

                ' support updating CORS rules through'

                ' cloudstorage.'.format(

                    driver_name=cs.driver_name

                )

            )

    def _initdb():

        drop_tables()

        create_tables()

        print("DB tables are reinitialized")

    def get_commands():

        return [cloudstorage]

ckanext/cloudstorage/commands.py

-Original file line number
+Diff line change
@@ -0,0 +1,56 @@
+    #!/usr/bin/env python
+    # -*- coding: utf-8 -*-
+    from __future__ import print_function
+    from ckan.lib.cli import CkanCommand
+    from docopt import docopt
+    import ckanext.cloudstorage.utils as utils
+    USAGE = """ckanext-cloudstorage
+    Commands:
+        - fix-cors       Update CORS rules where possible.
+        - migrate        Upload local storage to the remote.
+        - initdb         Reinitalize database tables.
+    Usage:
+        cloudstorage fix-cors <domains>... [--c=<config>]
+        cloudstorage migrate <path_to_storage> [<resource_id>] [--c=<config>]
+        cloudstorage initdb [--c=<config>]
+    Options:
+        -c=<config>       The CKAN configuration file.
+    """
+    class PasterCommand(CkanCommand):
+        summary = 'ckanext-cloudstorage maintence utilities.'
+        usage = USAGE
+        def command(self):
+            self._load_config()
+            args = docopt(USAGE, argv=self.args)
+            if args['fix-cors']:
+                _fix_cors(args)
+            elif args['migrate']:
+                _migrate(args)
+            elif args['initdb']:
+                _initdb()
+    def _migrate(args):
+        path = args['<path_to_storage>']
+        single_id = args['<resource_id>']
+        utils.migrate(path, single_id)
+    def _fix_cors(args):
+        msg, _ = utils.fix_cors(args['<domains>'])
+        print(msg)
+    def _initdb():
+        utils.initdb()
+        print("DB tables are reinitialized")

ckanext/cloudstorage/controller.py

-Original file line number
+Diff line change
@@ -1,62 +1,10 @@
     #!/usr/bin/env python
     # -*- coding: utf-8 -*-
-    import os.path
-    from pylons import c
-    from pylons.i18n import _
-    from ckan import logic, model
-    from ckan.lib import base, uploader
-    import ckan.lib.helpers as h
+    from ckan.lib import base
+    import ckanext.cloudstorage.utils as utils
     class StorageController(base.BaseController):
         def resource_download(self, id, resource_id, filename=None):
-            context = {
-                'model': model,
-                'session': model.Session,
-                'user': c.user or c.author,
-                'auth_user_obj': c.userobj
-            }
-            try:
-                resource = logic.get_action('resource_show')(
-                    context,
-                    {
-                        'id': resource_id
-                    }
-                )
-            except logic.NotFound:
-                base.abort(404, _('Resource not found'))
-            except logic.NotAuthorized:
-                base.abort(401, _('Unauthorized to read resource {0}'.format(id)))
-            # This isn't a file upload, so either redirect to the source
-            # (if available) or error out.
-            if resource.get('url_type') != 'upload':
-                url = resource.get('url')
-                if not url:
-                    base.abort(404, _('No download is available'))
-                h.redirect_to(url)
-            if filename is None:
-                # No filename was provided so we'll try to get one from the url.
-                filename = os.path.basename(resource['url'])
-            upload = uploader.get_resource_uploader(resource)
-            # if the client requests with a Content-Type header (e.g. Text preview)
-            # we have to add the header to the signature
-            try:
-                content_type = getattr(c.pylons.request, "content_type", None)
-            except AttributeError:
-                content_type = None
-            uploaded_url = upload.get_url_from_filename(resource['id'], filename,
-                                                        content_type=content_type)
-            # The uploaded file is missing for some reason, such as the
-            # provider being down.
-            if uploaded_url is None:
-                base.abort(404, _('No download is available'))
-            h.redirect_to(uploaded_url)
+            return utils.resource_download(id, resource_id, filename)

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

[WIP] Py3 #42

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!

[WIP] Py3 #42

Are you sure you want to change the base?

Uh oh!

[WIP] Py3 #42

Uh oh!

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!