(Not for review) split stor modules

.coveragerc

@@ -0,0 +1,10 @@
+[run]
+omit = stor/stor/tests/test_posix_path_compat.py,stor/stor/tests/test_integration.py,stor/stor/third_party/backoff.py,stor_dx/stor_dx/tests/test_integration_dx.py,stor_s3/stor_s3/tests/test_integration_s3.py,stor_swift/stor_swift/tests/test_integration_swift.py
+
+[report]
+exclude_lines =
+    # Have to re-enable the standard pragma
+    pragma: no cover
+    raise NotImplementedError
+show_missing=1
+fail_under=100

.gitignore

@@ -6,12 +6,17 @@
 *.orig
 *.DS_Store
 .tox
+stor*/*.egg-info/*
+stor*/.eggs/
+.*cache/
+stor*/.*cache/
 *.egg/*
 .eggs/
 *.egg-info/*
+stor*/build/*
 build/*
 docs/_build/*
-dist/*
+stor*/dist/*
 
 *.log
 nosetests*.xml

Makefile

@@ -1,6 +1,6 @@
 # Makefile utilities for running tests and publishing the package
 
-PACKAGE_NAME=stor
+PACKAGE_NAMES:=stor/ stor_dx/ stor_swift/ stor_s3/
 TEST_OUTPUT?=nosetests.xml
 PIP_INDEX_URL=https://pypi.python.org/simple/
 PYTHON?=$(shell which python)
@@ -13,7 +13,7 @@ endif
 
 .PHONY: default
 default:
-	python setup.py check build
+	cd stor; python setup.py check build; cd ..
 
 VENV_DIR?=.venv
 VENV_ACTIVATE=$(VENV_DIR)/bin/activate
@@ -23,42 +23,44 @@ WITH_PBR=$(WITH_VENV) PBR_REQUIREMENTS_FILES=requirements-pbr.txt
 .PHONY: venv
 venv: $(VENV_ACTIVATE)
 
-$(VENV_ACTIVATE): requirements*.txt
+$(VENV_ACTIVATE): stor*/requirements*.txt
 	test -f $@ || virtualenv --python=$(PYTHON) $(VENV_DIR)
 	$(WITH_VENV) echo "Within venv, running $$(python --version)"
-	$(WITH_VENV) pip install -r requirements-setup.txt --index-url=${PIP_INDEX_URL}
-	$(WITH_VENV) pip install -e . --index-url=${PIP_INDEX_URL}
-	$(WITH_VENV) pip install -r requirements-dev.txt  --index-url=${PIP_INDEX_URL}
-	$(WITH_VENV) pip install -r requirements-docs.txt --index-url=${PIP_INDEX_URL}
+	$(WITH_VENV) pip install -r stor/requirements-setup.txt --index-url=${PIP_INDEX_URL}
+	$(WITH_VENV) ./run_all.sh 'pip install -e . --index-url=${PIP_INDEX_URL}' $(PACKAGE_NAMES)
+	$(WITH_VENV) pip install -r stor/requirements-dev.txt  --index-url=${PIP_INDEX_URL}
+	$(WITH_VENV) pip install -r stor/requirements-docs.txt --index-url=${PIP_INDEX_URL}
 	touch $@
 
 develop: venv
-	$(WITH_VENV) python setup.py develop
+	$(WITH_VENV) ./run_all.sh 'python setup.py develop' $(PACKAGE_NAMES)
 
 .PHONY: docs
 docs: venv clean-docs
 	$(WITH_VENV) cd docs && make html
 
-
 .PHONY: setup
 setup: ##[setup] Run an arbitrary setup.py command
 setup: venv
 ifdef ARGS
-	$(WITH_PBR) python setup.py ${ARGS}
+	$(WITH_PBR) ./run_all.sh 'python setup.py ${ARGS}' $(PACKAGE_NAMES)
 else
 	@echo "Won't run 'python setup.py ${ARGS}' without ARGS set."
 endif
 
 .PHONY: clean
 clean:
-	$(PYTHON) setup.py clean
-	rm -rf build/
-	rm -rf dist/
-	rm -rf *.egg*/
+	./run_all.sh '$(PYTHON) setup.py clean' $(PACKAGE_NAMES)
 	rm -rf __pycache__/
+	rm -rf .*cache/
+	rm -rf *.egg*/
+	./run_all.sh 'rm -rf dist/' $(PACKAGE_NAMES)
+	./run_all.sh 'rm -rf build/' $(PACKAGE_NAMES)
+	./run_all.sh 'rm -rf __pycache__/' $(PACKAGE_NAMES)
+	./run_all.sh 'rm -rf .*cache/' $(PACKAGE_NAMES)
 	rm -f MANIFEST
 	rm -f $(TEST_OUTPUT)
-	find $(PACKAGE_NAME) -type f -name '*.pyc' -delete
+	./run_all.sh 'find . -type f -name '*.pyc' -delete' $(PACKAGE_NAMES)
 	rm -rf nosetests* "${TEST_OUTPUT}" coverage .coverage
 
 
@@ -73,7 +75,7 @@ teardown:
 
 .PHONY: lint
 lint: venv
-	$(WITH_VENV) flake8 $(PACKAGE_NAME)/
+	$(WITH_VENV) flake8 $(PACKAGE_NAMES)
 
 .PHONY: unit-test
 unit-test: venv
@@ -100,13 +102,13 @@ endif
 travis-test: venv
 	$(WITH_VENV) \
 	coverage erase; \
-	coverage run setup.py test; \
+	./run_all.sh 'coverage run setup.py test' $(PACKAGE_NAMES); \
 	status=$$?; \
 	coverage report && exit $$status;
 
 # Distribution
 
-VERSION=$(shell $(WITH_PBR) python setup.py --version | sed 's/\([0-9]*\.[0-9]*\.[0-9]*\).*$$/\1/')
+VERSION=$(cd stor; shell $(WITH_PBR) python setup.py --version | sed 's/\([0-9]*\.[0-9]*\.[0-9]*\).*$$/\1/'; cd ..)
 
 .PHONY: tag
 tag: ##[distribution] Tag the release.

docs/dx.rst

@@ -164,7 +164,7 @@ a folder or a file (i.e., `DXPath.listdir` will assume folder, `DXPath.stat` wil
 DXPath
 -------
 
-.. automodule:: stor.dx
+.. automodule:: stor_dx.dx
     :members:
 
 Copy and copytree by example

docs/s3.rst

@@ -3,5 +3,5 @@
 S3
 ==
 
-.. automodule:: stor.s3
+.. automodule:: stor_s3.s3
     :members:

docs/settings.rst

@@ -47,7 +47,7 @@ Settings can be configured in the following ways in order of precedence:
 Default Settings
 ----------------
 
-.. literalinclude:: ../stor/default.cfg
+.. literalinclude:: ../stor/stor/default.cfg
 
 Settings API
 ------------

docs/swift.rst

@@ -1,4 +1,4 @@
-.. module:: stor.swift
+.. module:: stor_swift.swift
 
 .. _swift:
 

docs/testing.rst

@@ -2,12 +2,23 @@
 
 Testing
 =======
-Stor provides a test case to use when testing against Swift storage. It also
+Stor provides a test case to use when testing against OBS storage. It also
 provides a mixin class that can be used when creating other base test classes
 
-.. automodule:: stor.test
-.. autoclass:: stor.test.SwiftTestMixin
+.. autoclass:: stor_swift.test.SwiftTestMixin
     :members:
 
-.. autoclass:: stor.test.SwiftTestCase
+.. autoclass:: stor_swift.test.SwiftTestCase
+    :members:
+
+.. autoclass:: stor_s3.test.S3TestMixin
+    :members:
+
+.. autoclass:: stor_s3.test.S3TestCase
+    :members:
+
+.. autoclass:: stor_dx.test.DXTestMixin
+    :members:
+
+.. autoclass:: stor_dx.test.DXTestCase
     :members:

run_all.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+CMD=$1
+shift
+DIRS=$*
+for directory in $DIRS; do
+    cd $directory; ${CMD}; cd ..
+done

setup.py

@@ -1,6 +0,0 @@
-from setuptools import setup
-
-setup(
-    pbr=True,
-    setup_requires=['pbr'],
-)

stor/requirements.txt

@@ -0,0 +1,4 @@
+# have to explicitly ban certain requests versions to match keystoneauth1 package for swift
+requests!=2.12.2,!=2.13.0,>=2.10.0
+six
+python-swiftclient

setup.cfg → stor/setup.cfg

@@ -20,19 +20,6 @@ classifier =
     Development Status :: 5 - Production/Stable
     Operating System :: OS Independent
 
-[coverage:run]
-branch = True
-source = stor
-omit = stor/tests/test_posix_path_compat.py,stor/tests/test_integration_swift.py,stor/tests/test_integration_s3.py,stor/tests/test_integration.py,stor/third_party/backoff.py,stor/tests/test_integration_dx.py
-
-[coverage:report]
-exclude_lines =
-    # Have to re-enable the standard pragma
-    pragma: no cover
-    raise NotImplementedError
-fail_under = 100
-show_missing = 1
-
 [files]
 packages = stor
 scripts = stor/stor-completion.bash

stor/setup.py

@@ -0,0 +1,6 @@
+from setuptools import setup
+
+setup(
+    pbr=True,
+    setup_requires=['pbr'],
+)

stor/__init__.py → stor/stor/__init__.py

@@ -18,14 +18,13 @@
 modified to avoid the need to know whether you have a Path or a string for most
 functions.
 
-See `stor.swift` for more information on Swift-specific functionality.
+See `stor_swift.swift` for more information on Swift-specific functionality.
 """
 import pkg_resources
 
 from stor.utils import copy
 from stor.utils import copytree
 from stor.utils import is_filesystem_path
-from stor.utils import is_swift_path
 from stor.utils import is_obs_path
 from stor.utils import NamedTemporaryDirectory
 from stor.base import Path
@@ -51,6 +50,7 @@ def wrapper(path, *args, **kwargs):
     wrapper.__name__ = name
     return wrapper
 
+
 # extra compat!
 open = _delegate_to_path('open')
 abspath = _delegate_to_path('abspath')
@@ -118,7 +118,6 @@ def glob(pth, pattern):  # pragma: no cover
     'rmtree',
     'walkfiles',
     'is_filesystem_path',
-    'is_swift_path',
     'is_obs_path',
     'NamedTemporaryDirectory',
 ]

stor/base.py → stor/stor/base.py

@@ -1,8 +1,10 @@
 import errno
 import fnmatch
 import glob
-import os
+import logging
 import ntpath
+import os
+import pkg_resources
 import posixpath
 import shutil
 import sys
@@ -14,7 +16,32 @@
 from six import PY3
 
 from stor import utils
-import six
+
+
+logger = logging.getLogger(__name__)
+
+
+def get_modules():
+    modules = {}
+    for entry_point in pkg_resources.iter_entry_points('stor.providers'):
+        try:
+            modules.update({entry_point.name: entry_point.load()})
+        except pkg_resources.DistributionNotFound as e:  # pragma: no cover
+            from stor import exceptions
+            print('Ignoring {entry_point} module as the requirement(s) '
+                          'for the module are not installed'.format(entry_point=entry_point.name))
+            pass
+    return modules
+
+
+def find_cls_for_path(path):
+    module_map = get_modules()
+    for k, v in module_map.items():
+        if path.startswith(k + '://'):
+            cls, pth = v(k, path)
+            if cls:
+                return cls, pth
+    return None, None
 
 
 class TreeWalkWarning(Warning):
@@ -45,26 +72,23 @@ def __new__(cls, path):
         if cls is Path:
             if not hasattr(path, 'startswith'):
                 raise TypeError('must be a string like')
-            if utils.is_dx_path(path):
-                cls = utils.find_dx_class(path)
-            elif utils.is_swift_path(path):
-                from stor.swift import SwiftPath
-
-                cls = SwiftPath
-            elif utils.is_s3_path(path):
-                from stor.s3 import S3Path
-
-                cls = S3Path
-            elif os.path == ntpath:
-                from stor.windows import WindowsPath
-
-                cls = WindowsPath
-            elif os.path == posixpath:
-                from stor.posix import PosixPath
-
-                cls = PosixPath
-            else:  # pragma: no cover
-                assert False, 'path is not compatible with stor'
+            cls, new_p = find_cls_for_path(path)
+            if cls is None:
+                if os.path == ntpath:
+                    from stor.windows import WindowsPath
+
+                    cls = WindowsPath
+                elif os.path == posixpath:
+                    from stor.posix import PosixPath
+
+                    cls = PosixPath
+                else:  # pragma: no cover
+                    assert False, 'path is not compatible with stor'
+            else:
+                if path != new_p:  # pragma: no cover
+                    path = new_p
+                    cls.path_converted = True
+                    cls.new_path = new_p
         return text_type.__new__(cls, path)
 
     def __init__(self, path):