Preparing the modules before final package separation

docs/testing.rst

@@ -2,12 +2,23 @@
 
 Testing
 =======
-Stor provides a test case to use when testing against Swift storage. It also
+Stor provides a test case to use when testing against OBS storage. It also
 provides a mixin class that can be used when creating other base test classes
 
-.. automodule:: stor.test
 .. autoclass:: stor_swift.test.SwiftTestMixin
     :members:
 
 .. autoclass:: stor_swift.test.SwiftTestCase
     :members:
+
+.. autoclass:: stor_s3.test.S3TestMixin
+    :members:
+
+.. autoclass:: stor_s3.test.S3TestCase
+    :members:
+
+.. autoclass:: stor_dx.test.DXTestMixin
+    :members:
+
+.. autoclass:: stor_dx.test.DXTestCase
+    :members:

stor/__init__.py

@@ -23,7 +23,6 @@
 import pkg_resources
 
 from stor.utils import is_filesystem_path
-from stor.utils import is_swift_path
 from stor.utils import is_obs_path
 from stor.utils import NamedTemporaryDirectory
 from stor.base import Path
@@ -118,7 +117,6 @@ def glob(pth, pattern):  # pragma: no cover
     'rmtree',
     'walkfiles',
     'is_filesystem_path',
-    'is_swift_path',
     'is_obs_path',
     'NamedTemporaryDirectory',
 ]

stor/base.py

@@ -4,7 +4,6 @@
 import os
 import ntpath
 import posixpath
-import shlex
 import shutil
 import sys
 import warnings
@@ -42,17 +41,19 @@ class Path(text_type):
     """
 
     def __new__(cls, path):
+        from stor_swift import utils as swift_utils
+        from stor_s3 import utils as s3_utils
+        from stor_dx import utils as dx_utils
         if cls is Path:
             if not hasattr(path, 'startswith'):
                 raise TypeError('must be a string like')
-            if utils.is_dx_path(path):
-                from stor_dx import utils as dx_utils
+            if dx_utils.is_dx_path(path):
                 cls = dx_utils.find_dx_class(path)
-            elif utils.is_swift_path(path):
+            elif swift_utils.is_swift_path(path):
                 from stor_swift.swift import SwiftPath
 
                 cls = SwiftPath
-            elif utils.is_s3_path(path):
+            elif s3_utils.is_s3_path(path):
                 from stor_s3.s3 import S3Path
 
                 cls = S3Path

stor/cli.py

@@ -277,10 +277,13 @@ def get_path(pth, mode=None):
 
 def _wrapped_list(path, **kwargs):
     """Use iterative walkfiles for DX paths, rather than trying to generate full list first"""
-    if utils.is_dx_path(path):
-        func = stor.walkfiles
-    else:
-        func = stor.list
+    func = stor.list
+    try:
+        from stor_dx import utils as dx_utils
+        if dx_utils.is_dx_path(path):
+            func = stor.walkfiles
+    except ImportError:  # pragma: no cover
+        pass
     return func(path, **kwargs)
 
 
@@ -292,15 +295,22 @@ def _to_url(path):
 
 def _convert_swiftstack(path, bucket=None):
     path = stor.Path(path)
-    if utils.is_swift_path(path):
-        if not bucket:
-            # TODO (jtratner): print help here
-            raise ValueError('--bucket is required for swift paths')
-        return swiftstack.swift_to_s3(path, bucket=bucket)
-    elif utils.is_s3_path(path):
-        return swiftstack.s3_to_swift(path)
-    else:
-        raise ValueError("invalid path for conversion: '%s'" % path)
+    try:
+        from stor_swift import utils as swift_utils
+        if swift_utils.is_swift_path(path):
+            if not bucket:
+                # TODO (jtratner): print help here
+                raise ValueError('--bucket is required for swift paths')
+            return swiftstack.swift_to_s3(path, bucket=bucket)
+    except ImportError:  # pragma: no cover
+        pass
+    try:
+        from stor_s3 import utils as s3_utils
+        if s3_utils.is_s3_path(path):
+            return swiftstack.s3_to_swift(path)
+    except ImportError:  # pragma: no cover
+        pass
+    raise ValueError("invalid path for conversion: '%s'" % path)
 
 
 def create_parser():

stor/utils.py

@@ -173,21 +173,6 @@ def validate_manifest_list(expected_objs, list_results):
     return set(expected_objs).issubset(listed_objs)
 
 
-def is_swift_path(p):
-    """Determines if the path is a Swift path.
-
-    All Swift paths start with swift://
-
-    Args:
-        p (str): The path string
-
-    Returns:
-        bool: True if p is a Swift path, False otherwise.
-    """
-    from stor_swift.swift import SwiftPath
-    return p.startswith(SwiftPath.drive)
-
-
 def is_filesystem_path(p):
     """Determines if the path is posix or windows filesystem.
 
@@ -197,24 +182,12 @@ def is_filesystem_path(p):
     Returns:
         bool: True if p is a Windows path, False otherwise.
     """
+    from stor_swift.utils import is_swift_path
+    from stor_s3.utils import is_s3_path
+    from stor_dx.utils import is_dx_path
     return not (is_swift_path(p) or is_s3_path(p) or is_dx_path(p))
 
 
-def is_s3_path(p):
-    """Determines if the path is a S3 path.
-
-    All S3 paths start with ``s3://``
-
-    Args:
-        p (str): The path string
-
-    Returns
-        bool: True if p is a S3 path, False otherwise.
-    """
-    from stor_s3.s3 import S3Path
-    return p.startswith(S3Path.drive)
-
-
 def is_obs_path(p):
     """Determines if the path is an OBS path (an S3 or Swift path).
 
@@ -224,22 +197,7 @@ def is_obs_path(p):
     Returns
         bool: True if p is an OBS path, False otherwise.
     """
-    return is_s3_path(p) or is_swift_path(p) or is_dx_path(p)
-
-
-def is_dx_path(p):
-    """Determines if the path is a DX path.
-
-    All DX paths start with ``dx://``
-
-    Args:
-        p (str): The path string
-
-    Returns
-        bool: True if p is a DX path, False otherwise.
-    """
-    from stor_dx.dx import DXPath
-    return p.startswith(DXPath.drive)
+    return not is_filesystem_path(p)
 
 
 def is_writeable(path, swift_retry_options=None):

stor_dx/dx.py

@@ -1302,4 +1302,4 @@ def _wait_on_close(self):
                 # ignore timeout because we want client code to deal with it
                 except dxpy.DXError as e:  # pragma: no cover
                     if 'timeout' not in str(e):
-                        raise
+                        raise

stor_swift/swift.py

@@ -50,14 +50,14 @@
 from swiftclient.utils import generate_temp_url
 
 from stor import exceptions as stor_exceptions
-from stor import is_swift_path
 from stor import settings
 from stor import utils
 from stor.base import Path
 from stor.obs import OBSPath
 from stor.obs import OBSUploadObject
 from stor.posix import PosixPath
 from stor.third_party.backoff import with_backoff
+from stor_swift.utils import is_swift_path
 
 
 logger = logging.getLogger(__name__)
@@ -1052,7 +1052,8 @@ def upload(self,
                to_upload,
                condition=None,
                use_manifest=False,
-               headers=None):
+               headers=None,
+               **kwargs):
         """Uploads a list of files and directories to swift.
 
         This method retries ``num_retries`` times if swift is unavailable or if