diff --git a/docs/testing.rst b/docs/testing.rst index 5abf9c91..be1f4797 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -2,12 +2,23 @@ Testing ======= -Stor provides a test case to use when testing against Swift storage. It also +Stor provides a test case to use when testing against OBS storage. It also provides a mixin class that can be used when creating other base test classes -.. automodule:: stor.test .. autoclass:: stor_swift.test.SwiftTestMixin :members: .. autoclass:: stor_swift.test.SwiftTestCase :members: + +.. autoclass:: stor_s3.test.S3TestMixin + :members: + +.. autoclass:: stor_s3.test.S3TestCase + :members: + +.. autoclass:: stor_dx.test.DXTestMixin + :members: + +.. autoclass:: stor_dx.test.DXTestCase + :members: diff --git a/stor/__init__.py b/stor/__init__.py index 71d9edd6..6054840e 100644 --- a/stor/__init__.py +++ b/stor/__init__.py @@ -23,7 +23,6 @@ import pkg_resources from stor.utils import is_filesystem_path -from stor.utils import is_swift_path from stor.utils import is_obs_path from stor.utils import NamedTemporaryDirectory from stor.base import Path @@ -118,7 +117,6 @@ def glob(pth, pattern): # pragma: no cover 'rmtree', 'walkfiles', 'is_filesystem_path', - 'is_swift_path', 'is_obs_path', 'NamedTemporaryDirectory', ] diff --git a/stor/base.py b/stor/base.py index 19ca805d..b9e394d3 100644 --- a/stor/base.py +++ b/stor/base.py @@ -4,7 +4,6 @@ import os import ntpath import posixpath -import shlex import shutil import sys import warnings @@ -42,17 +41,19 @@ class Path(text_type): """ def __new__(cls, path): + from stor_swift import utils as swift_utils + from stor_s3 import utils as s3_utils + from stor_dx import utils as dx_utils if cls is Path: if not hasattr(path, 'startswith'): raise TypeError('must be a string like') - if utils.is_dx_path(path): - from stor_dx import utils as dx_utils + if dx_utils.is_dx_path(path): cls = dx_utils.find_dx_class(path) - elif utils.is_swift_path(path): + elif swift_utils.is_swift_path(path): from stor_swift.swift import SwiftPath cls = SwiftPath - elif utils.is_s3_path(path): + elif s3_utils.is_s3_path(path): from stor_s3.s3 import S3Path cls = S3Path diff --git a/stor/cli.py b/stor/cli.py index 3f6c9ff0..39e1b623 100644 --- a/stor/cli.py +++ b/stor/cli.py @@ -277,10 +277,13 @@ def get_path(pth, mode=None): def _wrapped_list(path, **kwargs): """Use iterative walkfiles for DX paths, rather than trying to generate full list first""" - if utils.is_dx_path(path): - func = stor.walkfiles - else: - func = stor.list + func = stor.list + try: + from stor_dx import utils as dx_utils + if dx_utils.is_dx_path(path): + func = stor.walkfiles + except ImportError: # pragma: no cover + pass return func(path, **kwargs) @@ -292,15 +295,22 @@ def _to_url(path): def _convert_swiftstack(path, bucket=None): path = stor.Path(path) - if utils.is_swift_path(path): - if not bucket: - # TODO (jtratner): print help here - raise ValueError('--bucket is required for swift paths') - return swiftstack.swift_to_s3(path, bucket=bucket) - elif utils.is_s3_path(path): - return swiftstack.s3_to_swift(path) - else: - raise ValueError("invalid path for conversion: '%s'" % path) + try: + from stor_swift import utils as swift_utils + if swift_utils.is_swift_path(path): + if not bucket: + # TODO (jtratner): print help here + raise ValueError('--bucket is required for swift paths') + return swiftstack.swift_to_s3(path, bucket=bucket) + except ImportError: # pragma: no cover + pass + try: + from stor_s3 import utils as s3_utils + if s3_utils.is_s3_path(path): + return swiftstack.s3_to_swift(path) + except ImportError: # pragma: no cover + pass + raise ValueError("invalid path for conversion: '%s'" % path) def create_parser(): diff --git a/stor/test.py b/stor/test.py deleted file mode 100644 index e69de29b..00000000 diff --git a/stor/utils.py b/stor/utils.py index c2e7b72c..5322253f 100644 --- a/stor/utils.py +++ b/stor/utils.py @@ -173,21 +173,6 @@ def validate_manifest_list(expected_objs, list_results): return set(expected_objs).issubset(listed_objs) -def is_swift_path(p): - """Determines if the path is a Swift path. - - All Swift paths start with swift:// - - Args: - p (str): The path string - - Returns: - bool: True if p is a Swift path, False otherwise. - """ - from stor_swift.swift import SwiftPath - return p.startswith(SwiftPath.drive) - - def is_filesystem_path(p): """Determines if the path is posix or windows filesystem. @@ -197,24 +182,12 @@ def is_filesystem_path(p): Returns: bool: True if p is a Windows path, False otherwise. """ + from stor_swift.utils import is_swift_path + from stor_s3.utils import is_s3_path + from stor_dx.utils import is_dx_path return not (is_swift_path(p) or is_s3_path(p) or is_dx_path(p)) -def is_s3_path(p): - """Determines if the path is a S3 path. - - All S3 paths start with ``s3://`` - - Args: - p (str): The path string - - Returns - bool: True if p is a S3 path, False otherwise. - """ - from stor_s3.s3 import S3Path - return p.startswith(S3Path.drive) - - def is_obs_path(p): """Determines if the path is an OBS path (an S3 or Swift path). @@ -224,22 +197,7 @@ def is_obs_path(p): Returns bool: True if p is an OBS path, False otherwise. """ - return is_s3_path(p) or is_swift_path(p) or is_dx_path(p) - - -def is_dx_path(p): - """Determines if the path is a DX path. - - All DX paths start with ``dx://`` - - Args: - p (str): The path string - - Returns - bool: True if p is a DX path, False otherwise. - """ - from stor_dx.dx import DXPath - return p.startswith(DXPath.drive) + return not is_filesystem_path(p) def is_writeable(path, swift_retry_options=None): diff --git a/stor_dx/dx.py b/stor_dx/dx.py index ab18cc3b..06b5a796 100644 --- a/stor_dx/dx.py +++ b/stor_dx/dx.py @@ -1302,4 +1302,4 @@ def _wait_on_close(self): # ignore timeout because we want client code to deal with it except dxpy.DXError as e: # pragma: no cover if 'timeout' not in str(e): - raise \ No newline at end of file + raise diff --git a/stor_swift/swift.py b/stor_swift/swift.py index afc129e0..0bcee451 100644 --- a/stor_swift/swift.py +++ b/stor_swift/swift.py @@ -50,7 +50,6 @@ from swiftclient.utils import generate_temp_url from stor import exceptions as stor_exceptions -from stor import is_swift_path from stor import settings from stor import utils from stor.base import Path @@ -58,6 +57,7 @@ from stor.obs import OBSUploadObject from stor.posix import PosixPath from stor.third_party.backoff import with_backoff +from stor_swift.utils import is_swift_path logger = logging.getLogger(__name__) @@ -1052,7 +1052,8 @@ def upload(self, to_upload, condition=None, use_manifest=False, - headers=None): + headers=None, + **kwargs): """Uploads a list of files and directories to swift. This method retries ``num_retries`` times if swift is unavailable or if