Merge pull request #22 from longaccess/ignore

support .gitignore like files when uploading

Author: kouk

BigStash/filename.py

@@ -4,6 +4,7 @@
 import sys
 import os.path
 import posixpath
+import fnmatch
 from functools import partial
 from operator import contains
 
@@ -58,6 +59,20 @@
 }
 
 
+class FnMatches(object):
+    def __init__(self, patterns=[]):
+        self.patterns = patterns
+
+    def __call__(self, path):
+        return any(map(partial(fnmatch.fnmatch, path), self.patterns))
+
+
+def setup_user_ignore(ignorefile):
+    with open(os.path.expanduser(ignorefile)) as ignorefile:
+        p = list(l.strip() for l in ignorefile if not l.startswith('#'))
+        IGNORE_TESTS['is user ignored'] = FnMatches(p)
+
+
 def splitpath(path):
     _, path = os.path.splitdrive(path)
     folders = []
@@ -91,6 +106,6 @@ def get_validator(tests={}, search={}, match={}):
     return lambda path: ((m, f, p) for p in splitpath(path)
                          for m, f in validators if p != os.sep and f(p))
 
-should_ignore = get_validator(tests=IGNORE_TESTS)
-is_invalid = get_validator(
+should_ignore = lambda: get_validator(tests=IGNORE_TESTS)
+is_invalid = lambda: get_validator(
     search=SEARCH_PATTERNS, match=MATCH_PATTERNS)

BigStash/manifest.py

@@ -4,6 +4,7 @@
 import logging
 import collections
 import os.path
+import six
 from datetime import datetime
 from cached_property import cached_property
 from functools import partial
@@ -78,21 +79,30 @@ def files(self):
     @classmethod
     def from_paths(cls, paths, title=''):
         errors = []
+        ignored_files = []
 
         def ignored(path, reason, *args):
+            ignored_files.append((path, reason))
             log.debug("Ignoring {}: {}".format(path, reason))
+            return True
 
         def invalid(path, reason, *args):
             errors.append((path, reason))
             log.debug("Invalid file {}: {}".format(path, reason))
+            return True
+
+        validators = {
+            ignored: filename.should_ignore(),
+            invalid: filename.is_invalid()
+        }
 
         def _include_file(path):
             if not os.path.exists(path):
                 return invalid(path, "File doesn't exist")
+            validations = [starmap(partial(c, path), v(path))
+                           for c, v in six.iteritems(validators)]
 
-            return not any(chain(
-                starmap(partial(ignored, path), filename.should_ignore(path)),
-                starmap(partial(invalid, path), filename.is_invalid(path))))
+            return not any(chain(*validations))
 
         def _walk_dirs(paths):
             for path in paths:
@@ -115,4 +125,4 @@ def _tofile(path):
 
         files = map(_tofile, map(os.path.abspath, _walk_dirs(paths)))
 
-        return (cls(title=title, files=files), errors)
+        return (cls(title=title, files=files), errors, ignored_files)

BigStash/upload.py

@@ -1,7 +1,7 @@
 """bgst is a command line client to BigStash.co
 
 Usage:
-  bgst put [-t TITLE] [--silent] [--dont-wait] FILES...
+  bgst put [--ignore-file IGNORE] [-t TITLE] [--silent] [--dont-wait] FILES...
   bgst settings [--user=USERNAME] [--password=PASSWORD]
   bgst settings --reset
   bgst list [--limit=NUMBER]
@@ -22,6 +22,7 @@
   --reset                       Remove saved configuration, revoke
                                 authentication token.
   --limit=NUMBER                Show up to NUMBER results. [default: 10]
+  --ignore-file=IGNORE           Path to a .gitignore like file.
 """
 
 from __future__ import print_function
@@ -33,8 +34,10 @@
 import logging
 import posixpath
 import threading
+import inflect
 from wrapt import decorator
 from BigStash import __version__
+from BigStash.filename import setup_user_ignore
 from BigStash.auth import get_api_credentials
 from BigStash.conf import BigStashAPISettings
 from BigStash import BigStashAPI, BigStashError
@@ -45,6 +48,8 @@
 
 log = logging.getLogger('bigstash.upload')
 
+peng = inflect.engine()
+
 
 def smart_str(s):
     if isinstance(s, six.text_type):
@@ -195,16 +200,30 @@ def bgst_put(args, settings):
         opt_dont_wait = False if not args['--dont-wait'] else True
         upload = None
         filepaths = map(smart_str, args['FILES'])
-        manifest, errors = Manifest.from_paths(paths=filepaths, title=title)
+        ignorefile = args['--ignore-file']
+        if ignorefile:
+            setup_user_ignore(ignorefile)
+        manifest, errors, ignored = Manifest.from_paths(
+            paths=filepaths, title=title)
+        ignored_msg = ''
+        if ignored:
+            ignored_msg = "({} {} ignored)".format(
+                len(ignored), peng.plural("file", len(ignored)))
+        if len(manifest) == 0:
+            print(" ".join(["No files found", ignored_msg]))
+            sys.exit(5)
         if errors:
             errtext = [": ".join(e) for e in errors]
             print("\n".join(["There were errors:"] + errtext))
             sys.exit(4)
         k, s = get_api_credentials(settings)
         bigstash = BigStashAPI(key=k, secret=s, settings=settings)
         upload = bigstash.CreateUpload(manifest=manifest)
+        filecount = len(manifest)
         if not opt_silent:
-            print("Uploading {}..".format(upload.archive.key))
+            msg = "Uploading {} {} as archive {}..".format(
+                filecount, peng.plural("file", filecount), upload.archive.key)
+            print(" ".join([msg, ignored_msg]))
         s3 = boto3.resource(
             's3', region_name=upload.s3.region,
             aws_access_key_id=upload.s3.token_access_key,

requirements/base.txt

@@ -22,3 +22,5 @@ retrying==1.3.3
 boto3==0.0.17
 # sha256: SbOoJSgL1ms6qDWF71nEqMgvLIpSLb51SovI0IyFxJE
 docopt==0.6.2
+# sha256: DiNY9PRqVMyj13uYP6NwiwLcg6zY0oRmpn6Ity5cGzQ
+inflect==0.2.5

setup.py

@@ -24,7 +24,8 @@ def pep386adapt(version):
     'wrapt',
     'boto3',
     'cached_property',
-    'docopt'
+    'docopt',
+    'inflect'
 ]