Use os.walk for recursing+add ignored dirs

.coverage

@@ -1 +1 @@
-!coverage.py: This is a private format, don't read it directly!{"lines":{"/home/dustin/development/python/pyinotify/inotify/test_support.py":[1,2,3,4,5,7,9,11,13,14,16,17,19,21,22],"/home/dustin/development/python/pyinotify/inotify/adapters.py":[1,2,3,4,5,6,8,10,11,15,16,20,22,23,25,26,27,28,31,32,35,36,39,40,41,42,43,44,46,47,49,51,53,56,59,60,61,63,65,66,67,69,70,72,73,75,76,78,84,85,88,89,91,92,93,95,97,98,99,100,101,102,104,105,107,110,112,115,117,118,121,123,124,126,132,134,135,136,138,139,141,142,145,148,150,152,153,154,161,162,163,165,170,171,172,176,177,191,194,195,196,197,199,200,201,202,204,208,209,210,216,217,219,221,230,231,232,234,235,238,239,242,245,247,250,252,253,256,259,261,264,265,267,268,269,271,273,275,276,278,280,281,282,283,285,287,288,289,292,294,295,298,299,301,302,303,305,307,308,310,312,313,314,315,317,319,327,328],"/home/dustin/development/python/pyinotify/inotify/library.py":[8,1,2,4,5],"/home/dustin/development/python/pyinotify/inotify/calls.py":[1,2,4,6,8,11,12,18,25,32,33,37,39,40,41,43,45,46,47,49,51,52,53,55,56],"/home/dustin/development/python/pyinotify/inotify/__init__.py":[1],"/home/dustin/development/python/pyinotify/inotify/constants.py":[3,4,8,9,10,11,12,13,14,15,16,17,18,19,23,24,30,34,35,36,40,41,42,43,44,46,47,48,52,53,54,55,56,57,58,59,60,61,62,63,67,68,69,73,74,75,76,77]}}
+!coverage.py: This is a private format, don't read it directly!{"lines":{"/home/eohm/github/Elias481/PyInotify/inotify/calls.py":[1,2,4,6,8,11,12,13,14,16,18,25,32,33,34,35,37,39,40,41,43,45,46,47,49,51,52,53,55,56],"/home/dustin/development/python/pyinotify/inotify/adapters.py":[1,2,3,4,5,6,8,10,11,15,16,20,22,23,25,26,27,28,31,32,35,36,39,40,41,42,43,44,46,47,49,51,53,56,59,60,61,63,65,66,67,69,70,72,73,75,76,78,84,85,88,89,91,92,93,95,97,98,99,100,101,102,104,105,107,110,112,115,117,118,121,123,124,126,132,134,135,136,138,139,141,142,145,148,150,152,153,154,161,162,163,165,170,171,172,176,177,191,194,195,196,197,199,200,201,202,204,208,209,210,216,217,219,221,230,231,232,234,235,238,239,242,245,247,250,252,253,256,259,261,264,265,267,268,269,271,273,275,276,278,280,281,282,283,285,287,288,289,292,294,295,298,299,301,302,303,305,307,308,310,312,313,314,315,317,319,327,328],"/home/dustin/development/python/pyinotify/inotify/test_support.py":[1,2,3,4,5,7,9,11,13,14,16,17,19,21,22],"/home/eohm/github/Elias481/PyInotify/inotify/test_support.py":[1,2,3,4,5,7,9,11,13,14,16,17,19,21,22],"/home/eohm/github/Elias481/PyInotify/inotify/constants.py":[3,4,8,9,10,11,12,13,14,15,16,17,18,19,23,24,30,34,35,36,40,41,42,43,44,46,47,48,52,53,54,55,56,57,58,59,60,61,62,63,67,68,69,73,74,75,76,77],"/home/dustin/development/python/pyinotify/inotify/constants.py":[3,4,8,9,10,11,12,13,14,15,16,17,18,19,23,24,30,34,35,36,40,41,42,43,44,46,47,48,52,53,54,55,56,57,58,59,60,61,62,63,67,68,69,73,74,75,76,77],"/home/eohm/github/Elias481/PyInotify/inotify/__init__.py":[1],"/home/eohm/github/Elias481/PyInotify/inotify/library.py":[1,2,4,5,8],"/home/dustin/development/python/pyinotify/inotify/calls.py":[1,2,4,6,8,11,12,18,25,32,33,37,39,40,41,43,45,46,47,49,51,52,53,55,56],"/home/dustin/development/python/pyinotify/inotify/__init__.py":[1],"/home/dustin/development/python/pyinotify/inotify/library.py":[8,1,2,4,5],"/home/eohm/github/Elias481/PyInotify/inotify/adapters.py":[1,2,3,4,5,6,8,10,11,12,13,14,15,16,18,19,20,23,24,25,27,28,30,31,32,33,35,36,37,38,39,40,41,44,45,48,49,50,51,52,53,54,55,57,58,59,60,61,62,63,65,66,68,69,71,72,73,75,76,77,78,79,80,81,82,83,85,86,87,89,90,91,93,95,96,97,98,99,101,103,104,106,107,109,110,111,113,114,116,117,118,120,121,122,123,124,126,128,129,130,131,132,133,134,135,136,137,138,139,140,141,143,144,146,147,149,150,151,152,153,154,155,158,160,161,163,164,165,167,168,169,171,172,173,175,176,177,178,179,180,182,183,184,185,186,187,188,189,190,191,192,193,195,196,197,200,201,203,204,205,209,210,211,212,213,217,218,223,224,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,396,397,398,399]}}

inotify/adapters.py

@@ -5,6 +5,14 @@
 import collections
 import time
 
+if hasattr(os, 'scandir'):
+    from os import walk
+else:
+    try:
+        from scandir import walk
+    except ImportError:
+        from os import walk
+
 from errno import EINTR
 
 import inotify.constants
@@ -223,9 +231,13 @@ def event_gen(
 
             for fd, event_type in events:
                 # (fd) looks to always match the inotify FD.
-
-                names = self._get_event_names(event_type)
-                _LOGGER.debug("Events received from epoll: {}".format(names))
+
+                #names = self._get_event_names(event_type)
+                #_LOGGER.debug("Events received from epoll: {}".format(names))
+                #remove confusing event name... if resolved it should resolve to
+                #proper EPOLL* name (EPOLLIN/1 should be common case)
+                #but implement this just for this single debug line?
+                _LOGGER.debug("Events received from epoll (mask o%o)", event_type)
 
                 for (header, type_names, path, filename) \
                         in self._handle_inotify_event(fd):
@@ -257,7 +269,7 @@ def last_success_return(self):
 
 class _BaseTree(object):
     def __init__(self, mask=inotify.constants.IN_ALL_EVENTS,
-                 block_duration_s=_DEFAULT_EPOLL_BLOCK_DURATION_S):
+                 block_duration_s=_DEFAULT_EPOLL_BLOCK_DURATION_S, ignored_dirs=[]):
 
         # No matter what we actually received as the mask, make sure we have
         # the minimum that we require to curate our list of watches.
@@ -266,6 +278,16 @@ def __init__(self, mask=inotify.constants.IN_ALL_EVENTS,
                         inotify.constants.IN_CREATE | \
                         inotify.constants.IN_DELETE
 
+        ignored_dirs_lookup = {}
+        for parent, child in (os.path.split(ignored.rstrip('/')) for ignored in ignored_dirs):
+            if not parent:
+                parent = '.'
+            if parent in ignored_dirs_lookup:
+                ignored_dirs_lookup[parent].add(child)
+            else:
+                ignored_dirs_lookup[parent] = set((child,))
+        self._ignored_dirs = ignored_dirs_lookup
+
         self._i = Inotify(block_duration_s=block_duration_s)
 
     def event_gen(self, ignore_missing_new_folders=False, **kwargs):
@@ -291,13 +313,17 @@ def event_gen(self, ignore_missing_new_folders=False, **kwargs):
                        (
                         os.path.exists(full_path) is True or
                         ignore_missing_new_folders is False
+                       ) and \
+                       (
+                        path not in self._ignored_dirs or
+                        filename not in self._ignored_dirs[path]
                        ):
                         _LOGGER.debug("A directory has been created. We're "
                                       "adding a watch on it (because we're "
                                       "being recursive): [%s]", full_path)
 
 
-                        self._i.add_watch(full_path, self._mask)
+                        self._load_tree(full_path)
 
                     if header.mask & inotify.constants.IN_MOVED_FROM:
                         _LOGGER.debug("A directory has been removed. We're "
@@ -314,82 +340,60 @@ def event_gen(self, ignore_missing_new_folders=False, **kwargs):
                                       full_path)
 
                         self._i.remove_watch(full_path, superficial=True)
-                    elif header.mask & inotify.constants.IN_MOVED_TO:
-                        _LOGGER.debug("A directory has been renamed. We're "
-                                      "adding a watch on it (because we're "
-                                      "being recursive): [%s]", full_path)
-
-                        self._i.add_watch(full_path, self._mask)
 
             yield event
 
     @property
     def inotify(self):
         return self._i
 
+    def _load_tree(self, path):
+        def filter_dirs_add_watches_gen(inotify, mask, dirpath, subdirs, ignored_subdirs):
+            for subdir in subdirs:
+                if subdir in ignored_subdirs:
+                    continue
+                inotify.add_watch(os.path.join(dirpath, subdir), mask)
+                yield subdir
+
+        inotify = self._i
+        mask = self._mask
+        inotify.add_watch(path, mask)
+        ignored_dirs = self._ignored_dirs
+
+        for dirpath, subdirs, _f in walk(path):
+            ignored_subdirs = ignored_dirs.get(dirpath)
+            if ignored_subdirs:
+                subdirs[:] = filter_dirs_add_watches_gen(inotify, mask, dirpath, subdirs, ignored_subdirs)
+                continue
+            for subdir in subdirs:
+                inotify.add_watch(os.path.join(dirpath, subdir), mask)
 
 class InotifyTree(_BaseTree):
     """Recursively watch a path."""
 
     def __init__(self, path, mask=inotify.constants.IN_ALL_EVENTS,
-                 block_duration_s=_DEFAULT_EPOLL_BLOCK_DURATION_S):
-        super(InotifyTree, self).__init__(mask=mask, block_duration_s=block_duration_s)
-
-        self.__root_path = path
+                 block_duration_s=_DEFAULT_EPOLL_BLOCK_DURATION_S, ignored_dirs=[]):
+        super(InotifyTree, self).__init__(mask=mask, block_duration_s=block_duration_s,
+              ignored_dirs=ignored_dirs)
 
         self.__load_tree(path)
 
     def __load_tree(self, path):
         _LOGGER.debug("Adding initial watches on tree: [%s]", path)
-
-        paths = []
-
-        q = [path]
-        while q:
-            current_path = q[0]
-            del q[0]
-
-            paths.append(current_path)
-
-            for filename in os.listdir(current_path):
-                entry_filepath = os.path.join(current_path, filename)
-                if os.path.isdir(entry_filepath) is False:
-                    continue
-
-                q.append(entry_filepath)
-
-        for path in paths:
-            self._i.add_watch(path, self._mask)
+        self._load_tree(path)
 
 
 class InotifyTrees(_BaseTree):
     """Recursively watch over a list of trees."""
 
     def __init__(self, paths, mask=inotify.constants.IN_ALL_EVENTS,
-                 block_duration_s=_DEFAULT_EPOLL_BLOCK_DURATION_S):
-        super(InotifyTrees, self).__init__(mask=mask, block_duration_s=block_duration_s)
+                 block_duration_s=_DEFAULT_EPOLL_BLOCK_DURATION_S, ignored_dirs=[]):
+        super(InotifyTrees, self).__init__(mask=mask, block_duration_s=block_duration_s,
+              ignored_dirs=ignored_dirs)
 
         self.__load_trees(paths)
 
     def __load_trees(self, paths):
         _LOGGER.debug("Adding initial watches on trees: [%s]", ",".join(map(str, paths)))
-
-        found = []
-
-        q = paths
-        while q:
-            current_path = q[0]
-            del q[0]
-
-            found.append(current_path)
-
-            for filename in os.listdir(current_path):
-                entry_filepath = os.path.join(current_path, filename)
-                if os.path.isdir(entry_filepath) is False:
-                    continue
-
-                q.append(entry_filepath)
-
-
-        for path in found:
-            self._i.add_watch(path, self._mask)
+        for path in paths:
+            self._load_tree(path)

inotify/resources/README.rst

@@ -110,6 +110,7 @@ This will immediately recurse through the directory tree and add watches on all
 
 The other differences from the standard functionality:
 
+- You can ignore specific directories with the *ignored_dirs* parameter.
 - You can't remove a watch since watches are automatically managed.
 - Even if you provide a very restrictive mask that doesn't allow for directory create/delete events, the *IN_ISDIR*, *IN_CREATE*, and *IN_DELETE* flags will still be seen.
 
@@ -118,7 +119,9 @@ The other differences from the standard functionality:
 Notes
 =====
 
-- **IMPORTANT:** Recursively monitoring paths is **not** a functionality provided by the kernel. Rather, we artificially implement it. As directory-created events are received, we create watches for the child directories on-the-fly. This means that there is potential for a race condition: if a directory is created and a file or directory is created inside before you (using the `event_gen()` loop) have a chance to observe it, then you are going to have a problem: If it is a file, then you will miss the events related to its creation, but, if it is a directory, then not only will you miss those creation events but this library will also miss them and not be able to add a watch for them. If you are dealing with a **large number of hierarchical directory creations** and have the ability to be aware new directories via a secondary channel with some lead time before any files are populated *into* them, you can take advantage of this and call `add_watch()` manually. In this case there is limited value in using `InotifyTree()`/`InotifyTree()` instead of just `Inotify()` but this choice is left to you.
+- **IMPORTANT:** Recursively monitoring paths is **not** a functionality provided by the kernel. Rather, we artificially implement it. As directory-created events are received, we create watches for the child directories on-the-fly. This means that there is potential for a race condition: if a directory is created and files or directorie are created inside before you (using the `event_gen()` loop) have a chance to observe it, then you are going to miss the events related to them up to the point in time the directory is registered by `event_gen()` loop. If you are dealing with a **large number of hierarchical directory creations** and have the ability to be aware new directories via a secondary channel with some lead time before any files are populated *into* them, you can take advantage of this and call `add_watch()` manually. In this case there would be limited value in using `InotifyTree()`/`InotifyTree()` instead of just `Inotify()` so `add_watch()` is not provided by `InotifyTree()`/`InotifyTree()`.
+
+- For best performance on recursive paths monitoring it is recommended to install *scandir* module if You are using a pre 3.5 Python.
 
 - *epoll* is used to audit for *inotify* kernel events.