Merge pull request #799 from theupdateframework/longer_testing_delays_for_appveyor

test: Delay more when spawning test servers to avoid rare failures

Author: awwad

tests/test_proxy_use.py

@@ -124,7 +124,7 @@ def setUpClass(cls):
     # refused" errors. On the first test system. 0.1s was too short and 0.15s
     # was long enough. Use 0.5s to be safe, and if issues arise, increase it.
     # Observed some occasional AppVeyor failures, so increasing this to 1s.
-    time.sleep(1)
+    time.sleep(2)
 
 
 

tests/test_updater.py

@@ -105,9 +105,14 @@ def setUpClass(cls):
     logger.info('\tServing on port: '+str(cls.SERVER_PORT))
     cls.url = 'http://localhost:'+str(cls.SERVER_PORT) + os.path.sep
 
-    # NOTE: Following error is raised if a delay is not applied:
-    # <urlopen error [Errno 111] Connection refused>
-    time.sleep(1)
+    # NOTE: Following error is raised if a delay is not long enough to allow
+    # the server process to set up and start listening:
+    #     <urlopen error [Errno 111] Connection refused>
+    # or, on Windows:
+    #     Failed to establish a new connection: [Errno 111] Connection refused'
+    # While 0.3s has consistently worked on Travis and local builds, it led to
+    # occasional failures in AppVeyor builds, so increasing this to 2s, sadly.
+    time.sleep(2)
 
 
 
@@ -1036,9 +1041,13 @@ def test_6_get_one_valid_targetinfo(self):
     command = ['python', '-m', 'tuf.scripts.simple_server', str(SERVER_PORT)]
     server_process = subprocess.Popen(command, stderr=subprocess.PIPE)
 
-    # NOTE: Following error is raised if a delay is not applied:
+    # NOTE: Following error is raised if a delay is not long enough:
     # <urlopen error [Errno 111] Connection refused>
-    time.sleep(.3)
+    # or, on Windows:
+    # Failed to establish a new connection: [Errno 111] Connection refused'
+    # While 0.3s has consistently worked on Travis and local builds, it led to
+    # occasional failures in AppVeyor builds, so increasing this to 2s, sadly.
+    time.sleep(2)
 
     # 'path/to/tmp/repository' -> 'localhost:8001/tmp/repository'.
     repository_basepath = self.repository_directory[len(os.getcwd()):]
@@ -1289,9 +1298,14 @@ def test_7_updated_targets(self):
     command = ['python', '-m', 'tuf.scripts.simple_server', str(SERVER_PORT)]
     server_process = subprocess.Popen(command, stderr=subprocess.PIPE)
 
-    # NOTE: Following error is raised if a delay is not applied:
-    # <urlopen error [Errno 111] Connection refused>
-    time.sleep(.3)
+    # NOTE: Following error is raised if a delay is not long enough to allow
+    # the server process to set up and start listening:
+    #     <urlopen error [Errno 111] Connection refused>
+    # or, on Windows:
+    #     Failed to establish a new connection: [Errno 111] Connection refused'
+    # While 0.3s has consistently worked on Travis and local builds, it led to
+    # occasional failures in AppVeyor builds, so increasing this to 2s, sadly.
+    time.sleep(2)
 
     # 'path/to/tmp/repository' -> 'localhost:8001/tmp/repository'.
     repository_basepath = self.repository_directory[len(os.getcwd()):]
@@ -1416,9 +1430,14 @@ def test_8_remove_obsolete_targets(self):
     command = ['python', '-m', 'tuf.scripts.simple_server', str(SERVER_PORT)]
     server_process = subprocess.Popen(command, stderr=subprocess.PIPE)
 
-    # NOTE: Following error is raised if a delay is not applied:
-    # <urlopen error [Errno 111] Connection refused>
-    time.sleep(.3)
+    # NOTE: Following error is raised if a delay is not long enough to allow
+    # the server process to set up and start listening:
+    #     <urlopen error [Errno 111] Connection refused>
+    # or, on Windows:
+    #     Failed to establish a new connection: [Errno 111] Connection refused'
+    # While 0.3s has consistently worked on Travis and local builds, it led to
+    # occasional failures in AppVeyor builds, so increasing this to 2s, sadly.
+    time.sleep(2)
 
     # 'path/to/tmp/repository' -> 'localhost:8001/tmp/repository'.
     repository_basepath = self.repository_directory[len(os.getcwd()):]
@@ -1811,9 +1830,14 @@ def setUp(self):
     self.url = 'http://localhost:' + str(self.SERVER_PORT) + os.path.sep
     self.url2 = 'http://localhost:' + str(self.SERVER_PORT2) + os.path.sep
 
-    # NOTE: Following error is raised if a delay is not applied:
-    # <urlopen error [Errno 111] Connection refused>
-    time.sleep(.3)
+    # NOTE: Following error is raised if a delay is not long enough to allow
+    # the server process to set up and start listening:
+    #     <urlopen error [Errno 111] Connection refused>
+    # or, on Windows:
+    #     Failed to establish a new connection: [Errno 111] Connection refused'
+    # While 0.3s has consistently worked on Travis and local builds, it led to
+    # occasional failures in AppVeyor builds, so increasing this to 2s, sadly.
+    time.sleep(2)
 
     url_prefix = 'http://localhost:' + str(self.SERVER_PORT)
     url_prefix2 = 'http://localhost:' + str(self.SERVER_PORT2)