systemd: $MAINPID handover for NotifyAccess=main

This reverts dup-ing the socket fds on re-exec (and --bind=fd://3, and
systemd socket activation)

Author: pajod

docs/source/settings.rst

@@ -1595,6 +1595,11 @@ If the ``PORT`` environment variable is defined, the default
 is ``['0.0.0.0:$PORT']``. If it is not defined, the default
 is ``['127.0.0.1:8000']``.
 
+.. note::
+   Specifying any fd://FD socket or inheriting any socket from systemd
+   (LISTEN_FDS) results in other bind addresses to be skipped.
+   Do not mix fd://FD and systemd socket activation.
+
 .. _backlog:
 
 ``backlog``

gunicorn/arbiter.py

@@ -151,6 +151,7 @@ def start(self):
                 self.systemd = True
                 fds = range(systemd.SD_LISTEN_FDS_START,
                             systemd.SD_LISTEN_FDS_START + listen_fds)
+                self.log.debug("Inherited sockets from systemd: %r", fds)
 
             elif self.master_pid:
                 fds = []
@@ -172,6 +173,10 @@ def start(self):
 
         self.cfg.when_ready(self)
 
+        # # call `pkill --oldest -TERM -f "gunicorn: master "` instead
+        # if self.master_pid and self.systemd:
+        #     os.kill(self.master_pid, signal.SIGTERM)
+
     def init_signals(self):
         """\
         Initialize master signal handling. Most of the signals
@@ -350,7 +355,12 @@ def wakeup(self):
 
     def halt(self, reason=None, exit_status=0):
         """ halt arbiter """
-        systemd.sd_notify("STOPPING=1\nSTATUS=Gunicorn shutting down..\n", self.log)
+        if self.master_pid != 0:
+            # if NotifyAccess=main, systemd needs to know old master is in control
+            systemd.sd_notify("READY=1\nMAINPID=%d\nSTATUS=New arbiter shutdown\n" % (self.master_pid, ), self.log)
+        elif self.reexec_pid == 0:
+            # skip setting status if this is merely superseded master stopping
+            systemd.sd_notify("STOPPING=1\nSTATUS=Shutting down..\n", self.log)
 
         self.stop()
 
@@ -425,6 +435,10 @@ def reexec(self):
         master_pid = os.getpid()
         self.reexec_pid = os.fork()
         if self.reexec_pid != 0:
+            # let systemd know they will be in control after exec()
+            systemd.sd_notify(
+                "RELOADING=1\nMAINPID=%d\nSTATUS=Gunicorn arbiter re-exec in forked..\n" % (self.reexec_pid, ), self.log
+            )
             # old master
             return
 
@@ -437,6 +451,9 @@ def reexec(self):
         if self.systemd:
             environ['LISTEN_PID'] = str(os.getpid())
             environ['LISTEN_FDS'] = str(len(self.LISTENERS))
+            # move socket fds back to 3+N after we duped+closed them
+            # for idx, lnr in enumerate(self.LISTENERS):
+            #    os.dup2(lnr.fileno(), 3+idx)
         else:
             environ['GUNICORN_FD'] = ','.join(
                 str(lnr.fileno()) for lnr in self.LISTENERS)
@@ -445,8 +462,10 @@ def reexec(self):
 
         # exec the process using the original environment
         self.log.debug("exe=%r argv=%r" % (self.START_CTX[0], self.START_CTX['args']))
-        # let systemd know are are in control
-        systemd.sd_notify("READY=1\nMAINPID=%d\nSTATUS=Gunicorn arbiter re-exec\n" % (master_pid, ), self.log)
+        # let systemd know we will be in control after exec()
+        systemd.sd_notify(
+            "RELOADING=1\nMAINPID=%d\nSTATUS=Gunicorn arbiter re-exec in progress..\n" % (self.reexec_pid, ), self.log
+        )
         os.execve(self.START_CTX[0], self.START_CTX['args'], environ)
 
     def reload(self):
@@ -538,8 +557,7 @@ def reap_workers(self):
                     self.reexec_pid = 0
                     self.log.info("Master exited before promotion.")
                     # let systemd know we are (back) in control
-                    systemd.sd_notify("READY=1\nMAINPID=%d\nSTATUS=Gunicorn arbiter re-exec aborted\n" % (os.getpid(), ), self.log)
-                    continue
+                    systemd.sd_notify("READY=1\nMAINPID=%d\nSTATUS=Old arbiter promoted\n" % (os.getpid(), ), self.log)
                 else:
                     worker = self.WORKERS.pop(wpid, None)
                     if not worker:

gunicorn/config.py

@@ -616,6 +616,11 @@ class Bind(Setting):
         If the ``PORT`` environment variable is defined, the default
         is ``['0.0.0.0:$PORT']``. If it is not defined, the default
         is ``['127.0.0.1:8000']``.
+
+        .. note::
+           Specifying any fd://FD socket or inheriting any socket from systemd
+           (LISTEN_FDS) results in other bind addresses to be skipped.
+           Do not mix fd://FD and systemd socket activation.
         """
 
 

gunicorn/instrument/statsd.py

@@ -35,6 +35,7 @@ def __init__(self, cfg):
             self.sock = socket.socket(address_family, socket.SOCK_DGRAM)
             self.sock.connect(cfg.statsd_host)
         except Exception:
+            self.sock.close()
             self.sock = None
 
         self.dogstatsd_tags = cfg.dogstatsd_tags

gunicorn/sock.py

@@ -24,8 +24,8 @@ def __init__(self, address, conf, log, fd=None):
             sock = socket.socket(self.FAMILY, socket.SOCK_STREAM)
             bound = False
         else:
-            sock = socket.fromfd(fd, self.FAMILY, socket.SOCK_STREAM)
-            os.close(fd)
+            # does not duplicate the fd, this LISTEN_FDS stays at fds 3+N
+            sock = socket.socket(self.FAMILY, socket.SOCK_STREAM, fileno=fd)
             bound = True
 
         self.sock = self.set_options(sock, bound=bound)
@@ -156,6 +156,12 @@ def create_sockets(conf, log, fds=None):
         fdaddr += list(fds)
     laddr = [bind for bind in addr if not isinstance(bind, int)]
 
+    # LISTEN_FDS=1 + fd://3
+    uniq_fdaddr = set()
+    duped_fdaddr = {fd for fd in fdaddr if fd in uniq_fdaddr or uniq_fdaddr.add(fd)}
+    if duped_fdaddr:
+        log.warning("Binding with fd:// is unsupported with systemd/re-exec.")
+
     # check ssl config early to raise the error on startup
     # only the certfile is needed since it can contains the keyfile
     if conf.certfile and not os.path.exists(conf.certfile):
@@ -167,9 +173,11 @@ def create_sockets(conf, log, fds=None):
     # sockets are already bound
     if fdaddr:
         for fd in fdaddr:
-            sock = socket.fromfd(fd, socket.AF_UNIX, socket.SOCK_STREAM)
+            sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, fileno=fd)
             sock_name = sock.getsockname()
             sock_type = _sock_type(sock_name)
+            log.debug("listen: fd %d => fd %d for %s", fd, sock.fileno(), sock.getsockname())
+            sock.detach()  # only created to call getsockname(), will re-attach shorty
             listener = sock_type(sock_name, conf, log, fd=fd)
             listeners.append(listener)