systemd: send MAINPID updates on re-exec

Author: pajod

gunicorn/arbiter.py

@@ -67,8 +67,11 @@ def __init__(self, app):
 
         cwd = util.getcwd()
 
-        args = sys.argv[:]
-        args.insert(0, sys.executable)
+        if sys.version_info < (3, 10):
+            args = sys.argv[:]
+            args.insert(0, sys.executable)
+        else:
+            args = sys.orig_argv[:]
 
         # init start context
         self.START_CTX = {
@@ -159,7 +162,7 @@ def start(self):
         self.log.debug("Arbiter booted")
         self.log.info("Listening at: %s (%s)", listeners_str, self.pid)
         self.log.info("Using worker: %s", self.cfg.worker_class_str)
-        systemd.sd_notify("READY=1\nSTATUS=Gunicorn arbiter booted", self.log)
+        systemd.sd_notify("READY=1\nSTATUS=Gunicorn arbiter booted\n", self.log)
 
         # check worker class requirements
         if hasattr(self.worker_class, "check_config"):
@@ -251,7 +254,10 @@ def handle_hup(self):
         - Gracefully shutdown the old worker processes
         """
         self.log.info("Hang up: %s", self.master_name)
+        systemd.sd_notify("RELOADING=1\nSTATUS=Gunicorn arbiter reloading..\n", self.log)
         self.reload()
+        # possibly premature, newly launched workers might have failed
+        systemd.sd_notify("READY=1\nSTATUS=Gunicorn arbiter reloaded\n", self.log)
 
     def handle_term(self):
         "SIGTERM handling"
@@ -327,6 +333,8 @@ def maybe_promote_master(self):
                 self.pidfile.rename(self.cfg.pidfile)
             # reset proctitle
             util._setproctitle("master [%s]" % self.proc_name)
+            # MAINPID does not change here, it was already set on fork
+            systemd.sd_notify("READY=1\nMAINPID=%d\nSTATUS=Gunicorn arbiter promoted\n" % (os.getpid(), ), self.log)
 
     def wakeup(self):
         """\
@@ -432,7 +440,10 @@ def reexec(self):
         os.chdir(self.START_CTX['cwd'])
 
         # exec the process using the original environment
-        os.execvpe(self.START_CTX[0], self.START_CTX['args'], environ)
+        self.log.debug("exe=%r argv=%r" % (self.START_CTX[0], self.START_CTX['args']))
+        # let systemd know are are in control
+        systemd.sd_notify("READY=1\nMAINPID=%d\nSTATUS=Gunicorn arbiter re-exec\n" % (master_pid, ), self.log)
+        os.execve(self.START_CTX[0], self.START_CTX['args'], environ)
 
     def reload(self):
         old_address = self.cfg.address
@@ -522,6 +533,8 @@ def reap_workers(self):
                 if self.reexec_pid == wpid:
                     self.reexec_pid = 0
                     self.log.info("Master exited before promotion.")
+                    # let systemd know we are (back) in control
+                    systemd.sd_notify("READY=1\nMAINPID=%d\nSTATUS=Gunicorn arbiter re-exec aborted\n" % (os.getpid(), ), self.log)
                     continue
                 else:
                     worker = self.WORKERS.pop(wpid, None)

gunicorn/systemd.py

@@ -4,6 +4,7 @@
 
 import os
 import socket
+import time
 
 SD_LISTEN_FDS_START = 3
 
@@ -66,6 +67,13 @@ def sd_notify(state, logger, unset_environment=False):
         if addr[0] == '@':
             addr = '\0' + addr[1:]
         sock.connect(addr)
+        assert state.endswith("\n")
+        if "RELOADING" in state:  # broad, but systemd man promises tolerating
+            # wrong clock on some platforms.. but this is only needed on Linux
+            # nsec = 10**-9
+            # usec = 10**-6
+            state += "MONOTONIC_USEC=%d\n" % (1_000*time.monotonic_ns(), )
+        logger.debug("sd_notify: %r" % (state, ))
         sock.sendall(state.encode('utf-8'))
     except Exception:
         logger.debug("Exception while invoking sd_notify()", exc_info=True)

tests/test_arbiter.py

@@ -71,24 +71,27 @@ def test_arbiter_stop_does_not_unlink_when_using_reuse_port(close_sockets):
 
 @mock.patch('os.getpid')
 @mock.patch('os.fork')
-@mock.patch('os.execvpe')
-def test_arbiter_reexec_passing_systemd_sockets(execvpe, fork, getpid):
+@mock.patch('os.execve')
+@mock.patch('gunicorn.systemd.sd_notify')
+def test_arbiter_reexec_passing_systemd_sockets(sd_notify, execve, fork, getpid):
     arbiter = gunicorn.arbiter.Arbiter(DummyApplication())
     arbiter.LISTENERS = [mock.Mock(), mock.Mock()]
     arbiter.systemd = True
     fork.return_value = 0
+    sd_notify.return_value = None
     getpid.side_effect = [2, 3]
     arbiter.reexec()
-    environ = execvpe.call_args[0][2]
+    environ = execve.call_args[0][2]
     assert environ['GUNICORN_PID'] == '2'
     assert environ['LISTEN_FDS'] == '2'
     assert environ['LISTEN_PID'] == '3'
+    sd_notify.assert_called_once()
 
 
 @mock.patch('os.getpid')
 @mock.patch('os.fork')
-@mock.patch('os.execvpe')
-def test_arbiter_reexec_passing_gunicorn_sockets(execvpe, fork, getpid):
+@mock.patch('os.execve')
+def test_arbiter_reexec_passing_gunicorn_sockets(execve, fork, getpid):
     arbiter = gunicorn.arbiter.Arbiter(DummyApplication())
     listener1 = mock.Mock()
     listener2 = mock.Mock()
@@ -98,7 +101,7 @@ def test_arbiter_reexec_passing_gunicorn_sockets(execvpe, fork, getpid):
     fork.return_value = 0
     getpid.side_effect = [2, 3]
     arbiter.reexec()
-    environ = execvpe.call_args[0][2]
+    environ = execve.call_args[0][2]
     assert environ['GUNICORN_FD'] == '4,5'
     assert environ['GUNICORN_PID'] == '2'