template tutoring-admin-dashboard v1

quit_on_failure.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+"""Exit the container when either half of the app has failed for good.
+
+A container running two processes has a failure mode a single-process one does
+not: supervisord keeps running after it gives up on a child, so the container
+stays "up" with half an app in it. From the outside that is a healthy
+container. From a browser it is a total outage, and nothing restarts it because
+nothing has reported anything wrong.
+
+This subscribes to ``PROCESS_STATE_FATAL`` — the state supervisord puts a
+program in once it has stopped retrying — and shuts supervisord down, so the
+container stops instead.
+
+**The container exits 0, not non-zero**, and that is worth knowing before
+choosing a restart policy. SIGTERM asks supervisord for an orderly shutdown, it
+stops the surviving child properly, and it reports success at having done so.
+The signal to an orchestrator is therefore *the container stopped*, not the
+exit code. With ``restart: unless-stopped`` or ``always`` it comes back. With
+``restart: on-failure`` it will NOT, because 0 is not a failure — which is a
+defensible choice for an app that cannot start, but it should be a choice
+rather than a surprise.
+
+Making it exit non-zero would mean not running supervisord as PID 1, which
+costs correct signal handling on ``docker stop`` — a worse trade for a rarer
+event.
+
+Why a script and not a one-line ``sh -c`` in the config: the event listener
+protocol is a length-prefixed exchange on stdin/stdout, and getting it wrong
+does not raise. It leaves a listener that sits reading forever while the thing
+it was meant to watch stays broken — which is worse than not having it, because
+it looks like it is working. The shell version of this really did do that.
+"""
+
+from __future__ import annotations
+
+import os
+import signal
+import sys
+
+
+def main() -> None:
+    while True:
+        # 1. Say we are ready for one event.
+        sys.stdout.write("READY\n")
+        sys.stdout.flush()
+
+        # 2. The header is a single line of `key:value` pairs, and it tells us
+        #    how many bytes of payload follow. Reading the payload matters even
+        #    though it is unused: leaving it in the pipe desynchronises every
+        #    later exchange.
+        header = sys.stdin.readline()
+        if not header:
+            return  # supervisord closed the pipe; it is going away anyway
+        fields = dict(
+            pair.split(":", 1) for pair in header.split() if ":" in pair
+        )
+        length = int(fields.get("len", 0) or 0)
+        payload = sys.stdin.read(length) if length else ""
+
+        if fields.get("eventname") == "PROCESS_STATE_FATAL":
+            name = dict(
+                pair.split(":", 1) for pair in payload.split() if ":" in pair
+            ).get("processname", "a process")
+            sys.stderr.write(
+                f"quit_on_failure: {name} is FATAL — supervisord gave up "
+                f"restarting it. Stopping the container so this is visible as a "
+                f"failure rather than as a half-working app.\n"
+            )
+            sys.stderr.flush()
+            # 3. Answer before dying, so supervisord does not log a protocol
+            #    error on top of the real one.
+            sys.stdout.write("RESULT 2\nOK")
+            sys.stdout.flush()
+            # SIGTERM, not SIGQUIT: supervisord treats it as an orderly
+            # shutdown and stops the surviving child properly, so nginx gets a
+            # chance to finish in-flight requests.
+            os.kill(os.getppid(), signal.SIGTERM)
+            return
+
+        sys.stdout.write("RESULT 2\nOK")
+        sys.stdout.flush()
+
+
+if __name__ == "__main__":
+    main()