86 lines
3.6 KiB
Python
86 lines
3.6 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
"""Exit the container when either half of the app has failed for good.
|
||
|
|
|
||
|
|
A container running two processes has a failure mode a single-process one does
|
||
|
|
not: supervisord keeps running after it gives up on a child, so the container
|
||
|
|
stays "up" with half an app in it. From the outside that is a healthy
|
||
|
|
container. From a browser it is a total outage, and nothing restarts it because
|
||
|
|
nothing has reported anything wrong.
|
||
|
|
|
||
|
|
This subscribes to ``PROCESS_STATE_FATAL`` — the state supervisord puts a
|
||
|
|
program in once it has stopped retrying — and shuts supervisord down, so the
|
||
|
|
container stops instead.
|
||
|
|
|
||
|
|
**The container exits 0, not non-zero**, and that is worth knowing before
|
||
|
|
choosing a restart policy. SIGTERM asks supervisord for an orderly shutdown, it
|
||
|
|
stops the surviving child properly, and it reports success at having done so.
|
||
|
|
The signal to an orchestrator is therefore *the container stopped*, not the
|
||
|
|
exit code. With ``restart: unless-stopped`` or ``always`` it comes back. With
|
||
|
|
``restart: on-failure`` it will NOT, because 0 is not a failure — which is a
|
||
|
|
defensible choice for an app that cannot start, but it should be a choice
|
||
|
|
rather than a surprise.
|
||
|
|
|
||
|
|
Making it exit non-zero would mean not running supervisord as PID 1, which
|
||
|
|
costs correct signal handling on ``docker stop`` — a worse trade for a rarer
|
||
|
|
event.
|
||
|
|
|
||
|
|
Why a script and not a one-line ``sh -c`` in the config: the event listener
|
||
|
|
protocol is a length-prefixed exchange on stdin/stdout, and getting it wrong
|
||
|
|
does not raise. It leaves a listener that sits reading forever while the thing
|
||
|
|
it was meant to watch stays broken — which is worse than not having it, because
|
||
|
|
it looks like it is working. The shell version of this really did do that.
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import os
|
||
|
|
import signal
|
||
|
|
import sys
|
||
|
|
|
||
|
|
|
||
|
|
def main() -> None:
|
||
|
|
while True:
|
||
|
|
# 1. Say we are ready for one event.
|
||
|
|
sys.stdout.write("READY\n")
|
||
|
|
sys.stdout.flush()
|
||
|
|
|
||
|
|
# 2. The header is a single line of `key:value` pairs, and it tells us
|
||
|
|
# how many bytes of payload follow. Reading the payload matters even
|
||
|
|
# though it is unused: leaving it in the pipe desynchronises every
|
||
|
|
# later exchange.
|
||
|
|
header = sys.stdin.readline()
|
||
|
|
if not header:
|
||
|
|
return # supervisord closed the pipe; it is going away anyway
|
||
|
|
fields = dict(
|
||
|
|
pair.split(":", 1) for pair in header.split() if ":" in pair
|
||
|
|
)
|
||
|
|
length = int(fields.get("len", 0) or 0)
|
||
|
|
payload = sys.stdin.read(length) if length else ""
|
||
|
|
|
||
|
|
if fields.get("eventname") == "PROCESS_STATE_FATAL":
|
||
|
|
name = dict(
|
||
|
|
pair.split(":", 1) for pair in payload.split() if ":" in pair
|
||
|
|
).get("processname", "a process")
|
||
|
|
sys.stderr.write(
|
||
|
|
f"quit_on_failure: {name} is FATAL — supervisord gave up "
|
||
|
|
f"restarting it. Stopping the container so this is visible as a "
|
||
|
|
f"failure rather than as a half-working app.\n"
|
||
|
|
)
|
||
|
|
sys.stderr.flush()
|
||
|
|
# 3. Answer before dying, so supervisord does not log a protocol
|
||
|
|
# error on top of the real one.
|
||
|
|
sys.stdout.write("RESULT 2\nOK")
|
||
|
|
sys.stdout.flush()
|
||
|
|
# SIGTERM, not SIGQUIT: supervisord treats it as an orderly
|
||
|
|
# shutdown and stops the surviving child properly, so nginx gets a
|
||
|
|
# chance to finish in-flight requests.
|
||
|
|
os.kill(os.getppid(), signal.SIGTERM)
|
||
|
|
return
|
||
|
|
|
||
|
|
sys.stdout.write("RESULT 2\nOK")
|
||
|
|
sys.stdout.flush()
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
main()
|