Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 51 additions & 6 deletions .github/scripts/check_fly_document_workers.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,25 @@
"""Check that each document queue has a running Fly worker that auto-restarts."""

import argparse
import json
import sys


def main():
parser = argparse.ArgumentParser(description=__doc__)
modes = parser.add_mutually_exclusive_group()
modes.add_argument(
"--autoscale", action="store_true", help="Validate staging's opt-in worker pool"
)
modes.add_argument(
"--warm-transition",
action="store_true",
help="Verify workers are awake before leaving the pilot",
)
options = parser.parse_args()
machines = json.load(sys.stdin)
if options.autoscale:
return check_autoscaling(machines)
ready = True
for process_group in ("preparation_worker", "extraction_worker"):
running = []
Expand All @@ -16,22 +30,53 @@ def main():
if (
group == process_group
and machine["state"] == "started"
and policy == "always"
and (options.warm_transition or policy == "always")
):
running.append(machine["id"])

if running:
print(
f"{process_group} running with automatic restart: {', '.join(running)}"
)
print(f"{process_group} running: {', '.join(running)}")
else:
print(
f"::error::No {process_group} machine is started with restart policy 'always'."
required_policy = (
"" if options.warm_transition else " with restart policy 'always'"
)
print(f"::error::No {process_group} machine is started{required_policy}.")
ready = False

return 0 if ready else 1


def check_autoscaling(machines):
groups = {}
for machine in machines:
config = machine["config"]
group = (config.get("metadata") or {}).get("fly_process_group")
policy = (config.get("restart") or {}).get("policy")
if not config.get("standbys") and machine.get("host_status") in (None, "ok"):
groups.setdefault(group, []).append((machine["state"], policy))
coordinators = groups.get("document_worker_coordinator", [])
if coordinators != [("started", "always")]:
print(
"::error::The staging autoscaling pilot requires one running coordinator with restart policy 'always'."
)
return 1
for group in ("preparation_worker", "extraction_worker"):
candidates = [
state
for state, policy in groups.get(group, [])
if policy == "on-failure"
and state in {"started", "stopped", "created", "suspended"}
]
if len(candidates) < 2:
print(
f"::error::{group} needs two startable Machines with restart policy 'on-failure'."
)
return 1
print(
"Both staging document worker pools are startable and the coordinator is running."
)
return 0


if __name__ == "__main__":
sys.exit(main())
57 changes: 52 additions & 5 deletions .github/workflows/fly-deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,21 +42,68 @@ jobs:
fi
- uses: superfly/flyctl-actions/setup-flyctl@ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1
if: steps.current.outputs.deploy == 'true'
- name: Guard the return to warm workers
if: steps.current.outputs.deploy == 'true' && vars.LITEFILE_STAGING_WORKER_AUTOSCALING != 'true'
shell: bash
run: |
health=$(mktemp)
machines=$(mktemp)
trap 'rm -f "$health" "$machines"' EXIT
flyctl machine list --app litefile-staging --json > "$machines"
coordinator=$(python3 -c '
import json, sys
print("present" if any((m["config"].get("metadata") or {}).get("fly_process_group")
== "document_worker_coordinator" for m in json.load(sys.stdin)) else "absent")
' < "$machines")
if [[ "$coordinator" == "absent" ]]; then
exit 0
fi
status=$(curl --silent --show-error --max-time 10 --output "$health" --write-out '%{http_code}' \
https://litefile-staging.fly.dev/health/document-workers/)
if [[ "$status" != "200" && "$status" != "503" ]]; then
echo "::error::Could not verify staging's current document worker mode."
exit 1
fi
if ! python3 -c 'import json, sys; sys.exit(0 if json.load(sys.stdin).get("mode") == "warm" else 1)' < "$health"; then
echo "::error::Set the live staging coordinator to warm before removing it."
exit 1
fi
flyctl machine list --app litefile-staging --json |
python3 .github/scripts/check_fly_document_workers.py --warm-transition
env:
FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
- name: Deploy the tested commit
id: fly_deploy
if: steps.current.outputs.deploy == 'true'
run: >-
flyctl deploy --remote-only --config fly.toml
--build-arg LITEFILE_COMMIT_SHA="$(git rev-parse HEAD)"
--build-arg LITEFILE_COMMIT_DATE="$(git show -s --format=%cI HEAD)"
run: |
config=fly.toml
flags=()
if [[ "$STAGING_AUTOSCALE" == "true" ]]; then
config=fly.staging-autoscale.toml
flags+=(--ha=false)
fi
flyctl deploy --remote-only --config "$config" "${flags[@]}" \
--build-arg LITEFILE_COMMIT_SHA="$(git rev-parse HEAD)" \
--build-arg LITEFILE_COMMIT_DATE="$(git show -s --format=%cI HEAD)"
env:
FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
STAGING_AUTOSCALE: ${{ vars.LITEFILE_STAGING_WORKER_AUTOSCALING }}
- name: Check document workers
if: steps.current.outputs.deploy == 'true'
shell: bash
run: flyctl machine list --app litefile-staging --json | python3 .github/scripts/check_fly_document_workers.py
run: |
flags=()
if [[ "$STAGING_AUTOSCALE" == "true" ]]; then
flags+=(--autoscale)
curl --fail --silent --show-error --retry 6 --retry-all-errors --retry-delay 5 \
https://litefile-staging.fly.dev/health/document-workers/ |
python3 -c 'import json, sys; sys.exit(0 if json.load(sys.stdin).get("mode") == "autoscale" else 1)'
fi
flyctl machine list --app litefile-staging --json |
python3 .github/scripts/check_fly_document_workers.py "${flags[@]}"
env:
FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
STAGING_AUTOSCALE: ${{ vars.LITEFILE_STAGING_WORKER_AUTOSCALING }}
- name: Check the public site
if: ${{ !cancelled() && steps.fly_deploy.outcome == 'success' }}
run: curl --fail --silent --show-error --retry 5 --retry-all-errors --retry-delay 10 https://litefile-staging.fly.dev/about/ > /dev/null
60 changes: 60 additions & 0 deletions docs/developer-notes/capacity-observability.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,63 @@ calls, non-fatal telemetry, read-only aggregate queries, and sensitive-data
exclusion. The PostgreSQL safety suite additionally checks bounded provider
throttling retries and confirms that pending, processing and failed replacements
all block the actual submission endpoint before any court request.

## Staging document workers that sleep when idle

The normal Fly configuration keeps both document workers running. The optional
`fly.staging-autoscale.toml` profile adds a small coordinator to staging. Its
durable mode in PostgreSQL defaults to warm. In autoscale mode a worker exits
between jobs after five idle minutes. The coordinator reads committed work and
short upload wake reservations, then starts existing Machines through Fly. It
never chooses a running worker for scale-down. It can try another configured
Machine or restart an unresponsive one once there are no live claims to interrupt.

Before a new upload is stored, both supervisors must report after its wake
request. A failure returns HTTP 503 with the selected files still in the form.
Retries of an already accepted upload return the stored job. The web app
independently checks the age of work and returns a delayed message on affected
drafts. The top-level Fly probe polls `/health/document-workers/` independently
of the coordinator and does not route web traffic away from the app.

### Enable a staging pilot

1. Keep the worker configuration warm while preparing the pilot. Verify that
there are two independently startable Machines in each document process group.
Record their IDs in preference order in the staging
`DOCUMENT_WORKER_MACHINES` JSON secret, keyed by `preparation_worker` and
`extraction_worker`. Fly-managed hardware standbys have their own failover
behavior and should stay out of this list.
2. Set `DOCUMENT_WORKER_FLY_TOKEN` as a short-lived, app-scoped deploy token for
`litefile-staging`. The app-wide Fly secret is available to its web Machine,
so the token must never have access to production or the organization.
The coordinator is the only process that uses it. Renew it before it expires.
3. Deploy the pilot configuration to staging with `--ha=false` so it has one
coordinator. With the database mode still warm, verify health with
`fly checks list -a litefile-staging` and check that both workers are started.
Verify that the independent probe reports an unhealthy response when presented
with stale coordinator state in the test environment. Connect the failing
probe to an operator alert and verify delivery before enabling idle exits.
4. On the staging coordinator, run
`uv run python manage.py coordinate_document_workers --mode autoscale`.
After both workers have slept, upload a synthetic document and time it through
preparation, analysis and review. Compare this with the 79-second warm
measurement and the two-minute confirmation wait. Use three cold runs,
each with both groups initially stopped. Require both worker heartbeats
within 20 seconds and the upload marked Ready within 120 seconds on each
run. Verify primary start
failure, in-flight job recovery, and upload admission failure using an
isolated test or a mock.
5. Enable the `LITEFILE_STAGING_WORKER_AUTOSCALING` repository variable for
subsequent staging deploys only after the cold-start and outage validation.
The deployment checks the live coordinator, the queue-health response and
both Machine pools. Production continues to use its warm worker configuration.

### Return to warm workers

On the staging coordinator, run
`uv run python manage.py coordinate_document_workers --mode warm`. Confirm that
one worker in each group has started. Turn off
`LITEFILE_STAGING_WORKER_AUTOSCALING`, then deploy the normal staging
`fly.toml` or the previous warm release. Verify the live worker check after
the deploy. If the coordinator itself is unavailable, first start one worker
in each group manually and verify their processes before rolling back.
Loading