Release v1.0.0

scripts/alertmanager-smoke.mjs

@@ -0,0 +1,310 @@
+#!/usr/bin/env node
+
+import { randomUUID } from 'node:crypto'
+import { resolve } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const scriptPath = fileURLToPath(import.meta.url)
+const acknowledgement = 'I_ACCEPT_TEST_ALERT_NOTIFICATIONS'
+const defaultReceiver = 'frameflow-operator-webhook'
+
+export class AlertmanagerSmokeError extends Error {
+  constructor(message, options = {}) {
+    super(message, options)
+    this.name = 'AlertmanagerSmokeError'
+  }
+}
+
+function required(value, name) {
+  if (typeof value !== 'string' || value.trim() === '') throw new AlertmanagerSmokeError(`${name} is required`)
+  return value.trim()
+}
+
+function integerValue(value, name, minimum, maximum) {
+  const parsed = Number(value)
+  if (!Number.isInteger(parsed) || parsed < minimum || parsed > maximum) {
+    throw new AlertmanagerSmokeError(`${name} must be an integer from ${minimum} to ${maximum}`)
+  }
+  return parsed
+}
+
+export function resolveAlertmanagerUrl(value) {
+  let url
+  try {
+    url = new URL(required(value, 'FRAMEFLOW_ALERTMANAGER_URL'))
+  } catch (error) {
+    if (error instanceof AlertmanagerSmokeError) throw error
+    throw new AlertmanagerSmokeError('FRAMEFLOW_ALERTMANAGER_URL must be a valid URL')
+  }
+  if (url.username || url.password || url.search || url.hash) {
+    throw new AlertmanagerSmokeError('Alertmanager URL cannot contain credentials, a query, or a fragment')
+  }
+  if (!['http:', 'https:'].includes(url.protocol)) {
+    throw new AlertmanagerSmokeError('Alertmanager URL must use HTTP or HTTPS')
+  }
+  const localHostnames = new Set(['localhost', '127.0.0.1', '::1', '[::1]'])
+  if (url.protocol === 'http:' && !localHostnames.has(url.hostname)) {
+    throw new AlertmanagerSmokeError('HTTP is allowed only for a loopback Alertmanager URL')
+  }
+  if (url.pathname !== '/' && url.pathname !== '') {
+    throw new AlertmanagerSmokeError('Alertmanager URL must not contain a path')
+  }
+  return url.origin
+}
+
+export function usage() {
+  return `Usage: npm run smoke:alertmanager -- [options]
+
+Sends a real FrameFlowDeliveryTest alert and resolution through Alertmanager,
+then verifies successful webhook notification counters for both events.
+
+Required environment:
+  FRAMEFLOW_ALERT_TEST_ACK=${acknowledgement}
+
+Optional environment:
+  FRAMEFLOW_ALERTMANAGER_URL=http://127.0.0.1:9093
+  FRAMEFLOW_ALERT_TEST_RECEIVER=${defaultReceiver}
+  FRAMEFLOW_ALERT_TEST_TIMEOUT_SECONDS=60
+
+Options:
+  --url <origin>             Alertmanager origin; HTTP is loopback-only
+  --receiver <name>         Receiver label used by notification metrics
+  --timeout-seconds <15-300>
+  --json
+  --help
+
+This command causes real firing and resolved webhook notifications. A passing
+report proves that the configured webhook returned success to Alertmanager; it
+does not prove downstream processing after that acknowledgement.
+`
+}
+
+export function parseAlertmanagerSmokeOptions(argv = process.argv.slice(2), environment = process.env) {
+  const values = {}
+  const flags = new Set()
+  const valueOptions = new Set(['--url', '--receiver', '--timeout-seconds'])
+  const flagOptions = new Set(['--json', '--help'])
+  for (let index = 0; index < argv.length; index += 1) {
+    const argument = argv[index]
+    if (valueOptions.has(argument)) {
+      const value = argv[index + 1]
+      if (!value || value.startsWith('--')) throw new AlertmanagerSmokeError(`${argument} requires a value`)
+      values[argument] = value
+      index += 1
+    } else if (flagOptions.has(argument)) {
+      flags.add(argument)
+    } else {
+      throw new AlertmanagerSmokeError(`Unknown option: ${argument}`)
+    }
+  }
+  if (flags.has('--help')) return { help: true }
+  if (environment.FRAMEFLOW_ALERT_TEST_ACK !== acknowledgement) {
+    throw new AlertmanagerSmokeError(`Alert smoke requires FRAMEFLOW_ALERT_TEST_ACK=${acknowledgement}`)
+  }
+  const receiver = required(values['--receiver'] ?? environment.FRAMEFLOW_ALERT_TEST_RECEIVER ?? defaultReceiver, 'receiver')
+  if (!/^[a-zA-Z0-9_.-]{1,100}$/.test(receiver)) {
+    throw new AlertmanagerSmokeError('receiver must contain only letters, digits, dot, underscore, or hyphen')
+  }
+  const timeoutSeconds = integerValue(values['--timeout-seconds'] ?? environment.FRAMEFLOW_ALERT_TEST_TIMEOUT_SECONDS ?? '60', 'timeout-seconds', 15, 300)
+  return {
+    alertmanagerUrl: resolveAlertmanagerUrl(values['--url'] ?? environment.FRAMEFLOW_ALERTMANAGER_URL ?? 'http://127.0.0.1:9093'),
+    receiver,
+    timeoutMs: timeoutSeconds * 1000,
+    requestTimeoutMs: Math.min(10_000, timeoutSeconds * 1000),
+    pollIntervalMs: 500,
+    json: flags.has('--json'),
+  }
+}
+
+function unescapePrometheusLabel(value) {
+  return value.replace(/\\([\\"n])/g, (_match, escaped) => escaped === 'n' ? '\n' : escaped)
+}
+
+function parsePrometheusLabels(value) {
+  const labels = {}
+  const matcher = /([a-zA-Z_][a-zA-Z0-9_]*)="((?:\\.|[^"\\])*)"/g
+  for (const match of value.matchAll(matcher)) labels[match[1]] = unescapePrometheusLabel(match[2])
+  return labels
+}
+
+export function notificationCounters(metricsText, receiver) {
+  const counters = { total: 0, failed: 0 }
+  for (const line of String(metricsText).split('\n')) {
+    const match = line.match(/^(alertmanager_notifications(?:_failed)?_total)\{([^}]*)\}\s+([^\s]+)$/)
+    if (!match) continue
+    const labels = parsePrometheusLabels(match[2])
+    if (labels.integration !== 'webhook' || labels.receiver_name !== receiver) continue
+    const value = Number(match[3])
+    if (!Number.isFinite(value)) continue
+    if (match[1] === 'alertmanager_notifications_failed_total') counters.failed += value
+    else counters.total += value
+  }
+  return counters
+}
+
+function safeResponseText(value) {
+  return String(value ?? '').replace(/[\r\n\t]+/g, ' ').slice(0, 500)
+}
+
+async function fetchWithTimeout(fetchImpl, url, options, timeoutMs) {
+  const controller = new AbortController()
+  const timer = setTimeout(() => controller.abort(new Error(`request timed out after ${timeoutMs}ms`)), timeoutMs)
+  try {
+    return await fetchImpl(url, { ...options, signal: controller.signal })
+  } finally {
+    clearTimeout(timer)
+  }
+}
+
+async function expectOk(fetchImpl, url, options, timeoutMs) {
+  const response = await fetchWithTimeout(fetchImpl, url, options, timeoutMs)
+  if (!response.ok) {
+    throw new AlertmanagerSmokeError(`${options.method ?? 'GET'} ${new URL(url).pathname} failed (${response.status}): ${safeResponseText(await response.text())}`)
+  }
+  return response
+}
+
+async function postAlert(fetchImpl, options, alert) {
+  await expectOk(fetchImpl, `${options.alertmanagerUrl}/api/v2/alerts`, {
+    method: 'POST',
+    headers: { 'content-type': 'application/json' },
+    body: JSON.stringify([alert]),
+  }, options.requestTimeoutMs)
+}
+
+async function listTestAlerts(fetchImpl, options, testId) {
+  const matcher = encodeURIComponent(`test_id="${testId}"`)
+  const response = await expectOk(fetchImpl, `${options.alertmanagerUrl}/api/v2/alerts?active=true&silenced=true&inhibited=true&unprocessed=true&filter=${matcher}`, {}, options.requestTimeoutMs)
+  const body = await response.json()
+  if (!Array.isArray(body)) throw new AlertmanagerSmokeError('Alertmanager returned an invalid alert list')
+  return body.filter((alert) => alert?.labels?.test_id === testId)
+}
+
+async function readNotificationCounters(fetchImpl, options) {
+  const response = await expectOk(fetchImpl, `${options.alertmanagerUrl}/metrics`, {}, options.requestTimeoutMs)
+  return notificationCounters(await response.text(), options.receiver)
+}
+
+async function waitUntil(check, description, options, dependencies) {
+  const deadline = dependencies.now() + options.timeoutMs
+  let lastError = null
+  do {
+    try {
+      const result = await check()
+      if (result?.fatal) throw result.error
+      if (result?.done) return result.value
+    } catch (error) {
+      lastError = error
+    }
+    if (dependencies.now() >= deadline) break
+    await dependencies.sleep(Math.min(options.pollIntervalMs, Math.max(1, deadline - dependencies.now())))
+  } while (dependencies.now() <= deadline)
+  const suffix = lastError instanceof Error ? `: ${lastError.message}` : ''
+  throw new AlertmanagerSmokeError(`Timed out waiting for ${description}${suffix}`)
+}
+
+async function waitForAlertState(fetchImpl, options, testId, expectedActive, dependencies) {
+  return waitUntil(async () => {
+    const alerts = await listTestAlerts(fetchImpl, options, testId)
+    const active = alerts.some((alert) => Date.parse(alert.endsAt) > dependencies.now())
+    return active === expectedActive ? { done: true, value: alerts } : { done: false }
+  }, expectedActive ? 'the test alert to become active' : 'the test alert to resolve', options, dependencies)
+}
+
+async function waitForSuccessfulNotification(fetchImpl, options, baseline, phase, dependencies) {
+  return waitUntil(async () => {
+    const counters = await readNotificationCounters(fetchImpl, options)
+    if (counters.failed > baseline.failed) {
+      return {
+        fatal: true,
+        error: new AlertmanagerSmokeError(`${phase} webhook notification failed according to Alertmanager metrics`),
+      }
+    }
+    return counters.total > baseline.total ? { done: true, value: counters } : { done: false }
+  }, `a successful ${phase} webhook notification`, options, dependencies)
+}
+
+export async function runAlertmanagerSmoke(options, dependencies = {}) {
+  const fetchImpl = dependencies.fetchImpl ?? fetch
+  const now = dependencies.now ?? Date.now
+  const sleep = dependencies.sleep ?? ((milliseconds) => new Promise((resolvePromise) => setTimeout(resolvePromise, milliseconds)))
+  const createId = dependencies.createId ?? randomUUID
+  const runtime = { now, sleep }
+  const testId = createId()
+  const startsAt = new Date(now()).toISOString()
+  const firingAlert = {
+    labels: {
+      alertname: 'FrameFlowDeliveryTest',
+      severity: 'info',
+      service: 'frameflow',
+      test_id: testId,
+    },
+    annotations: {
+      summary: 'FrameFlow controlled Alertmanager delivery test',
+      description: `Controlled firing notification for deployment acceptance test ${testId}.`,
+    },
+    startsAt,
+    endsAt: new Date(now() + 15 * 60_000).toISOString(),
+    generatorURL: `${options.alertmanagerUrl}/#/alerts`,
+  }
+  const baseline = await readNotificationCounters(fetchImpl, options)
+  let firingPosted = false
+  let resolutionPosted = false
+
+  try {
+    await postAlert(fetchImpl, options, firingAlert)
+    firingPosted = true
+    await waitForAlertState(fetchImpl, options, testId, true, runtime)
+    const afterFiring = await waitForSuccessfulNotification(fetchImpl, options, baseline, 'firing', runtime)
+
+    const resolvedAt = new Date(now()).toISOString()
+    await postAlert(fetchImpl, options, { ...firingAlert, endsAt: resolvedAt })
+    resolutionPosted = true
+    await waitForAlertState(fetchImpl, options, testId, false, runtime)
+    const afterResolved = await waitForSuccessfulNotification(fetchImpl, options, afterFiring, 'resolved', runtime)
+
+    return {
+      status: 'passed',
+      testId,
+      receiver: options.receiver,
+      alertmanagerUrl: options.alertmanagerUrl,
+      startsAt,
+      resolvedAt,
+      firingWebhookAccepted: true,
+      resolvedWebhookAccepted: true,
+      notificationDelta: afterResolved.total - baseline.total,
+      failedNotificationDelta: afterResolved.failed - baseline.failed,
+      downstreamProcessingProven: false,
+    }
+  } catch (error) {
+    if (firingPosted && !resolutionPosted) {
+      try {
+        await postAlert(fetchImpl, options, { ...firingAlert, endsAt: new Date(now()).toISOString() })
+      } catch (cleanupError) {
+        throw new AlertmanagerSmokeError(`${error instanceof Error ? error.message : String(error)}; cleanup resolution also failed: ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}`, { cause: error })
+      }
+    }
+    throw error
+  }
+}
+
+async function main() {
+  const options = parseAlertmanagerSmokeOptions()
+  if (options.help) {
+    process.stdout.write(usage())
+    return
+  }
+  const report = await runAlertmanagerSmoke(options)
+  if (options.json) {
+    process.stdout.write(`${JSON.stringify(report, null, 2)}\n`)
+    return
+  }
+  process.stdout.write(`Alertmanager webhook smoke passed.\nTest ID: ${report.testId}\nReceiver: ${report.receiver}\nFiring and resolved webhooks returned success; confirm downstream processing with the test ID.\n`)
+}
+
+if (process.argv[1] && resolve(process.argv[1]) === scriptPath) {
+  main().catch((error) => {
+    process.stderr.write(`Alertmanager smoke failed: ${error instanceof Error ? error.message : String(error)}\n`)
+    process.exitCode = 1
+  })
+}

scripts/backup.sh

@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+repo_dir="$(cd "${script_dir}/.." && pwd)"
+env_file="${FRAMEFLOW_ENV_FILE:-${repo_dir}/.env.production}"
+backup_parent="${1:-${repo_dir}/backups}"
+metrics_dir="${FRAMEFLOW_BACKUP_METRICS_DIR:-/var/lib/frameflow/metrics}"
+timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
+backup_dir="${backup_parent%/}/frameflow-${timestamp}"
+started_at_seconds="$(date +%s)"
+metrics_tmp=""
+
+cleanup() {
+  if [[ -n "${metrics_tmp}" && -f "${metrics_tmp}" ]]; then
+    rm -f "${metrics_tmp}"
+  fi
+}
+trap cleanup EXIT
+
+if [[ ! -f "${env_file}" ]]; then
+  echo "Production environment file not found: ${env_file}" >&2
+  exit 1
+fi
+
+compose=(docker compose --env-file "${env_file}" -f "${repo_dir}/docker-compose.prod.yml")
+mkdir -p "${backup_parent}"
+if ! mkdir "${backup_dir}"; then
+  echo "Refusing to overwrite an existing backup directory: ${backup_dir}" >&2
+  exit 1
+fi
+
+postgres_container="$("${compose[@]}" ps -q postgres)"
+redis_container="$("${compose[@]}" ps -q redis)"
+minio_container="$("${compose[@]}" ps -q minio)"
+
+for container in "${postgres_container}" "${redis_container}" "${minio_container}"; do
+  if [[ -z "${container}" ]]; then
+    echo "PostgreSQL, Redis, and MinIO must be running before backup." >&2
+    exit 1
+  fi
+done
+
+echo "Creating PostgreSQL logical backup..."
+"${compose[@]}" exec -T postgres sh -c 'exec pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Fc' > "${backup_dir}/postgres.dump"
+
+echo "Flushing and archiving Redis persistence..."
+"${compose[@]}" exec -T redis sh -c '
+  if [ -n "${REDIS_PASSWORD:-}" ]; then
+    redis-cli --no-auth-warning -a "$REDIS_PASSWORD" SAVE >/dev/null
+  else
+    redis-cli SAVE >/dev/null
+  fi
+'
+docker run --rm --volumes-from "${redis_container}" -v "${backup_dir}:/backup" alpine:3.20 \
+  tar -czf /backup/redis-data.tar.gz -C /data .
+
+echo "Archiving MinIO object data..."
+docker run --rm --volumes-from "${minio_container}" -v "${backup_dir}:/backup" alpine:3.20 \
+  tar -czf /backup/minio-data.tar.gz -C /data .
+
+{
+  echo "created_at=${timestamp}"
+  echo "compose_project=$(docker inspect --format '{{ index .Config.Labels "com.docker.compose.project" }}' "${postgres_container}")"
+  echo "postgres_container=${postgres_container}"
+  echo "redis_container=${redis_container}"
+  echo "minio_container=${minio_container}"
+} > "${backup_dir}/manifest.txt"
+
+if command -v sha256sum >/dev/null 2>&1; then
+  (cd "${backup_dir}" && sha256sum postgres.dump redis-data.tar.gz minio-data.tar.gz > SHA256SUMS)
+else
+  (cd "${backup_dir}" && shasum -a 256 postgres.dump redis-data.tar.gz minio-data.tar.gz > SHA256SUMS)
+fi
+
+"${repo_dir}/scripts/verify-backup.sh" "${backup_dir}"
+
+completed_at_seconds="$(date +%s)"
+backup_size_bytes=0
+for backup_file in postgres.dump redis-data.tar.gz minio-data.tar.gz manifest.txt SHA256SUMS; do
+  file_size="$(wc -c < "${backup_dir}/${backup_file}" | tr -d '[:space:]')"
+  backup_size_bytes=$((backup_size_bytes + file_size))
+done
+
+mkdir -p "${metrics_dir}"
+metrics_tmp="$(mktemp "${metrics_dir%/}/frameflow_backup.prom.tmp.XXXXXX")"
+cat > "${metrics_tmp}" <<EOF
+# HELP frameflow_backup_last_success_timestamp_seconds Unix timestamp of the last complete FrameFlow backup.
+# TYPE frameflow_backup_last_success_timestamp_seconds gauge
+frameflow_backup_last_success_timestamp_seconds ${completed_at_seconds}
+# HELP frameflow_backup_last_size_bytes Total size of files in the last complete FrameFlow backup.
+# TYPE frameflow_backup_last_size_bytes gauge
+frameflow_backup_last_size_bytes ${backup_size_bytes}
+# HELP frameflow_backup_last_duration_seconds Duration of the last complete FrameFlow backup.
+# TYPE frameflow_backup_last_duration_seconds gauge
+frameflow_backup_last_duration_seconds $((completed_at_seconds - started_at_seconds))
+EOF
+chmod 0644 "${metrics_tmp}"
+mv -f "${metrics_tmp}" "${metrics_dir%/}/frameflow_backup.prom"
+metrics_tmp=""
+
+echo "Backup complete: ${backup_dir}"
+echo "Backup metrics updated: ${metrics_dir%/}/frameflow_backup.prom"

scripts/rehearse-restore.sh

@@ -0,0 +1,360 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+usage() {
+  cat >&2 <<'EOF'
+Usage: rehearse-restore.sh [--keep-on-failure] <frameflow-backup-directory>
+
+Restores a verified FrameFlow backup into isolated, randomly named Docker
+resources. No host ports are published. Resources are removed automatically
+unless --keep-on-failure is supplied and the rehearsal fails.
+EOF
+  exit "${1:-2}"
+}
+
+fail() {
+  echo "Restore rehearsal failed: $1" >&2
+  exit 1
+}
+
+keep_on_failure=0
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --keep-on-failure)
+      keep_on_failure=1
+      shift
+      ;;
+    --help|-h)
+      usage 0
+      ;;
+    --)
+      shift
+      break
+      ;;
+    -*)
+      usage
+      ;;
+    *)
+      break
+      ;;
+  esac
+done
+
+[[ $# -eq 1 ]] || usage
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+backup_input="$1"
+timeout_seconds="${FRAMEFLOW_RESTORE_TIMEOUT_SECONDS:-120}"
+metrics_dir="${FRAMEFLOW_RESTORE_METRICS_DIR:-${FRAMEFLOW_BACKUP_METRICS_DIR:-/var/lib/frameflow/metrics}}"
+started_at_seconds="$(date +%s)"
+metrics_tmp=""
+
+[[ "${timeout_seconds}" =~ ^[0-9]+$ ]] || fail "FRAMEFLOW_RESTORE_TIMEOUT_SECONDS must be an integer"
+(( timeout_seconds >= 5 && timeout_seconds <= 3600 )) \
+  || fail "FRAMEFLOW_RESTORE_TIMEOUT_SECONDS must be between 5 and 3600"
+
+# Verification deliberately precedes every Docker call.
+"${script_dir}/verify-backup.sh" "${backup_input}"
+backup_dir="$(cd "${backup_input}" && pwd -P)"
+
+command -v docker >/dev/null 2>&1 || fail "docker is required"
+docker version >/dev/null 2>&1 || fail "the Docker engine is unavailable"
+
+random_hex() {
+  local bytes="$1"
+  local value
+  value="$(LC_ALL=C od -An -N "${bytes}" -tx1 /dev/urandom | tr -d '[:space:]')"
+  [[ "${value}" =~ ^[0-9a-f]+$ ]] || fail "could not generate a random resource identifier"
+  printf '%s' "${value}"
+}
+
+timestamp="$(date -u +%Y%m%d%H%M%S)"
+suffix="$(random_hex 4)"
+run_id="${timestamp}-${suffix}"
+prefix="frameflow-restore-${run_id}"
+label_flag="com.frameflow.restore-rehearsal"
+label_id="com.frameflow.restore-rehearsal.id"
+expected_labels="true|${run_id}"
+labels=(--label "${label_flag}=true" --label "${label_id}=${run_id}")
+
+network_name="${prefix}-network"
+postgres_volume="${prefix}-postgres"
+redis_volume="${prefix}-redis"
+minio_volume="${prefix}-minio"
+postgres_container="${prefix}-postgres"
+redis_container="${prefix}-redis"
+minio_container="${prefix}-minio"
+redis_extract_container="${prefix}-redis-extract"
+minio_extract_container="${prefix}-minio-extract"
+minio_check_container="${prefix}-minio-check"
+
+postgres_image="postgres:16-alpine"
+redis_image="redis:7.4-alpine"
+alpine_image="alpine:3.20"
+minio_image="minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e"
+mc_image="minio/mc@sha256:a7fe349ef4bd8521fb8497f55c6042871b2ae640607cf99d9bede5e9bdf11727"
+
+postgres_db="frameflow"
+postgres_user="frameflow_restore"
+postgres_password="$(random_hex 24)"
+redis_password="$(random_hex 24)"
+minio_access_key="restore${suffix}"
+minio_secret_key="$(random_hex 32)"
+
+resources_cleaned=0
+
+resource_labels() {
+  local kind="$1"
+  local name="$2"
+  case "${kind}" in
+    container)
+      docker container inspect --format "{{ index .Config.Labels \"${label_flag}\" }}|{{ index .Config.Labels \"${label_id}\" }}" "${name}" 2>/dev/null
+      ;;
+    network|volume)
+      docker "${kind}" inspect --format "{{ index .Labels \"${label_flag}\" }}|{{ index .Labels \"${label_id}\" }}" "${name}" 2>/dev/null
+      ;;
+    *)
+      return 2
+      ;;
+  esac
+}
+
+remove_labeled_resource() {
+  local kind="$1"
+  local name="$2"
+  local actual
+
+  if ! actual="$(resource_labels "${kind}" "${name}")"; then
+    return 0
+  fi
+  if [[ "${actual}" != "${expected_labels}" ]]; then
+    echo "Refusing to remove ${kind} ${name}: rehearsal label mismatch" >&2
+    return 1
+  fi
+
+  case "${kind}" in
+    container) docker rm -f "${name}" >/dev/null ;;
+    network) docker network rm "${name}" >/dev/null ;;
+    volume) docker volume rm "${name}" >/dev/null ;;
+  esac
+}
+
+cleanup_resources() {
+  local result=0
+  local name
+
+  for name in \
+    "${minio_check_container}" \
+    "${minio_extract_container}" \
+    "${redis_extract_container}" \
+    "${minio_container}" \
+    "${redis_container}" \
+    "${postgres_container}"; do
+    if ! remove_labeled_resource container "${name}"; then
+      result=1
+    fi
+  done
+
+  if ! remove_labeled_resource network "${network_name}"; then
+    result=1
+  fi
+  for name in "${minio_volume}" "${redis_volume}" "${postgres_volume}"; do
+    if ! remove_labeled_resource volume "${name}"; then
+      result=1
+    fi
+  done
+  return "${result}"
+}
+
+cleanup_on_exit() {
+  local status=$?
+  trap - EXIT
+
+  if [[ -n "${metrics_tmp}" && -f "${metrics_tmp}" ]]; then
+    rm -f "${metrics_tmp}"
+  fi
+
+  if (( status != 0 && keep_on_failure == 1 )); then
+    cat >&2 <<EOF
+Restore rehearsal resources retained for diagnosis:
+  network: ${network_name}
+  volumes: ${postgres_volume}, ${redis_volume}, ${minio_volume}
+  containers: ${postgres_container}, ${redis_container}, ${minio_container}
+Remove them only after confirming both labels ${label_flag}=true and ${label_id}=${run_id}.
+EOF
+    exit "${status}"
+  fi
+
+  if (( resources_cleaned == 0 )); then
+    if ! cleanup_resources && (( status == 0 )); then
+      status=1
+    fi
+  fi
+  exit "${status}"
+}
+
+trap cleanup_on_exit EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+wait_for_postgres() {
+  local deadline=$((SECONDS + timeout_seconds))
+  until docker exec "${postgres_container}" pg_isready -U "${postgres_user}" -d "${postgres_db}" >/dev/null 2>&1; do
+    (( SECONDS < deadline )) || fail "PostgreSQL did not become ready within ${timeout_seconds} seconds"
+    sleep 1
+  done
+}
+
+wait_for_redis() {
+  local deadline=$((SECONDS + timeout_seconds))
+  until docker exec "${redis_container}" redis-cli --no-auth-warning -a "${redis_password}" ping 2>/dev/null | grep -q '^PONG$'; do
+    (( SECONDS < deadline )) || fail "Redis did not become ready within ${timeout_seconds} seconds"
+    sleep 1
+  done
+}
+
+wait_for_minio() {
+  local deadline=$((SECONDS + timeout_seconds))
+  until docker exec "${minio_container}" curl -fsS http://127.0.0.1:9000/minio/health/live >/dev/null 2>&1; do
+    (( SECONDS < deadline )) || fail "MinIO did not become ready within ${timeout_seconds} seconds"
+    sleep 1
+  done
+}
+
+echo "Creating isolated restore resources for rehearsal ${run_id}..."
+docker network create --internal "${labels[@]}" "${network_name}" >/dev/null
+docker volume create "${labels[@]}" "${postgres_volume}" >/dev/null
+docker volume create "${labels[@]}" "${redis_volume}" >/dev/null
+docker volume create "${labels[@]}" "${minio_volume}" >/dev/null
+
+echo "Restoring PostgreSQL into an empty rehearsal volume..."
+docker run -d \
+  --name "${postgres_container}" \
+  --network "${network_name}" \
+  "${labels[@]}" \
+  -e POSTGRES_DB="${postgres_db}" \
+  -e POSTGRES_USER="${postgres_user}" \
+  -e POSTGRES_PASSWORD="${postgres_password}" \
+  -v "${postgres_volume}:/var/lib/postgresql/data" \
+  "${postgres_image}" >/dev/null
+wait_for_postgres
+docker exec -i "${postgres_container}" \
+  pg_restore -U "${postgres_user}" -d "${postgres_db}" \
+  --exit-on-error --no-owner --no-privileges \
+  < "${backup_dir}/postgres.dump"
+
+core_tables_ok="$(docker exec "${postgres_container}" psql -v ON_ERROR_STOP=1 -U "${postgres_user}" -d "${postgres_db}" -Atc \
+  "select case when to_regclass('drizzle.__drizzle_migrations') is not null
+    and to_regclass('public.users') is not null
+    and to_regclass('public.workspaces') is not null
+    and to_regclass('public.projects') is not null
+    and to_regclass('public.episodes') is not null
+    and to_regclass('public.assets') is not null
+    and to_regclass('public.asset_versions') is not null
+    and to_regclass('public.shots') is not null
+    and to_regclass('public.shot_versions') is not null
+    and to_regclass('public.generation_jobs') is not null
+    and to_regclass('public.renders') is not null
+    and to_regclass('public.provider_verifications') is not null
+    then 'ok' else 'missing' end")"
+[[ "${core_tables_ok}" == "ok" ]] || fail "PostgreSQL restore is missing required tables"
+
+docker exec "${postgres_container}" psql -v ON_ERROR_STOP=1 -U "${postgres_user}" -d "${postgres_db}" -Atc \
+  "select count(*) from users;
+   select count(*) from workspaces;
+   select count(*) from projects;
+   select count(*) from episodes;
+   select count(*) from assets;
+   select count(*) from shots;
+   select count(*) from generation_jobs;
+   select count(*) from renders;
+   select count(*) from provider_verifications;" >/dev/null
+
+migration_count="$(docker exec "${postgres_container}" psql -v ON_ERROR_STOP=1 -U "${postgres_user}" -d "${postgres_db}" -Atc \
+  'select count(*) from drizzle.__drizzle_migrations')"
+table_count="$(docker exec "${postgres_container}" psql -v ON_ERROR_STOP=1 -U "${postgres_user}" -d "${postgres_db}" -Atc \
+  "select count(*) from pg_tables where schemaname in ('public', 'drizzle')")"
+[[ "${migration_count}" =~ ^[0-9]+$ && "${table_count}" =~ ^[0-9]+$ ]] \
+  || fail "PostgreSQL verification returned invalid counts"
+(( migration_count > 0 )) || fail "PostgreSQL migration history is empty"
+
+echo "Restoring Redis persistence into an empty rehearsal volume..."
+docker run --rm \
+  --name "${redis_extract_container}" \
+  "${labels[@]}" \
+  -v "${redis_volume}:/target" \
+  --mount "type=bind,src=${backup_dir},dst=/backup,readonly" \
+  --entrypoint /bin/sh \
+  "${alpine_image}" -ec \
+  'test -z "$(find /target -mindepth 1 -maxdepth 1 -print -quit)" && tar -xzf /backup/redis-data.tar.gz -C /target'
+docker run -d \
+  --name "${redis_container}" \
+  --network "${network_name}" \
+  "${labels[@]}" \
+  -v "${redis_volume}:/data" \
+  "${redis_image}" redis-server --appendonly yes --requirepass "${redis_password}" >/dev/null
+wait_for_redis
+redis_key_count="$(docker exec "${redis_container}" redis-cli --no-auth-warning -a "${redis_password}" --raw DBSIZE)"
+redis_persistence="$(docker exec "${redis_container}" redis-cli --no-auth-warning -a "${redis_password}" --raw INFO persistence | tr -d '\r')"
+[[ "${redis_key_count}" =~ ^[0-9]+$ ]] || fail "Redis verification returned an invalid key count"
+grep -q '^loading:0$' <<< "${redis_persistence}" || fail "Redis is still loading restored persistence"
+grep -q '^aof_last_write_status:ok$' <<< "${redis_persistence}" || fail "Redis AOF persistence is not healthy"
+
+echo "Restoring MinIO data into an empty rehearsal volume..."
+docker run --rm \
+  --name "${minio_extract_container}" \
+  "${labels[@]}" \
+  -v "${minio_volume}:/target" \
+  --mount "type=bind,src=${backup_dir},dst=/backup,readonly" \
+  --entrypoint /bin/sh \
+  "${alpine_image}" -ec \
+  'test -z "$(find /target -mindepth 1 -maxdepth 1 -print -quit)" && tar -xzf /backup/minio-data.tar.gz -C /target'
+docker run -d \
+  --name "${minio_container}" \
+  --network "${network_name}" \
+  "${labels[@]}" \
+  -e MINIO_ROOT_USER="${minio_access_key}" \
+  -e MINIO_ROOT_PASSWORD="${minio_secret_key}" \
+  -v "${minio_volume}:/data" \
+  "${minio_image}" server /data --console-address :9001 >/dev/null
+wait_for_minio
+minio_object_count="$(docker run --rm \
+  --name "${minio_check_container}" \
+  --network "${network_name}" \
+  "${labels[@]}" \
+  -e MC_HOST_RESTORE="http://${minio_access_key}:${minio_secret_key}@${minio_container}:9000" \
+  --entrypoint /bin/sh \
+  "${mc_image}" -ec \
+  'mc admin info RESTORE >/dev/null && mc ls RESTORE >/dev/null && mc ls --recursive RESTORE | wc -l | tr -d "[:space:]"')"
+[[ "${minio_object_count}" =~ ^[0-9]+$ ]] || fail "MinIO verification returned an invalid object count"
+
+echo "Cleaning isolated restore resources..."
+cleanup_resources || fail "one or more rehearsal resources failed label verification or cleanup"
+resources_cleaned=1
+
+completed_at_seconds="$(date +%s)"
+mkdir -p "${metrics_dir}"
+metrics_tmp="$(mktemp "${metrics_dir%/}/frameflow_restore.prom.tmp.XXXXXX")"
+cat > "${metrics_tmp}" <<EOF
+# HELP frameflow_restore_rehearsal_last_success_timestamp_seconds Unix timestamp of the last successful isolated restore rehearsal.
+# TYPE frameflow_restore_rehearsal_last_success_timestamp_seconds gauge
+frameflow_restore_rehearsal_last_success_timestamp_seconds ${completed_at_seconds}
+# HELP frameflow_restore_rehearsal_last_duration_seconds Duration of the last successful isolated restore rehearsal.
+# TYPE frameflow_restore_rehearsal_last_duration_seconds gauge
+frameflow_restore_rehearsal_last_duration_seconds $((completed_at_seconds - started_at_seconds))
+# HELP frameflow_restore_rehearsal_postgres_table_count Number of restored PostgreSQL application and migration tables.
+# TYPE frameflow_restore_rehearsal_postgres_table_count gauge
+frameflow_restore_rehearsal_postgres_table_count ${table_count}
+# HELP frameflow_restore_rehearsal_redis_key_count Number of Redis keys observed after restore.
+# TYPE frameflow_restore_rehearsal_redis_key_count gauge
+frameflow_restore_rehearsal_redis_key_count ${redis_key_count}
+# HELP frameflow_restore_rehearsal_minio_object_count Number of MinIO objects listed after restore.
+# TYPE frameflow_restore_rehearsal_minio_object_count gauge
+frameflow_restore_rehearsal_minio_object_count ${minio_object_count}
+EOF
+chmod 0644 "${metrics_tmp}"
+mv -f "${metrics_tmp}" "${metrics_dir%/}/frameflow_restore.prom"
+metrics_tmp=""
+
+echo "Restore rehearsal passed: PostgreSQL ${table_count} tables, Redis ${redis_key_count} keys, MinIO ${minio_object_count} objects."
+echo "Restore rehearsal metrics updated: ${metrics_dir%/}/frameflow_restore.prom"

scripts/scheduled-maintenance.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+usage() {
+  cat >&2 <<'EOF'
+Usage: scheduled-maintenance.sh <backup|restore-latest>
+
+Requires FRAMEFLOW_BACKUP_MOUNTPOINT and FRAMEFLOW_BACKUP_DESTINATION.
+The destination must be a writable directory inside the active mount point.
+EOF
+  exit "${1:-2}"
+}
+
+fail() {
+  echo "Scheduled maintenance failed: $1" >&2
+  exit 1
+}
+
+[[ $# -eq 1 ]] || usage
+case "$1" in
+  backup|restore-latest) mode="$1" ;;
+  --help|-h) usage 0 ;;
+  *) usage ;;
+esac
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+mount_point="${FRAMEFLOW_BACKUP_MOUNTPOINT:-}"
+backup_destination="${FRAMEFLOW_BACKUP_DESTINATION:-}"
+
+[[ -n "${mount_point}" ]] || fail "FRAMEFLOW_BACKUP_MOUNTPOINT is required"
+[[ -n "${backup_destination}" ]] || fail "FRAMEFLOW_BACKUP_DESTINATION is required"
+
+for path in "${mount_point}" "${backup_destination}"; do
+  [[ "${path}" == /* ]] || fail "backup paths must be absolute: ${path}"
+  [[ "${path}" != "/" ]] || fail "the filesystem root cannot be used for backups"
+  [[ ! "${path}" =~ [[:space:]] ]] || fail "backup paths must not contain whitespace: ${path}"
+  [[ -d "${path}" ]] || fail "backup directory does not exist: ${path}"
+  [[ ! -L "${path}" ]] || fail "backup paths must not be symbolic links: ${path}"
+done
+
+mount_point="$(cd "${mount_point}" && pwd -P)"
+backup_destination="$(cd "${backup_destination}" && pwd -P)"
+if [[ "${backup_destination}" != "${mount_point}" && "${backup_destination}" != "${mount_point}/"* ]]; then
+  fail "backup destination must be inside FRAMEFLOW_BACKUP_MOUNTPOINT"
+fi
+[[ -w "${backup_destination}" ]] || fail "backup destination is not writable: ${backup_destination}"
+
+command -v findmnt >/dev/null 2>&1 || fail "findmnt is required to verify the backup mount"
+mounted_target="$(findmnt -rn -M "${mount_point}" -o TARGET 2>/dev/null)" \
+  || fail "backup mount is not active: ${mount_point}"
+[[ "${mounted_target}" == "${mount_point}" ]] \
+  || fail "backup mount resolved to an unexpected target: ${mounted_target}"
+
+case "${mode}" in
+  backup)
+    exec "${script_dir}/backup.sh" "${backup_destination}"
+    ;;
+  restore-latest)
+    latest_name=""
+    latest_path=""
+    while IFS= read -r -d '' candidate; do
+      [[ ! -L "${candidate}" ]] || continue
+      candidate_name="${candidate##*/}"
+      [[ "${candidate_name}" =~ ^frameflow-[0-9]{8}T[0-9]{6}Z$ ]] || continue
+      if [[ -z "${latest_name}" || "${candidate_name}" > "${latest_name}" ]]; then
+        latest_name="${candidate_name}"
+        latest_path="${candidate}"
+      fi
+    done < <(find "${backup_destination}" -mindepth 1 -maxdepth 1 -type d -name 'frameflow-*' -print0)
+
+    [[ -n "${latest_path}" ]] || fail "no timestamped FrameFlow backup was found"
+    echo "Selected latest backup for restore rehearsal: ${latest_path}"
+    "${script_dir}/verify-backup.sh" "${latest_path}"
+    exec "${script_dir}/rehearse-restore.sh" "${latest_path}"
+    ;;
+esac

scripts/smoke-test.mjs

@@ -0,0 +1,927 @@
+#!/usr/bin/env node
+
+import { createHash, randomUUID } from 'node:crypto'
+import { readFile } from 'node:fs/promises'
+import { basename, extname, resolve } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const scriptPath = fileURLToPath(import.meta.url)
+const defaultImagePath = fileURLToPath(new URL('../public/assets/scene-rooftop.jpg', import.meta.url))
+const terminalJobStatuses = new Set(['SUCCEEDED', 'FAILED', 'CANCELLED'])
+const billableAiAcknowledgement = 'I_ACCEPT_BILLABLE_AI_CHARGES'
+export const requiredLiveAiWorkflowIds = ['script', 'storyboard', 'image-openai', 'image-replicate', 'tts', 'video', 'lipsync']
+
+export class SmokeTestError extends Error {
+  constructor(message, options = {}) {
+    super(message, options)
+    this.name = 'SmokeTestError'
+    this.report = options.report
+  }
+}
+
+function required(value, name) {
+  if (typeof value !== 'string' || value.trim() === '') throw new SmokeTestError(`${name} is required`)
+  return value.trim()
+}
+
+function booleanValue(value) {
+  return ['1', 'true', 'yes', 'on'].includes(String(value ?? '').trim().toLowerCase())
+}
+
+function integerValue(value, name, minimum, maximum) {
+  const parsed = Number(value)
+  if (!Number.isInteger(parsed) || parsed < minimum || parsed > maximum) {
+    throw new SmokeTestError(`${name} must be an integer from ${minimum} to ${maximum}`)
+  }
+  return parsed
+}
+
+function jsonObjectValue(value, name) {
+  if (value === undefined || value === null || String(value).trim() === '') return {}
+  let parsed
+  try {
+    parsed = JSON.parse(String(value))
+  } catch {
+    throw new SmokeTestError(`${name} must be valid JSON`)
+  }
+  if (!parsed || Array.isArray(parsed) || typeof parsed !== 'object') {
+    throw new SmokeTestError(`${name} must be a JSON object`)
+  }
+  return parsed
+}
+
+export function resolveEndpoints(value, options = {}) {
+  const input = required(value, 'FRAMEFLOW_SMOKE_BASE_URL')
+  let url
+  try {
+    url = new URL(input)
+  } catch {
+    throw new SmokeTestError('FRAMEFLOW_SMOKE_BASE_URL must be a valid URL')
+  }
+  if (url.username || url.password || url.search || url.hash) {
+    throw new SmokeTestError('Smoke-test base URL cannot contain credentials, a query, or a fragment')
+  }
+  if (!['http:', 'https:'].includes(url.protocol)) throw new SmokeTestError('Smoke-test base URL must use HTTP or HTTPS')
+  const localHostnames = new Set(['localhost', '127.0.0.1', '::1', '[::1]'])
+  if (url.protocol === 'http:' && (!options.allowHttp || !localHostnames.has(url.hostname))) {
+    throw new SmokeTestError('HTTPS is required; HTTP can only be enabled explicitly for localhost')
+  }
+  const path = url.pathname.replace(/\/+$/, '')
+  if (path && path !== '/api/v1') {
+    throw new SmokeTestError('Smoke-test base URL path must be empty or /api/v1')
+  }
+  const origin = url.origin
+  return { origin, apiBase: `${origin}/api/v1` }
+}
+
+export function parseSmokeOptions(argv = process.argv.slice(2), environment = process.env) {
+  const values = {}
+  const flags = new Set()
+  const valueOptions = new Set(['--base-url', '--email', '--workspace-id', '--image', '--timeout-seconds', '--ai-video-seconds'])
+  const flagOptions = new Set(['--allow-http', '--keep-project', '--include-ai', '--json', '--help'])
+  for (let index = 0; index < argv.length; index += 1) {
+    const argument = argv[index]
+    if (valueOptions.has(argument)) {
+      const value = argv[index + 1]
+      if (!value || value.startsWith('--')) throw new SmokeTestError(`${argument} requires a value`)
+      values[argument] = value
+      index += 1
+    } else if (flagOptions.has(argument)) {
+      flags.add(argument)
+    } else {
+      throw new SmokeTestError(`Unknown option: ${argument}`)
+    }
+  }
+  if (flags.has('--help')) return { help: true }
+
+  const allowHttp = flags.has('--allow-http') || booleanValue(environment.FRAMEFLOW_SMOKE_ALLOW_HTTP)
+  const endpoints = resolveEndpoints(values['--base-url'] ?? environment.FRAMEFLOW_SMOKE_BASE_URL, { allowHttp })
+  const email = required(values['--email'] ?? environment.FRAMEFLOW_SMOKE_EMAIL, 'FRAMEFLOW_SMOKE_EMAIL').toLowerCase()
+  const password = required(environment.FRAMEFLOW_SMOKE_PASSWORD, 'FRAMEFLOW_SMOKE_PASSWORD')
+  const timeoutSeconds = integerValue(values['--timeout-seconds'] ?? environment.FRAMEFLOW_SMOKE_TIMEOUT_SECONDS ?? '180', 'timeout-seconds', 30, 900)
+  const includeAi = flags.has('--include-ai')
+  if (includeAi && environment.FRAMEFLOW_SMOKE_BILLABLE_AI_ACK !== billableAiAcknowledgement) {
+    throw new SmokeTestError(`--include-ai requires FRAMEFLOW_SMOKE_BILLABLE_AI_ACK=${billableAiAcknowledgement}`)
+  }
+  return {
+    ...endpoints,
+    email,
+    password,
+    workspaceId: values['--workspace-id'] ?? environment.FRAMEFLOW_SMOKE_WORKSPACE_ID ?? null,
+    imagePath: resolve(values['--image'] ?? environment.FRAMEFLOW_SMOKE_IMAGE ?? defaultImagePath),
+    timeoutMs: timeoutSeconds * 1000,
+    requestTimeoutMs: Math.min(60_000, timeoutSeconds * 1000),
+    keepProject: flags.has('--keep-project') || booleanValue(environment.FRAMEFLOW_SMOKE_KEEP_PROJECT),
+    includeAi,
+    aiVideoSeconds: integerValue(values['--ai-video-seconds'] ?? environment.FRAMEFLOW_SMOKE_AI_VIDEO_SECONDS ?? '5', 'ai-video-seconds', 1, 30),
+    aiVideoParams: jsonObjectValue(environment.FRAMEFLOW_SMOKE_VIDEO_PARAMS_JSON, 'FRAMEFLOW_SMOKE_VIDEO_PARAMS_JSON'),
+    aiLipsyncParams: jsonObjectValue(environment.FRAMEFLOW_SMOKE_LIPSYNC_PARAMS_JSON, 'FRAMEFLOW_SMOKE_LIPSYNC_PARAMS_JSON'),
+    json: flags.has('--json'),
+  }
+}
+
+function safeError(error) {
+  const message = error instanceof Error ? error.message : String(error)
+  return message.replace(/([?&](?:X-Amz-[^=\s]+|token|signature)=)[^&\s]+/gi, '$1[redacted]').slice(0, 2000)
+}
+
+function assertCondition(condition, message) {
+  if (!condition) throw new SmokeTestError(message)
+}
+
+function sha256(value) {
+  return createHash('sha256').update(value).digest('hex')
+}
+
+function imageContentType(path) {
+  const extension = extname(path).toLowerCase()
+  if (extension === '.jpg' || extension === '.jpeg') return 'image/jpeg'
+  if (extension === '.png') return 'image/png'
+  if (extension === '.webp') return 'image/webp'
+  throw new SmokeTestError('Smoke-test image must be JPEG, PNG, or WebP')
+}
+
+function responseBody(response) {
+  if (response.status === 204) return Promise.resolve(null)
+  const contentType = response.headers.get('content-type') ?? ''
+  if (contentType.includes('application/json')) return response.json()
+  return response.text()
+}
+
+async function fetchWithTimeout(fetchImpl, url, options, timeoutMs) {
+  const controller = new AbortController()
+  const timer = setTimeout(() => controller.abort(new Error(`Request timed out after ${timeoutMs}ms`)), timeoutMs)
+  try {
+    return await fetchImpl(url, { ...options, signal: controller.signal })
+  } finally {
+    clearTimeout(timer)
+  }
+}
+
+function createApiClient(options, fetchImpl) {
+  let accessToken = null
+  const cookies = new Map()
+
+  function captureCookies(headers) {
+    const values = typeof headers.getSetCookie === 'function'
+      ? headers.getSetCookie()
+      : headers.get('set-cookie') ? [headers.get('set-cookie')] : []
+    for (const value of values) {
+      const pair = value?.split(';', 1)[0]
+      const separator = pair?.indexOf('=') ?? -1
+      if (!pair || separator < 1) continue
+      const name = pair.slice(0, separator).trim()
+      const cookieValue = pair.slice(separator + 1).trim()
+      if (!cookieValue || /(?:^|;)\s*max-age=0(?:;|$)/i.test(value)) cookies.delete(name)
+      else cookies.set(name, cookieValue)
+    }
+  }
+
+  async function raw(path, request = {}) {
+    const headers = new Headers(request.headers)
+    const url = path.startsWith('http://') || path.startsWith('https://') ? path : `${options.apiBase}${path}`
+    const authenticatedApiRequest = request.auth !== false && (url === options.apiBase || url.startsWith(`${options.apiBase}/`))
+    if (accessToken && authenticatedApiRequest) headers.set('authorization', `Bearer ${accessToken}`)
+    if (authenticatedApiRequest && cookies.size > 0) {
+      headers.set('cookie', [...cookies].map(([name, value]) => `${name}=${value}`).join('; '))
+    }
+    let body = request.body
+    if (request.json !== undefined) {
+      headers.set('content-type', 'application/json')
+      body = JSON.stringify(request.json)
+    }
+    const response = await fetchWithTimeout(fetchImpl, url, {
+      method: request.method ?? 'GET',
+      headers,
+      body,
+      redirect: request.redirect ?? 'follow',
+    }, request.timeoutMs ?? options.requestTimeoutMs)
+    captureCookies(response.headers)
+    return response
+  }
+  async function request(path, input = {}) {
+    const response = await raw(path, input)
+    const body = await responseBody(response)
+    if (!response.ok) {
+      const message = body?.error?.message ?? (typeof body === 'string' && body) ?? `HTTP ${response.status}`
+      throw new SmokeTestError(`${input.method ?? 'GET'} ${path} failed (${response.status}): ${message}`)
+    }
+    return body
+  }
+  return {
+    raw,
+    request,
+    setAccessToken(value) { accessToken = value },
+    clearSession() {
+      accessToken = null
+      cookies.clear()
+    },
+  }
+}
+
+export async function waitForJob(getJob, jobId, options = {}) {
+  const timeoutMs = options.timeoutMs ?? 180_000
+  const pollIntervalMs = options.pollIntervalMs ?? 500
+  const sleep = options.sleep ?? ((milliseconds) => new Promise((resolvePromise) => setTimeout(resolvePromise, milliseconds)))
+  const now = options.now ?? Date.now
+  const startedAt = now()
+  let lastStatus = null
+  while (now() - startedAt <= timeoutMs) {
+    const job = await getJob(jobId)
+    lastStatus = job?.status ?? null
+    if (lastStatus === 'SUCCEEDED') return job
+    if (terminalJobStatuses.has(lastStatus)) {
+      throw new SmokeTestError(`Job ${jobId} ended in ${lastStatus}: ${job.errorMessage ?? job.errorCode ?? 'unknown error'}`)
+    }
+    await sleep(pollIntervalMs)
+  }
+  throw new SmokeTestError(`Job ${jobId} did not finish within ${timeoutMs}ms (last status: ${lastStatus ?? 'unknown'})`)
+}
+
+export async function waitForNotifications(getNotifications, resourceIds, options = {}) {
+  const timeoutMs = options.timeoutMs ?? 10_000
+  const pollIntervalMs = options.pollIntervalMs ?? 250
+  const sleep = options.sleep ?? ((milliseconds) => new Promise((resolvePromise) => setTimeout(resolvePromise, milliseconds)))
+  const now = options.now ?? Date.now
+  const expected = new Set(resourceIds)
+  const startedAt = now()
+  let lastItems = []
+  while (now() - startedAt <= timeoutMs) {
+    const page = await getNotifications()
+    lastItems = Array.isArray(page?.items) ? page.items : []
+    const matched = lastItems.filter((notification) => expected.has(notification.resourceId))
+    if (new Set(matched.map((notification) => notification.resourceId)).size === expected.size) return matched
+    await sleep(pollIntervalMs)
+  }
+  const matchedIds = new Set(lastItems.map((notification) => notification.resourceId))
+  const missing = [...expected].filter((resourceId) => !matchedIds.has(resourceId))
+  throw new SmokeTestError(`Notifications did not arrive within ${timeoutMs}ms (missing resources: ${missing.join(', ')})`)
+}
+
+function selectWorkspace(workspaces, requestedId) {
+  assertCondition(Array.isArray(workspaces) && workspaces.length > 0, 'Smoke account has no workspace membership')
+  const workspace = requestedId
+    ? workspaces.find((candidate) => candidate.id === requestedId)
+    : workspaces.length === 1 ? workspaces[0] : null
+  if (!workspace) {
+    throw new SmokeTestError(requestedId
+      ? `Smoke account is not a member of workspace ${requestedId}`
+      : 'Smoke account belongs to multiple workspaces; set FRAMEFLOW_SMOKE_WORKSPACE_ID')
+  }
+  assertCondition(['OWNER', 'ADMIN'].includes(workspace.role), 'Full smoke test requires an OWNER or ADMIN workspace role')
+  return workspace
+}
+
+async function downloadBytes(fetchImpl, url, timeoutMs, expectedType) {
+  const response = await fetchWithTimeout(fetchImpl, url, { redirect: 'follow' }, timeoutMs)
+  assertCondition(response.ok, `Media download failed with HTTP ${response.status}`)
+  const contentType = response.headers.get('content-type') ?? ''
+  if (expectedType) assertCondition(contentType.toLowerCase().includes(expectedType), `Unexpected media content type: ${contentType || 'missing'}`)
+  return Buffer.from(await response.arrayBuffer())
+}
+
+function readinessSummary(readiness) {
+  return {
+    infrastructure: readiness.infrastructure,
+    ai: {
+      status: readiness.ai?.status ?? 'UNKNOWN',
+      verificationMode: readiness.ai?.verificationMode ?? 'UNKNOWN',
+      summary: readiness.ai?.summary ?? null,
+      workflows: (readiness.ai?.workflows ?? []).map((workflow) => ({
+        id: workflow.id,
+        provider: workflow.provider,
+        model: workflow.model,
+        status: workflow.status,
+      })),
+      providers: (readiness.ai?.providers ?? []).map((provider) => ({
+        id: provider.id,
+        status: provider.status,
+        verification: provider.verification,
+        configuredCapabilities: provider.configuredCapabilities,
+        totalCapabilities: provider.totalCapabilities,
+      })),
+    },
+  }
+}
+
+export function assertLiveAiReady(readiness, minimumCredits = 700, billing = null) {
+  const capabilityRows = [
+    ...(readiness?.ai?.providers ?? []).flatMap((provider) => provider.capabilities ?? []),
+    ...(readiness?.ai?.workflows ?? []),
+  ]
+  const workflows = new Map(capabilityRows.map((workflow) => [workflow.id, workflow]))
+  const missing = requiredLiveAiWorkflowIds.filter((id) => workflows.get(id)?.status !== 'CONFIGURED')
+  if (missing.length > 0) {
+    const details = missing.map((id) => {
+      const workflow = workflows.get(id)
+      const requirements = workflow?.missingRequirements?.join(', ')
+      return requirements ? `${id} (${requirements})` : id
+    })
+    throw new SmokeTestError(`Live AI smoke test requires configured workflows: ${details.join('; ')}`)
+  }
+  if (billing && Number(billing.remainingCredits) < minimumCredits) {
+    throw new SmokeTestError(`Live AI smoke test requires at least ${minimumCredits} remaining credits`)
+  }
+  return requiredLiveAiWorkflowIds.map((id) => {
+    const workflow = workflows.get(id)
+    return { id, provider: workflow.provider, model: workflow.model }
+  })
+}
+
+export async function runProductionSmoke(options, dependencies = {}) {
+  const fetchImpl = dependencies.fetch ?? fetch
+  const readFileImpl = dependencies.readFile ?? readFile
+  const sleep = dependencies.sleep ?? ((milliseconds) => new Promise((resolvePromise) => setTimeout(resolvePromise, milliseconds)))
+  const now = dependencies.now ?? Date.now
+  const log = dependencies.log ?? ((message) => console.error(message))
+  const api = createApiClient(options, fetchImpl)
+  const startedAt = now()
+  const runId = randomUUID()
+  const marker = `FRAMEFLOW_SMOKE_${runId}`
+  const report = {
+    status: 'running',
+    runId,
+    baseUrl: options.origin,
+    startedAt: new Date(startedAt).toISOString(),
+    workspaceId: null,
+    projectId: null,
+    projectArchived: false,
+    checks: [],
+    services: null,
+    billing: null,
+    aiVerification: {
+      mode: options.includeAi ? 'LIVE_PROVIDER_CALLS' : 'CONFIGURATION_ONLY',
+      jobs: [],
+      artifacts: {},
+    },
+    artifacts: {},
+  }
+  let projectId = null
+  let loggedIn = false
+  let smokeSessionId = null
+  let primaryError = null
+
+  async function step(id, label, action, summarize) {
+    const stepStartedAt = now()
+    log(`[...] ${label}`)
+    try {
+      const value = await action()
+      const check = { id, label, status: 'passed', durationMs: Math.max(0, now() - stepStartedAt) }
+      const details = summarize?.(value)
+      if (details !== undefined) check.details = details
+      report.checks.push(check)
+      log(`[ ok] ${label} (${check.durationMs}ms)`)
+      return value
+    } catch (error) {
+      const check = { id, label, status: 'failed', durationMs: Math.max(0, now() - stepStartedAt), error: safeError(error) }
+      report.checks.push(check)
+      log(`[fail] ${label}: ${check.error}`)
+      throw error
+    }
+  }
+
+  async function runAiJob(projectIdForJob, episodeIdForJob, type, input) {
+    const queued = await api.request(`/projects/${projectIdForJob}/jobs`, {
+      method: 'POST',
+      json: {
+        type,
+        episodeId: episodeIdForJob,
+        idempotencyKey: `smoke-ai-${runId}-${report.aiVerification.jobs.length}-${type.toLowerCase()}`,
+        input,
+        maxAttempts: 1,
+      },
+    })
+    const job = await waitForJob((jobId) => api.request(`/jobs/${jobId}`), queued.job.id, {
+      timeoutMs: options.timeoutMs,
+      sleep,
+    })
+    assertCondition(['openai', 'replicate'].includes(job.provider), `${type} did not record a live AI provider`)
+    assertCondition(job.providerModel, `${type} did not record a provider model`)
+    report.aiVerification.jobs.push({ id: job.id, type, provider: job.provider, model: job.providerModel })
+    return job
+  }
+
+  try {
+    await step('liveness', 'API liveness', async () => {
+      const response = await fetchWithTimeout(fetchImpl, `${options.origin}/health/live`, {}, options.requestTimeoutMs)
+      const body = await responseBody(response)
+      assertCondition(response.ok && body?.status === 'ok', `Liveness failed with HTTP ${response.status}`)
+      return body
+    })
+
+    await step('readiness', 'API dependency readiness', async () => {
+      const response = await fetchWithTimeout(fetchImpl, `${options.origin}/health/ready`, {}, options.requestTimeoutMs)
+      const body = await responseBody(response)
+      assertCondition(response.ok && body?.status === 'ready', `Readiness failed with HTTP ${response.status}`)
+      const unavailable = Object.entries(body.dependencies ?? {}).filter(([, status]) => status !== 'ok')
+      assertCondition(unavailable.length === 0, `Unavailable dependencies: ${unavailable.map(([name]) => name).join(', ')}`)
+      return body
+    }, (body) => body.dependencies)
+
+    const session = await step('login', 'Dedicated smoke-account login', async () => {
+      const value = await api.request('/auth/login', {
+        method: 'POST',
+        auth: false,
+        json: { email: options.email, password: options.password },
+      })
+      assertCondition(value?.accessToken && value?.user?.id, 'Login response is missing the authenticated session')
+      api.setAccessToken(value.accessToken)
+      loggedIn = true
+      const sessions = await api.request('/auth/sessions')
+      const currentSession = sessions.find((candidate) => candidate.current)
+      assertCondition(currentSession?.id, 'Login refresh session cookie was not retained by the smoke runner')
+      smokeSessionId = currentSession.id
+      return value
+    }, (value) => ({ userId: value.user.id, sessionId: smokeSessionId, workspaceCount: value.workspaces.length }))
+
+    const workspace = selectWorkspace(session.workspaces, options.workspaceId)
+    report.workspaceId = workspace.id
+    const serviceReadiness = await step('service-readiness', 'Worker, storage, FFmpeg, and AI configuration', async () => {
+      const value = await api.request(`/workspaces/${workspace.id}/service-readiness`)
+      assertCondition(value?.infrastructure?.status === 'AVAILABLE', 'Authenticated service readiness is degraded')
+      assertCondition(value.infrastructure.available === value.infrastructure.total, 'Not every production service is available')
+      return value
+    }, (value) => ({
+      infrastructure: value.infrastructure.services.map((service) => ({ id: service.id, status: service.status, latencyMs: service.latencyMs })),
+      ai: value.ai.summary,
+    }))
+    report.services = readinessSummary(serviceReadiness)
+
+    const billingReadModel = await step('billing', 'Billing and usage read model', async () => {
+      const value = await api.request(`/workspaces/${workspace.id}/billing/summary`)
+      assertCondition(value?.plan && value?.provider?.status, 'Billing summary is incomplete')
+      return value
+    }, (value) => ({ plan: value.plan, provider: value.provider.status, webhookConfigured: value.provider.webhookConfigured, remainingCredits: value.remainingCredits }))
+    report.billing = {
+      plan: billingReadModel.plan,
+      provider: billingReadModel.provider.status,
+      webhookConfigured: billingReadModel.provider.webhookConfigured,
+      remainingCredits: billingReadModel.remainingCredits,
+    }
+
+    if (options.includeAi) {
+      await step('ai-preflight', 'Billable AI authorization, configuration, and credit preflight', async () => {
+        const workflows = assertLiveAiReady(serviceReadiness, 700, billingReadModel)
+        return { acknowledgement: 'accepted', minimumCredits: 700, workflows }
+      }, (value) => value)
+    }
+
+    const project = await step('project', 'Project persistence', async () => api.request(`/workspaces/${workspace.id}/projects`, {
+      method: 'POST',
+      json: {
+        name: `[SMOKE] ${new Date(startedAt).toISOString()} ${runId.slice(0, 8)}`,
+        logline: 'Automated production smoke test. Safe to archive after completion.',
+        genre: 'SMOKE_TEST',
+        visualBible: { smokeTest: { runId, createdAt: new Date(startedAt).toISOString() } },
+      },
+    }), (value) => ({ projectId: value.id }))
+    projectId = project.id
+    report.projectId = projectId
+
+    const episode = await step('script', 'Episode and immutable script persistence', async () => {
+      const createdEpisode = await api.request(`/projects/${projectId}/episodes`, {
+        method: 'POST',
+        json: { episodeNumber: 1, title: 'Production smoke episode', synopsis: marker, targetDurationSeconds: 15 },
+      })
+      const script = await api.request(`/episodes/${createdEpisode.id}/scripts`, {
+        method: 'POST',
+        json: {
+          title: 'Production smoke script',
+          content: { smokeTest: true, runId, scenes: [{ number: 1, text: marker }] },
+          plainText: `SCENE 1 - SMOKE TEST\n${marker}`,
+          changeSummary: 'Production smoke test',
+        },
+      })
+      const persisted = await api.request(`/episodes/${createdEpisode.id}`)
+      assertCondition(persisted.currentScript?.id === script.id, 'Current script version was not persisted')
+      assertCondition(persisted.currentScript?.plainText?.includes(marker), 'Persisted script marker is missing')
+      return { episode: createdEpisode, script }
+    }, (value) => ({ episodeId: value.episode.id, scriptVersion: value.script.version }))
+
+    const sourceImage = await readFileImpl(options.imagePath)
+    const sourceHash = sha256(sourceImage)
+    const contentType = imageContentType(options.imagePath)
+    const imageAssetVersion = await step('upload', 'Object upload and signed download integrity', async () => {
+      const form = new FormData()
+      form.append('file', new Blob([sourceImage], { type: contentType }), basename(options.imagePath))
+      const upload = await api.request(`/projects/${projectId}/uploads`, { method: 'POST', body: form })
+      assertCondition(upload.byteSize === sourceImage.length, 'Uploaded object byte size does not match the source')
+      const asset = await api.request(`/projects/${projectId}/assets`, {
+        method: 'POST',
+        json: { type: 'SHOT', name: `Smoke source ${runId.slice(0, 8)}`, description: marker, tags: ['smoke-test'], metadata: { smokeRunId: runId } },
+      })
+      const version = await api.request(`/assets/${asset.id}/versions`, {
+        method: 'POST',
+        json: { storageKey: upload.key, mimeType: upload.contentType, byteSize: upload.byteSize, prompt: marker, provider: 'smoke-test', providerModel: 'uploaded-source-v1' },
+      })
+      const signed = await api.request(`/assets/${asset.id}/versions/${version.id}/url`)
+      const downloaded = await downloadBytes(fetchImpl, signed.url, options.requestTimeoutMs, contentType)
+      assertCondition(sha256(downloaded) === sourceHash, 'Signed object download does not match the uploaded image')
+      return { asset, version, upload }
+    }, (value) => ({ assetId: value.asset.id, assetVersionId: value.version.id, byteSize: value.upload.byteSize, sha256: sourceHash }))
+
+    const shot = await step('shot', 'Shot and immutable media-version persistence', async () => {
+      const created = await api.request(`/episodes/${episode.episode.id}/shots`, {
+        method: 'POST',
+        json: { shotNumber: 1, sceneNumber: 1, title: 'Smoke render shot', description: marker, shotType: '全景', cameraMotion: '静止', durationMs: 2000, characterAssetIds: [], sortOrder: 100 },
+      })
+      const version = await api.request(`/shots/${created.id}/versions`, {
+        method: 'POST',
+        json: { imageAssetVersionId: imageAssetVersion.version.id, subtitleText: marker, generationParams: { smokeRunId: runId } },
+      })
+      const persisted = await api.request(`/shots/${created.id}`)
+      assertCondition(persisted.versions?.[0]?.id === version.id, 'Current shot version was not persisted')
+      return { shot: created, version }
+    }, (value) => ({ shotId: value.shot.id, shotVersion: value.version.version }))
+
+    await step('queue', 'Redis/BullMQ worker execution', async () => {
+      const queued = await api.request(`/projects/${projectId}/jobs`, {
+        method: 'POST',
+        json: { type: 'PIPELINE_HEALTHCHECK', episodeId: episode.episode.id, idempotencyKey: `smoke-health-${runId}`, input: { marker }, maxAttempts: 1 },
+      })
+      const job = await waitForJob((jobId) => api.request(`/jobs/${jobId}`), queued.job.id, { timeoutMs: options.timeoutMs, sleep })
+      assertCondition(job.output?.ok === true && job.output?.echo?.marker === marker, 'Worker healthcheck output is invalid')
+      return job
+    }, (job) => ({ jobId: job.id, provider: job.provider, providerModel: job.providerModel }))
+
+    const renderResult = await step('render', 'FFmpeg render and immutable source manifest', async () => {
+      const queued = await api.request(`/episodes/${episode.episode.id}/renders`, {
+        method: 'POST',
+        json: {
+          idempotencyKey: `smoke-render-${runId}`,
+          settings: { width: 640, height: 360, fps: 24, quality: 'draft', burnSubtitles: false, normalizeAudio: false, musicVolume: 0, subtitleStyle: { fontSize: 20, textColor: '#FFFFFF', outlineColor: '#000000', outlineWidth: 2, bottomMargin: 24, bold: true } },
+        },
+      })
+      const job = await waitForJob((jobId) => api.request(`/jobs/${jobId}`), queued.job.id, { timeoutMs: options.timeoutMs, sleep })
+      const render = await api.request(`/renders/${queued.render.id}`)
+      assertCondition(render.status === 'REVIEW', `Render ended in unexpected status ${render.status}`)
+      assertCondition(render.sourceShotCount === 1, 'Render source manifest does not contain exactly one shot')
+      assertCondition(render.subtitleFormats?.includes('srt') && render.subtitleFormats?.includes('vtt'), 'Render subtitle sidecars are missing')
+      assertCondition(render.downloadUrl, 'Render download URL is missing')
+      const video = await downloadBytes(fetchImpl, render.downloadUrl, options.requestTimeoutMs, 'video/mp4')
+      assertCondition(video.length > 1000, 'Rendered video is unexpectedly small')
+      return { render, job, videoHash: sha256(video), videoBytes: video.length }
+    }, (value) => ({ renderId: value.render.id, jobId: value.job.id, byteSize: value.videoBytes, subtitleFormats: value.render.subtitleFormats, sourceShotCount: value.render.sourceShotCount }))
+    report.artifacts.renderId = renderResult.render.id
+    report.artifacts.videoSha256 = renderResult.videoHash
+
+    await step('subtitles', 'SRT and VTT sidecar downloads', async () => {
+      const results = {}
+      for (const format of ['srt', 'vtt']) {
+        const subtitle = await api.request(`/renders/${renderResult.render.id}/subtitles/${format}`)
+        const bytes = await downloadBytes(fetchImpl, subtitle.downloadUrl, options.requestTimeoutMs)
+        const text = bytes.toString('utf8')
+        assertCondition(text.includes(marker), `${format.toUpperCase()} subtitle does not contain the smoke marker`)
+        results[format] = { bytes: bytes.length, sha256: sha256(bytes) }
+      }
+      return results
+    }, (value) => value)
+
+    await step('review', 'Render review and approval', async () => {
+      const review = await api.request(`/workspaces/${workspace.id}/reviews`, {
+        method: 'POST',
+        json: { targetType: 'RENDER', targetId: renderResult.render.id, assignedToId: session.user.id, summary: `Automated approval for ${marker}` },
+      })
+      const decision = await api.request(`/reviews/${review.id}/decision`, {
+        method: 'POST',
+        json: { status: 'APPROVED', summary: 'Production smoke render passed automated checks.' },
+      })
+      const approved = await api.request(`/renders/${renderResult.render.id}`)
+      assertCondition(decision.status === 'APPROVED' && approved.status === 'APPROVED', 'Render approval was not persisted')
+      return { review, approved }
+    }, (value) => ({ reviewId: value.review.id, renderStatus: value.approved.status }))
+
+    const deliveryResult = await step('delivery', 'Approved download delivery through the worker', async () => {
+      const queued = await api.request(`/renders/${renderResult.render.id}/deliveries`, {
+        method: 'POST',
+        json: { platform: 'DOWNLOAD', destination: { smokeRunId: runId } },
+      })
+      const job = await waitForJob((jobId) => api.request(`/jobs/${jobId}`), queued.job.id, { timeoutMs: options.timeoutMs, sleep })
+      const delivery = await api.request(`/deliveries/${queued.delivery.id}`)
+      assertCondition(delivery.status === 'PUBLISHED' && delivery.downloadUrl, 'Download delivery was not published')
+      const redirect = await api.raw(`/deliveries/${delivery.id}/download`, { redirect: 'manual' })
+      assertCondition([301, 302, 303, 307, 308].includes(redirect.status), `Delivery endpoint returned HTTP ${redirect.status} instead of a redirect`)
+      const location = redirect.headers.get('location')
+      assertCondition(location, 'Delivery redirect is missing its signed location')
+      const video = await downloadBytes(fetchImpl, location, options.requestTimeoutMs, 'video/mp4')
+      assertCondition(sha256(video) === renderResult.videoHash, 'Delivered video differs from the approved render')
+      return { delivery, job, bytes: video.length }
+    }, (value) => ({ deliveryId: value.delivery.id, jobId: value.job.id, status: value.delivery.status, byteSize: value.bytes }))
+    report.artifacts.deliveryId = deliveryResult.delivery.id
+
+    await step('notifications', 'Persistent job notifications and read state', async () => {
+      const expectedJobIds = [renderResult.job.id, deliveryResult.job.id]
+      const rows = await waitForNotifications(
+        () => api.request(`/workspaces/${workspace.id}/notifications?limit=100`),
+        expectedJobIds,
+        { timeoutMs: Math.min(options.timeoutMs, 10_000), sleep, now },
+      )
+      for (const jobId of expectedJobIds) {
+        const notification = rows.find((candidate) => candidate.resourceId === jobId)
+        assertCondition(notification?.kind === 'JOB_SUCCEEDED' && notification.tone === 'SUCCESS', `Successful job notification is invalid for ${jobId}`)
+        const expectedPage = jobId === deliveryResult.job.id ? 'publish' : 'jobs'
+        assertCondition(notification.page === expectedPage, `Job notification points to ${notification.page} instead of ${expectedPage}`)
+        const read = await api.request(`/notifications/${notification.id}`, { method: 'PATCH', json: { read: true } })
+        assertCondition(read.readAt, `Notification ${notification.id} did not persist its read state`)
+      }
+      return rows
+    }, (rows) => ({ notifications: rows.length, resources: rows.map((notification) => notification.resourceId) }))
+
+    await step('audit', 'Audit-log persistence', async () => {
+      const page = await api.request(`/workspaces/${workspace.id}/audit-logs?limit=100&actorId=${session.user.id}`)
+      const actions = new Set((page.items ?? []).map((item) => item.action))
+      for (const action of ['project.create', 'upload.create', 'render.create', 'review.decide', 'delivery.create']) {
+        assertCondition(actions.has(action), `Audit log is missing ${action}`)
+      }
+      return { actions: actions.size }
+    }, (value) => value)
+
+    if (options.includeAi) {
+      const aiSetup = await step('ai-setup', 'Live AI validation workspace data', async () => {
+        const aiEpisode = await api.request(`/projects/${projectId}/episodes`, {
+          method: 'POST',
+          json: {
+            episodeNumber: 2,
+            title: 'Live provider validation',
+            synopsis: `A single character says a short line in a controlled studio. ${marker}`,
+            targetDurationSeconds: 15,
+          },
+        })
+        const character = await api.request(`/projects/${projectId}/assets`, {
+          method: 'POST',
+          json: {
+            type: 'CHARACTER',
+            name: `Provider test character ${runId.slice(0, 8)}`,
+            description: 'Adult presenter, centered portrait, neutral studio lighting, production-safe wardrobe.',
+            tags: ['smoke-test', 'provider-validation'],
+            metadata: { smokeRunId: runId },
+          },
+        })
+        return { episode: aiEpisode, character }
+      }, (value) => ({ episodeId: value.episode.id, characterAssetId: value.character.id }))
+
+      const openAiImage = await step('ai-image-openai', 'OpenAI character image generation', async () => {
+        const job = await runAiJob(projectId, aiSetup.episode.id, 'ASSET_GENERATE', {
+          assetId: aiSetup.character.id,
+          prompt: `Clean comic-drama character reference sheet. Adult presenter, centered bust portrait, neutral gray studio, consistent facial landmarks. ${marker}`,
+          negativePrompt: 'text, watermark, extra limbs, child',
+          width: 1024,
+          height: 1024,
+          provider: 'openai',
+          referenceAssetVersionIds: [],
+          params: { consistencyStrength: 82 },
+        })
+        const asset = await api.request(`/assets/${aiSetup.character.id}`)
+        const version = asset.versions?.find((candidate) => candidate.id === job.output?.assetVersionId)
+        assertCondition(version?.storageKey && version.provider === 'openai', 'OpenAI image version was not persisted as current asset media')
+        const signed = await api.request(`/assets/${asset.id}/versions/${version.id}/url`)
+        const bytes = await downloadBytes(fetchImpl, signed.url, options.requestTimeoutMs, 'image/')
+        assertCondition(bytes.length > 1000, 'OpenAI image output is unexpectedly small')
+        return { job, asset, version, bytes }
+      }, (value) => ({ jobId: value.job.id, assetVersionId: value.version.id, model: value.job.providerModel, byteSize: value.bytes.length }))
+      report.aiVerification.artifacts.openAiImageAssetVersionId = openAiImage.version.id
+
+      const replicateImage = await step('ai-image-replicate', 'Replicate character-consistency variant', async () => {
+        const profile = await api.request(`/assets/${aiSetup.character.id}/consistency-profile`, {
+          method: 'PUT',
+          json: {
+            referenceAssetVersionIds: [openAiImage.version.id],
+            identityPrompt: 'Preserve facial proportions, eye shape, hairstyle, and adult age.',
+            costumePrompt: 'Preserve the dark production jacket and simple collar.',
+            stylePrompt: 'Polished cinematic comic-drama illustration.',
+            negativePrompt: 'identity drift, age change, child, text, watermark',
+            consistencyStrength: 90,
+            identityLocked: true,
+          },
+        })
+        assertCondition(profile.consistencyProfile?.identityLocked, 'Character identity profile was not locked')
+        const job = await runAiJob(projectId, aiSetup.episode.id, 'ASSET_GENERATE', {
+          assetId: aiSetup.character.id,
+          prompt: `The same adult presenter turns slightly toward camera under warm key light. ${marker}`,
+          negativePrompt: 'text, watermark',
+          width: 1024,
+          height: 1024,
+          provider: 'replicate',
+          referenceAssetVersionIds: [openAiImage.version.id],
+          params: { consistencyStrength: 90 },
+        })
+        const asset = await api.request(`/assets/${aiSetup.character.id}`)
+        const version = asset.versions?.find((candidate) => candidate.id === job.output?.assetVersionId)
+        const appliedProfiles = version?.generationParams?.appliedCharacterConsistencyProfiles
+        assertCondition(version?.storageKey && version.provider === 'replicate', 'Replicate image version was not persisted')
+        assertCondition(Array.isArray(appliedProfiles) && appliedProfiles.some((snapshot) => snapshot.assetId === aiSetup.character.id && snapshot.identityLocked), 'Replicate variant did not capture the locked character profile')
+        assertCondition(version.generationParams?.referenceAssetVersionIds?.includes(openAiImage.version.id), 'Replicate variant did not record its identity reference')
+        const signed = await api.request(`/assets/${asset.id}/versions/${version.id}/url`)
+        const bytes = await downloadBytes(fetchImpl, signed.url, options.requestTimeoutMs, 'image/')
+        assertCondition(bytes.length > 1000, 'Replicate image output is unexpectedly small')
+        return { job, asset, version, bytes }
+      }, (value) => ({ jobId: value.job.id, assetVersionId: value.version.id, model: value.job.providerModel, byteSize: value.bytes.length }))
+      report.aiVerification.artifacts.replicateImageAssetVersionId = replicateImage.version.id
+
+      await step('ai-script', 'OpenAI script generation and immutable persistence', async () => {
+        const job = await runAiJob(projectId, aiSetup.episode.id, 'SCRIPT_GENERATE', {
+          provider: 'openai',
+          includeExistingScript: false,
+          instruction: 'Write one production-safe interior scene for a 15-second Chinese comic drama. Use one adult presenter, one short spoken line, and a clear visual action.',
+        })
+        const episode = await api.request(`/episodes/${aiSetup.episode.id}`)
+        assertCondition(episode.currentScript?.id === job.output?.scriptVersionId, 'Generated script was not activated as the current version')
+        assertCondition(episode.currentScript?.plainText?.trim(), 'Generated script has no plain text')
+        return { job, script: episode.currentScript }
+      }, (value) => ({ jobId: value.job.id, scriptVersionId: value.script.id, version: value.script.version, model: value.job.providerModel }))
+
+      const storyboard = await step('ai-storyboard', 'OpenAI storyboard generation and shot persistence', async () => {
+        const job = await runAiJob(projectId, aiSetup.episode.id, 'STORYBOARD_GENERATE', {
+          provider: 'openai',
+          replaceExisting: false,
+          instruction: `Create the smallest viable storyboard for provider validation. Prefer exactly one shot featuring ${aiSetup.character.name}, with one short spoken line and a total duration near five seconds.`,
+        })
+        const rows = await api.request(`/episodes/${aiSetup.episode.id}/shots`)
+        assertCondition(Array.isArray(rows) && rows.length > 0, 'Generated storyboard contains no shots')
+        assertCondition(job.output?.shotCount === rows.length, 'Storyboard job output does not match persisted shots')
+        const [selected, ...extra] = rows
+        for (const row of extra) await api.request(`/shots/${row.shot.id}`, { method: 'DELETE' })
+        const validationText = 'FrameFlow AI provider validation passed.'
+        const version = await api.request(`/shots/${selected.shot.id}/versions`, {
+          method: 'POST',
+          json: {
+            imageAssetVersionId: replicateImage.version.id,
+            subtitleText: validationText,
+            generationParams: { smokeRunId: runId, source: 'live-provider-smoke' },
+          },
+        })
+        return { job, shot: selected.shot, version, removedShots: extra.length, validationText }
+      }, (value) => ({ jobId: value.job.id, shotId: value.shot.id, generatedShots: value.job.output.shotCount, removedShots: value.removedShots, model: value.job.providerModel }))
+
+      await step('ai-tts', 'OpenAI TTS generation and audio storage', async () => {
+        const job = await runAiJob(projectId, aiSetup.episode.id, 'TTS_GENERATE', {
+          shotId: storyboard.shot.id,
+          text: storyboard.validationText,
+          voice: 'alloy',
+          speed: 1,
+          provider: 'openai',
+        })
+        const media = await api.request(`/shots/${storyboard.shot.id}/media`)
+        assertCondition(job.output?.audioStorageKey && media.audioUrl, 'TTS output was not bound to the current shot version')
+        const bytes = await downloadBytes(fetchImpl, media.audioUrl, options.requestTimeoutMs, 'audio/')
+        assertCondition(bytes.length > 500, 'TTS audio output is unexpectedly small')
+        return { job, media, bytes }
+      }, (value) => ({ jobId: value.job.id, model: value.job.providerModel, byteSize: value.bytes.length }))
+
+      await step('ai-video', 'Replicate image-to-video generation and storage', async () => {
+        const job = await runAiJob(projectId, aiSetup.episode.id, 'VIDEO_GENERATE', {
+          shotId: storyboard.shot.id,
+          prompt: 'Subtle natural head turn and blinking, locked camera, stable adult character identity.',
+          imageAssetVersionId: replicateImage.version.id,
+          durationSeconds: options.aiVideoSeconds,
+          provider: 'replicate',
+          params: options.aiVideoParams,
+        })
+        const media = await api.request(`/shots/${storyboard.shot.id}/media`)
+        assertCondition(job.output?.videoStorageKey && media.videoUrl, 'Video output was not bound to the current shot version')
+        const bytes = await downloadBytes(fetchImpl, media.videoUrl, options.requestTimeoutMs, 'video/')
+        assertCondition(bytes.length > 1000, 'Generated video output is unexpectedly small')
+        return { job, media, bytes }
+      }, (value) => ({ jobId: value.job.id, model: value.job.providerModel, durationSeconds: options.aiVideoSeconds, byteSize: value.bytes.length }))
+
+      const lipsync = await step('ai-lipsync', 'Replicate lip-sync generation and media continuity', async () => {
+        const job = await runAiJob(projectId, aiSetup.episode.id, 'LIPSYNC_GENERATE', {
+          shotId: storyboard.shot.id,
+          provider: 'replicate',
+          params: options.aiLipsyncParams,
+        })
+        const media = await api.request(`/shots/${storyboard.shot.id}/media`)
+        assertCondition(job.output?.videoStorageKey && media.videoUrl && media.audioUrl, 'Lip-sync output did not preserve video and audio continuity')
+        const [video, audio] = await Promise.all([
+          downloadBytes(fetchImpl, media.videoUrl, options.requestTimeoutMs, 'video/'),
+          downloadBytes(fetchImpl, media.audioUrl, options.requestTimeoutMs, 'audio/'),
+        ])
+        assertCondition(video.length > 1000 && audio.length > 500, 'Lip-sync media outputs are unexpectedly small')
+        return { job, media, video, audio }
+      }, (value) => ({ jobId: value.job.id, model: value.job.providerModel, videoBytes: value.video.length, audioBytes: value.audio.length }))
+      report.aiVerification.artifacts.lipsyncShotVersionId = lipsync.media.versionId
+
+      const aiRender = await step('ai-render', 'FFmpeg render from live AI media', async () => {
+        const queued = await api.request(`/episodes/${aiSetup.episode.id}/renders`, {
+          method: 'POST',
+          json: {
+            idempotencyKey: `smoke-ai-render-${runId}`,
+            settings: { width: 640, height: 360, fps: 24, quality: 'draft', burnSubtitles: false, normalizeAudio: false, musicVolume: 0, subtitleStyle: { fontSize: 20, textColor: '#FFFFFF', outlineColor: '#000000', outlineWidth: 2, bottomMargin: 24, bold: true } },
+          },
+        })
+        const job = await waitForJob((jobId) => api.request(`/jobs/${jobId}`), queued.job.id, { timeoutMs: options.timeoutMs, sleep })
+        const render = await api.request(`/renders/${queued.render.id}`)
+        assertCondition(job.status === 'SUCCEEDED' && render.status === 'REVIEW', 'AI media render did not finish in review state')
+        assertCondition(render.sourceShotCount === 1 && render.downloadUrl, 'AI media render source manifest is invalid')
+        const video = await downloadBytes(fetchImpl, render.downloadUrl, options.requestTimeoutMs, 'video/mp4')
+        assertCondition(video.length > 1000, 'AI media render is unexpectedly small')
+        return { job, render, video }
+      }, (value) => ({ jobId: value.job.id, renderId: value.render.id, sourceShotCount: value.render.sourceShotCount, byteSize: value.video.length }))
+      report.aiVerification.artifacts.renderId = aiRender.render.id
+      report.aiVerification.artifacts.videoSha256 = sha256(aiRender.video)
+
+      await step('ai-usage', 'Live AI usage-ledger reconciliation', async () => {
+        const billingAfter = await api.request(`/workspaces/${workspace.id}/billing/summary`)
+        const providerJobIds = new Set(report.aiVerification.jobs.map((job) => job.id))
+        const billedJobIds = new Set((billingAfter.recent ?? []).filter((entry) => providerJobIds.has(entry.jobId)).map((entry) => entry.jobId))
+        const missingJobIds = [...providerJobIds].filter((jobId) => !billedJobIds.has(jobId))
+        assertCondition(missingJobIds.length === 0, `Usage ledger is missing ${missingJobIds.length} live AI jobs`)
+        const creditsUsed = Number(billingAfter.usedCredits) - Number(billingReadModel.usedCredits)
+        assertCondition(creditsUsed > 0, 'Live AI provider calls did not increase recorded usage')
+        report.aiVerification.creditsUsed = creditsUsed
+        return { jobs: providerJobIds.size, ledgerJobs: billedJobIds.size, creditsUsed }
+      }, (value) => value)
+    }
+  } catch (error) {
+    primaryError = error
+  } finally {
+    if (projectId && !options.keepProject) {
+      try {
+        await step('cleanup', 'Archive smoke-test project', async () => {
+          const archived = await api.request(`/projects/${projectId}`, { method: 'DELETE' })
+          assertCondition(archived.status === 'ARCHIVED', 'Smoke project was not archived')
+          report.projectArchived = true
+          return archived
+        }, (value) => ({ projectId: value.id, status: value.status }))
+      } catch (cleanupError) {
+        if (!primaryError) primaryError = cleanupError
+      }
+    }
+    if (loggedIn) {
+      try {
+        await step('logout', 'Revoke smoke-account refresh session', async () => {
+          await api.request('/auth/logout', { method: 'POST' })
+          const sessions = await api.request('/auth/sessions')
+          assertCondition(!sessions.some((candidate) => candidate.id === smokeSessionId), 'Smoke refresh session remains active after logout')
+          api.clearSession()
+          return { sessionId: smokeSessionId, revoked: true }
+        }, (value) => value)
+      } catch (logoutError) {
+        if (!primaryError) primaryError = logoutError
+      }
+    }
+  }
+
+  report.status = primaryError ? 'failed' : 'passed'
+  report.completedAt = new Date(now()).toISOString()
+  report.durationMs = Math.max(0, now() - startedAt)
+  if (options.keepProject && projectId) report.projectArchived = false
+  if (primaryError) {
+    report.error = safeError(primaryError)
+    throw new SmokeTestError(report.error, { cause: primaryError, report })
+  }
+  return report
+}
+
+export function usage() {
+  return `FrameFlow production smoke test
+
+Usage:
+  FRAMEFLOW_SMOKE_BASE_URL=https://studio.example.com \\
+  FRAMEFLOW_SMOKE_EMAIL=smoke@example.com \\
+  FRAMEFLOW_SMOKE_PASSWORD='...' \\
+    npm run smoke:production -- [options]
+
+Options:
+  --base-url URL       Application origin or /api/v1 URL
+  --email EMAIL        Dedicated verified smoke-account email
+  --workspace-id UUID  Required when the account has multiple workspaces
+  --image PATH         JPEG, PNG, or WebP render source
+  --timeout-seconds N  Per-job timeout from 30 to 900 (default: 180)
+  --include-ai         Run live, billable OpenAI and Replicate validation
+  --ai-video-seconds N Live AI video duration from 1 to 30 (default: 5)
+  --keep-project       Keep the generated smoke project instead of archiving it
+  --allow-http         Allow HTTP only for localhost development
+  --json               Print the complete report as JSON
+  --help               Show this help
+
+The password is accepted only through FRAMEFLOW_SMOKE_PASSWORD so it is not exposed in the process list.
+Default mode never invokes AI provider jobs; configuration-only readiness is included in the report.
+
+--include-ai starts real provider jobs that can incur charges. It is accepted only when
+FRAMEFLOW_SMOKE_BILLABLE_AI_ACK=I_ACCEPT_BILLABLE_AI_CHARGES is also set, all seven AI
+workflows are configured, and the workspace has at least 700 remaining credits. Provider
+jobs use maxAttempts=1. Optional model-specific JSON objects can be supplied through
+FRAMEFLOW_SMOKE_VIDEO_PARAMS_JSON and FRAMEFLOW_SMOKE_LIPSYNC_PARAMS_JSON.`
+}
+
+async function main() {
+  let options
+  try {
+    options = parseSmokeOptions()
+    if (options.help) {
+      console.log(usage())
+      return
+    }
+    const report = await runProductionSmoke(options)
+    if (options.json) console.log(JSON.stringify(report, null, 2))
+    else console.log(`Production smoke test passed in ${report.durationMs}ms. Project ${report.projectId}${report.projectArchived ? ' was archived' : ' was retained'}.`)
+  } catch (error) {
+    const report = error instanceof SmokeTestError ? error.report : null
+    if (options?.json && report) console.error(JSON.stringify(report, null, 2))
+    else console.error(`Production smoke test failed: ${safeError(error)}`)
+    process.exitCode = 1
+  }
+}
+
+if (process.argv[1] && resolve(process.argv[1]) === scriptPath) await main()

scripts/stripe-smoke.mjs

@@ -0,0 +1,305 @@
+#!/usr/bin/env node
+
+import { resolve } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import Stripe from 'stripe'
+
+const scriptPath = fileURLToPath(import.meta.url)
+export const requiredStripeWebhookEvents = [
+  'checkout.session.completed',
+  'customer.subscription.created',
+  'customer.subscription.updated',
+  'customer.subscription.deleted',
+]
+
+export class StripeSmokeError extends Error {
+  constructor(message, options = {}) {
+    super(message, options)
+    this.name = 'StripeSmokeError'
+  }
+}
+
+function required(value, name) {
+  if (typeof value !== 'string' || value.trim() === '') throw new StripeSmokeError(`${name} is required`)
+  return value.trim()
+}
+
+function integerValue(value, name, minimum, maximum) {
+  const parsed = Number(value)
+  if (!Number.isInteger(parsed) || parsed < minimum || parsed > maximum) {
+    throw new StripeSmokeError(`${name} must be an integer from ${minimum} to ${maximum}`)
+  }
+  return parsed
+}
+
+function booleanValue(value) {
+  return ['1', 'true', 'yes', 'on'].includes(String(value ?? '').trim().toLowerCase())
+}
+
+export function resolveStripeAppOrigin(value, options = {}) {
+  let url
+  try {
+    url = new URL(required(value, 'FRAMEFLOW_STRIPE_APP_ORIGIN'))
+  } catch (error) {
+    if (error instanceof StripeSmokeError) throw error
+    throw new StripeSmokeError('FRAMEFLOW_STRIPE_APP_ORIGIN must be a valid URL')
+  }
+  if (url.username || url.password || url.search || url.hash || url.pathname !== '/') {
+    throw new StripeSmokeError('Stripe app origin must be a bare origin without credentials, path, query, or fragment')
+  }
+  if (!['http:', 'https:'].includes(url.protocol)) throw new StripeSmokeError('Stripe app origin must use HTTP or HTTPS')
+  const localHostnames = new Set(['localhost', '127.0.0.1', '::1', '[::1]'])
+  if (url.protocol === 'http:' && (!options.allowHttp || !localHostnames.has(url.hostname))) {
+    throw new StripeSmokeError('HTTPS is required; HTTP can only be enabled explicitly for localhost')
+  }
+  return url.origin
+}
+
+export function usage() {
+  return `Usage: npm run smoke:stripe -- [options]
+
+Performs read-only Stripe account, recurring Price, Product, Customer Portal,
+and webhook-endpoint configuration checks. It never creates a customer,
+Checkout Session, Portal Session, charge, subscription, or webhook event.
+
+Required environment:
+  STRIPE_SECRET_KEY
+  STRIPE_WEBHOOK_SECRET
+  STRIPE_PRO_PRICE_ID
+  STRIPE_STUDIO_PRICE_ID
+  FRAMEFLOW_STRIPE_APP_ORIGIN (falls back to API_PUBLIC_ORIGIN or WEB_ORIGIN)
+
+Optional environment:
+  STRIPE_PORTAL_CONFIGURATION_ID
+  FRAMEFLOW_STRIPE_SMOKE_TIMEOUT_SECONDS=30
+
+Options:
+  --app-origin <origin>
+  --timeout-seconds <10-120>
+  --allow-http              Loopback development only
+  --require-live            Reject test keys and test-mode Stripe resources
+  --json
+  --help
+
+The signing secret is never sent or printed. Stripe does not expose webhook
+signing secrets through its API, so a passing report proves endpoint/event
+configuration but not that STRIPE_WEBHOOK_SECRET matches that endpoint.
+`
+}
+
+export function parseStripeSmokeOptions(argv = process.argv.slice(2), environment = process.env) {
+  const values = {}
+  const flags = new Set()
+  const valueOptions = new Set(['--app-origin', '--timeout-seconds'])
+  const flagOptions = new Set(['--allow-http', '--require-live', '--json', '--help'])
+  for (let index = 0; index < argv.length; index += 1) {
+    const argument = argv[index]
+    if (valueOptions.has(argument)) {
+      const value = argv[index + 1]
+      if (!value || value.startsWith('--')) throw new StripeSmokeError(`${argument} requires a value`)
+      values[argument] = value
+      index += 1
+    } else if (flagOptions.has(argument)) {
+      flags.add(argument)
+    } else {
+      throw new StripeSmokeError(`Unknown option: ${argument}`)
+    }
+  }
+  if (flags.has('--help')) return { help: true }
+
+  const secretKey = required(environment.STRIPE_SECRET_KEY, 'STRIPE_SECRET_KEY')
+  const keyMatch = /^(?:sk|rk)_(test|live)_/.exec(secretKey)
+  if (!keyMatch) throw new StripeSmokeError('STRIPE_SECRET_KEY must be a Stripe secret or restricted key')
+  const mode = keyMatch[1]
+  const requireLive = flags.has('--require-live') || booleanValue(environment.FRAMEFLOW_STRIPE_REQUIRE_LIVE)
+  if (requireLive && mode !== 'live') throw new StripeSmokeError('--require-live requires a live-mode Stripe key')
+
+  const webhookSecret = required(environment.STRIPE_WEBHOOK_SECRET, 'STRIPE_WEBHOOK_SECRET')
+  if (!webhookSecret.startsWith('whsec_')) throw new StripeSmokeError('STRIPE_WEBHOOK_SECRET must use the Stripe whsec_ format')
+  const proPriceId = required(environment.STRIPE_PRO_PRICE_ID, 'STRIPE_PRO_PRICE_ID')
+  const studioPriceId = required(environment.STRIPE_STUDIO_PRICE_ID, 'STRIPE_STUDIO_PRICE_ID')
+  if (!proPriceId.startsWith('price_') || !studioPriceId.startsWith('price_')) {
+    throw new StripeSmokeError('Stripe plan IDs must use the price_ format')
+  }
+  if (proPriceId === studioPriceId) throw new StripeSmokeError('PRO and STUDIO must use different Stripe Price IDs')
+  const portalConfigurationId = environment.STRIPE_PORTAL_CONFIGURATION_ID?.trim() || null
+  if (portalConfigurationId && !portalConfigurationId.startsWith('bpc_')) {
+    throw new StripeSmokeError('STRIPE_PORTAL_CONFIGURATION_ID must use the bpc_ format')
+  }
+
+  const allowHttp = flags.has('--allow-http') || booleanValue(environment.FRAMEFLOW_STRIPE_ALLOW_HTTP)
+  const originValue = values['--app-origin']
+    ?? environment.FRAMEFLOW_STRIPE_APP_ORIGIN
+    ?? environment.API_PUBLIC_ORIGIN
+    ?? environment.WEB_ORIGIN?.split(',')[0]?.trim()
+  const timeoutSeconds = integerValue(values['--timeout-seconds'] ?? environment.FRAMEFLOW_STRIPE_SMOKE_TIMEOUT_SECONDS ?? '30', 'timeout-seconds', 10, 120)
+  return {
+    secretKey,
+    webhookSecretConfigured: true,
+    proPriceId,
+    studioPriceId,
+    portalConfigurationId,
+    appOrigin: resolveStripeAppOrigin(originValue, { allowHttp }),
+    mode,
+    requireLive,
+    timeoutMs: timeoutSeconds * 1000,
+    json: flags.has('--json'),
+  }
+}
+
+async function listStripeCollection(fetchPage, parameters = {}) {
+  const rows = []
+  let startingAfter
+  for (let page = 0; page < 20; page += 1) {
+    const response = await fetchPage({ ...parameters, limit: 100, ...(startingAfter ? { starting_after: startingAfter } : {}) })
+    if (!response || !Array.isArray(response.data)) throw new StripeSmokeError('Stripe returned an invalid list response')
+    rows.push(...response.data)
+    if (!response.has_more) return rows
+    startingAfter = response.data.at(-1)?.id
+    if (!startingAfter) throw new StripeSmokeError('Stripe pagination returned no continuation ID')
+  }
+  throw new StripeSmokeError('Stripe list exceeded the 2,000-resource verification limit')
+}
+
+function assertMode(resource, mode, label) {
+  if (typeof resource?.livemode !== 'boolean') throw new StripeSmokeError(`${label} did not report livemode`)
+  if (resource.livemode !== (mode === 'live')) throw new StripeSmokeError(`${label} does not match the Stripe key mode`)
+}
+
+async function verifyPrice(client, priceId, plan, mode) {
+  const price = await client.prices.retrieve(priceId, { expand: ['product'] })
+  if (price?.object !== 'price' || price.id !== priceId) throw new StripeSmokeError(`${plan} Stripe Price could not be retrieved`)
+  assertMode(price, mode, `${plan} Price`)
+  if (price.active !== true) throw new StripeSmokeError(`${plan} Stripe Price is not active`)
+  if (price.type !== 'recurring' || !price.recurring) throw new StripeSmokeError(`${plan} Stripe Price must be recurring`)
+  if (price.recurring.usage_type !== 'licensed') throw new StripeSmokeError(`${plan} Stripe Price must use licensed recurring usage`)
+  const product = price.product
+  if (!product || typeof product === 'string' || product.deleted || product.object !== 'product') {
+    throw new StripeSmokeError(`${plan} Stripe Product was not expanded or has been deleted`)
+  }
+  if (product.active !== true) throw new StripeSmokeError(`${plan} Stripe Product is not active`)
+  assertMode(product, mode, `${plan} Product`)
+  return {
+    plan,
+    priceId: price.id,
+    productId: product.id,
+    productName: product.name,
+    currency: price.currency,
+    unitAmount: price.unit_amount,
+    billingScheme: price.billing_scheme,
+    interval: price.recurring.interval,
+    intervalCount: price.recurring.interval_count,
+  }
+}
+
+async function verifyPortal(client, configurationId, mode) {
+  let configuration
+  if (configurationId) {
+    configuration = await client.billingPortal.configurations.retrieve(configurationId)
+  } else {
+    const configurations = await listStripeCollection(
+      (parameters) => client.billingPortal.configurations.list(parameters),
+      { active: true },
+    )
+    configuration = configurations.find((candidate) => candidate.is_default === true)
+  }
+  if (!configuration || configuration.object !== 'billing_portal.configuration') {
+    throw new StripeSmokeError('No active default Stripe Customer Portal configuration was found')
+  }
+  if (configuration.active !== true) throw new StripeSmokeError('Stripe Customer Portal configuration is not active')
+  assertMode(configuration, mode, 'Customer Portal configuration')
+  return {
+    id: configuration.id,
+    isDefault: configuration.is_default === true,
+    active: true,
+  }
+}
+
+async function verifyWebhook(client, appOrigin, mode) {
+  const expectedUrl = `${appOrigin}/api/v1/billing/webhooks/stripe`
+  const endpoints = await listStripeCollection((parameters) => client.webhookEndpoints.list(parameters))
+  const endpoint = endpoints.find((candidate) => candidate.url === expectedUrl && candidate.status === 'enabled')
+  if (!endpoint) throw new StripeSmokeError(`No enabled Stripe webhook endpoint matches ${expectedUrl}`)
+  assertMode(endpoint, mode, 'Webhook endpoint')
+  const enabledEvents = new Set(endpoint.enabled_events ?? [])
+  const missingEvents = enabledEvents.has('*')
+    ? []
+    : requiredStripeWebhookEvents.filter((event) => !enabledEvents.has(event))
+  if (missingEvents.length > 0) {
+    throw new StripeSmokeError(`Stripe webhook endpoint is missing events: ${missingEvents.join(', ')}`)
+  }
+  return {
+    id: endpoint.id,
+    url: endpoint.url,
+    status: endpoint.status,
+    requiredEvents: requiredStripeWebhookEvents,
+  }
+}
+
+export async function runStripeSmoke(options, dependencies = {}) {
+  const client = dependencies.client ?? new Stripe(options.secretKey, {
+    maxNetworkRetries: 1,
+    timeout: options.timeoutMs,
+  })
+  const account = await client.accounts.retrieve()
+  if (!account || account.object !== 'account' || typeof account.id !== 'string') {
+    throw new StripeSmokeError('Stripe account could not be retrieved')
+  }
+  if (options.requireLive && account.charges_enabled !== true) {
+    throw new StripeSmokeError('Stripe live charges are not enabled for this account')
+  }
+  if (options.requireLive && account.details_submitted !== true) {
+    throw new StripeSmokeError('Stripe account onboarding details are incomplete')
+  }
+
+  const [pro, studio, portal, webhook] = await Promise.all([
+    verifyPrice(client, options.proPriceId, 'PRO', options.mode),
+    verifyPrice(client, options.studioPriceId, 'STUDIO', options.mode),
+    verifyPortal(client, options.portalConfigurationId, options.mode),
+    verifyWebhook(client, options.appOrigin, options.mode),
+  ])
+  return {
+    status: 'passed',
+    mode: options.mode,
+    account: {
+      id: account.id,
+      country: account.country ?? null,
+      defaultCurrency: account.default_currency ?? null,
+      chargesEnabled: account.charges_enabled === true,
+      payoutsEnabled: account.payouts_enabled === true,
+      detailsSubmitted: account.details_submitted === true,
+    },
+    prices: { PRO: pro, STUDIO: studio },
+    portal,
+    webhook,
+    webhookSigningSecretConfigured: options.webhookSecretConfigured,
+    webhookSigningSecretMatchedToEndpoint: false,
+    sideEffectsCreated: false,
+  }
+}
+
+function safeError(error) {
+  const message = error instanceof Error ? error.message : String(error)
+  return message.replace(/(?:sk|rk)_(?:test|live)_[a-zA-Z0-9]+|whsec_[a-zA-Z0-9]+/g, '[redacted]').slice(0, 1000)
+}
+
+async function main() {
+  const options = parseStripeSmokeOptions()
+  if (options.help) {
+    process.stdout.write(usage())
+    return
+  }
+  const report = await runStripeSmoke(options)
+  if (options.json) {
+    process.stdout.write(`${JSON.stringify(report, null, 2)}\n`)
+    return
+  }
+  process.stdout.write(`Stripe read-only smoke passed in ${report.mode} mode.\nAccount: ${report.account.id}\nWebhook: ${report.webhook.url}\nNo Stripe resources were created. Run a signed webhook acceptance event before launch.\n`)
+}
+
+if (process.argv[1] && resolve(process.argv[1]) === scriptPath) {
+  main().catch((error) => {
+    process.stderr.write(`Stripe smoke failed: ${safeError(error)}\n`)
+    process.exitCode = 1
+  })
+}

scripts/verify-backup.sh

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+usage() {
+  echo "Usage: $0 <frameflow-backup-directory>" >&2
+  exit 2
+}
+
+fail() {
+  echo "Backup verification failed: $1" >&2
+  exit 1
+}
+
+[[ $# -eq 1 ]] || usage
+[[ -d "$1" ]] || fail "directory not found: $1"
+[[ ! -L "$1" ]] || fail "backup directory must not be a symbolic link"
+
+backup_dir="$(cd "$1" && pwd -P)"
+required_files=(postgres.dump redis-data.tar.gz minio-data.tar.gz manifest.txt SHA256SUMS)
+
+for filename in "${required_files[@]}"; do
+  path="${backup_dir}/${filename}"
+  [[ -f "${path}" ]] || fail "missing ${filename}"
+  [[ ! -L "${path}" ]] || fail "${filename} must not be a symbolic link"
+  [[ -s "${path}" ]] || fail "${filename} is empty"
+done
+
+created_at="$(sed -n 's/^created_at=//p' "${backup_dir}/manifest.txt")"
+compose_project="$(sed -n 's/^compose_project=//p' "${backup_dir}/manifest.txt")"
+[[ "${created_at}" =~ ^[0-9]{8}T[0-9]{6}Z$ ]] || fail "manifest has an invalid created_at value"
+[[ "${compose_project}" =~ ^[a-zA-Z0-9][a-zA-Z0-9_.-]*$ ]] || fail "manifest has an invalid compose_project value"
+
+postgres_seen=0
+redis_seen=0
+minio_seen=0
+checksum_rows=0
+while read -r digest filename extra; do
+  [[ -z "${extra:-}" ]] || fail "SHA256SUMS contains an invalid row"
+  [[ "${digest}" =~ ^[[:xdigit:]]{64}$ ]] || fail "SHA256SUMS contains an invalid digest"
+  filename="${filename#\*}"
+  case "${filename}" in
+    postgres.dump) postgres_seen=$((postgres_seen + 1)) ;;
+    redis-data.tar.gz) redis_seen=$((redis_seen + 1)) ;;
+    minio-data.tar.gz) minio_seen=$((minio_seen + 1)) ;;
+    *) fail "SHA256SUMS references an unexpected file: ${filename}" ;;
+  esac
+  checksum_rows=$((checksum_rows + 1))
+done < "${backup_dir}/SHA256SUMS"
+
+[[ ${checksum_rows} -eq 3 && ${postgres_seen} -eq 1 && ${redis_seen} -eq 1 && ${minio_seen} -eq 1 ]] \
+  || fail "SHA256SUMS must reference each data artifact exactly once"
+
+if command -v sha256sum >/dev/null 2>&1; then
+  (cd "${backup_dir}" && sha256sum -c SHA256SUMS) >/dev/null \
+    || fail "artifact checksum mismatch"
+else
+  (cd "${backup_dir}" && shasum -a 256 -c SHA256SUMS) >/dev/null \
+    || fail "artifact checksum mismatch"
+fi
+
+postgres_magic="$(LC_ALL=C head -c 5 "${backup_dir}/postgres.dump")"
+[[ "${postgres_magic}" == "PGDMP" ]] || fail "PostgreSQL dump does not use the required custom format"
+
+verify_archive_paths() {
+  local archive_name="$1"
+  local listing_file
+  local entry
+  local normalized
+  listing_file="$(mktemp "${TMPDIR:-/tmp}/frameflow-archive-list.XXXXXX")"
+  if ! tar -tzf "${backup_dir}/${archive_name}" > "${listing_file}"; then
+    rm -f "${listing_file}"
+    fail "${archive_name} is not a readable gzip tar archive"
+  fi
+  if [[ ! -s "${listing_file}" ]]; then
+    rm -f "${listing_file}"
+    fail "${archive_name} contains no entries"
+  fi
+  while IFS= read -r entry; do
+    [[ "${entry}" != /* ]] || {
+      rm -f "${listing_file}"
+      fail "${archive_name} contains an absolute path"
+    }
+    normalized="${entry#./}"
+    case "/${normalized}/" in
+      *"/../"*)
+        rm -f "${listing_file}"
+        fail "${archive_name} contains a parent-directory path"
+        ;;
+    esac
+  done < "${listing_file}"
+  rm -f "${listing_file}"
+}
+
+verify_archive_paths redis-data.tar.gz
+verify_archive_paths minio-data.tar.gz
+
+echo "Backup verification passed: ${backup_dir}"
+echo "Source Compose project: ${compose_project}; created at: ${created_at}"