Release v1.0.0
Some checks failed
CI / Migrations, tests, build, and audit (push) Failing after 10m7s
CI / Production gate and container images (push) Successful in 20m35s

This commit is contained in:
2026-07-31 14:21:43 +08:00
commit 36f466ba11
187 changed files with 98907 additions and 0 deletions

927
scripts/smoke-test.mjs Executable file
View File

@@ -0,0 +1,927 @@
#!/usr/bin/env node
import { createHash, randomUUID } from 'node:crypto'
import { readFile } from 'node:fs/promises'
import { basename, extname, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
const scriptPath = fileURLToPath(import.meta.url)
const defaultImagePath = fileURLToPath(new URL('../public/assets/scene-rooftop.jpg', import.meta.url))
const terminalJobStatuses = new Set(['SUCCEEDED', 'FAILED', 'CANCELLED'])
const billableAiAcknowledgement = 'I_ACCEPT_BILLABLE_AI_CHARGES'
export const requiredLiveAiWorkflowIds = ['script', 'storyboard', 'image-openai', 'image-replicate', 'tts', 'video', 'lipsync']
export class SmokeTestError extends Error {
constructor(message, options = {}) {
super(message, options)
this.name = 'SmokeTestError'
this.report = options.report
}
}
function required(value, name) {
if (typeof value !== 'string' || value.trim() === '') throw new SmokeTestError(`${name} is required`)
return value.trim()
}
function booleanValue(value) {
return ['1', 'true', 'yes', 'on'].includes(String(value ?? '').trim().toLowerCase())
}
function integerValue(value, name, minimum, maximum) {
const parsed = Number(value)
if (!Number.isInteger(parsed) || parsed < minimum || parsed > maximum) {
throw new SmokeTestError(`${name} must be an integer from ${minimum} to ${maximum}`)
}
return parsed
}
function jsonObjectValue(value, name) {
if (value === undefined || value === null || String(value).trim() === '') return {}
let parsed
try {
parsed = JSON.parse(String(value))
} catch {
throw new SmokeTestError(`${name} must be valid JSON`)
}
if (!parsed || Array.isArray(parsed) || typeof parsed !== 'object') {
throw new SmokeTestError(`${name} must be a JSON object`)
}
return parsed
}
export function resolveEndpoints(value, options = {}) {
const input = required(value, 'FRAMEFLOW_SMOKE_BASE_URL')
let url
try {
url = new URL(input)
} catch {
throw new SmokeTestError('FRAMEFLOW_SMOKE_BASE_URL must be a valid URL')
}
if (url.username || url.password || url.search || url.hash) {
throw new SmokeTestError('Smoke-test base URL cannot contain credentials, a query, or a fragment')
}
if (!['http:', 'https:'].includes(url.protocol)) throw new SmokeTestError('Smoke-test base URL must use HTTP or HTTPS')
const localHostnames = new Set(['localhost', '127.0.0.1', '::1', '[::1]'])
if (url.protocol === 'http:' && (!options.allowHttp || !localHostnames.has(url.hostname))) {
throw new SmokeTestError('HTTPS is required; HTTP can only be enabled explicitly for localhost')
}
const path = url.pathname.replace(/\/+$/, '')
if (path && path !== '/api/v1') {
throw new SmokeTestError('Smoke-test base URL path must be empty or /api/v1')
}
const origin = url.origin
return { origin, apiBase: `${origin}/api/v1` }
}
export function parseSmokeOptions(argv = process.argv.slice(2), environment = process.env) {
const values = {}
const flags = new Set()
const valueOptions = new Set(['--base-url', '--email', '--workspace-id', '--image', '--timeout-seconds', '--ai-video-seconds'])
const flagOptions = new Set(['--allow-http', '--keep-project', '--include-ai', '--json', '--help'])
for (let index = 0; index < argv.length; index += 1) {
const argument = argv[index]
if (valueOptions.has(argument)) {
const value = argv[index + 1]
if (!value || value.startsWith('--')) throw new SmokeTestError(`${argument} requires a value`)
values[argument] = value
index += 1
} else if (flagOptions.has(argument)) {
flags.add(argument)
} else {
throw new SmokeTestError(`Unknown option: ${argument}`)
}
}
if (flags.has('--help')) return { help: true }
const allowHttp = flags.has('--allow-http') || booleanValue(environment.FRAMEFLOW_SMOKE_ALLOW_HTTP)
const endpoints = resolveEndpoints(values['--base-url'] ?? environment.FRAMEFLOW_SMOKE_BASE_URL, { allowHttp })
const email = required(values['--email'] ?? environment.FRAMEFLOW_SMOKE_EMAIL, 'FRAMEFLOW_SMOKE_EMAIL').toLowerCase()
const password = required(environment.FRAMEFLOW_SMOKE_PASSWORD, 'FRAMEFLOW_SMOKE_PASSWORD')
const timeoutSeconds = integerValue(values['--timeout-seconds'] ?? environment.FRAMEFLOW_SMOKE_TIMEOUT_SECONDS ?? '180', 'timeout-seconds', 30, 900)
const includeAi = flags.has('--include-ai')
if (includeAi && environment.FRAMEFLOW_SMOKE_BILLABLE_AI_ACK !== billableAiAcknowledgement) {
throw new SmokeTestError(`--include-ai requires FRAMEFLOW_SMOKE_BILLABLE_AI_ACK=${billableAiAcknowledgement}`)
}
return {
...endpoints,
email,
password,
workspaceId: values['--workspace-id'] ?? environment.FRAMEFLOW_SMOKE_WORKSPACE_ID ?? null,
imagePath: resolve(values['--image'] ?? environment.FRAMEFLOW_SMOKE_IMAGE ?? defaultImagePath),
timeoutMs: timeoutSeconds * 1000,
requestTimeoutMs: Math.min(60_000, timeoutSeconds * 1000),
keepProject: flags.has('--keep-project') || booleanValue(environment.FRAMEFLOW_SMOKE_KEEP_PROJECT),
includeAi,
aiVideoSeconds: integerValue(values['--ai-video-seconds'] ?? environment.FRAMEFLOW_SMOKE_AI_VIDEO_SECONDS ?? '5', 'ai-video-seconds', 1, 30),
aiVideoParams: jsonObjectValue(environment.FRAMEFLOW_SMOKE_VIDEO_PARAMS_JSON, 'FRAMEFLOW_SMOKE_VIDEO_PARAMS_JSON'),
aiLipsyncParams: jsonObjectValue(environment.FRAMEFLOW_SMOKE_LIPSYNC_PARAMS_JSON, 'FRAMEFLOW_SMOKE_LIPSYNC_PARAMS_JSON'),
json: flags.has('--json'),
}
}
function safeError(error) {
const message = error instanceof Error ? error.message : String(error)
return message.replace(/([?&](?:X-Amz-[^=\s]+|token|signature)=)[^&\s]+/gi, '$1[redacted]').slice(0, 2000)
}
function assertCondition(condition, message) {
if (!condition) throw new SmokeTestError(message)
}
function sha256(value) {
return createHash('sha256').update(value).digest('hex')
}
function imageContentType(path) {
const extension = extname(path).toLowerCase()
if (extension === '.jpg' || extension === '.jpeg') return 'image/jpeg'
if (extension === '.png') return 'image/png'
if (extension === '.webp') return 'image/webp'
throw new SmokeTestError('Smoke-test image must be JPEG, PNG, or WebP')
}
function responseBody(response) {
if (response.status === 204) return Promise.resolve(null)
const contentType = response.headers.get('content-type') ?? ''
if (contentType.includes('application/json')) return response.json()
return response.text()
}
async function fetchWithTimeout(fetchImpl, url, options, timeoutMs) {
const controller = new AbortController()
const timer = setTimeout(() => controller.abort(new Error(`Request timed out after ${timeoutMs}ms`)), timeoutMs)
try {
return await fetchImpl(url, { ...options, signal: controller.signal })
} finally {
clearTimeout(timer)
}
}
function createApiClient(options, fetchImpl) {
let accessToken = null
const cookies = new Map()
function captureCookies(headers) {
const values = typeof headers.getSetCookie === 'function'
? headers.getSetCookie()
: headers.get('set-cookie') ? [headers.get('set-cookie')] : []
for (const value of values) {
const pair = value?.split(';', 1)[0]
const separator = pair?.indexOf('=') ?? -1
if (!pair || separator < 1) continue
const name = pair.slice(0, separator).trim()
const cookieValue = pair.slice(separator + 1).trim()
if (!cookieValue || /(?:^|;)\s*max-age=0(?:;|$)/i.test(value)) cookies.delete(name)
else cookies.set(name, cookieValue)
}
}
async function raw(path, request = {}) {
const headers = new Headers(request.headers)
const url = path.startsWith('http://') || path.startsWith('https://') ? path : `${options.apiBase}${path}`
const authenticatedApiRequest = request.auth !== false && (url === options.apiBase || url.startsWith(`${options.apiBase}/`))
if (accessToken && authenticatedApiRequest) headers.set('authorization', `Bearer ${accessToken}`)
if (authenticatedApiRequest && cookies.size > 0) {
headers.set('cookie', [...cookies].map(([name, value]) => `${name}=${value}`).join('; '))
}
let body = request.body
if (request.json !== undefined) {
headers.set('content-type', 'application/json')
body = JSON.stringify(request.json)
}
const response = await fetchWithTimeout(fetchImpl, url, {
method: request.method ?? 'GET',
headers,
body,
redirect: request.redirect ?? 'follow',
}, request.timeoutMs ?? options.requestTimeoutMs)
captureCookies(response.headers)
return response
}
async function request(path, input = {}) {
const response = await raw(path, input)
const body = await responseBody(response)
if (!response.ok) {
const message = body?.error?.message ?? (typeof body === 'string' && body) ?? `HTTP ${response.status}`
throw new SmokeTestError(`${input.method ?? 'GET'} ${path} failed (${response.status}): ${message}`)
}
return body
}
return {
raw,
request,
setAccessToken(value) { accessToken = value },
clearSession() {
accessToken = null
cookies.clear()
},
}
}
export async function waitForJob(getJob, jobId, options = {}) {
const timeoutMs = options.timeoutMs ?? 180_000
const pollIntervalMs = options.pollIntervalMs ?? 500
const sleep = options.sleep ?? ((milliseconds) => new Promise((resolvePromise) => setTimeout(resolvePromise, milliseconds)))
const now = options.now ?? Date.now
const startedAt = now()
let lastStatus = null
while (now() - startedAt <= timeoutMs) {
const job = await getJob(jobId)
lastStatus = job?.status ?? null
if (lastStatus === 'SUCCEEDED') return job
if (terminalJobStatuses.has(lastStatus)) {
throw new SmokeTestError(`Job ${jobId} ended in ${lastStatus}: ${job.errorMessage ?? job.errorCode ?? 'unknown error'}`)
}
await sleep(pollIntervalMs)
}
throw new SmokeTestError(`Job ${jobId} did not finish within ${timeoutMs}ms (last status: ${lastStatus ?? 'unknown'})`)
}
export async function waitForNotifications(getNotifications, resourceIds, options = {}) {
const timeoutMs = options.timeoutMs ?? 10_000
const pollIntervalMs = options.pollIntervalMs ?? 250
const sleep = options.sleep ?? ((milliseconds) => new Promise((resolvePromise) => setTimeout(resolvePromise, milliseconds)))
const now = options.now ?? Date.now
const expected = new Set(resourceIds)
const startedAt = now()
let lastItems = []
while (now() - startedAt <= timeoutMs) {
const page = await getNotifications()
lastItems = Array.isArray(page?.items) ? page.items : []
const matched = lastItems.filter((notification) => expected.has(notification.resourceId))
if (new Set(matched.map((notification) => notification.resourceId)).size === expected.size) return matched
await sleep(pollIntervalMs)
}
const matchedIds = new Set(lastItems.map((notification) => notification.resourceId))
const missing = [...expected].filter((resourceId) => !matchedIds.has(resourceId))
throw new SmokeTestError(`Notifications did not arrive within ${timeoutMs}ms (missing resources: ${missing.join(', ')})`)
}
function selectWorkspace(workspaces, requestedId) {
assertCondition(Array.isArray(workspaces) && workspaces.length > 0, 'Smoke account has no workspace membership')
const workspace = requestedId
? workspaces.find((candidate) => candidate.id === requestedId)
: workspaces.length === 1 ? workspaces[0] : null
if (!workspace) {
throw new SmokeTestError(requestedId
? `Smoke account is not a member of workspace ${requestedId}`
: 'Smoke account belongs to multiple workspaces; set FRAMEFLOW_SMOKE_WORKSPACE_ID')
}
assertCondition(['OWNER', 'ADMIN'].includes(workspace.role), 'Full smoke test requires an OWNER or ADMIN workspace role')
return workspace
}
async function downloadBytes(fetchImpl, url, timeoutMs, expectedType) {
const response = await fetchWithTimeout(fetchImpl, url, { redirect: 'follow' }, timeoutMs)
assertCondition(response.ok, `Media download failed with HTTP ${response.status}`)
const contentType = response.headers.get('content-type') ?? ''
if (expectedType) assertCondition(contentType.toLowerCase().includes(expectedType), `Unexpected media content type: ${contentType || 'missing'}`)
return Buffer.from(await response.arrayBuffer())
}
function readinessSummary(readiness) {
return {
infrastructure: readiness.infrastructure,
ai: {
status: readiness.ai?.status ?? 'UNKNOWN',
verificationMode: readiness.ai?.verificationMode ?? 'UNKNOWN',
summary: readiness.ai?.summary ?? null,
workflows: (readiness.ai?.workflows ?? []).map((workflow) => ({
id: workflow.id,
provider: workflow.provider,
model: workflow.model,
status: workflow.status,
})),
providers: (readiness.ai?.providers ?? []).map((provider) => ({
id: provider.id,
status: provider.status,
verification: provider.verification,
configuredCapabilities: provider.configuredCapabilities,
totalCapabilities: provider.totalCapabilities,
})),
},
}
}
export function assertLiveAiReady(readiness, minimumCredits = 700, billing = null) {
const capabilityRows = [
...(readiness?.ai?.providers ?? []).flatMap((provider) => provider.capabilities ?? []),
...(readiness?.ai?.workflows ?? []),
]
const workflows = new Map(capabilityRows.map((workflow) => [workflow.id, workflow]))
const missing = requiredLiveAiWorkflowIds.filter((id) => workflows.get(id)?.status !== 'CONFIGURED')
if (missing.length > 0) {
const details = missing.map((id) => {
const workflow = workflows.get(id)
const requirements = workflow?.missingRequirements?.join(', ')
return requirements ? `${id} (${requirements})` : id
})
throw new SmokeTestError(`Live AI smoke test requires configured workflows: ${details.join('; ')}`)
}
if (billing && Number(billing.remainingCredits) < minimumCredits) {
throw new SmokeTestError(`Live AI smoke test requires at least ${minimumCredits} remaining credits`)
}
return requiredLiveAiWorkflowIds.map((id) => {
const workflow = workflows.get(id)
return { id, provider: workflow.provider, model: workflow.model }
})
}
export async function runProductionSmoke(options, dependencies = {}) {
const fetchImpl = dependencies.fetch ?? fetch
const readFileImpl = dependencies.readFile ?? readFile
const sleep = dependencies.sleep ?? ((milliseconds) => new Promise((resolvePromise) => setTimeout(resolvePromise, milliseconds)))
const now = dependencies.now ?? Date.now
const log = dependencies.log ?? ((message) => console.error(message))
const api = createApiClient(options, fetchImpl)
const startedAt = now()
const runId = randomUUID()
const marker = `FRAMEFLOW_SMOKE_${runId}`
const report = {
status: 'running',
runId,
baseUrl: options.origin,
startedAt: new Date(startedAt).toISOString(),
workspaceId: null,
projectId: null,
projectArchived: false,
checks: [],
services: null,
billing: null,
aiVerification: {
mode: options.includeAi ? 'LIVE_PROVIDER_CALLS' : 'CONFIGURATION_ONLY',
jobs: [],
artifacts: {},
},
artifacts: {},
}
let projectId = null
let loggedIn = false
let smokeSessionId = null
let primaryError = null
async function step(id, label, action, summarize) {
const stepStartedAt = now()
log(`[...] ${label}`)
try {
const value = await action()
const check = { id, label, status: 'passed', durationMs: Math.max(0, now() - stepStartedAt) }
const details = summarize?.(value)
if (details !== undefined) check.details = details
report.checks.push(check)
log(`[ ok] ${label} (${check.durationMs}ms)`)
return value
} catch (error) {
const check = { id, label, status: 'failed', durationMs: Math.max(0, now() - stepStartedAt), error: safeError(error) }
report.checks.push(check)
log(`[fail] ${label}: ${check.error}`)
throw error
}
}
async function runAiJob(projectIdForJob, episodeIdForJob, type, input) {
const queued = await api.request(`/projects/${projectIdForJob}/jobs`, {
method: 'POST',
json: {
type,
episodeId: episodeIdForJob,
idempotencyKey: `smoke-ai-${runId}-${report.aiVerification.jobs.length}-${type.toLowerCase()}`,
input,
maxAttempts: 1,
},
})
const job = await waitForJob((jobId) => api.request(`/jobs/${jobId}`), queued.job.id, {
timeoutMs: options.timeoutMs,
sleep,
})
assertCondition(['openai', 'replicate'].includes(job.provider), `${type} did not record a live AI provider`)
assertCondition(job.providerModel, `${type} did not record a provider model`)
report.aiVerification.jobs.push({ id: job.id, type, provider: job.provider, model: job.providerModel })
return job
}
try {
await step('liveness', 'API liveness', async () => {
const response = await fetchWithTimeout(fetchImpl, `${options.origin}/health/live`, {}, options.requestTimeoutMs)
const body = await responseBody(response)
assertCondition(response.ok && body?.status === 'ok', `Liveness failed with HTTP ${response.status}`)
return body
})
await step('readiness', 'API dependency readiness', async () => {
const response = await fetchWithTimeout(fetchImpl, `${options.origin}/health/ready`, {}, options.requestTimeoutMs)
const body = await responseBody(response)
assertCondition(response.ok && body?.status === 'ready', `Readiness failed with HTTP ${response.status}`)
const unavailable = Object.entries(body.dependencies ?? {}).filter(([, status]) => status !== 'ok')
assertCondition(unavailable.length === 0, `Unavailable dependencies: ${unavailable.map(([name]) => name).join(', ')}`)
return body
}, (body) => body.dependencies)
const session = await step('login', 'Dedicated smoke-account login', async () => {
const value = await api.request('/auth/login', {
method: 'POST',
auth: false,
json: { email: options.email, password: options.password },
})
assertCondition(value?.accessToken && value?.user?.id, 'Login response is missing the authenticated session')
api.setAccessToken(value.accessToken)
loggedIn = true
const sessions = await api.request('/auth/sessions')
const currentSession = sessions.find((candidate) => candidate.current)
assertCondition(currentSession?.id, 'Login refresh session cookie was not retained by the smoke runner')
smokeSessionId = currentSession.id
return value
}, (value) => ({ userId: value.user.id, sessionId: smokeSessionId, workspaceCount: value.workspaces.length }))
const workspace = selectWorkspace(session.workspaces, options.workspaceId)
report.workspaceId = workspace.id
const serviceReadiness = await step('service-readiness', 'Worker, storage, FFmpeg, and AI configuration', async () => {
const value = await api.request(`/workspaces/${workspace.id}/service-readiness`)
assertCondition(value?.infrastructure?.status === 'AVAILABLE', 'Authenticated service readiness is degraded')
assertCondition(value.infrastructure.available === value.infrastructure.total, 'Not every production service is available')
return value
}, (value) => ({
infrastructure: value.infrastructure.services.map((service) => ({ id: service.id, status: service.status, latencyMs: service.latencyMs })),
ai: value.ai.summary,
}))
report.services = readinessSummary(serviceReadiness)
const billingReadModel = await step('billing', 'Billing and usage read model', async () => {
const value = await api.request(`/workspaces/${workspace.id}/billing/summary`)
assertCondition(value?.plan && value?.provider?.status, 'Billing summary is incomplete')
return value
}, (value) => ({ plan: value.plan, provider: value.provider.status, webhookConfigured: value.provider.webhookConfigured, remainingCredits: value.remainingCredits }))
report.billing = {
plan: billingReadModel.plan,
provider: billingReadModel.provider.status,
webhookConfigured: billingReadModel.provider.webhookConfigured,
remainingCredits: billingReadModel.remainingCredits,
}
if (options.includeAi) {
await step('ai-preflight', 'Billable AI authorization, configuration, and credit preflight', async () => {
const workflows = assertLiveAiReady(serviceReadiness, 700, billingReadModel)
return { acknowledgement: 'accepted', minimumCredits: 700, workflows }
}, (value) => value)
}
const project = await step('project', 'Project persistence', async () => api.request(`/workspaces/${workspace.id}/projects`, {
method: 'POST',
json: {
name: `[SMOKE] ${new Date(startedAt).toISOString()} ${runId.slice(0, 8)}`,
logline: 'Automated production smoke test. Safe to archive after completion.',
genre: 'SMOKE_TEST',
visualBible: { smokeTest: { runId, createdAt: new Date(startedAt).toISOString() } },
},
}), (value) => ({ projectId: value.id }))
projectId = project.id
report.projectId = projectId
const episode = await step('script', 'Episode and immutable script persistence', async () => {
const createdEpisode = await api.request(`/projects/${projectId}/episodes`, {
method: 'POST',
json: { episodeNumber: 1, title: 'Production smoke episode', synopsis: marker, targetDurationSeconds: 15 },
})
const script = await api.request(`/episodes/${createdEpisode.id}/scripts`, {
method: 'POST',
json: {
title: 'Production smoke script',
content: { smokeTest: true, runId, scenes: [{ number: 1, text: marker }] },
plainText: `SCENE 1 - SMOKE TEST\n${marker}`,
changeSummary: 'Production smoke test',
},
})
const persisted = await api.request(`/episodes/${createdEpisode.id}`)
assertCondition(persisted.currentScript?.id === script.id, 'Current script version was not persisted')
assertCondition(persisted.currentScript?.plainText?.includes(marker), 'Persisted script marker is missing')
return { episode: createdEpisode, script }
}, (value) => ({ episodeId: value.episode.id, scriptVersion: value.script.version }))
const sourceImage = await readFileImpl(options.imagePath)
const sourceHash = sha256(sourceImage)
const contentType = imageContentType(options.imagePath)
const imageAssetVersion = await step('upload', 'Object upload and signed download integrity', async () => {
const form = new FormData()
form.append('file', new Blob([sourceImage], { type: contentType }), basename(options.imagePath))
const upload = await api.request(`/projects/${projectId}/uploads`, { method: 'POST', body: form })
assertCondition(upload.byteSize === sourceImage.length, 'Uploaded object byte size does not match the source')
const asset = await api.request(`/projects/${projectId}/assets`, {
method: 'POST',
json: { type: 'SHOT', name: `Smoke source ${runId.slice(0, 8)}`, description: marker, tags: ['smoke-test'], metadata: { smokeRunId: runId } },
})
const version = await api.request(`/assets/${asset.id}/versions`, {
method: 'POST',
json: { storageKey: upload.key, mimeType: upload.contentType, byteSize: upload.byteSize, prompt: marker, provider: 'smoke-test', providerModel: 'uploaded-source-v1' },
})
const signed = await api.request(`/assets/${asset.id}/versions/${version.id}/url`)
const downloaded = await downloadBytes(fetchImpl, signed.url, options.requestTimeoutMs, contentType)
assertCondition(sha256(downloaded) === sourceHash, 'Signed object download does not match the uploaded image')
return { asset, version, upload }
}, (value) => ({ assetId: value.asset.id, assetVersionId: value.version.id, byteSize: value.upload.byteSize, sha256: sourceHash }))
const shot = await step('shot', 'Shot and immutable media-version persistence', async () => {
const created = await api.request(`/episodes/${episode.episode.id}/shots`, {
method: 'POST',
json: { shotNumber: 1, sceneNumber: 1, title: 'Smoke render shot', description: marker, shotType: '全景', cameraMotion: '静止', durationMs: 2000, characterAssetIds: [], sortOrder: 100 },
})
const version = await api.request(`/shots/${created.id}/versions`, {
method: 'POST',
json: { imageAssetVersionId: imageAssetVersion.version.id, subtitleText: marker, generationParams: { smokeRunId: runId } },
})
const persisted = await api.request(`/shots/${created.id}`)
assertCondition(persisted.versions?.[0]?.id === version.id, 'Current shot version was not persisted')
return { shot: created, version }
}, (value) => ({ shotId: value.shot.id, shotVersion: value.version.version }))
await step('queue', 'Redis/BullMQ worker execution', async () => {
const queued = await api.request(`/projects/${projectId}/jobs`, {
method: 'POST',
json: { type: 'PIPELINE_HEALTHCHECK', episodeId: episode.episode.id, idempotencyKey: `smoke-health-${runId}`, input: { marker }, maxAttempts: 1 },
})
const job = await waitForJob((jobId) => api.request(`/jobs/${jobId}`), queued.job.id, { timeoutMs: options.timeoutMs, sleep })
assertCondition(job.output?.ok === true && job.output?.echo?.marker === marker, 'Worker healthcheck output is invalid')
return job
}, (job) => ({ jobId: job.id, provider: job.provider, providerModel: job.providerModel }))
const renderResult = await step('render', 'FFmpeg render and immutable source manifest', async () => {
const queued = await api.request(`/episodes/${episode.episode.id}/renders`, {
method: 'POST',
json: {
idempotencyKey: `smoke-render-${runId}`,
settings: { width: 640, height: 360, fps: 24, quality: 'draft', burnSubtitles: false, normalizeAudio: false, musicVolume: 0, subtitleStyle: { fontSize: 20, textColor: '#FFFFFF', outlineColor: '#000000', outlineWidth: 2, bottomMargin: 24, bold: true } },
},
})
const job = await waitForJob((jobId) => api.request(`/jobs/${jobId}`), queued.job.id, { timeoutMs: options.timeoutMs, sleep })
const render = await api.request(`/renders/${queued.render.id}`)
assertCondition(render.status === 'REVIEW', `Render ended in unexpected status ${render.status}`)
assertCondition(render.sourceShotCount === 1, 'Render source manifest does not contain exactly one shot')
assertCondition(render.subtitleFormats?.includes('srt') && render.subtitleFormats?.includes('vtt'), 'Render subtitle sidecars are missing')
assertCondition(render.downloadUrl, 'Render download URL is missing')
const video = await downloadBytes(fetchImpl, render.downloadUrl, options.requestTimeoutMs, 'video/mp4')
assertCondition(video.length > 1000, 'Rendered video is unexpectedly small')
return { render, job, videoHash: sha256(video), videoBytes: video.length }
}, (value) => ({ renderId: value.render.id, jobId: value.job.id, byteSize: value.videoBytes, subtitleFormats: value.render.subtitleFormats, sourceShotCount: value.render.sourceShotCount }))
report.artifacts.renderId = renderResult.render.id
report.artifacts.videoSha256 = renderResult.videoHash
await step('subtitles', 'SRT and VTT sidecar downloads', async () => {
const results = {}
for (const format of ['srt', 'vtt']) {
const subtitle = await api.request(`/renders/${renderResult.render.id}/subtitles/${format}`)
const bytes = await downloadBytes(fetchImpl, subtitle.downloadUrl, options.requestTimeoutMs)
const text = bytes.toString('utf8')
assertCondition(text.includes(marker), `${format.toUpperCase()} subtitle does not contain the smoke marker`)
results[format] = { bytes: bytes.length, sha256: sha256(bytes) }
}
return results
}, (value) => value)
await step('review', 'Render review and approval', async () => {
const review = await api.request(`/workspaces/${workspace.id}/reviews`, {
method: 'POST',
json: { targetType: 'RENDER', targetId: renderResult.render.id, assignedToId: session.user.id, summary: `Automated approval for ${marker}` },
})
const decision = await api.request(`/reviews/${review.id}/decision`, {
method: 'POST',
json: { status: 'APPROVED', summary: 'Production smoke render passed automated checks.' },
})
const approved = await api.request(`/renders/${renderResult.render.id}`)
assertCondition(decision.status === 'APPROVED' && approved.status === 'APPROVED', 'Render approval was not persisted')
return { review, approved }
}, (value) => ({ reviewId: value.review.id, renderStatus: value.approved.status }))
const deliveryResult = await step('delivery', 'Approved download delivery through the worker', async () => {
const queued = await api.request(`/renders/${renderResult.render.id}/deliveries`, {
method: 'POST',
json: { platform: 'DOWNLOAD', destination: { smokeRunId: runId } },
})
const job = await waitForJob((jobId) => api.request(`/jobs/${jobId}`), queued.job.id, { timeoutMs: options.timeoutMs, sleep })
const delivery = await api.request(`/deliveries/${queued.delivery.id}`)
assertCondition(delivery.status === 'PUBLISHED' && delivery.downloadUrl, 'Download delivery was not published')
const redirect = await api.raw(`/deliveries/${delivery.id}/download`, { redirect: 'manual' })
assertCondition([301, 302, 303, 307, 308].includes(redirect.status), `Delivery endpoint returned HTTP ${redirect.status} instead of a redirect`)
const location = redirect.headers.get('location')
assertCondition(location, 'Delivery redirect is missing its signed location')
const video = await downloadBytes(fetchImpl, location, options.requestTimeoutMs, 'video/mp4')
assertCondition(sha256(video) === renderResult.videoHash, 'Delivered video differs from the approved render')
return { delivery, job, bytes: video.length }
}, (value) => ({ deliveryId: value.delivery.id, jobId: value.job.id, status: value.delivery.status, byteSize: value.bytes }))
report.artifacts.deliveryId = deliveryResult.delivery.id
await step('notifications', 'Persistent job notifications and read state', async () => {
const expectedJobIds = [renderResult.job.id, deliveryResult.job.id]
const rows = await waitForNotifications(
() => api.request(`/workspaces/${workspace.id}/notifications?limit=100`),
expectedJobIds,
{ timeoutMs: Math.min(options.timeoutMs, 10_000), sleep, now },
)
for (const jobId of expectedJobIds) {
const notification = rows.find((candidate) => candidate.resourceId === jobId)
assertCondition(notification?.kind === 'JOB_SUCCEEDED' && notification.tone === 'SUCCESS', `Successful job notification is invalid for ${jobId}`)
const expectedPage = jobId === deliveryResult.job.id ? 'publish' : 'jobs'
assertCondition(notification.page === expectedPage, `Job notification points to ${notification.page} instead of ${expectedPage}`)
const read = await api.request(`/notifications/${notification.id}`, { method: 'PATCH', json: { read: true } })
assertCondition(read.readAt, `Notification ${notification.id} did not persist its read state`)
}
return rows
}, (rows) => ({ notifications: rows.length, resources: rows.map((notification) => notification.resourceId) }))
await step('audit', 'Audit-log persistence', async () => {
const page = await api.request(`/workspaces/${workspace.id}/audit-logs?limit=100&actorId=${session.user.id}`)
const actions = new Set((page.items ?? []).map((item) => item.action))
for (const action of ['project.create', 'upload.create', 'render.create', 'review.decide', 'delivery.create']) {
assertCondition(actions.has(action), `Audit log is missing ${action}`)
}
return { actions: actions.size }
}, (value) => value)
if (options.includeAi) {
const aiSetup = await step('ai-setup', 'Live AI validation workspace data', async () => {
const aiEpisode = await api.request(`/projects/${projectId}/episodes`, {
method: 'POST',
json: {
episodeNumber: 2,
title: 'Live provider validation',
synopsis: `A single character says a short line in a controlled studio. ${marker}`,
targetDurationSeconds: 15,
},
})
const character = await api.request(`/projects/${projectId}/assets`, {
method: 'POST',
json: {
type: 'CHARACTER',
name: `Provider test character ${runId.slice(0, 8)}`,
description: 'Adult presenter, centered portrait, neutral studio lighting, production-safe wardrobe.',
tags: ['smoke-test', 'provider-validation'],
metadata: { smokeRunId: runId },
},
})
return { episode: aiEpisode, character }
}, (value) => ({ episodeId: value.episode.id, characterAssetId: value.character.id }))
const openAiImage = await step('ai-image-openai', 'OpenAI character image generation', async () => {
const job = await runAiJob(projectId, aiSetup.episode.id, 'ASSET_GENERATE', {
assetId: aiSetup.character.id,
prompt: `Clean comic-drama character reference sheet. Adult presenter, centered bust portrait, neutral gray studio, consistent facial landmarks. ${marker}`,
negativePrompt: 'text, watermark, extra limbs, child',
width: 1024,
height: 1024,
provider: 'openai',
referenceAssetVersionIds: [],
params: { consistencyStrength: 82 },
})
const asset = await api.request(`/assets/${aiSetup.character.id}`)
const version = asset.versions?.find((candidate) => candidate.id === job.output?.assetVersionId)
assertCondition(version?.storageKey && version.provider === 'openai', 'OpenAI image version was not persisted as current asset media')
const signed = await api.request(`/assets/${asset.id}/versions/${version.id}/url`)
const bytes = await downloadBytes(fetchImpl, signed.url, options.requestTimeoutMs, 'image/')
assertCondition(bytes.length > 1000, 'OpenAI image output is unexpectedly small')
return { job, asset, version, bytes }
}, (value) => ({ jobId: value.job.id, assetVersionId: value.version.id, model: value.job.providerModel, byteSize: value.bytes.length }))
report.aiVerification.artifacts.openAiImageAssetVersionId = openAiImage.version.id
const replicateImage = await step('ai-image-replicate', 'Replicate character-consistency variant', async () => {
const profile = await api.request(`/assets/${aiSetup.character.id}/consistency-profile`, {
method: 'PUT',
json: {
referenceAssetVersionIds: [openAiImage.version.id],
identityPrompt: 'Preserve facial proportions, eye shape, hairstyle, and adult age.',
costumePrompt: 'Preserve the dark production jacket and simple collar.',
stylePrompt: 'Polished cinematic comic-drama illustration.',
negativePrompt: 'identity drift, age change, child, text, watermark',
consistencyStrength: 90,
identityLocked: true,
},
})
assertCondition(profile.consistencyProfile?.identityLocked, 'Character identity profile was not locked')
const job = await runAiJob(projectId, aiSetup.episode.id, 'ASSET_GENERATE', {
assetId: aiSetup.character.id,
prompt: `The same adult presenter turns slightly toward camera under warm key light. ${marker}`,
negativePrompt: 'text, watermark',
width: 1024,
height: 1024,
provider: 'replicate',
referenceAssetVersionIds: [openAiImage.version.id],
params: { consistencyStrength: 90 },
})
const asset = await api.request(`/assets/${aiSetup.character.id}`)
const version = asset.versions?.find((candidate) => candidate.id === job.output?.assetVersionId)
const appliedProfiles = version?.generationParams?.appliedCharacterConsistencyProfiles
assertCondition(version?.storageKey && version.provider === 'replicate', 'Replicate image version was not persisted')
assertCondition(Array.isArray(appliedProfiles) && appliedProfiles.some((snapshot) => snapshot.assetId === aiSetup.character.id && snapshot.identityLocked), 'Replicate variant did not capture the locked character profile')
assertCondition(version.generationParams?.referenceAssetVersionIds?.includes(openAiImage.version.id), 'Replicate variant did not record its identity reference')
const signed = await api.request(`/assets/${asset.id}/versions/${version.id}/url`)
const bytes = await downloadBytes(fetchImpl, signed.url, options.requestTimeoutMs, 'image/')
assertCondition(bytes.length > 1000, 'Replicate image output is unexpectedly small')
return { job, asset, version, bytes }
}, (value) => ({ jobId: value.job.id, assetVersionId: value.version.id, model: value.job.providerModel, byteSize: value.bytes.length }))
report.aiVerification.artifacts.replicateImageAssetVersionId = replicateImage.version.id
await step('ai-script', 'OpenAI script generation and immutable persistence', async () => {
const job = await runAiJob(projectId, aiSetup.episode.id, 'SCRIPT_GENERATE', {
provider: 'openai',
includeExistingScript: false,
instruction: 'Write one production-safe interior scene for a 15-second Chinese comic drama. Use one adult presenter, one short spoken line, and a clear visual action.',
})
const episode = await api.request(`/episodes/${aiSetup.episode.id}`)
assertCondition(episode.currentScript?.id === job.output?.scriptVersionId, 'Generated script was not activated as the current version')
assertCondition(episode.currentScript?.plainText?.trim(), 'Generated script has no plain text')
return { job, script: episode.currentScript }
}, (value) => ({ jobId: value.job.id, scriptVersionId: value.script.id, version: value.script.version, model: value.job.providerModel }))
const storyboard = await step('ai-storyboard', 'OpenAI storyboard generation and shot persistence', async () => {
const job = await runAiJob(projectId, aiSetup.episode.id, 'STORYBOARD_GENERATE', {
provider: 'openai',
replaceExisting: false,
instruction: `Create the smallest viable storyboard for provider validation. Prefer exactly one shot featuring ${aiSetup.character.name}, with one short spoken line and a total duration near five seconds.`,
})
const rows = await api.request(`/episodes/${aiSetup.episode.id}/shots`)
assertCondition(Array.isArray(rows) && rows.length > 0, 'Generated storyboard contains no shots')
assertCondition(job.output?.shotCount === rows.length, 'Storyboard job output does not match persisted shots')
const [selected, ...extra] = rows
for (const row of extra) await api.request(`/shots/${row.shot.id}`, { method: 'DELETE' })
const validationText = 'FrameFlow AI provider validation passed.'
const version = await api.request(`/shots/${selected.shot.id}/versions`, {
method: 'POST',
json: {
imageAssetVersionId: replicateImage.version.id,
subtitleText: validationText,
generationParams: { smokeRunId: runId, source: 'live-provider-smoke' },
},
})
return { job, shot: selected.shot, version, removedShots: extra.length, validationText }
}, (value) => ({ jobId: value.job.id, shotId: value.shot.id, generatedShots: value.job.output.shotCount, removedShots: value.removedShots, model: value.job.providerModel }))
await step('ai-tts', 'OpenAI TTS generation and audio storage', async () => {
const job = await runAiJob(projectId, aiSetup.episode.id, 'TTS_GENERATE', {
shotId: storyboard.shot.id,
text: storyboard.validationText,
voice: 'alloy',
speed: 1,
provider: 'openai',
})
const media = await api.request(`/shots/${storyboard.shot.id}/media`)
assertCondition(job.output?.audioStorageKey && media.audioUrl, 'TTS output was not bound to the current shot version')
const bytes = await downloadBytes(fetchImpl, media.audioUrl, options.requestTimeoutMs, 'audio/')
assertCondition(bytes.length > 500, 'TTS audio output is unexpectedly small')
return { job, media, bytes }
}, (value) => ({ jobId: value.job.id, model: value.job.providerModel, byteSize: value.bytes.length }))
await step('ai-video', 'Replicate image-to-video generation and storage', async () => {
const job = await runAiJob(projectId, aiSetup.episode.id, 'VIDEO_GENERATE', {
shotId: storyboard.shot.id,
prompt: 'Subtle natural head turn and blinking, locked camera, stable adult character identity.',
imageAssetVersionId: replicateImage.version.id,
durationSeconds: options.aiVideoSeconds,
provider: 'replicate',
params: options.aiVideoParams,
})
const media = await api.request(`/shots/${storyboard.shot.id}/media`)
assertCondition(job.output?.videoStorageKey && media.videoUrl, 'Video output was not bound to the current shot version')
const bytes = await downloadBytes(fetchImpl, media.videoUrl, options.requestTimeoutMs, 'video/')
assertCondition(bytes.length > 1000, 'Generated video output is unexpectedly small')
return { job, media, bytes }
}, (value) => ({ jobId: value.job.id, model: value.job.providerModel, durationSeconds: options.aiVideoSeconds, byteSize: value.bytes.length }))
const lipsync = await step('ai-lipsync', 'Replicate lip-sync generation and media continuity', async () => {
const job = await runAiJob(projectId, aiSetup.episode.id, 'LIPSYNC_GENERATE', {
shotId: storyboard.shot.id,
provider: 'replicate',
params: options.aiLipsyncParams,
})
const media = await api.request(`/shots/${storyboard.shot.id}/media`)
assertCondition(job.output?.videoStorageKey && media.videoUrl && media.audioUrl, 'Lip-sync output did not preserve video and audio continuity')
const [video, audio] = await Promise.all([
downloadBytes(fetchImpl, media.videoUrl, options.requestTimeoutMs, 'video/'),
downloadBytes(fetchImpl, media.audioUrl, options.requestTimeoutMs, 'audio/'),
])
assertCondition(video.length > 1000 && audio.length > 500, 'Lip-sync media outputs are unexpectedly small')
return { job, media, video, audio }
}, (value) => ({ jobId: value.job.id, model: value.job.providerModel, videoBytes: value.video.length, audioBytes: value.audio.length }))
report.aiVerification.artifacts.lipsyncShotVersionId = lipsync.media.versionId
const aiRender = await step('ai-render', 'FFmpeg render from live AI media', async () => {
const queued = await api.request(`/episodes/${aiSetup.episode.id}/renders`, {
method: 'POST',
json: {
idempotencyKey: `smoke-ai-render-${runId}`,
settings: { width: 640, height: 360, fps: 24, quality: 'draft', burnSubtitles: false, normalizeAudio: false, musicVolume: 0, subtitleStyle: { fontSize: 20, textColor: '#FFFFFF', outlineColor: '#000000', outlineWidth: 2, bottomMargin: 24, bold: true } },
},
})
const job = await waitForJob((jobId) => api.request(`/jobs/${jobId}`), queued.job.id, { timeoutMs: options.timeoutMs, sleep })
const render = await api.request(`/renders/${queued.render.id}`)
assertCondition(job.status === 'SUCCEEDED' && render.status === 'REVIEW', 'AI media render did not finish in review state')
assertCondition(render.sourceShotCount === 1 && render.downloadUrl, 'AI media render source manifest is invalid')
const video = await downloadBytes(fetchImpl, render.downloadUrl, options.requestTimeoutMs, 'video/mp4')
assertCondition(video.length > 1000, 'AI media render is unexpectedly small')
return { job, render, video }
}, (value) => ({ jobId: value.job.id, renderId: value.render.id, sourceShotCount: value.render.sourceShotCount, byteSize: value.video.length }))
report.aiVerification.artifacts.renderId = aiRender.render.id
report.aiVerification.artifacts.videoSha256 = sha256(aiRender.video)
await step('ai-usage', 'Live AI usage-ledger reconciliation', async () => {
const billingAfter = await api.request(`/workspaces/${workspace.id}/billing/summary`)
const providerJobIds = new Set(report.aiVerification.jobs.map((job) => job.id))
const billedJobIds = new Set((billingAfter.recent ?? []).filter((entry) => providerJobIds.has(entry.jobId)).map((entry) => entry.jobId))
const missingJobIds = [...providerJobIds].filter((jobId) => !billedJobIds.has(jobId))
assertCondition(missingJobIds.length === 0, `Usage ledger is missing ${missingJobIds.length} live AI jobs`)
const creditsUsed = Number(billingAfter.usedCredits) - Number(billingReadModel.usedCredits)
assertCondition(creditsUsed > 0, 'Live AI provider calls did not increase recorded usage')
report.aiVerification.creditsUsed = creditsUsed
return { jobs: providerJobIds.size, ledgerJobs: billedJobIds.size, creditsUsed }
}, (value) => value)
}
} catch (error) {
primaryError = error
} finally {
if (projectId && !options.keepProject) {
try {
await step('cleanup', 'Archive smoke-test project', async () => {
const archived = await api.request(`/projects/${projectId}`, { method: 'DELETE' })
assertCondition(archived.status === 'ARCHIVED', 'Smoke project was not archived')
report.projectArchived = true
return archived
}, (value) => ({ projectId: value.id, status: value.status }))
} catch (cleanupError) {
if (!primaryError) primaryError = cleanupError
}
}
if (loggedIn) {
try {
await step('logout', 'Revoke smoke-account refresh session', async () => {
await api.request('/auth/logout', { method: 'POST' })
const sessions = await api.request('/auth/sessions')
assertCondition(!sessions.some((candidate) => candidate.id === smokeSessionId), 'Smoke refresh session remains active after logout')
api.clearSession()
return { sessionId: smokeSessionId, revoked: true }
}, (value) => value)
} catch (logoutError) {
if (!primaryError) primaryError = logoutError
}
}
}
report.status = primaryError ? 'failed' : 'passed'
report.completedAt = new Date(now()).toISOString()
report.durationMs = Math.max(0, now() - startedAt)
if (options.keepProject && projectId) report.projectArchived = false
if (primaryError) {
report.error = safeError(primaryError)
throw new SmokeTestError(report.error, { cause: primaryError, report })
}
return report
}
export function usage() {
return `FrameFlow production smoke test
Usage:
FRAMEFLOW_SMOKE_BASE_URL=https://studio.example.com \\
FRAMEFLOW_SMOKE_EMAIL=smoke@example.com \\
FRAMEFLOW_SMOKE_PASSWORD='...' \\
npm run smoke:production -- [options]
Options:
--base-url URL Application origin or /api/v1 URL
--email EMAIL Dedicated verified smoke-account email
--workspace-id UUID Required when the account has multiple workspaces
--image PATH JPEG, PNG, or WebP render source
--timeout-seconds N Per-job timeout from 30 to 900 (default: 180)
--include-ai Run live, billable OpenAI and Replicate validation
--ai-video-seconds N Live AI video duration from 1 to 30 (default: 5)
--keep-project Keep the generated smoke project instead of archiving it
--allow-http Allow HTTP only for localhost development
--json Print the complete report as JSON
--help Show this help
The password is accepted only through FRAMEFLOW_SMOKE_PASSWORD so it is not exposed in the process list.
Default mode never invokes AI provider jobs; configuration-only readiness is included in the report.
--include-ai starts real provider jobs that can incur charges. It is accepted only when
FRAMEFLOW_SMOKE_BILLABLE_AI_ACK=I_ACCEPT_BILLABLE_AI_CHARGES is also set, all seven AI
workflows are configured, and the workspace has at least 700 remaining credits. Provider
jobs use maxAttempts=1. Optional model-specific JSON objects can be supplied through
FRAMEFLOW_SMOKE_VIDEO_PARAMS_JSON and FRAMEFLOW_SMOKE_LIPSYNC_PARAMS_JSON.`
}
async function main() {
let options
try {
options = parseSmokeOptions()
if (options.help) {
console.log(usage())
return
}
const report = await runProductionSmoke(options)
if (options.json) console.log(JSON.stringify(report, null, 2))
else console.log(`Production smoke test passed in ${report.durationMs}ms. Project ${report.projectId}${report.projectArchived ? ' was archived' : ' was retained'}.`)
} catch (error) {
const report = error instanceof SmokeTestError ? error.report : null
if (options?.json && report) console.error(JSON.stringify(report, null, 2))
else console.error(`Production smoke test failed: ${safeError(error)}`)
process.exitCode = 1
}
}
if (process.argv[1] && resolve(process.argv[1]) === scriptPath) await main()