#!/usr/bin/env node import { createHash, randomUUID } from 'node:crypto' import { readFile } from 'node:fs/promises' import { basename, extname, resolve } from 'node:path' import { fileURLToPath } from 'node:url' const scriptPath = fileURLToPath(import.meta.url) const defaultImagePath = fileURLToPath(new URL('../public/assets/scene-rooftop.jpg', import.meta.url)) const terminalJobStatuses = new Set(['SUCCEEDED', 'FAILED', 'CANCELLED']) const billableAiAcknowledgement = 'I_ACCEPT_BILLABLE_AI_CHARGES' export const requiredLiveAiWorkflowIds = ['script', 'storyboard', 'image-openai', 'image-replicate', 'tts', 'video', 'lipsync'] export class SmokeTestError extends Error { constructor(message, options = {}) { super(message, options) this.name = 'SmokeTestError' this.report = options.report } } function required(value, name) { if (typeof value !== 'string' || value.trim() === '') throw new SmokeTestError(`${name} is required`) return value.trim() } function booleanValue(value) { return ['1', 'true', 'yes', 'on'].includes(String(value ?? '').trim().toLowerCase()) } function integerValue(value, name, minimum, maximum) { const parsed = Number(value) if (!Number.isInteger(parsed) || parsed < minimum || parsed > maximum) { throw new SmokeTestError(`${name} must be an integer from ${minimum} to ${maximum}`) } return parsed } function jsonObjectValue(value, name) { if (value === undefined || value === null || String(value).trim() === '') return {} let parsed try { parsed = JSON.parse(String(value)) } catch { throw new SmokeTestError(`${name} must be valid JSON`) } if (!parsed || Array.isArray(parsed) || typeof parsed !== 'object') { throw new SmokeTestError(`${name} must be a JSON object`) } return parsed } export function resolveEndpoints(value, options = {}) { const input = required(value, 'FRAMEFLOW_SMOKE_BASE_URL') let url try { url = new URL(input) } catch { throw new SmokeTestError('FRAMEFLOW_SMOKE_BASE_URL must be a valid URL') } if (url.username || url.password || url.search || url.hash) { throw new SmokeTestError('Smoke-test base URL cannot contain credentials, a query, or a fragment') } if (!['http:', 'https:'].includes(url.protocol)) throw new SmokeTestError('Smoke-test base URL must use HTTP or HTTPS') const localHostnames = new Set(['localhost', '127.0.0.1', '::1', '[::1]']) if (url.protocol === 'http:' && (!options.allowHttp || !localHostnames.has(url.hostname))) { throw new SmokeTestError('HTTPS is required; HTTP can only be enabled explicitly for localhost') } const path = url.pathname.replace(/\/+$/, '') if (path && path !== '/api/v1') { throw new SmokeTestError('Smoke-test base URL path must be empty or /api/v1') } const origin = url.origin return { origin, apiBase: `${origin}/api/v1` } } export function parseSmokeOptions(argv = process.argv.slice(2), environment = process.env) { const values = {} const flags = new Set() const valueOptions = new Set(['--base-url', '--email', '--workspace-id', '--image', '--timeout-seconds', '--ai-video-seconds']) const flagOptions = new Set(['--allow-http', '--keep-project', '--include-ai', '--json', '--help']) for (let index = 0; index < argv.length; index += 1) { const argument = argv[index] if (valueOptions.has(argument)) { const value = argv[index + 1] if (!value || value.startsWith('--')) throw new SmokeTestError(`${argument} requires a value`) values[argument] = value index += 1 } else if (flagOptions.has(argument)) { flags.add(argument) } else { throw new SmokeTestError(`Unknown option: ${argument}`) } } if (flags.has('--help')) return { help: true } const allowHttp = flags.has('--allow-http') || booleanValue(environment.FRAMEFLOW_SMOKE_ALLOW_HTTP) const endpoints = resolveEndpoints(values['--base-url'] ?? environment.FRAMEFLOW_SMOKE_BASE_URL, { allowHttp }) const email = required(values['--email'] ?? environment.FRAMEFLOW_SMOKE_EMAIL, 'FRAMEFLOW_SMOKE_EMAIL').toLowerCase() const password = required(environment.FRAMEFLOW_SMOKE_PASSWORD, 'FRAMEFLOW_SMOKE_PASSWORD') const timeoutSeconds = integerValue(values['--timeout-seconds'] ?? environment.FRAMEFLOW_SMOKE_TIMEOUT_SECONDS ?? '180', 'timeout-seconds', 30, 900) const includeAi = flags.has('--include-ai') if (includeAi && environment.FRAMEFLOW_SMOKE_BILLABLE_AI_ACK !== billableAiAcknowledgement) { throw new SmokeTestError(`--include-ai requires FRAMEFLOW_SMOKE_BILLABLE_AI_ACK=${billableAiAcknowledgement}`) } return { ...endpoints, email, password, workspaceId: values['--workspace-id'] ?? environment.FRAMEFLOW_SMOKE_WORKSPACE_ID ?? null, imagePath: resolve(values['--image'] ?? environment.FRAMEFLOW_SMOKE_IMAGE ?? defaultImagePath), timeoutMs: timeoutSeconds * 1000, requestTimeoutMs: Math.min(60_000, timeoutSeconds * 1000), keepProject: flags.has('--keep-project') || booleanValue(environment.FRAMEFLOW_SMOKE_KEEP_PROJECT), includeAi, aiVideoSeconds: integerValue(values['--ai-video-seconds'] ?? environment.FRAMEFLOW_SMOKE_AI_VIDEO_SECONDS ?? '5', 'ai-video-seconds', 1, 30), aiVideoParams: jsonObjectValue(environment.FRAMEFLOW_SMOKE_VIDEO_PARAMS_JSON, 'FRAMEFLOW_SMOKE_VIDEO_PARAMS_JSON'), aiLipsyncParams: jsonObjectValue(environment.FRAMEFLOW_SMOKE_LIPSYNC_PARAMS_JSON, 'FRAMEFLOW_SMOKE_LIPSYNC_PARAMS_JSON'), json: flags.has('--json'), } } function safeError(error) { const message = error instanceof Error ? error.message : String(error) return message.replace(/([?&](?:X-Amz-[^=\s]+|token|signature)=)[^&\s]+/gi, '$1[redacted]').slice(0, 2000) } function assertCondition(condition, message) { if (!condition) throw new SmokeTestError(message) } function sha256(value) { return createHash('sha256').update(value).digest('hex') } function imageContentType(path) { const extension = extname(path).toLowerCase() if (extension === '.jpg' || extension === '.jpeg') return 'image/jpeg' if (extension === '.png') return 'image/png' if (extension === '.webp') return 'image/webp' throw new SmokeTestError('Smoke-test image must be JPEG, PNG, or WebP') } function responseBody(response) { if (response.status === 204) return Promise.resolve(null) const contentType = response.headers.get('content-type') ?? '' if (contentType.includes('application/json')) return response.json() return response.text() } async function fetchWithTimeout(fetchImpl, url, options, timeoutMs) { const controller = new AbortController() const timer = setTimeout(() => controller.abort(new Error(`Request timed out after ${timeoutMs}ms`)), timeoutMs) try { return await fetchImpl(url, { ...options, signal: controller.signal }) } finally { clearTimeout(timer) } } function createApiClient(options, fetchImpl) { let accessToken = null const cookies = new Map() function captureCookies(headers) { const values = typeof headers.getSetCookie === 'function' ? headers.getSetCookie() : headers.get('set-cookie') ? [headers.get('set-cookie')] : [] for (const value of values) { const pair = value?.split(';', 1)[0] const separator = pair?.indexOf('=') ?? -1 if (!pair || separator < 1) continue const name = pair.slice(0, separator).trim() const cookieValue = pair.slice(separator + 1).trim() if (!cookieValue || /(?:^|;)\s*max-age=0(?:;|$)/i.test(value)) cookies.delete(name) else cookies.set(name, cookieValue) } } async function raw(path, request = {}) { const headers = new Headers(request.headers) const url = path.startsWith('http://') || path.startsWith('https://') ? path : `${options.apiBase}${path}` const authenticatedApiRequest = request.auth !== false && (url === options.apiBase || url.startsWith(`${options.apiBase}/`)) if (accessToken && authenticatedApiRequest) headers.set('authorization', `Bearer ${accessToken}`) if (authenticatedApiRequest && cookies.size > 0) { headers.set('cookie', [...cookies].map(([name, value]) => `${name}=${value}`).join('; ')) } let body = request.body if (request.json !== undefined) { headers.set('content-type', 'application/json') body = JSON.stringify(request.json) } const response = await fetchWithTimeout(fetchImpl, url, { method: request.method ?? 'GET', headers, body, redirect: request.redirect ?? 'follow', }, request.timeoutMs ?? options.requestTimeoutMs) captureCookies(response.headers) return response } async function request(path, input = {}) { const response = await raw(path, input) const body = await responseBody(response) if (!response.ok) { const message = body?.error?.message ?? (typeof body === 'string' && body) ?? `HTTP ${response.status}` throw new SmokeTestError(`${input.method ?? 'GET'} ${path} failed (${response.status}): ${message}`) } return body } return { raw, request, setAccessToken(value) { accessToken = value }, clearSession() { accessToken = null cookies.clear() }, } } export async function waitForJob(getJob, jobId, options = {}) { const timeoutMs = options.timeoutMs ?? 180_000 const pollIntervalMs = options.pollIntervalMs ?? 500 const sleep = options.sleep ?? ((milliseconds) => new Promise((resolvePromise) => setTimeout(resolvePromise, milliseconds))) const now = options.now ?? Date.now const startedAt = now() let lastStatus = null while (now() - startedAt <= timeoutMs) { const job = await getJob(jobId) lastStatus = job?.status ?? null if (lastStatus === 'SUCCEEDED') return job if (terminalJobStatuses.has(lastStatus)) { throw new SmokeTestError(`Job ${jobId} ended in ${lastStatus}: ${job.errorMessage ?? job.errorCode ?? 'unknown error'}`) } await sleep(pollIntervalMs) } throw new SmokeTestError(`Job ${jobId} did not finish within ${timeoutMs}ms (last status: ${lastStatus ?? 'unknown'})`) } export async function waitForNotifications(getNotifications, resourceIds, options = {}) { const timeoutMs = options.timeoutMs ?? 10_000 const pollIntervalMs = options.pollIntervalMs ?? 250 const sleep = options.sleep ?? ((milliseconds) => new Promise((resolvePromise) => setTimeout(resolvePromise, milliseconds))) const now = options.now ?? Date.now const expected = new Set(resourceIds) const startedAt = now() let lastItems = [] while (now() - startedAt <= timeoutMs) { const page = await getNotifications() lastItems = Array.isArray(page?.items) ? page.items : [] const matched = lastItems.filter((notification) => expected.has(notification.resourceId)) if (new Set(matched.map((notification) => notification.resourceId)).size === expected.size) return matched await sleep(pollIntervalMs) } const matchedIds = new Set(lastItems.map((notification) => notification.resourceId)) const missing = [...expected].filter((resourceId) => !matchedIds.has(resourceId)) throw new SmokeTestError(`Notifications did not arrive within ${timeoutMs}ms (missing resources: ${missing.join(', ')})`) } function selectWorkspace(workspaces, requestedId) { assertCondition(Array.isArray(workspaces) && workspaces.length > 0, 'Smoke account has no workspace membership') const workspace = requestedId ? workspaces.find((candidate) => candidate.id === requestedId) : workspaces.length === 1 ? workspaces[0] : null if (!workspace) { throw new SmokeTestError(requestedId ? `Smoke account is not a member of workspace ${requestedId}` : 'Smoke account belongs to multiple workspaces; set FRAMEFLOW_SMOKE_WORKSPACE_ID') } assertCondition(['OWNER', 'ADMIN'].includes(workspace.role), 'Full smoke test requires an OWNER or ADMIN workspace role') return workspace } async function downloadBytes(fetchImpl, url, timeoutMs, expectedType) { const response = await fetchWithTimeout(fetchImpl, url, { redirect: 'follow' }, timeoutMs) assertCondition(response.ok, `Media download failed with HTTP ${response.status}`) const contentType = response.headers.get('content-type') ?? '' if (expectedType) assertCondition(contentType.toLowerCase().includes(expectedType), `Unexpected media content type: ${contentType || 'missing'}`) return Buffer.from(await response.arrayBuffer()) } function readinessSummary(readiness) { return { infrastructure: readiness.infrastructure, ai: { status: readiness.ai?.status ?? 'UNKNOWN', verificationMode: readiness.ai?.verificationMode ?? 'UNKNOWN', summary: readiness.ai?.summary ?? null, workflows: (readiness.ai?.workflows ?? []).map((workflow) => ({ id: workflow.id, provider: workflow.provider, model: workflow.model, status: workflow.status, })), providers: (readiness.ai?.providers ?? []).map((provider) => ({ id: provider.id, status: provider.status, verification: provider.verification, configuredCapabilities: provider.configuredCapabilities, totalCapabilities: provider.totalCapabilities, })), }, } } export function assertLiveAiReady(readiness, minimumCredits = 700, billing = null) { const capabilityRows = [ ...(readiness?.ai?.providers ?? []).flatMap((provider) => provider.capabilities ?? []), ...(readiness?.ai?.workflows ?? []), ] const workflows = new Map(capabilityRows.map((workflow) => [workflow.id, workflow])) const missing = requiredLiveAiWorkflowIds.filter((id) => workflows.get(id)?.status !== 'CONFIGURED') if (missing.length > 0) { const details = missing.map((id) => { const workflow = workflows.get(id) const requirements = workflow?.missingRequirements?.join(', ') return requirements ? `${id} (${requirements})` : id }) throw new SmokeTestError(`Live AI smoke test requires configured workflows: ${details.join('; ')}`) } if (billing && Number(billing.remainingCredits) < minimumCredits) { throw new SmokeTestError(`Live AI smoke test requires at least ${minimumCredits} remaining credits`) } return requiredLiveAiWorkflowIds.map((id) => { const workflow = workflows.get(id) return { id, provider: workflow.provider, model: workflow.model } }) } export async function runProductionSmoke(options, dependencies = {}) { const fetchImpl = dependencies.fetch ?? fetch const readFileImpl = dependencies.readFile ?? readFile const sleep = dependencies.sleep ?? ((milliseconds) => new Promise((resolvePromise) => setTimeout(resolvePromise, milliseconds))) const now = dependencies.now ?? Date.now const log = dependencies.log ?? ((message) => console.error(message)) const api = createApiClient(options, fetchImpl) const startedAt = now() const runId = randomUUID() const marker = `FRAMEFLOW_SMOKE_${runId}` const report = { status: 'running', runId, baseUrl: options.origin, startedAt: new Date(startedAt).toISOString(), workspaceId: null, projectId: null, projectArchived: false, checks: [], services: null, billing: null, aiVerification: { mode: options.includeAi ? 'LIVE_PROVIDER_CALLS' : 'CONFIGURATION_ONLY', jobs: [], artifacts: {}, }, artifacts: {}, } let projectId = null let loggedIn = false let smokeSessionId = null let primaryError = null async function step(id, label, action, summarize) { const stepStartedAt = now() log(`[...] ${label}`) try { const value = await action() const check = { id, label, status: 'passed', durationMs: Math.max(0, now() - stepStartedAt) } const details = summarize?.(value) if (details !== undefined) check.details = details report.checks.push(check) log(`[ ok] ${label} (${check.durationMs}ms)`) return value } catch (error) { const check = { id, label, status: 'failed', durationMs: Math.max(0, now() - stepStartedAt), error: safeError(error) } report.checks.push(check) log(`[fail] ${label}: ${check.error}`) throw error } } async function runAiJob(projectIdForJob, episodeIdForJob, type, input) { const queued = await api.request(`/projects/${projectIdForJob}/jobs`, { method: 'POST', json: { type, episodeId: episodeIdForJob, idempotencyKey: `smoke-ai-${runId}-${report.aiVerification.jobs.length}-${type.toLowerCase()}`, input, maxAttempts: 1, }, }) const job = await waitForJob((jobId) => api.request(`/jobs/${jobId}`), queued.job.id, { timeoutMs: options.timeoutMs, sleep, }) assertCondition(['openai', 'replicate'].includes(job.provider), `${type} did not record a live AI provider`) assertCondition(job.providerModel, `${type} did not record a provider model`) report.aiVerification.jobs.push({ id: job.id, type, provider: job.provider, model: job.providerModel }) return job } try { await step('liveness', 'API liveness', async () => { const response = await fetchWithTimeout(fetchImpl, `${options.origin}/health/live`, {}, options.requestTimeoutMs) const body = await responseBody(response) assertCondition(response.ok && body?.status === 'ok', `Liveness failed with HTTP ${response.status}`) return body }) await step('readiness', 'API dependency readiness', async () => { const response = await fetchWithTimeout(fetchImpl, `${options.origin}/health/ready`, {}, options.requestTimeoutMs) const body = await responseBody(response) assertCondition(response.ok && body?.status === 'ready', `Readiness failed with HTTP ${response.status}`) const unavailable = Object.entries(body.dependencies ?? {}).filter(([, status]) => status !== 'ok') assertCondition(unavailable.length === 0, `Unavailable dependencies: ${unavailable.map(([name]) => name).join(', ')}`) return body }, (body) => body.dependencies) const session = await step('login', 'Dedicated smoke-account login', async () => { const value = await api.request('/auth/login', { method: 'POST', auth: false, json: { email: options.email, password: options.password }, }) assertCondition(value?.accessToken && value?.user?.id, 'Login response is missing the authenticated session') api.setAccessToken(value.accessToken) loggedIn = true const sessions = await api.request('/auth/sessions') const currentSession = sessions.find((candidate) => candidate.current) assertCondition(currentSession?.id, 'Login refresh session cookie was not retained by the smoke runner') smokeSessionId = currentSession.id return value }, (value) => ({ userId: value.user.id, sessionId: smokeSessionId, workspaceCount: value.workspaces.length })) const workspace = selectWorkspace(session.workspaces, options.workspaceId) report.workspaceId = workspace.id const serviceReadiness = await step('service-readiness', 'Worker, storage, FFmpeg, and AI configuration', async () => { const value = await api.request(`/workspaces/${workspace.id}/service-readiness`) assertCondition(value?.infrastructure?.status === 'AVAILABLE', 'Authenticated service readiness is degraded') assertCondition(value.infrastructure.available === value.infrastructure.total, 'Not every production service is available') return value }, (value) => ({ infrastructure: value.infrastructure.services.map((service) => ({ id: service.id, status: service.status, latencyMs: service.latencyMs })), ai: value.ai.summary, })) report.services = readinessSummary(serviceReadiness) const billingReadModel = await step('billing', 'Billing and usage read model', async () => { const value = await api.request(`/workspaces/${workspace.id}/billing/summary`) assertCondition(value?.plan && value?.provider?.status, 'Billing summary is incomplete') return value }, (value) => ({ plan: value.plan, provider: value.provider.status, webhookConfigured: value.provider.webhookConfigured, remainingCredits: value.remainingCredits })) report.billing = { plan: billingReadModel.plan, provider: billingReadModel.provider.status, webhookConfigured: billingReadModel.provider.webhookConfigured, remainingCredits: billingReadModel.remainingCredits, } if (options.includeAi) { await step('ai-preflight', 'Billable AI authorization, configuration, and credit preflight', async () => { const workflows = assertLiveAiReady(serviceReadiness, 700, billingReadModel) return { acknowledgement: 'accepted', minimumCredits: 700, workflows } }, (value) => value) } const project = await step('project', 'Project persistence', async () => api.request(`/workspaces/${workspace.id}/projects`, { method: 'POST', json: { name: `[SMOKE] ${new Date(startedAt).toISOString()} ${runId.slice(0, 8)}`, logline: 'Automated production smoke test. Safe to archive after completion.', genre: 'SMOKE_TEST', visualBible: { smokeTest: { runId, createdAt: new Date(startedAt).toISOString() } }, }, }), (value) => ({ projectId: value.id })) projectId = project.id report.projectId = projectId const episode = await step('script', 'Episode and immutable script persistence', async () => { const createdEpisode = await api.request(`/projects/${projectId}/episodes`, { method: 'POST', json: { episodeNumber: 1, title: 'Production smoke episode', synopsis: marker, targetDurationSeconds: 15 }, }) const script = await api.request(`/episodes/${createdEpisode.id}/scripts`, { method: 'POST', json: { title: 'Production smoke script', content: { smokeTest: true, runId, scenes: [{ number: 1, text: marker }] }, plainText: `SCENE 1 - SMOKE TEST\n${marker}`, changeSummary: 'Production smoke test', }, }) const persisted = await api.request(`/episodes/${createdEpisode.id}`) assertCondition(persisted.currentScript?.id === script.id, 'Current script version was not persisted') assertCondition(persisted.currentScript?.plainText?.includes(marker), 'Persisted script marker is missing') return { episode: createdEpisode, script } }, (value) => ({ episodeId: value.episode.id, scriptVersion: value.script.version })) const sourceImage = await readFileImpl(options.imagePath) const sourceHash = sha256(sourceImage) const contentType = imageContentType(options.imagePath) const imageAssetVersion = await step('upload', 'Object upload and signed download integrity', async () => { const form = new FormData() form.append('file', new Blob([sourceImage], { type: contentType }), basename(options.imagePath)) const upload = await api.request(`/projects/${projectId}/uploads`, { method: 'POST', body: form }) assertCondition(upload.byteSize === sourceImage.length, 'Uploaded object byte size does not match the source') const asset = await api.request(`/projects/${projectId}/assets`, { method: 'POST', json: { type: 'SHOT', name: `Smoke source ${runId.slice(0, 8)}`, description: marker, tags: ['smoke-test'], metadata: { smokeRunId: runId } }, }) const version = await api.request(`/assets/${asset.id}/versions`, { method: 'POST', json: { storageKey: upload.key, mimeType: upload.contentType, byteSize: upload.byteSize, prompt: marker, provider: 'smoke-test', providerModel: 'uploaded-source-v1' }, }) const signed = await api.request(`/assets/${asset.id}/versions/${version.id}/url`) const downloaded = await downloadBytes(fetchImpl, signed.url, options.requestTimeoutMs, contentType) assertCondition(sha256(downloaded) === sourceHash, 'Signed object download does not match the uploaded image') return { asset, version, upload } }, (value) => ({ assetId: value.asset.id, assetVersionId: value.version.id, byteSize: value.upload.byteSize, sha256: sourceHash })) const shot = await step('shot', 'Shot and immutable media-version persistence', async () => { const created = await api.request(`/episodes/${episode.episode.id}/shots`, { method: 'POST', json: { shotNumber: 1, sceneNumber: 1, title: 'Smoke render shot', description: marker, shotType: '全景', cameraMotion: '静止', durationMs: 2000, characterAssetIds: [], sortOrder: 100 }, }) const version = await api.request(`/shots/${created.id}/versions`, { method: 'POST', json: { imageAssetVersionId: imageAssetVersion.version.id, subtitleText: marker, generationParams: { smokeRunId: runId } }, }) const persisted = await api.request(`/shots/${created.id}`) assertCondition(persisted.versions?.[0]?.id === version.id, 'Current shot version was not persisted') return { shot: created, version } }, (value) => ({ shotId: value.shot.id, shotVersion: value.version.version })) await step('queue', 'Redis/BullMQ worker execution', async () => { const queued = await api.request(`/projects/${projectId}/jobs`, { method: 'POST', json: { type: 'PIPELINE_HEALTHCHECK', episodeId: episode.episode.id, idempotencyKey: `smoke-health-${runId}`, input: { marker }, maxAttempts: 1 }, }) const job = await waitForJob((jobId) => api.request(`/jobs/${jobId}`), queued.job.id, { timeoutMs: options.timeoutMs, sleep }) assertCondition(job.output?.ok === true && job.output?.echo?.marker === marker, 'Worker healthcheck output is invalid') return job }, (job) => ({ jobId: job.id, provider: job.provider, providerModel: job.providerModel })) const renderResult = await step('render', 'FFmpeg render and immutable source manifest', async () => { const queued = await api.request(`/episodes/${episode.episode.id}/renders`, { method: 'POST', json: { idempotencyKey: `smoke-render-${runId}`, settings: { width: 640, height: 360, fps: 24, quality: 'draft', burnSubtitles: false, normalizeAudio: false, musicVolume: 0, subtitleStyle: { fontSize: 20, textColor: '#FFFFFF', outlineColor: '#000000', outlineWidth: 2, bottomMargin: 24, bold: true } }, }, }) const job = await waitForJob((jobId) => api.request(`/jobs/${jobId}`), queued.job.id, { timeoutMs: options.timeoutMs, sleep }) const render = await api.request(`/renders/${queued.render.id}`) assertCondition(render.status === 'REVIEW', `Render ended in unexpected status ${render.status}`) assertCondition(render.sourceShotCount === 1, 'Render source manifest does not contain exactly one shot') assertCondition(render.subtitleFormats?.includes('srt') && render.subtitleFormats?.includes('vtt'), 'Render subtitle sidecars are missing') assertCondition(render.downloadUrl, 'Render download URL is missing') const video = await downloadBytes(fetchImpl, render.downloadUrl, options.requestTimeoutMs, 'video/mp4') assertCondition(video.length > 1000, 'Rendered video is unexpectedly small') return { render, job, videoHash: sha256(video), videoBytes: video.length } }, (value) => ({ renderId: value.render.id, jobId: value.job.id, byteSize: value.videoBytes, subtitleFormats: value.render.subtitleFormats, sourceShotCount: value.render.sourceShotCount })) report.artifacts.renderId = renderResult.render.id report.artifacts.videoSha256 = renderResult.videoHash await step('subtitles', 'SRT and VTT sidecar downloads', async () => { const results = {} for (const format of ['srt', 'vtt']) { const subtitle = await api.request(`/renders/${renderResult.render.id}/subtitles/${format}`) const bytes = await downloadBytes(fetchImpl, subtitle.downloadUrl, options.requestTimeoutMs) const text = bytes.toString('utf8') assertCondition(text.includes(marker), `${format.toUpperCase()} subtitle does not contain the smoke marker`) results[format] = { bytes: bytes.length, sha256: sha256(bytes) } } return results }, (value) => value) await step('review', 'Render review and approval', async () => { const review = await api.request(`/workspaces/${workspace.id}/reviews`, { method: 'POST', json: { targetType: 'RENDER', targetId: renderResult.render.id, assignedToId: session.user.id, summary: `Automated approval for ${marker}` }, }) const decision = await api.request(`/reviews/${review.id}/decision`, { method: 'POST', json: { status: 'APPROVED', summary: 'Production smoke render passed automated checks.' }, }) const approved = await api.request(`/renders/${renderResult.render.id}`) assertCondition(decision.status === 'APPROVED' && approved.status === 'APPROVED', 'Render approval was not persisted') return { review, approved } }, (value) => ({ reviewId: value.review.id, renderStatus: value.approved.status })) const deliveryResult = await step('delivery', 'Approved download delivery through the worker', async () => { const queued = await api.request(`/renders/${renderResult.render.id}/deliveries`, { method: 'POST', json: { platform: 'DOWNLOAD', destination: { smokeRunId: runId } }, }) const job = await waitForJob((jobId) => api.request(`/jobs/${jobId}`), queued.job.id, { timeoutMs: options.timeoutMs, sleep }) const delivery = await api.request(`/deliveries/${queued.delivery.id}`) assertCondition(delivery.status === 'PUBLISHED' && delivery.downloadUrl, 'Download delivery was not published') const redirect = await api.raw(`/deliveries/${delivery.id}/download`, { redirect: 'manual' }) assertCondition([301, 302, 303, 307, 308].includes(redirect.status), `Delivery endpoint returned HTTP ${redirect.status} instead of a redirect`) const location = redirect.headers.get('location') assertCondition(location, 'Delivery redirect is missing its signed location') const video = await downloadBytes(fetchImpl, location, options.requestTimeoutMs, 'video/mp4') assertCondition(sha256(video) === renderResult.videoHash, 'Delivered video differs from the approved render') return { delivery, job, bytes: video.length } }, (value) => ({ deliveryId: value.delivery.id, jobId: value.job.id, status: value.delivery.status, byteSize: value.bytes })) report.artifacts.deliveryId = deliveryResult.delivery.id await step('notifications', 'Persistent job notifications and read state', async () => { const expectedJobIds = [renderResult.job.id, deliveryResult.job.id] const rows = await waitForNotifications( () => api.request(`/workspaces/${workspace.id}/notifications?limit=100`), expectedJobIds, { timeoutMs: Math.min(options.timeoutMs, 10_000), sleep, now }, ) for (const jobId of expectedJobIds) { const notification = rows.find((candidate) => candidate.resourceId === jobId) assertCondition(notification?.kind === 'JOB_SUCCEEDED' && notification.tone === 'SUCCESS', `Successful job notification is invalid for ${jobId}`) const expectedPage = jobId === deliveryResult.job.id ? 'publish' : 'jobs' assertCondition(notification.page === expectedPage, `Job notification points to ${notification.page} instead of ${expectedPage}`) const read = await api.request(`/notifications/${notification.id}`, { method: 'PATCH', json: { read: true } }) assertCondition(read.readAt, `Notification ${notification.id} did not persist its read state`) } return rows }, (rows) => ({ notifications: rows.length, resources: rows.map((notification) => notification.resourceId) })) await step('audit', 'Audit-log persistence', async () => { const page = await api.request(`/workspaces/${workspace.id}/audit-logs?limit=100&actorId=${session.user.id}`) const actions = new Set((page.items ?? []).map((item) => item.action)) for (const action of ['project.create', 'upload.create', 'render.create', 'review.decide', 'delivery.create']) { assertCondition(actions.has(action), `Audit log is missing ${action}`) } return { actions: actions.size } }, (value) => value) if (options.includeAi) { const aiSetup = await step('ai-setup', 'Live AI validation workspace data', async () => { const aiEpisode = await api.request(`/projects/${projectId}/episodes`, { method: 'POST', json: { episodeNumber: 2, title: 'Live provider validation', synopsis: `A single character says a short line in a controlled studio. ${marker}`, targetDurationSeconds: 15, }, }) const character = await api.request(`/projects/${projectId}/assets`, { method: 'POST', json: { type: 'CHARACTER', name: `Provider test character ${runId.slice(0, 8)}`, description: 'Adult presenter, centered portrait, neutral studio lighting, production-safe wardrobe.', tags: ['smoke-test', 'provider-validation'], metadata: { smokeRunId: runId }, }, }) return { episode: aiEpisode, character } }, (value) => ({ episodeId: value.episode.id, characterAssetId: value.character.id })) const openAiImage = await step('ai-image-openai', 'OpenAI character image generation', async () => { const job = await runAiJob(projectId, aiSetup.episode.id, 'ASSET_GENERATE', { assetId: aiSetup.character.id, prompt: `Clean comic-drama character reference sheet. Adult presenter, centered bust portrait, neutral gray studio, consistent facial landmarks. ${marker}`, negativePrompt: 'text, watermark, extra limbs, child', width: 1024, height: 1024, provider: 'openai', referenceAssetVersionIds: [], params: { consistencyStrength: 82 }, }) const asset = await api.request(`/assets/${aiSetup.character.id}`) const version = asset.versions?.find((candidate) => candidate.id === job.output?.assetVersionId) assertCondition(version?.storageKey && version.provider === 'openai', 'OpenAI image version was not persisted as current asset media') const signed = await api.request(`/assets/${asset.id}/versions/${version.id}/url`) const bytes = await downloadBytes(fetchImpl, signed.url, options.requestTimeoutMs, 'image/') assertCondition(bytes.length > 1000, 'OpenAI image output is unexpectedly small') return { job, asset, version, bytes } }, (value) => ({ jobId: value.job.id, assetVersionId: value.version.id, model: value.job.providerModel, byteSize: value.bytes.length })) report.aiVerification.artifacts.openAiImageAssetVersionId = openAiImage.version.id const replicateImage = await step('ai-image-replicate', 'Replicate character-consistency variant', async () => { const profile = await api.request(`/assets/${aiSetup.character.id}/consistency-profile`, { method: 'PUT', json: { referenceAssetVersionIds: [openAiImage.version.id], identityPrompt: 'Preserve facial proportions, eye shape, hairstyle, and adult age.', costumePrompt: 'Preserve the dark production jacket and simple collar.', stylePrompt: 'Polished cinematic comic-drama illustration.', negativePrompt: 'identity drift, age change, child, text, watermark', consistencyStrength: 90, identityLocked: true, }, }) assertCondition(profile.consistencyProfile?.identityLocked, 'Character identity profile was not locked') const job = await runAiJob(projectId, aiSetup.episode.id, 'ASSET_GENERATE', { assetId: aiSetup.character.id, prompt: `The same adult presenter turns slightly toward camera under warm key light. ${marker}`, negativePrompt: 'text, watermark', width: 1024, height: 1024, provider: 'replicate', referenceAssetVersionIds: [openAiImage.version.id], params: { consistencyStrength: 90 }, }) const asset = await api.request(`/assets/${aiSetup.character.id}`) const version = asset.versions?.find((candidate) => candidate.id === job.output?.assetVersionId) const appliedProfiles = version?.generationParams?.appliedCharacterConsistencyProfiles assertCondition(version?.storageKey && version.provider === 'replicate', 'Replicate image version was not persisted') assertCondition(Array.isArray(appliedProfiles) && appliedProfiles.some((snapshot) => snapshot.assetId === aiSetup.character.id && snapshot.identityLocked), 'Replicate variant did not capture the locked character profile') assertCondition(version.generationParams?.referenceAssetVersionIds?.includes(openAiImage.version.id), 'Replicate variant did not record its identity reference') const signed = await api.request(`/assets/${asset.id}/versions/${version.id}/url`) const bytes = await downloadBytes(fetchImpl, signed.url, options.requestTimeoutMs, 'image/') assertCondition(bytes.length > 1000, 'Replicate image output is unexpectedly small') return { job, asset, version, bytes } }, (value) => ({ jobId: value.job.id, assetVersionId: value.version.id, model: value.job.providerModel, byteSize: value.bytes.length })) report.aiVerification.artifacts.replicateImageAssetVersionId = replicateImage.version.id await step('ai-script', 'OpenAI script generation and immutable persistence', async () => { const job = await runAiJob(projectId, aiSetup.episode.id, 'SCRIPT_GENERATE', { provider: 'openai', includeExistingScript: false, instruction: 'Write one production-safe interior scene for a 15-second Chinese comic drama. Use one adult presenter, one short spoken line, and a clear visual action.', }) const episode = await api.request(`/episodes/${aiSetup.episode.id}`) assertCondition(episode.currentScript?.id === job.output?.scriptVersionId, 'Generated script was not activated as the current version') assertCondition(episode.currentScript?.plainText?.trim(), 'Generated script has no plain text') return { job, script: episode.currentScript } }, (value) => ({ jobId: value.job.id, scriptVersionId: value.script.id, version: value.script.version, model: value.job.providerModel })) const storyboard = await step('ai-storyboard', 'OpenAI storyboard generation and shot persistence', async () => { const job = await runAiJob(projectId, aiSetup.episode.id, 'STORYBOARD_GENERATE', { provider: 'openai', replaceExisting: false, instruction: `Create the smallest viable storyboard for provider validation. Prefer exactly one shot featuring ${aiSetup.character.name}, with one short spoken line and a total duration near five seconds.`, }) const rows = await api.request(`/episodes/${aiSetup.episode.id}/shots`) assertCondition(Array.isArray(rows) && rows.length > 0, 'Generated storyboard contains no shots') assertCondition(job.output?.shotCount === rows.length, 'Storyboard job output does not match persisted shots') const [selected, ...extra] = rows for (const row of extra) await api.request(`/shots/${row.shot.id}`, { method: 'DELETE' }) const validationText = 'FrameFlow AI provider validation passed.' const version = await api.request(`/shots/${selected.shot.id}/versions`, { method: 'POST', json: { imageAssetVersionId: replicateImage.version.id, subtitleText: validationText, generationParams: { smokeRunId: runId, source: 'live-provider-smoke' }, }, }) return { job, shot: selected.shot, version, removedShots: extra.length, validationText } }, (value) => ({ jobId: value.job.id, shotId: value.shot.id, generatedShots: value.job.output.shotCount, removedShots: value.removedShots, model: value.job.providerModel })) await step('ai-tts', 'OpenAI TTS generation and audio storage', async () => { const job = await runAiJob(projectId, aiSetup.episode.id, 'TTS_GENERATE', { shotId: storyboard.shot.id, text: storyboard.validationText, voice: 'alloy', speed: 1, provider: 'openai', }) const media = await api.request(`/shots/${storyboard.shot.id}/media`) assertCondition(job.output?.audioStorageKey && media.audioUrl, 'TTS output was not bound to the current shot version') const bytes = await downloadBytes(fetchImpl, media.audioUrl, options.requestTimeoutMs, 'audio/') assertCondition(bytes.length > 500, 'TTS audio output is unexpectedly small') return { job, media, bytes } }, (value) => ({ jobId: value.job.id, model: value.job.providerModel, byteSize: value.bytes.length })) await step('ai-video', 'Replicate image-to-video generation and storage', async () => { const job = await runAiJob(projectId, aiSetup.episode.id, 'VIDEO_GENERATE', { shotId: storyboard.shot.id, prompt: 'Subtle natural head turn and blinking, locked camera, stable adult character identity.', imageAssetVersionId: replicateImage.version.id, durationSeconds: options.aiVideoSeconds, provider: 'replicate', params: options.aiVideoParams, }) const media = await api.request(`/shots/${storyboard.shot.id}/media`) assertCondition(job.output?.videoStorageKey && media.videoUrl, 'Video output was not bound to the current shot version') const bytes = await downloadBytes(fetchImpl, media.videoUrl, options.requestTimeoutMs, 'video/') assertCondition(bytes.length > 1000, 'Generated video output is unexpectedly small') return { job, media, bytes } }, (value) => ({ jobId: value.job.id, model: value.job.providerModel, durationSeconds: options.aiVideoSeconds, byteSize: value.bytes.length })) const lipsync = await step('ai-lipsync', 'Replicate lip-sync generation and media continuity', async () => { const job = await runAiJob(projectId, aiSetup.episode.id, 'LIPSYNC_GENERATE', { shotId: storyboard.shot.id, provider: 'replicate', params: options.aiLipsyncParams, }) const media = await api.request(`/shots/${storyboard.shot.id}/media`) assertCondition(job.output?.videoStorageKey && media.videoUrl && media.audioUrl, 'Lip-sync output did not preserve video and audio continuity') const [video, audio] = await Promise.all([ downloadBytes(fetchImpl, media.videoUrl, options.requestTimeoutMs, 'video/'), downloadBytes(fetchImpl, media.audioUrl, options.requestTimeoutMs, 'audio/'), ]) assertCondition(video.length > 1000 && audio.length > 500, 'Lip-sync media outputs are unexpectedly small') return { job, media, video, audio } }, (value) => ({ jobId: value.job.id, model: value.job.providerModel, videoBytes: value.video.length, audioBytes: value.audio.length })) report.aiVerification.artifacts.lipsyncShotVersionId = lipsync.media.versionId const aiRender = await step('ai-render', 'FFmpeg render from live AI media', async () => { const queued = await api.request(`/episodes/${aiSetup.episode.id}/renders`, { method: 'POST', json: { idempotencyKey: `smoke-ai-render-${runId}`, settings: { width: 640, height: 360, fps: 24, quality: 'draft', burnSubtitles: false, normalizeAudio: false, musicVolume: 0, subtitleStyle: { fontSize: 20, textColor: '#FFFFFF', outlineColor: '#000000', outlineWidth: 2, bottomMargin: 24, bold: true } }, }, }) const job = await waitForJob((jobId) => api.request(`/jobs/${jobId}`), queued.job.id, { timeoutMs: options.timeoutMs, sleep }) const render = await api.request(`/renders/${queued.render.id}`) assertCondition(job.status === 'SUCCEEDED' && render.status === 'REVIEW', 'AI media render did not finish in review state') assertCondition(render.sourceShotCount === 1 && render.downloadUrl, 'AI media render source manifest is invalid') const video = await downloadBytes(fetchImpl, render.downloadUrl, options.requestTimeoutMs, 'video/mp4') assertCondition(video.length > 1000, 'AI media render is unexpectedly small') return { job, render, video } }, (value) => ({ jobId: value.job.id, renderId: value.render.id, sourceShotCount: value.render.sourceShotCount, byteSize: value.video.length })) report.aiVerification.artifacts.renderId = aiRender.render.id report.aiVerification.artifacts.videoSha256 = sha256(aiRender.video) await step('ai-usage', 'Live AI usage-ledger reconciliation', async () => { const billingAfter = await api.request(`/workspaces/${workspace.id}/billing/summary`) const providerJobIds = new Set(report.aiVerification.jobs.map((job) => job.id)) const billedJobIds = new Set((billingAfter.recent ?? []).filter((entry) => providerJobIds.has(entry.jobId)).map((entry) => entry.jobId)) const missingJobIds = [...providerJobIds].filter((jobId) => !billedJobIds.has(jobId)) assertCondition(missingJobIds.length === 0, `Usage ledger is missing ${missingJobIds.length} live AI jobs`) const creditsUsed = Number(billingAfter.usedCredits) - Number(billingReadModel.usedCredits) assertCondition(creditsUsed > 0, 'Live AI provider calls did not increase recorded usage') report.aiVerification.creditsUsed = creditsUsed return { jobs: providerJobIds.size, ledgerJobs: billedJobIds.size, creditsUsed } }, (value) => value) } } catch (error) { primaryError = error } finally { if (projectId && !options.keepProject) { try { await step('cleanup', 'Archive smoke-test project', async () => { const archived = await api.request(`/projects/${projectId}`, { method: 'DELETE' }) assertCondition(archived.status === 'ARCHIVED', 'Smoke project was not archived') report.projectArchived = true return archived }, (value) => ({ projectId: value.id, status: value.status })) } catch (cleanupError) { if (!primaryError) primaryError = cleanupError } } if (loggedIn) { try { await step('logout', 'Revoke smoke-account refresh session', async () => { await api.request('/auth/logout', { method: 'POST' }) const sessions = await api.request('/auth/sessions') assertCondition(!sessions.some((candidate) => candidate.id === smokeSessionId), 'Smoke refresh session remains active after logout') api.clearSession() return { sessionId: smokeSessionId, revoked: true } }, (value) => value) } catch (logoutError) { if (!primaryError) primaryError = logoutError } } } report.status = primaryError ? 'failed' : 'passed' report.completedAt = new Date(now()).toISOString() report.durationMs = Math.max(0, now() - startedAt) if (options.keepProject && projectId) report.projectArchived = false if (primaryError) { report.error = safeError(primaryError) throw new SmokeTestError(report.error, { cause: primaryError, report }) } return report } export function usage() { return `FrameFlow production smoke test Usage: FRAMEFLOW_SMOKE_BASE_URL=https://studio.example.com \\ FRAMEFLOW_SMOKE_EMAIL=smoke@example.com \\ FRAMEFLOW_SMOKE_PASSWORD='...' \\ npm run smoke:production -- [options] Options: --base-url URL Application origin or /api/v1 URL --email EMAIL Dedicated verified smoke-account email --workspace-id UUID Required when the account has multiple workspaces --image PATH JPEG, PNG, or WebP render source --timeout-seconds N Per-job timeout from 30 to 900 (default: 180) --include-ai Run live, billable OpenAI and Replicate validation --ai-video-seconds N Live AI video duration from 1 to 30 (default: 5) --keep-project Keep the generated smoke project instead of archiving it --allow-http Allow HTTP only for localhost development --json Print the complete report as JSON --help Show this help The password is accepted only through FRAMEFLOW_SMOKE_PASSWORD so it is not exposed in the process list. Default mode never invokes AI provider jobs; configuration-only readiness is included in the report. --include-ai starts real provider jobs that can incur charges. It is accepted only when FRAMEFLOW_SMOKE_BILLABLE_AI_ACK=I_ACCEPT_BILLABLE_AI_CHARGES is also set, all seven AI workflows are configured, and the workspace has at least 700 remaining credits. Provider jobs use maxAttempts=1. Optional model-specific JSON objects can be supplied through FRAMEFLOW_SMOKE_VIDEO_PARAMS_JSON and FRAMEFLOW_SMOKE_LIPSYNC_PARAMS_JSON.` } async function main() { let options try { options = parseSmokeOptions() if (options.help) { console.log(usage()) return } const report = await runProductionSmoke(options) if (options.json) console.log(JSON.stringify(report, null, 2)) else console.log(`Production smoke test passed in ${report.durationMs}ms. Project ${report.projectId}${report.projectArchived ? ' was archived' : ' was retained'}.`) } catch (error) { const report = error instanceof SmokeTestError ? error.report : null if (options?.json && report) console.error(JSON.stringify(report, null, 2)) else console.error(`Production smoke test failed: ${safeError(error)}`) process.exitCode = 1 } } if (process.argv[1] && resolve(process.argv[1]) === scriptPath) await main()