928 lines
49 KiB
JavaScript
Executable File
928 lines
49 KiB
JavaScript
Executable File
#!/usr/bin/env node
|
|
|
|
import { createHash, randomUUID } from 'node:crypto'
|
|
import { readFile } from 'node:fs/promises'
|
|
import { basename, extname, resolve } from 'node:path'
|
|
import { fileURLToPath } from 'node:url'
|
|
|
|
const scriptPath = fileURLToPath(import.meta.url)
|
|
const defaultImagePath = fileURLToPath(new URL('../public/assets/scene-rooftop.jpg', import.meta.url))
|
|
const terminalJobStatuses = new Set(['SUCCEEDED', 'FAILED', 'CANCELLED'])
|
|
const billableAiAcknowledgement = 'I_ACCEPT_BILLABLE_AI_CHARGES'
|
|
export const requiredLiveAiWorkflowIds = ['script', 'storyboard', 'image-openai', 'image-replicate', 'tts', 'video', 'lipsync']
|
|
|
|
export class SmokeTestError extends Error {
|
|
constructor(message, options = {}) {
|
|
super(message, options)
|
|
this.name = 'SmokeTestError'
|
|
this.report = options.report
|
|
}
|
|
}
|
|
|
|
function required(value, name) {
|
|
if (typeof value !== 'string' || value.trim() === '') throw new SmokeTestError(`${name} is required`)
|
|
return value.trim()
|
|
}
|
|
|
|
function booleanValue(value) {
|
|
return ['1', 'true', 'yes', 'on'].includes(String(value ?? '').trim().toLowerCase())
|
|
}
|
|
|
|
function integerValue(value, name, minimum, maximum) {
|
|
const parsed = Number(value)
|
|
if (!Number.isInteger(parsed) || parsed < minimum || parsed > maximum) {
|
|
throw new SmokeTestError(`${name} must be an integer from ${minimum} to ${maximum}`)
|
|
}
|
|
return parsed
|
|
}
|
|
|
|
function jsonObjectValue(value, name) {
|
|
if (value === undefined || value === null || String(value).trim() === '') return {}
|
|
let parsed
|
|
try {
|
|
parsed = JSON.parse(String(value))
|
|
} catch {
|
|
throw new SmokeTestError(`${name} must be valid JSON`)
|
|
}
|
|
if (!parsed || Array.isArray(parsed) || typeof parsed !== 'object') {
|
|
throw new SmokeTestError(`${name} must be a JSON object`)
|
|
}
|
|
return parsed
|
|
}
|
|
|
|
export function resolveEndpoints(value, options = {}) {
|
|
const input = required(value, 'FRAMEFLOW_SMOKE_BASE_URL')
|
|
let url
|
|
try {
|
|
url = new URL(input)
|
|
} catch {
|
|
throw new SmokeTestError('FRAMEFLOW_SMOKE_BASE_URL must be a valid URL')
|
|
}
|
|
if (url.username || url.password || url.search || url.hash) {
|
|
throw new SmokeTestError('Smoke-test base URL cannot contain credentials, a query, or a fragment')
|
|
}
|
|
if (!['http:', 'https:'].includes(url.protocol)) throw new SmokeTestError('Smoke-test base URL must use HTTP or HTTPS')
|
|
const localHostnames = new Set(['localhost', '127.0.0.1', '::1', '[::1]'])
|
|
if (url.protocol === 'http:' && (!options.allowHttp || !localHostnames.has(url.hostname))) {
|
|
throw new SmokeTestError('HTTPS is required; HTTP can only be enabled explicitly for localhost')
|
|
}
|
|
const path = url.pathname.replace(/\/+$/, '')
|
|
if (path && path !== '/api/v1') {
|
|
throw new SmokeTestError('Smoke-test base URL path must be empty or /api/v1')
|
|
}
|
|
const origin = url.origin
|
|
return { origin, apiBase: `${origin}/api/v1` }
|
|
}
|
|
|
|
export function parseSmokeOptions(argv = process.argv.slice(2), environment = process.env) {
|
|
const values = {}
|
|
const flags = new Set()
|
|
const valueOptions = new Set(['--base-url', '--email', '--workspace-id', '--image', '--timeout-seconds', '--ai-video-seconds'])
|
|
const flagOptions = new Set(['--allow-http', '--keep-project', '--include-ai', '--json', '--help'])
|
|
for (let index = 0; index < argv.length; index += 1) {
|
|
const argument = argv[index]
|
|
if (valueOptions.has(argument)) {
|
|
const value = argv[index + 1]
|
|
if (!value || value.startsWith('--')) throw new SmokeTestError(`${argument} requires a value`)
|
|
values[argument] = value
|
|
index += 1
|
|
} else if (flagOptions.has(argument)) {
|
|
flags.add(argument)
|
|
} else {
|
|
throw new SmokeTestError(`Unknown option: ${argument}`)
|
|
}
|
|
}
|
|
if (flags.has('--help')) return { help: true }
|
|
|
|
const allowHttp = flags.has('--allow-http') || booleanValue(environment.FRAMEFLOW_SMOKE_ALLOW_HTTP)
|
|
const endpoints = resolveEndpoints(values['--base-url'] ?? environment.FRAMEFLOW_SMOKE_BASE_URL, { allowHttp })
|
|
const email = required(values['--email'] ?? environment.FRAMEFLOW_SMOKE_EMAIL, 'FRAMEFLOW_SMOKE_EMAIL').toLowerCase()
|
|
const password = required(environment.FRAMEFLOW_SMOKE_PASSWORD, 'FRAMEFLOW_SMOKE_PASSWORD')
|
|
const timeoutSeconds = integerValue(values['--timeout-seconds'] ?? environment.FRAMEFLOW_SMOKE_TIMEOUT_SECONDS ?? '180', 'timeout-seconds', 30, 900)
|
|
const includeAi = flags.has('--include-ai')
|
|
if (includeAi && environment.FRAMEFLOW_SMOKE_BILLABLE_AI_ACK !== billableAiAcknowledgement) {
|
|
throw new SmokeTestError(`--include-ai requires FRAMEFLOW_SMOKE_BILLABLE_AI_ACK=${billableAiAcknowledgement}`)
|
|
}
|
|
return {
|
|
...endpoints,
|
|
email,
|
|
password,
|
|
workspaceId: values['--workspace-id'] ?? environment.FRAMEFLOW_SMOKE_WORKSPACE_ID ?? null,
|
|
imagePath: resolve(values['--image'] ?? environment.FRAMEFLOW_SMOKE_IMAGE ?? defaultImagePath),
|
|
timeoutMs: timeoutSeconds * 1000,
|
|
requestTimeoutMs: Math.min(60_000, timeoutSeconds * 1000),
|
|
keepProject: flags.has('--keep-project') || booleanValue(environment.FRAMEFLOW_SMOKE_KEEP_PROJECT),
|
|
includeAi,
|
|
aiVideoSeconds: integerValue(values['--ai-video-seconds'] ?? environment.FRAMEFLOW_SMOKE_AI_VIDEO_SECONDS ?? '5', 'ai-video-seconds', 1, 30),
|
|
aiVideoParams: jsonObjectValue(environment.FRAMEFLOW_SMOKE_VIDEO_PARAMS_JSON, 'FRAMEFLOW_SMOKE_VIDEO_PARAMS_JSON'),
|
|
aiLipsyncParams: jsonObjectValue(environment.FRAMEFLOW_SMOKE_LIPSYNC_PARAMS_JSON, 'FRAMEFLOW_SMOKE_LIPSYNC_PARAMS_JSON'),
|
|
json: flags.has('--json'),
|
|
}
|
|
}
|
|
|
|
function safeError(error) {
|
|
const message = error instanceof Error ? error.message : String(error)
|
|
return message.replace(/([?&](?:X-Amz-[^=\s]+|token|signature)=)[^&\s]+/gi, '$1[redacted]').slice(0, 2000)
|
|
}
|
|
|
|
function assertCondition(condition, message) {
|
|
if (!condition) throw new SmokeTestError(message)
|
|
}
|
|
|
|
function sha256(value) {
|
|
return createHash('sha256').update(value).digest('hex')
|
|
}
|
|
|
|
function imageContentType(path) {
|
|
const extension = extname(path).toLowerCase()
|
|
if (extension === '.jpg' || extension === '.jpeg') return 'image/jpeg'
|
|
if (extension === '.png') return 'image/png'
|
|
if (extension === '.webp') return 'image/webp'
|
|
throw new SmokeTestError('Smoke-test image must be JPEG, PNG, or WebP')
|
|
}
|
|
|
|
function responseBody(response) {
|
|
if (response.status === 204) return Promise.resolve(null)
|
|
const contentType = response.headers.get('content-type') ?? ''
|
|
if (contentType.includes('application/json')) return response.json()
|
|
return response.text()
|
|
}
|
|
|
|
async function fetchWithTimeout(fetchImpl, url, options, timeoutMs) {
|
|
const controller = new AbortController()
|
|
const timer = setTimeout(() => controller.abort(new Error(`Request timed out after ${timeoutMs}ms`)), timeoutMs)
|
|
try {
|
|
return await fetchImpl(url, { ...options, signal: controller.signal })
|
|
} finally {
|
|
clearTimeout(timer)
|
|
}
|
|
}
|
|
|
|
function createApiClient(options, fetchImpl) {
|
|
let accessToken = null
|
|
const cookies = new Map()
|
|
|
|
function captureCookies(headers) {
|
|
const values = typeof headers.getSetCookie === 'function'
|
|
? headers.getSetCookie()
|
|
: headers.get('set-cookie') ? [headers.get('set-cookie')] : []
|
|
for (const value of values) {
|
|
const pair = value?.split(';', 1)[0]
|
|
const separator = pair?.indexOf('=') ?? -1
|
|
if (!pair || separator < 1) continue
|
|
const name = pair.slice(0, separator).trim()
|
|
const cookieValue = pair.slice(separator + 1).trim()
|
|
if (!cookieValue || /(?:^|;)\s*max-age=0(?:;|$)/i.test(value)) cookies.delete(name)
|
|
else cookies.set(name, cookieValue)
|
|
}
|
|
}
|
|
|
|
async function raw(path, request = {}) {
|
|
const headers = new Headers(request.headers)
|
|
const url = path.startsWith('http://') || path.startsWith('https://') ? path : `${options.apiBase}${path}`
|
|
const authenticatedApiRequest = request.auth !== false && (url === options.apiBase || url.startsWith(`${options.apiBase}/`))
|
|
if (accessToken && authenticatedApiRequest) headers.set('authorization', `Bearer ${accessToken}`)
|
|
if (authenticatedApiRequest && cookies.size > 0) {
|
|
headers.set('cookie', [...cookies].map(([name, value]) => `${name}=${value}`).join('; '))
|
|
}
|
|
let body = request.body
|
|
if (request.json !== undefined) {
|
|
headers.set('content-type', 'application/json')
|
|
body = JSON.stringify(request.json)
|
|
}
|
|
const response = await fetchWithTimeout(fetchImpl, url, {
|
|
method: request.method ?? 'GET',
|
|
headers,
|
|
body,
|
|
redirect: request.redirect ?? 'follow',
|
|
}, request.timeoutMs ?? options.requestTimeoutMs)
|
|
captureCookies(response.headers)
|
|
return response
|
|
}
|
|
async function request(path, input = {}) {
|
|
const response = await raw(path, input)
|
|
const body = await responseBody(response)
|
|
if (!response.ok) {
|
|
const message = body?.error?.message ?? (typeof body === 'string' && body) ?? `HTTP ${response.status}`
|
|
throw new SmokeTestError(`${input.method ?? 'GET'} ${path} failed (${response.status}): ${message}`)
|
|
}
|
|
return body
|
|
}
|
|
return {
|
|
raw,
|
|
request,
|
|
setAccessToken(value) { accessToken = value },
|
|
clearSession() {
|
|
accessToken = null
|
|
cookies.clear()
|
|
},
|
|
}
|
|
}
|
|
|
|
export async function waitForJob(getJob, jobId, options = {}) {
|
|
const timeoutMs = options.timeoutMs ?? 180_000
|
|
const pollIntervalMs = options.pollIntervalMs ?? 500
|
|
const sleep = options.sleep ?? ((milliseconds) => new Promise((resolvePromise) => setTimeout(resolvePromise, milliseconds)))
|
|
const now = options.now ?? Date.now
|
|
const startedAt = now()
|
|
let lastStatus = null
|
|
while (now() - startedAt <= timeoutMs) {
|
|
const job = await getJob(jobId)
|
|
lastStatus = job?.status ?? null
|
|
if (lastStatus === 'SUCCEEDED') return job
|
|
if (terminalJobStatuses.has(lastStatus)) {
|
|
throw new SmokeTestError(`Job ${jobId} ended in ${lastStatus}: ${job.errorMessage ?? job.errorCode ?? 'unknown error'}`)
|
|
}
|
|
await sleep(pollIntervalMs)
|
|
}
|
|
throw new SmokeTestError(`Job ${jobId} did not finish within ${timeoutMs}ms (last status: ${lastStatus ?? 'unknown'})`)
|
|
}
|
|
|
|
export async function waitForNotifications(getNotifications, resourceIds, options = {}) {
|
|
const timeoutMs = options.timeoutMs ?? 10_000
|
|
const pollIntervalMs = options.pollIntervalMs ?? 250
|
|
const sleep = options.sleep ?? ((milliseconds) => new Promise((resolvePromise) => setTimeout(resolvePromise, milliseconds)))
|
|
const now = options.now ?? Date.now
|
|
const expected = new Set(resourceIds)
|
|
const startedAt = now()
|
|
let lastItems = []
|
|
while (now() - startedAt <= timeoutMs) {
|
|
const page = await getNotifications()
|
|
lastItems = Array.isArray(page?.items) ? page.items : []
|
|
const matched = lastItems.filter((notification) => expected.has(notification.resourceId))
|
|
if (new Set(matched.map((notification) => notification.resourceId)).size === expected.size) return matched
|
|
await sleep(pollIntervalMs)
|
|
}
|
|
const matchedIds = new Set(lastItems.map((notification) => notification.resourceId))
|
|
const missing = [...expected].filter((resourceId) => !matchedIds.has(resourceId))
|
|
throw new SmokeTestError(`Notifications did not arrive within ${timeoutMs}ms (missing resources: ${missing.join(', ')})`)
|
|
}
|
|
|
|
function selectWorkspace(workspaces, requestedId) {
|
|
assertCondition(Array.isArray(workspaces) && workspaces.length > 0, 'Smoke account has no workspace membership')
|
|
const workspace = requestedId
|
|
? workspaces.find((candidate) => candidate.id === requestedId)
|
|
: workspaces.length === 1 ? workspaces[0] : null
|
|
if (!workspace) {
|
|
throw new SmokeTestError(requestedId
|
|
? `Smoke account is not a member of workspace ${requestedId}`
|
|
: 'Smoke account belongs to multiple workspaces; set FRAMEFLOW_SMOKE_WORKSPACE_ID')
|
|
}
|
|
assertCondition(['OWNER', 'ADMIN'].includes(workspace.role), 'Full smoke test requires an OWNER or ADMIN workspace role')
|
|
return workspace
|
|
}
|
|
|
|
async function downloadBytes(fetchImpl, url, timeoutMs, expectedType) {
|
|
const response = await fetchWithTimeout(fetchImpl, url, { redirect: 'follow' }, timeoutMs)
|
|
assertCondition(response.ok, `Media download failed with HTTP ${response.status}`)
|
|
const contentType = response.headers.get('content-type') ?? ''
|
|
if (expectedType) assertCondition(contentType.toLowerCase().includes(expectedType), `Unexpected media content type: ${contentType || 'missing'}`)
|
|
return Buffer.from(await response.arrayBuffer())
|
|
}
|
|
|
|
function readinessSummary(readiness) {
|
|
return {
|
|
infrastructure: readiness.infrastructure,
|
|
ai: {
|
|
status: readiness.ai?.status ?? 'UNKNOWN',
|
|
verificationMode: readiness.ai?.verificationMode ?? 'UNKNOWN',
|
|
summary: readiness.ai?.summary ?? null,
|
|
workflows: (readiness.ai?.workflows ?? []).map((workflow) => ({
|
|
id: workflow.id,
|
|
provider: workflow.provider,
|
|
model: workflow.model,
|
|
status: workflow.status,
|
|
})),
|
|
providers: (readiness.ai?.providers ?? []).map((provider) => ({
|
|
id: provider.id,
|
|
status: provider.status,
|
|
verification: provider.verification,
|
|
configuredCapabilities: provider.configuredCapabilities,
|
|
totalCapabilities: provider.totalCapabilities,
|
|
})),
|
|
},
|
|
}
|
|
}
|
|
|
|
export function assertLiveAiReady(readiness, minimumCredits = 700, billing = null) {
|
|
const capabilityRows = [
|
|
...(readiness?.ai?.providers ?? []).flatMap((provider) => provider.capabilities ?? []),
|
|
...(readiness?.ai?.workflows ?? []),
|
|
]
|
|
const workflows = new Map(capabilityRows.map((workflow) => [workflow.id, workflow]))
|
|
const missing = requiredLiveAiWorkflowIds.filter((id) => workflows.get(id)?.status !== 'CONFIGURED')
|
|
if (missing.length > 0) {
|
|
const details = missing.map((id) => {
|
|
const workflow = workflows.get(id)
|
|
const requirements = workflow?.missingRequirements?.join(', ')
|
|
return requirements ? `${id} (${requirements})` : id
|
|
})
|
|
throw new SmokeTestError(`Live AI smoke test requires configured workflows: ${details.join('; ')}`)
|
|
}
|
|
if (billing && Number(billing.remainingCredits) < minimumCredits) {
|
|
throw new SmokeTestError(`Live AI smoke test requires at least ${minimumCredits} remaining credits`)
|
|
}
|
|
return requiredLiveAiWorkflowIds.map((id) => {
|
|
const workflow = workflows.get(id)
|
|
return { id, provider: workflow.provider, model: workflow.model }
|
|
})
|
|
}
|
|
|
|
export async function runProductionSmoke(options, dependencies = {}) {
|
|
const fetchImpl = dependencies.fetch ?? fetch
|
|
const readFileImpl = dependencies.readFile ?? readFile
|
|
const sleep = dependencies.sleep ?? ((milliseconds) => new Promise((resolvePromise) => setTimeout(resolvePromise, milliseconds)))
|
|
const now = dependencies.now ?? Date.now
|
|
const log = dependencies.log ?? ((message) => console.error(message))
|
|
const api = createApiClient(options, fetchImpl)
|
|
const startedAt = now()
|
|
const runId = randomUUID()
|
|
const marker = `FRAMEFLOW_SMOKE_${runId}`
|
|
const report = {
|
|
status: 'running',
|
|
runId,
|
|
baseUrl: options.origin,
|
|
startedAt: new Date(startedAt).toISOString(),
|
|
workspaceId: null,
|
|
projectId: null,
|
|
projectArchived: false,
|
|
checks: [],
|
|
services: null,
|
|
billing: null,
|
|
aiVerification: {
|
|
mode: options.includeAi ? 'LIVE_PROVIDER_CALLS' : 'CONFIGURATION_ONLY',
|
|
jobs: [],
|
|
artifacts: {},
|
|
},
|
|
artifacts: {},
|
|
}
|
|
let projectId = null
|
|
let loggedIn = false
|
|
let smokeSessionId = null
|
|
let primaryError = null
|
|
|
|
async function step(id, label, action, summarize) {
|
|
const stepStartedAt = now()
|
|
log(`[...] ${label}`)
|
|
try {
|
|
const value = await action()
|
|
const check = { id, label, status: 'passed', durationMs: Math.max(0, now() - stepStartedAt) }
|
|
const details = summarize?.(value)
|
|
if (details !== undefined) check.details = details
|
|
report.checks.push(check)
|
|
log(`[ ok] ${label} (${check.durationMs}ms)`)
|
|
return value
|
|
} catch (error) {
|
|
const check = { id, label, status: 'failed', durationMs: Math.max(0, now() - stepStartedAt), error: safeError(error) }
|
|
report.checks.push(check)
|
|
log(`[fail] ${label}: ${check.error}`)
|
|
throw error
|
|
}
|
|
}
|
|
|
|
async function runAiJob(projectIdForJob, episodeIdForJob, type, input) {
|
|
const queued = await api.request(`/projects/${projectIdForJob}/jobs`, {
|
|
method: 'POST',
|
|
json: {
|
|
type,
|
|
episodeId: episodeIdForJob,
|
|
idempotencyKey: `smoke-ai-${runId}-${report.aiVerification.jobs.length}-${type.toLowerCase()}`,
|
|
input,
|
|
maxAttempts: 1,
|
|
},
|
|
})
|
|
const job = await waitForJob((jobId) => api.request(`/jobs/${jobId}`), queued.job.id, {
|
|
timeoutMs: options.timeoutMs,
|
|
sleep,
|
|
})
|
|
assertCondition(['openai', 'replicate'].includes(job.provider), `${type} did not record a live AI provider`)
|
|
assertCondition(job.providerModel, `${type} did not record a provider model`)
|
|
report.aiVerification.jobs.push({ id: job.id, type, provider: job.provider, model: job.providerModel })
|
|
return job
|
|
}
|
|
|
|
try {
|
|
await step('liveness', 'API liveness', async () => {
|
|
const response = await fetchWithTimeout(fetchImpl, `${options.origin}/health/live`, {}, options.requestTimeoutMs)
|
|
const body = await responseBody(response)
|
|
assertCondition(response.ok && body?.status === 'ok', `Liveness failed with HTTP ${response.status}`)
|
|
return body
|
|
})
|
|
|
|
await step('readiness', 'API dependency readiness', async () => {
|
|
const response = await fetchWithTimeout(fetchImpl, `${options.origin}/health/ready`, {}, options.requestTimeoutMs)
|
|
const body = await responseBody(response)
|
|
assertCondition(response.ok && body?.status === 'ready', `Readiness failed with HTTP ${response.status}`)
|
|
const unavailable = Object.entries(body.dependencies ?? {}).filter(([, status]) => status !== 'ok')
|
|
assertCondition(unavailable.length === 0, `Unavailable dependencies: ${unavailable.map(([name]) => name).join(', ')}`)
|
|
return body
|
|
}, (body) => body.dependencies)
|
|
|
|
const session = await step('login', 'Dedicated smoke-account login', async () => {
|
|
const value = await api.request('/auth/login', {
|
|
method: 'POST',
|
|
auth: false,
|
|
json: { email: options.email, password: options.password },
|
|
})
|
|
assertCondition(value?.accessToken && value?.user?.id, 'Login response is missing the authenticated session')
|
|
api.setAccessToken(value.accessToken)
|
|
loggedIn = true
|
|
const sessions = await api.request('/auth/sessions')
|
|
const currentSession = sessions.find((candidate) => candidate.current)
|
|
assertCondition(currentSession?.id, 'Login refresh session cookie was not retained by the smoke runner')
|
|
smokeSessionId = currentSession.id
|
|
return value
|
|
}, (value) => ({ userId: value.user.id, sessionId: smokeSessionId, workspaceCount: value.workspaces.length }))
|
|
|
|
const workspace = selectWorkspace(session.workspaces, options.workspaceId)
|
|
report.workspaceId = workspace.id
|
|
const serviceReadiness = await step('service-readiness', 'Worker, storage, FFmpeg, and AI configuration', async () => {
|
|
const value = await api.request(`/workspaces/${workspace.id}/service-readiness`)
|
|
assertCondition(value?.infrastructure?.status === 'AVAILABLE', 'Authenticated service readiness is degraded')
|
|
assertCondition(value.infrastructure.available === value.infrastructure.total, 'Not every production service is available')
|
|
return value
|
|
}, (value) => ({
|
|
infrastructure: value.infrastructure.services.map((service) => ({ id: service.id, status: service.status, latencyMs: service.latencyMs })),
|
|
ai: value.ai.summary,
|
|
}))
|
|
report.services = readinessSummary(serviceReadiness)
|
|
|
|
const billingReadModel = await step('billing', 'Billing and usage read model', async () => {
|
|
const value = await api.request(`/workspaces/${workspace.id}/billing/summary`)
|
|
assertCondition(value?.plan && value?.provider?.status, 'Billing summary is incomplete')
|
|
return value
|
|
}, (value) => ({ plan: value.plan, provider: value.provider.status, webhookConfigured: value.provider.webhookConfigured, remainingCredits: value.remainingCredits }))
|
|
report.billing = {
|
|
plan: billingReadModel.plan,
|
|
provider: billingReadModel.provider.status,
|
|
webhookConfigured: billingReadModel.provider.webhookConfigured,
|
|
remainingCredits: billingReadModel.remainingCredits,
|
|
}
|
|
|
|
if (options.includeAi) {
|
|
await step('ai-preflight', 'Billable AI authorization, configuration, and credit preflight', async () => {
|
|
const workflows = assertLiveAiReady(serviceReadiness, 700, billingReadModel)
|
|
return { acknowledgement: 'accepted', minimumCredits: 700, workflows }
|
|
}, (value) => value)
|
|
}
|
|
|
|
const project = await step('project', 'Project persistence', async () => api.request(`/workspaces/${workspace.id}/projects`, {
|
|
method: 'POST',
|
|
json: {
|
|
name: `[SMOKE] ${new Date(startedAt).toISOString()} ${runId.slice(0, 8)}`,
|
|
logline: 'Automated production smoke test. Safe to archive after completion.',
|
|
genre: 'SMOKE_TEST',
|
|
visualBible: { smokeTest: { runId, createdAt: new Date(startedAt).toISOString() } },
|
|
},
|
|
}), (value) => ({ projectId: value.id }))
|
|
projectId = project.id
|
|
report.projectId = projectId
|
|
|
|
const episode = await step('script', 'Episode and immutable script persistence', async () => {
|
|
const createdEpisode = await api.request(`/projects/${projectId}/episodes`, {
|
|
method: 'POST',
|
|
json: { episodeNumber: 1, title: 'Production smoke episode', synopsis: marker, targetDurationSeconds: 15 },
|
|
})
|
|
const script = await api.request(`/episodes/${createdEpisode.id}/scripts`, {
|
|
method: 'POST',
|
|
json: {
|
|
title: 'Production smoke script',
|
|
content: { smokeTest: true, runId, scenes: [{ number: 1, text: marker }] },
|
|
plainText: `SCENE 1 - SMOKE TEST\n${marker}`,
|
|
changeSummary: 'Production smoke test',
|
|
},
|
|
})
|
|
const persisted = await api.request(`/episodes/${createdEpisode.id}`)
|
|
assertCondition(persisted.currentScript?.id === script.id, 'Current script version was not persisted')
|
|
assertCondition(persisted.currentScript?.plainText?.includes(marker), 'Persisted script marker is missing')
|
|
return { episode: createdEpisode, script }
|
|
}, (value) => ({ episodeId: value.episode.id, scriptVersion: value.script.version }))
|
|
|
|
const sourceImage = await readFileImpl(options.imagePath)
|
|
const sourceHash = sha256(sourceImage)
|
|
const contentType = imageContentType(options.imagePath)
|
|
const imageAssetVersion = await step('upload', 'Object upload and signed download integrity', async () => {
|
|
const form = new FormData()
|
|
form.append('file', new Blob([sourceImage], { type: contentType }), basename(options.imagePath))
|
|
const upload = await api.request(`/projects/${projectId}/uploads`, { method: 'POST', body: form })
|
|
assertCondition(upload.byteSize === sourceImage.length, 'Uploaded object byte size does not match the source')
|
|
const asset = await api.request(`/projects/${projectId}/assets`, {
|
|
method: 'POST',
|
|
json: { type: 'SHOT', name: `Smoke source ${runId.slice(0, 8)}`, description: marker, tags: ['smoke-test'], metadata: { smokeRunId: runId } },
|
|
})
|
|
const version = await api.request(`/assets/${asset.id}/versions`, {
|
|
method: 'POST',
|
|
json: { storageKey: upload.key, mimeType: upload.contentType, byteSize: upload.byteSize, prompt: marker, provider: 'smoke-test', providerModel: 'uploaded-source-v1' },
|
|
})
|
|
const signed = await api.request(`/assets/${asset.id}/versions/${version.id}/url`)
|
|
const downloaded = await downloadBytes(fetchImpl, signed.url, options.requestTimeoutMs, contentType)
|
|
assertCondition(sha256(downloaded) === sourceHash, 'Signed object download does not match the uploaded image')
|
|
return { asset, version, upload }
|
|
}, (value) => ({ assetId: value.asset.id, assetVersionId: value.version.id, byteSize: value.upload.byteSize, sha256: sourceHash }))
|
|
|
|
const shot = await step('shot', 'Shot and immutable media-version persistence', async () => {
|
|
const created = await api.request(`/episodes/${episode.episode.id}/shots`, {
|
|
method: 'POST',
|
|
json: { shotNumber: 1, sceneNumber: 1, title: 'Smoke render shot', description: marker, shotType: '全景', cameraMotion: '静止', durationMs: 2000, characterAssetIds: [], sortOrder: 100 },
|
|
})
|
|
const version = await api.request(`/shots/${created.id}/versions`, {
|
|
method: 'POST',
|
|
json: { imageAssetVersionId: imageAssetVersion.version.id, subtitleText: marker, generationParams: { smokeRunId: runId } },
|
|
})
|
|
const persisted = await api.request(`/shots/${created.id}`)
|
|
assertCondition(persisted.versions?.[0]?.id === version.id, 'Current shot version was not persisted')
|
|
return { shot: created, version }
|
|
}, (value) => ({ shotId: value.shot.id, shotVersion: value.version.version }))
|
|
|
|
await step('queue', 'Redis/BullMQ worker execution', async () => {
|
|
const queued = await api.request(`/projects/${projectId}/jobs`, {
|
|
method: 'POST',
|
|
json: { type: 'PIPELINE_HEALTHCHECK', episodeId: episode.episode.id, idempotencyKey: `smoke-health-${runId}`, input: { marker }, maxAttempts: 1 },
|
|
})
|
|
const job = await waitForJob((jobId) => api.request(`/jobs/${jobId}`), queued.job.id, { timeoutMs: options.timeoutMs, sleep })
|
|
assertCondition(job.output?.ok === true && job.output?.echo?.marker === marker, 'Worker healthcheck output is invalid')
|
|
return job
|
|
}, (job) => ({ jobId: job.id, provider: job.provider, providerModel: job.providerModel }))
|
|
|
|
const renderResult = await step('render', 'FFmpeg render and immutable source manifest', async () => {
|
|
const queued = await api.request(`/episodes/${episode.episode.id}/renders`, {
|
|
method: 'POST',
|
|
json: {
|
|
idempotencyKey: `smoke-render-${runId}`,
|
|
settings: { width: 640, height: 360, fps: 24, quality: 'draft', burnSubtitles: false, normalizeAudio: false, musicVolume: 0, subtitleStyle: { fontSize: 20, textColor: '#FFFFFF', outlineColor: '#000000', outlineWidth: 2, bottomMargin: 24, bold: true } },
|
|
},
|
|
})
|
|
const job = await waitForJob((jobId) => api.request(`/jobs/${jobId}`), queued.job.id, { timeoutMs: options.timeoutMs, sleep })
|
|
const render = await api.request(`/renders/${queued.render.id}`)
|
|
assertCondition(render.status === 'REVIEW', `Render ended in unexpected status ${render.status}`)
|
|
assertCondition(render.sourceShotCount === 1, 'Render source manifest does not contain exactly one shot')
|
|
assertCondition(render.subtitleFormats?.includes('srt') && render.subtitleFormats?.includes('vtt'), 'Render subtitle sidecars are missing')
|
|
assertCondition(render.downloadUrl, 'Render download URL is missing')
|
|
const video = await downloadBytes(fetchImpl, render.downloadUrl, options.requestTimeoutMs, 'video/mp4')
|
|
assertCondition(video.length > 1000, 'Rendered video is unexpectedly small')
|
|
return { render, job, videoHash: sha256(video), videoBytes: video.length }
|
|
}, (value) => ({ renderId: value.render.id, jobId: value.job.id, byteSize: value.videoBytes, subtitleFormats: value.render.subtitleFormats, sourceShotCount: value.render.sourceShotCount }))
|
|
report.artifacts.renderId = renderResult.render.id
|
|
report.artifacts.videoSha256 = renderResult.videoHash
|
|
|
|
await step('subtitles', 'SRT and VTT sidecar downloads', async () => {
|
|
const results = {}
|
|
for (const format of ['srt', 'vtt']) {
|
|
const subtitle = await api.request(`/renders/${renderResult.render.id}/subtitles/${format}`)
|
|
const bytes = await downloadBytes(fetchImpl, subtitle.downloadUrl, options.requestTimeoutMs)
|
|
const text = bytes.toString('utf8')
|
|
assertCondition(text.includes(marker), `${format.toUpperCase()} subtitle does not contain the smoke marker`)
|
|
results[format] = { bytes: bytes.length, sha256: sha256(bytes) }
|
|
}
|
|
return results
|
|
}, (value) => value)
|
|
|
|
await step('review', 'Render review and approval', async () => {
|
|
const review = await api.request(`/workspaces/${workspace.id}/reviews`, {
|
|
method: 'POST',
|
|
json: { targetType: 'RENDER', targetId: renderResult.render.id, assignedToId: session.user.id, summary: `Automated approval for ${marker}` },
|
|
})
|
|
const decision = await api.request(`/reviews/${review.id}/decision`, {
|
|
method: 'POST',
|
|
json: { status: 'APPROVED', summary: 'Production smoke render passed automated checks.' },
|
|
})
|
|
const approved = await api.request(`/renders/${renderResult.render.id}`)
|
|
assertCondition(decision.status === 'APPROVED' && approved.status === 'APPROVED', 'Render approval was not persisted')
|
|
return { review, approved }
|
|
}, (value) => ({ reviewId: value.review.id, renderStatus: value.approved.status }))
|
|
|
|
const deliveryResult = await step('delivery', 'Approved download delivery through the worker', async () => {
|
|
const queued = await api.request(`/renders/${renderResult.render.id}/deliveries`, {
|
|
method: 'POST',
|
|
json: { platform: 'DOWNLOAD', destination: { smokeRunId: runId } },
|
|
})
|
|
const job = await waitForJob((jobId) => api.request(`/jobs/${jobId}`), queued.job.id, { timeoutMs: options.timeoutMs, sleep })
|
|
const delivery = await api.request(`/deliveries/${queued.delivery.id}`)
|
|
assertCondition(delivery.status === 'PUBLISHED' && delivery.downloadUrl, 'Download delivery was not published')
|
|
const redirect = await api.raw(`/deliveries/${delivery.id}/download`, { redirect: 'manual' })
|
|
assertCondition([301, 302, 303, 307, 308].includes(redirect.status), `Delivery endpoint returned HTTP ${redirect.status} instead of a redirect`)
|
|
const location = redirect.headers.get('location')
|
|
assertCondition(location, 'Delivery redirect is missing its signed location')
|
|
const video = await downloadBytes(fetchImpl, location, options.requestTimeoutMs, 'video/mp4')
|
|
assertCondition(sha256(video) === renderResult.videoHash, 'Delivered video differs from the approved render')
|
|
return { delivery, job, bytes: video.length }
|
|
}, (value) => ({ deliveryId: value.delivery.id, jobId: value.job.id, status: value.delivery.status, byteSize: value.bytes }))
|
|
report.artifacts.deliveryId = deliveryResult.delivery.id
|
|
|
|
await step('notifications', 'Persistent job notifications and read state', async () => {
|
|
const expectedJobIds = [renderResult.job.id, deliveryResult.job.id]
|
|
const rows = await waitForNotifications(
|
|
() => api.request(`/workspaces/${workspace.id}/notifications?limit=100`),
|
|
expectedJobIds,
|
|
{ timeoutMs: Math.min(options.timeoutMs, 10_000), sleep, now },
|
|
)
|
|
for (const jobId of expectedJobIds) {
|
|
const notification = rows.find((candidate) => candidate.resourceId === jobId)
|
|
assertCondition(notification?.kind === 'JOB_SUCCEEDED' && notification.tone === 'SUCCESS', `Successful job notification is invalid for ${jobId}`)
|
|
const expectedPage = jobId === deliveryResult.job.id ? 'publish' : 'jobs'
|
|
assertCondition(notification.page === expectedPage, `Job notification points to ${notification.page} instead of ${expectedPage}`)
|
|
const read = await api.request(`/notifications/${notification.id}`, { method: 'PATCH', json: { read: true } })
|
|
assertCondition(read.readAt, `Notification ${notification.id} did not persist its read state`)
|
|
}
|
|
return rows
|
|
}, (rows) => ({ notifications: rows.length, resources: rows.map((notification) => notification.resourceId) }))
|
|
|
|
await step('audit', 'Audit-log persistence', async () => {
|
|
const page = await api.request(`/workspaces/${workspace.id}/audit-logs?limit=100&actorId=${session.user.id}`)
|
|
const actions = new Set((page.items ?? []).map((item) => item.action))
|
|
for (const action of ['project.create', 'upload.create', 'render.create', 'review.decide', 'delivery.create']) {
|
|
assertCondition(actions.has(action), `Audit log is missing ${action}`)
|
|
}
|
|
return { actions: actions.size }
|
|
}, (value) => value)
|
|
|
|
if (options.includeAi) {
|
|
const aiSetup = await step('ai-setup', 'Live AI validation workspace data', async () => {
|
|
const aiEpisode = await api.request(`/projects/${projectId}/episodes`, {
|
|
method: 'POST',
|
|
json: {
|
|
episodeNumber: 2,
|
|
title: 'Live provider validation',
|
|
synopsis: `A single character says a short line in a controlled studio. ${marker}`,
|
|
targetDurationSeconds: 15,
|
|
},
|
|
})
|
|
const character = await api.request(`/projects/${projectId}/assets`, {
|
|
method: 'POST',
|
|
json: {
|
|
type: 'CHARACTER',
|
|
name: `Provider test character ${runId.slice(0, 8)}`,
|
|
description: 'Adult presenter, centered portrait, neutral studio lighting, production-safe wardrobe.',
|
|
tags: ['smoke-test', 'provider-validation'],
|
|
metadata: { smokeRunId: runId },
|
|
},
|
|
})
|
|
return { episode: aiEpisode, character }
|
|
}, (value) => ({ episodeId: value.episode.id, characterAssetId: value.character.id }))
|
|
|
|
const openAiImage = await step('ai-image-openai', 'OpenAI character image generation', async () => {
|
|
const job = await runAiJob(projectId, aiSetup.episode.id, 'ASSET_GENERATE', {
|
|
assetId: aiSetup.character.id,
|
|
prompt: `Clean comic-drama character reference sheet. Adult presenter, centered bust portrait, neutral gray studio, consistent facial landmarks. ${marker}`,
|
|
negativePrompt: 'text, watermark, extra limbs, child',
|
|
width: 1024,
|
|
height: 1024,
|
|
provider: 'openai',
|
|
referenceAssetVersionIds: [],
|
|
params: { consistencyStrength: 82 },
|
|
})
|
|
const asset = await api.request(`/assets/${aiSetup.character.id}`)
|
|
const version = asset.versions?.find((candidate) => candidate.id === job.output?.assetVersionId)
|
|
assertCondition(version?.storageKey && version.provider === 'openai', 'OpenAI image version was not persisted as current asset media')
|
|
const signed = await api.request(`/assets/${asset.id}/versions/${version.id}/url`)
|
|
const bytes = await downloadBytes(fetchImpl, signed.url, options.requestTimeoutMs, 'image/')
|
|
assertCondition(bytes.length > 1000, 'OpenAI image output is unexpectedly small')
|
|
return { job, asset, version, bytes }
|
|
}, (value) => ({ jobId: value.job.id, assetVersionId: value.version.id, model: value.job.providerModel, byteSize: value.bytes.length }))
|
|
report.aiVerification.artifacts.openAiImageAssetVersionId = openAiImage.version.id
|
|
|
|
const replicateImage = await step('ai-image-replicate', 'Replicate character-consistency variant', async () => {
|
|
const profile = await api.request(`/assets/${aiSetup.character.id}/consistency-profile`, {
|
|
method: 'PUT',
|
|
json: {
|
|
referenceAssetVersionIds: [openAiImage.version.id],
|
|
identityPrompt: 'Preserve facial proportions, eye shape, hairstyle, and adult age.',
|
|
costumePrompt: 'Preserve the dark production jacket and simple collar.',
|
|
stylePrompt: 'Polished cinematic comic-drama illustration.',
|
|
negativePrompt: 'identity drift, age change, child, text, watermark',
|
|
consistencyStrength: 90,
|
|
identityLocked: true,
|
|
},
|
|
})
|
|
assertCondition(profile.consistencyProfile?.identityLocked, 'Character identity profile was not locked')
|
|
const job = await runAiJob(projectId, aiSetup.episode.id, 'ASSET_GENERATE', {
|
|
assetId: aiSetup.character.id,
|
|
prompt: `The same adult presenter turns slightly toward camera under warm key light. ${marker}`,
|
|
negativePrompt: 'text, watermark',
|
|
width: 1024,
|
|
height: 1024,
|
|
provider: 'replicate',
|
|
referenceAssetVersionIds: [openAiImage.version.id],
|
|
params: { consistencyStrength: 90 },
|
|
})
|
|
const asset = await api.request(`/assets/${aiSetup.character.id}`)
|
|
const version = asset.versions?.find((candidate) => candidate.id === job.output?.assetVersionId)
|
|
const appliedProfiles = version?.generationParams?.appliedCharacterConsistencyProfiles
|
|
assertCondition(version?.storageKey && version.provider === 'replicate', 'Replicate image version was not persisted')
|
|
assertCondition(Array.isArray(appliedProfiles) && appliedProfiles.some((snapshot) => snapshot.assetId === aiSetup.character.id && snapshot.identityLocked), 'Replicate variant did not capture the locked character profile')
|
|
assertCondition(version.generationParams?.referenceAssetVersionIds?.includes(openAiImage.version.id), 'Replicate variant did not record its identity reference')
|
|
const signed = await api.request(`/assets/${asset.id}/versions/${version.id}/url`)
|
|
const bytes = await downloadBytes(fetchImpl, signed.url, options.requestTimeoutMs, 'image/')
|
|
assertCondition(bytes.length > 1000, 'Replicate image output is unexpectedly small')
|
|
return { job, asset, version, bytes }
|
|
}, (value) => ({ jobId: value.job.id, assetVersionId: value.version.id, model: value.job.providerModel, byteSize: value.bytes.length }))
|
|
report.aiVerification.artifacts.replicateImageAssetVersionId = replicateImage.version.id
|
|
|
|
await step('ai-script', 'OpenAI script generation and immutable persistence', async () => {
|
|
const job = await runAiJob(projectId, aiSetup.episode.id, 'SCRIPT_GENERATE', {
|
|
provider: 'openai',
|
|
includeExistingScript: false,
|
|
instruction: 'Write one production-safe interior scene for a 15-second Chinese comic drama. Use one adult presenter, one short spoken line, and a clear visual action.',
|
|
})
|
|
const episode = await api.request(`/episodes/${aiSetup.episode.id}`)
|
|
assertCondition(episode.currentScript?.id === job.output?.scriptVersionId, 'Generated script was not activated as the current version')
|
|
assertCondition(episode.currentScript?.plainText?.trim(), 'Generated script has no plain text')
|
|
return { job, script: episode.currentScript }
|
|
}, (value) => ({ jobId: value.job.id, scriptVersionId: value.script.id, version: value.script.version, model: value.job.providerModel }))
|
|
|
|
const storyboard = await step('ai-storyboard', 'OpenAI storyboard generation and shot persistence', async () => {
|
|
const job = await runAiJob(projectId, aiSetup.episode.id, 'STORYBOARD_GENERATE', {
|
|
provider: 'openai',
|
|
replaceExisting: false,
|
|
instruction: `Create the smallest viable storyboard for provider validation. Prefer exactly one shot featuring ${aiSetup.character.name}, with one short spoken line and a total duration near five seconds.`,
|
|
})
|
|
const rows = await api.request(`/episodes/${aiSetup.episode.id}/shots`)
|
|
assertCondition(Array.isArray(rows) && rows.length > 0, 'Generated storyboard contains no shots')
|
|
assertCondition(job.output?.shotCount === rows.length, 'Storyboard job output does not match persisted shots')
|
|
const [selected, ...extra] = rows
|
|
for (const row of extra) await api.request(`/shots/${row.shot.id}`, { method: 'DELETE' })
|
|
const validationText = 'FrameFlow AI provider validation passed.'
|
|
const version = await api.request(`/shots/${selected.shot.id}/versions`, {
|
|
method: 'POST',
|
|
json: {
|
|
imageAssetVersionId: replicateImage.version.id,
|
|
subtitleText: validationText,
|
|
generationParams: { smokeRunId: runId, source: 'live-provider-smoke' },
|
|
},
|
|
})
|
|
return { job, shot: selected.shot, version, removedShots: extra.length, validationText }
|
|
}, (value) => ({ jobId: value.job.id, shotId: value.shot.id, generatedShots: value.job.output.shotCount, removedShots: value.removedShots, model: value.job.providerModel }))
|
|
|
|
await step('ai-tts', 'OpenAI TTS generation and audio storage', async () => {
|
|
const job = await runAiJob(projectId, aiSetup.episode.id, 'TTS_GENERATE', {
|
|
shotId: storyboard.shot.id,
|
|
text: storyboard.validationText,
|
|
voice: 'alloy',
|
|
speed: 1,
|
|
provider: 'openai',
|
|
})
|
|
const media = await api.request(`/shots/${storyboard.shot.id}/media`)
|
|
assertCondition(job.output?.audioStorageKey && media.audioUrl, 'TTS output was not bound to the current shot version')
|
|
const bytes = await downloadBytes(fetchImpl, media.audioUrl, options.requestTimeoutMs, 'audio/')
|
|
assertCondition(bytes.length > 500, 'TTS audio output is unexpectedly small')
|
|
return { job, media, bytes }
|
|
}, (value) => ({ jobId: value.job.id, model: value.job.providerModel, byteSize: value.bytes.length }))
|
|
|
|
await step('ai-video', 'Replicate image-to-video generation and storage', async () => {
|
|
const job = await runAiJob(projectId, aiSetup.episode.id, 'VIDEO_GENERATE', {
|
|
shotId: storyboard.shot.id,
|
|
prompt: 'Subtle natural head turn and blinking, locked camera, stable adult character identity.',
|
|
imageAssetVersionId: replicateImage.version.id,
|
|
durationSeconds: options.aiVideoSeconds,
|
|
provider: 'replicate',
|
|
params: options.aiVideoParams,
|
|
})
|
|
const media = await api.request(`/shots/${storyboard.shot.id}/media`)
|
|
assertCondition(job.output?.videoStorageKey && media.videoUrl, 'Video output was not bound to the current shot version')
|
|
const bytes = await downloadBytes(fetchImpl, media.videoUrl, options.requestTimeoutMs, 'video/')
|
|
assertCondition(bytes.length > 1000, 'Generated video output is unexpectedly small')
|
|
return { job, media, bytes }
|
|
}, (value) => ({ jobId: value.job.id, model: value.job.providerModel, durationSeconds: options.aiVideoSeconds, byteSize: value.bytes.length }))
|
|
|
|
const lipsync = await step('ai-lipsync', 'Replicate lip-sync generation and media continuity', async () => {
|
|
const job = await runAiJob(projectId, aiSetup.episode.id, 'LIPSYNC_GENERATE', {
|
|
shotId: storyboard.shot.id,
|
|
provider: 'replicate',
|
|
params: options.aiLipsyncParams,
|
|
})
|
|
const media = await api.request(`/shots/${storyboard.shot.id}/media`)
|
|
assertCondition(job.output?.videoStorageKey && media.videoUrl && media.audioUrl, 'Lip-sync output did not preserve video and audio continuity')
|
|
const [video, audio] = await Promise.all([
|
|
downloadBytes(fetchImpl, media.videoUrl, options.requestTimeoutMs, 'video/'),
|
|
downloadBytes(fetchImpl, media.audioUrl, options.requestTimeoutMs, 'audio/'),
|
|
])
|
|
assertCondition(video.length > 1000 && audio.length > 500, 'Lip-sync media outputs are unexpectedly small')
|
|
return { job, media, video, audio }
|
|
}, (value) => ({ jobId: value.job.id, model: value.job.providerModel, videoBytes: value.video.length, audioBytes: value.audio.length }))
|
|
report.aiVerification.artifacts.lipsyncShotVersionId = lipsync.media.versionId
|
|
|
|
const aiRender = await step('ai-render', 'FFmpeg render from live AI media', async () => {
|
|
const queued = await api.request(`/episodes/${aiSetup.episode.id}/renders`, {
|
|
method: 'POST',
|
|
json: {
|
|
idempotencyKey: `smoke-ai-render-${runId}`,
|
|
settings: { width: 640, height: 360, fps: 24, quality: 'draft', burnSubtitles: false, normalizeAudio: false, musicVolume: 0, subtitleStyle: { fontSize: 20, textColor: '#FFFFFF', outlineColor: '#000000', outlineWidth: 2, bottomMargin: 24, bold: true } },
|
|
},
|
|
})
|
|
const job = await waitForJob((jobId) => api.request(`/jobs/${jobId}`), queued.job.id, { timeoutMs: options.timeoutMs, sleep })
|
|
const render = await api.request(`/renders/${queued.render.id}`)
|
|
assertCondition(job.status === 'SUCCEEDED' && render.status === 'REVIEW', 'AI media render did not finish in review state')
|
|
assertCondition(render.sourceShotCount === 1 && render.downloadUrl, 'AI media render source manifest is invalid')
|
|
const video = await downloadBytes(fetchImpl, render.downloadUrl, options.requestTimeoutMs, 'video/mp4')
|
|
assertCondition(video.length > 1000, 'AI media render is unexpectedly small')
|
|
return { job, render, video }
|
|
}, (value) => ({ jobId: value.job.id, renderId: value.render.id, sourceShotCount: value.render.sourceShotCount, byteSize: value.video.length }))
|
|
report.aiVerification.artifacts.renderId = aiRender.render.id
|
|
report.aiVerification.artifacts.videoSha256 = sha256(aiRender.video)
|
|
|
|
await step('ai-usage', 'Live AI usage-ledger reconciliation', async () => {
|
|
const billingAfter = await api.request(`/workspaces/${workspace.id}/billing/summary`)
|
|
const providerJobIds = new Set(report.aiVerification.jobs.map((job) => job.id))
|
|
const billedJobIds = new Set((billingAfter.recent ?? []).filter((entry) => providerJobIds.has(entry.jobId)).map((entry) => entry.jobId))
|
|
const missingJobIds = [...providerJobIds].filter((jobId) => !billedJobIds.has(jobId))
|
|
assertCondition(missingJobIds.length === 0, `Usage ledger is missing ${missingJobIds.length} live AI jobs`)
|
|
const creditsUsed = Number(billingAfter.usedCredits) - Number(billingReadModel.usedCredits)
|
|
assertCondition(creditsUsed > 0, 'Live AI provider calls did not increase recorded usage')
|
|
report.aiVerification.creditsUsed = creditsUsed
|
|
return { jobs: providerJobIds.size, ledgerJobs: billedJobIds.size, creditsUsed }
|
|
}, (value) => value)
|
|
}
|
|
} catch (error) {
|
|
primaryError = error
|
|
} finally {
|
|
if (projectId && !options.keepProject) {
|
|
try {
|
|
await step('cleanup', 'Archive smoke-test project', async () => {
|
|
const archived = await api.request(`/projects/${projectId}`, { method: 'DELETE' })
|
|
assertCondition(archived.status === 'ARCHIVED', 'Smoke project was not archived')
|
|
report.projectArchived = true
|
|
return archived
|
|
}, (value) => ({ projectId: value.id, status: value.status }))
|
|
} catch (cleanupError) {
|
|
if (!primaryError) primaryError = cleanupError
|
|
}
|
|
}
|
|
if (loggedIn) {
|
|
try {
|
|
await step('logout', 'Revoke smoke-account refresh session', async () => {
|
|
await api.request('/auth/logout', { method: 'POST' })
|
|
const sessions = await api.request('/auth/sessions')
|
|
assertCondition(!sessions.some((candidate) => candidate.id === smokeSessionId), 'Smoke refresh session remains active after logout')
|
|
api.clearSession()
|
|
return { sessionId: smokeSessionId, revoked: true }
|
|
}, (value) => value)
|
|
} catch (logoutError) {
|
|
if (!primaryError) primaryError = logoutError
|
|
}
|
|
}
|
|
}
|
|
|
|
report.status = primaryError ? 'failed' : 'passed'
|
|
report.completedAt = new Date(now()).toISOString()
|
|
report.durationMs = Math.max(0, now() - startedAt)
|
|
if (options.keepProject && projectId) report.projectArchived = false
|
|
if (primaryError) {
|
|
report.error = safeError(primaryError)
|
|
throw new SmokeTestError(report.error, { cause: primaryError, report })
|
|
}
|
|
return report
|
|
}
|
|
|
|
export function usage() {
|
|
return `FrameFlow production smoke test
|
|
|
|
Usage:
|
|
FRAMEFLOW_SMOKE_BASE_URL=https://studio.example.com \\
|
|
FRAMEFLOW_SMOKE_EMAIL=smoke@example.com \\
|
|
FRAMEFLOW_SMOKE_PASSWORD='...' \\
|
|
npm run smoke:production -- [options]
|
|
|
|
Options:
|
|
--base-url URL Application origin or /api/v1 URL
|
|
--email EMAIL Dedicated verified smoke-account email
|
|
--workspace-id UUID Required when the account has multiple workspaces
|
|
--image PATH JPEG, PNG, or WebP render source
|
|
--timeout-seconds N Per-job timeout from 30 to 900 (default: 180)
|
|
--include-ai Run live, billable OpenAI and Replicate validation
|
|
--ai-video-seconds N Live AI video duration from 1 to 30 (default: 5)
|
|
--keep-project Keep the generated smoke project instead of archiving it
|
|
--allow-http Allow HTTP only for localhost development
|
|
--json Print the complete report as JSON
|
|
--help Show this help
|
|
|
|
The password is accepted only through FRAMEFLOW_SMOKE_PASSWORD so it is not exposed in the process list.
|
|
Default mode never invokes AI provider jobs; configuration-only readiness is included in the report.
|
|
|
|
--include-ai starts real provider jobs that can incur charges. It is accepted only when
|
|
FRAMEFLOW_SMOKE_BILLABLE_AI_ACK=I_ACCEPT_BILLABLE_AI_CHARGES is also set, all seven AI
|
|
workflows are configured, and the workspace has at least 700 remaining credits. Provider
|
|
jobs use maxAttempts=1. Optional model-specific JSON objects can be supplied through
|
|
FRAMEFLOW_SMOKE_VIDEO_PARAMS_JSON and FRAMEFLOW_SMOKE_LIPSYNC_PARAMS_JSON.`
|
|
}
|
|
|
|
async function main() {
|
|
let options
|
|
try {
|
|
options = parseSmokeOptions()
|
|
if (options.help) {
|
|
console.log(usage())
|
|
return
|
|
}
|
|
const report = await runProductionSmoke(options)
|
|
if (options.json) console.log(JSON.stringify(report, null, 2))
|
|
else console.log(`Production smoke test passed in ${report.durationMs}ms. Project ${report.projectId}${report.projectArchived ? ' was archived' : ' was retained'}.`)
|
|
} catch (error) {
|
|
const report = error instanceof SmokeTestError ? error.report : null
|
|
if (options?.json && report) console.error(JSON.stringify(report, null, 2))
|
|
else console.error(`Production smoke test failed: ${safeError(error)}`)
|
|
process.exitCode = 1
|
|
}
|
|
}
|
|
|
|
if (process.argv[1] && resolve(process.argv[1]) === scriptPath) await main()
|