import { Client } from 'pg'; import * as dotenv from 'dotenv'; import * as path from 'path'; import * as fs from 'fs'; // Load env variables dotenv.config({ path: path.join(__dirname, '../.env.local') }); const connectionString = process.env.DATABASE_URL; if (!connectionString) { console.error("DATABASE_URL is not set in .env.local"); process.exit(1); } // Curation target message IDs from the live tests: const curatedMessagePairs = [ // 1. Success - Hardened fs_edit (Navbar title change) { user: 'acc489f6-808c-4a1c-9686-7f6145f2fa48', assistant: '0c71e628-f6f3-4bce-9ccf-7a1631e7537a', tag: 'WORK: Hardened fs_edit (Navbar Title Change)' }, // 2. Success - Port-reaper dev server restart { user: 'b304d55c-7d8a-4e1a-9823-58d521b1796d', assistant: 'a244bc69-b0e7-4977-89fb-d39949a7e7ed', tag: 'WORK: Socket-Inode Port Reaper dev_server_start' }, // 3. Failure - Prisma error and AI Success Hallucination { user: '6c98dd1f-ba1a-4cd8-a654-8e6da4eecace', assistant: '46acdaad-3ea7-4fd1-9fde-6aff1d33e896', tag: 'FAIL: Prisma DB Error and AI Success Hallucination' } ]; async function main() { const client = new Client({ connectionString }); await client.connect(); console.log("Connected to PostgreSQL DB..."); const projectId = 'be169fe8-d381-422b-8e9c-d2e513a8f902'; const threadId = 'a584c700-7ae2-4fad-a906-b8daf80fcace'; const turns = []; for (const pair of curatedMessagePairs) { const userRes = await client.query("SELECT id, created_at, data FROM fs_chat_messages WHERE id = $1", [pair.user]); const assistantRes = await client.query("SELECT id, created_at, data FROM fs_chat_messages WHERE id = $1", [pair.assistant]); if (userRes.rows.length === 0 || assistantRes.rows.length === 0) { console.warn(`Could not find pair: ${pair.user} -> ${pair.assistant}`); continue; } const userMsg = userRes.rows[0]; const assistantMsg = assistantRes.rows[0]; const rawToolResults = assistantMsg.data._rawToolResults || []; const actionsRun = rawToolResults.map((tr: any) => { let stdout = tr.result; let ok = true; let status = "success"; try { const parsedRes = JSON.parse(tr.result); if (parsedRes.ok === false || (parsedRes.errors && parsedRes.errors.length > 0) || parsedRes.error) { ok = false; status = "error"; } } catch (e) {} return { tool_name: tr.name, tool_call_id: tr.id || `tc-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`, input_args: tr.args || {}, execution_outcome: { ok, status, stdout } }; }); turns.push({ turn_metadata: { message_id: userMsg.id, timestamp_utc: userMsg.created_at.toISOString(), conversation_id: threadId, audit_tag: pair.tag }, "1_user_interaction": { prompt_text: userMsg.data.content }, "2_payload_sent_to_google": { endpoint_url: "https://us-central1-aiplatform.googleapis.com/v1/projects/gen-lang-client-0980079410/locations/us-central1/publishers/google/models/gemini-3.1-pro-preview:generateContent", system_instruction: "Configured via VIBN Coder System Prompt (coder.ts / buildSystemPrompt)", contents: [ { role: "user", parts: [{ text: userMsg.data.content }] } ] }, "3_payload_received_from_google": { timestamp_utc: assistantMsg.created_at.toISOString(), raw_candidates: { content: { role: "model", parts: [{ thought: null, text: assistantMsg.data.content }] } } }, "4_platform_executions_and_telemetry": { actions_run: actionsRun }, "5_git_version_control_diffs": [] }); } const dataset = { dataset_metadata: { title: "VIBN Production QA Telemetry Dataset (Hardening Validation)", purpose: "Telemetry audit tracking the exact performance of Task-1, Task-3, and Task-4 fixes, plus isolating the database-related AI success-hallucination error.", source_project: { id: projectId, name: "GetAcquired 2.0", slug: "getacquired-2-0" }, compiled_at: new Date().toISOString(), total_turns_audited: turns.length }, turns }; // We date the file dynamically to track your QA cycles const currentDate = new Date().toISOString().split('T')[0]; // YYYY-MM-DD const outputPath = path.join(__dirname, `../../opus_telemetry_audit_dataset_${currentDate}.json`); fs.writeFileSync(outputPath, JSON.stringify(dataset, null, 2)); console.log(`\nšŸŽ‰ New dated QA telemetry dataset written to: ${outputPath}`); await client.end(); } main().catch(console.error);