97 lines
3.4 KiB
TypeScript
97 lines
3.4 KiB
TypeScript
/**
|
|
* Smoke test: validate VIBN_TOOL_DEFINITIONS against the live Gemini API.
|
|
*
|
|
* Sends the full tool list with a trivial prompt and checks whether Gemini
|
|
* accepts the schemas. Catches schema validation errors (ARRAY without items,
|
|
* free OBJECT params, etc.) before we deploy.
|
|
*
|
|
* Usage: GOOGLE_API_KEY=... npx tsx scripts/smoke-chat-tools.ts
|
|
* (also picks up GOOGLE_API_KEY from .env.local automatically)
|
|
*/
|
|
import { readFileSync } from 'node:fs';
|
|
import { join } from 'node:path';
|
|
import { VIBN_TOOL_DEFINITIONS } from '../lib/ai/vibn-tools';
|
|
|
|
// Load .env.local manually to avoid needing dotenv as a dep
|
|
try {
|
|
const envPath = join(process.cwd(), '.env.local');
|
|
const envText = readFileSync(envPath, 'utf-8');
|
|
for (const line of envText.split('\n')) {
|
|
const m = line.match(/^([A-Z_][A-Z0-9_]*)=(.*)$/);
|
|
if (m && !process.env[m[1]]) process.env[m[1]] = m[2].trim();
|
|
}
|
|
} catch {}
|
|
|
|
const API_KEY = process.env.GOOGLE_API_KEY;
|
|
const MODEL = process.env.VIBN_CHAT_MODEL || 'gemini-3.1-pro-preview';
|
|
|
|
if (!API_KEY) {
|
|
console.error('Missing GOOGLE_API_KEY');
|
|
process.exit(1);
|
|
}
|
|
|
|
async function validateAll() {
|
|
console.log(`Validating ${VIBN_TOOL_DEFINITIONS.length} tool definitions against ${MODEL}...\n`);
|
|
|
|
const url = `https://generativelanguage.googleapis.com/v1beta/models/${MODEL}:generateContent?key=${API_KEY}`;
|
|
const body = {
|
|
contents: [{ role: 'user', parts: [{ text: 'Show me what is running in my workspace.' }] }],
|
|
tools: [{ functionDeclarations: VIBN_TOOL_DEFINITIONS }],
|
|
generationConfig: { temperature: 0.0, maxOutputTokens: 200 },
|
|
};
|
|
|
|
const res = await fetch(url, {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify(body),
|
|
});
|
|
|
|
const data = await res.json();
|
|
|
|
if (!res.ok) {
|
|
console.error(`\n❌ FAIL: HTTP ${res.status}`);
|
|
console.error(JSON.stringify(data, null, 2));
|
|
|
|
// Try to identify which tools are bad by sending them one at a time
|
|
console.log('\n🔍 Bisecting to find broken tools...\n');
|
|
const bad: { name: string; error: string }[] = [];
|
|
for (const tool of VIBN_TOOL_DEFINITIONS) {
|
|
const r = await fetch(url, {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({
|
|
contents: [{ role: 'user', parts: [{ text: 'hi' }] }],
|
|
tools: [{ functionDeclarations: [tool] }],
|
|
generationConfig: { temperature: 0.0, maxOutputTokens: 10 },
|
|
}),
|
|
});
|
|
if (!r.ok) {
|
|
const err = await r.json();
|
|
const msg = err?.error?.message || JSON.stringify(err).slice(0, 200);
|
|
bad.push({ name: tool.name, error: msg });
|
|
console.log(` ❌ ${tool.name}: ${msg.slice(0, 150)}`);
|
|
} else {
|
|
console.log(` ✓ ${tool.name}`);
|
|
}
|
|
}
|
|
console.log(`\n${bad.length} broken tool(s) out of ${VIBN_TOOL_DEFINITIONS.length}`);
|
|
process.exit(1);
|
|
}
|
|
|
|
console.log('✅ All tool definitions accepted by Gemini.');
|
|
|
|
const calls = data?.candidates?.[0]?.content?.parts
|
|
?.filter((p: any) => p.functionCall)
|
|
.map((p: any) => `${p.functionCall.name}(${Object.keys(p.functionCall.args || {}).join(',')})`);
|
|
if (calls?.length) {
|
|
console.log(`\nGemini chose to call: ${calls.join(', ')}`);
|
|
} else {
|
|
console.log('\n(No tool calls produced — model responded with text)');
|
|
}
|
|
}
|
|
|
|
validateAll().catch((e) => {
|
|
console.error('Test crashed:', e);
|
|
process.exit(1);
|
|
});
|