VIBN Frontend for Coolify deployment

2026-02-15 19:25:52 -08:00
commit 40bf8428cd
398 changed files with 76513 additions and 0 deletions
--- a/lib/db/alloydb.ts
+++ b/lib/db/alloydb.ts
@@ -0,0 +1,220 @@
+/**
+ * AlloyDB PostgreSQL client for vector semantic memory
+ * 
+ * Uses pgvector extension for semantic search over knowledge_chunks.
+ * Connection pooling ensures efficient resource usage in Next.js API routes.
+ */
+
+import { Pool, PoolClient, QueryResult, QueryResultRow } from 'pg';
+import { GoogleAuth } from 'google-auth-library';
+
+let pool: Pool | null = null;
+let cachedToken: { token: string; expiresAt: number } | null = null;
+
+interface AlloyDBConfig {
+  host: string;
+  port: number;
+  user: string;
+  password: string;
+  database: string;
+  ssl?: boolean;
+  maxConnections?: number;
+  idleTimeoutMillis?: number;
+}
+
+/**
+ * Get an OAuth2 access token for IAM authentication
+ */
+async function getAccessToken(): Promise<string> {
+  // Check if we have a cached token that's still valid (with 5 min buffer)
+  if (cachedToken && cachedToken.expiresAt > Date.now() + 5 * 60 * 1000) {
+    return cachedToken.token;
+  }
+
+  const auth = new GoogleAuth({
+    scopes: ['https://www.googleapis.com/auth/cloud-platform'],
+  });
+
+  const client = await auth.getClient();
+  const tokenResponse = await client.getAccessToken();
+
+  if (!tokenResponse.token) {
+    throw new Error('Failed to get access token for AlloyDB IAM authentication');
+  }
+
+  // Cache the token (Google tokens typically expire in 1 hour)
+  cachedToken = {
+    token: tokenResponse.token,
+    expiresAt: Date.now() + 55 * 60 * 1000, // 55 minutes (safe buffer)
+  };
+
+  return tokenResponse.token;
+}
+
+/**
+ * Load AlloyDB configuration from environment variables
+ */
+async function loadConfig(): Promise<AlloyDBConfig> {
+  const host = process.env.ALLOYDB_HOST;
+  const port = process.env.ALLOYDB_PORT;
+  const user = process.env.ALLOYDB_USER;
+  const password = process.env.ALLOYDB_PASSWORD;
+  const database = process.env.ALLOYDB_DATABASE;
+
+  if (!host || !port || !user || !database) {
+    throw new Error(
+      'Missing required AlloyDB environment variables. Required: ALLOYDB_HOST, ALLOYDB_PORT, ALLOYDB_USER, ALLOYDB_DATABASE'
+    );
+  }
+
+  // When using AlloyDB Auth Proxy, no password is needed (proxy handles IAM)
+  // For direct connections with IAM, generate an OAuth token
+  let finalPassword = password;
+  if (!finalPassword && !host.startsWith('/')) {
+    // Only generate token for direct IP connections, not Unix sockets
+    finalPassword = await getAccessToken();
+  }
+
+  return {
+    host,
+    port: parseInt(port, 10),
+    user,
+    password: finalPassword || '', // Empty string for proxy connections
+    database,
+    ssl: process.env.ALLOYDB_SSL ? process.env.ALLOYDB_SSL !== 'false' : false, // Enable SSL if set to anything other than 'false'
+    maxConnections: process.env.ALLOYDB_MAX_CONNECTIONS 
+      ? parseInt(process.env.ALLOYDB_MAX_CONNECTIONS, 10) 
+      : 10,
+    idleTimeoutMillis: 30000, // 30 seconds
+  };
+}
+
+/**
+ * Initialize the connection pool (singleton pattern)
+ */
+async function initializePool(): Promise<Pool> {
+  if (pool) {
+    return pool;
+  }
+
+  const config = await loadConfig();
+
+  pool = new Pool({
+    host: config.host,
+    port: config.port,
+    user: config.user,
+    password: config.password,
+    database: config.database,
+    ssl: config.ssl ? { rejectUnauthorized: false } : false,
+    max: config.maxConnections,
+    idleTimeoutMillis: config.idleTimeoutMillis,
+    connectionTimeoutMillis: 10000, // 10 seconds to establish connection
+  });
+
+  // Log pool errors
+  pool.on('error', (err) => {
+    console.error('[AlloyDB Pool] Unexpected error on idle client:', err);
+  });
+
+  console.log(`[AlloyDB] Connection pool initialized (max: ${config.maxConnections})`);
+
+  return pool;
+}
+
+/**
+ * Get a pooled client for AlloyDB
+ * 
+ * This is the primary interface for accessing AlloyDB from Next.js API routes.
+ * The pool handles connection reuse automatically.
+ * 
+ * @returns Pool instance for executing queries
+ * 
+ * @example
+ * ```typescript
+ * const pool = await getAlloyDbClient();
+ * const result = await pool.query('SELECT * FROM knowledge_chunks WHERE project_id = $1', [projectId]);
+ * ```
+ */
+export async function getAlloyDbClient(): Promise<Pool> {
+  if (!pool) {
+    return await initializePool();
+  }
+  return pool;
+}
+
+/**
+ * Execute a query with automatic error logging
+ * 
+ * @param text SQL query text (use $1, $2, etc. for parameters)
+ * @param params Query parameters
+ * @returns Query result
+ */
+export async function executeQuery<T extends QueryResultRow = any>(
+  text: string,
+  params?: any[]
+): Promise<QueryResult<T>> {
+  const client = await getAlloyDbClient();
+  try {
+    const result = await client.query<T>(text, params);
+    return result;
+  } catch (error) {
+    console.error('[AlloyDB Query Error]', {
+      query: text,
+      params,
+      error: error instanceof Error ? error.message : String(error),
+    });
+    throw error;
+  }
+}
+
+/**
+ * Get a dedicated client from the pool for transactions
+ * 
+ * Remember to call client.release() when done!
+ * 
+ * @example
+ * ```typescript
+ * const client = await getPooledClient();
+ * try {
+ *   await client.query('BEGIN');
+ *   await client.query('INSERT INTO ...');
+ *   await client.query('COMMIT');
+ * } catch (e) {
+ *   await client.query('ROLLBACK');
+ *   throw e;
+ * } finally {
+ *   client.release();
+ * }
+ * ```
+ */
+export async function getPooledClient(): Promise<PoolClient> {
+  const pool = await getAlloyDbClient();
+  return await pool.connect();
+}
+
+/**
+ * Close the connection pool gracefully
+ * 
+ * Should be called during application shutdown (e.g., in tests or cleanup)
+ */
+export async function closeAlloyDbPool(): Promise<void> {
+  if (pool) {
+    await pool.end();
+    pool = null;
+    console.log('[AlloyDB] Connection pool closed');
+  }
+}
+
+/**
+ * Health check - verify AlloyDB connection is working
+ */
+export async function checkAlloyDbHealth(): Promise<boolean> {
+  try {
+    const result = await executeQuery('SELECT 1 as health_check');
+    return result.rows.length > 0 && result.rows[0].health_check === 1;
+  } catch (error) {
+    console.error('[AlloyDB Health Check] Failed:', error);
+    return false;
+  }
+}
+
--- a/lib/db/knowledge-chunks-schema.sql
+++ b/lib/db/knowledge-chunks-schema.sql
@@ -0,0 +1,117 @@
+-- =====================================================================
+-- knowledge_chunks table: Stores chunked content with vector embeddings
+-- =====================================================================
+-- 
+-- This table stores semantic chunks of knowledge_items for vector search.
+-- Each chunk is embedded using an LLM embedding model (e.g., Gemini embeddings)
+-- and stored with pgvector for efficient similarity search.
+--
+-- Prerequisites:
+-- 1. Enable pgvector extension: CREATE EXTENSION IF NOT EXISTS vector;
+-- 2. Enable uuid generation: CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
+--
+
+-- Enable required extensions
+CREATE EXTENSION IF NOT EXISTS vector;
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
+
+-- Create the knowledge_chunks table
+CREATE TABLE IF NOT EXISTS knowledge_chunks (
+  -- Primary key (UUID auto-generated)
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  
+  -- References to parent entities (Firestore IDs stored as TEXT)
+  project_id TEXT NOT NULL,
+  knowledge_item_id TEXT NOT NULL,
+  
+  -- Chunk metadata
+  chunk_index INT NOT NULL,
+  content TEXT NOT NULL,
+  
+  -- Vector embedding (768 dimensions for Gemini text-embedding-004)
+  -- NOTE: OpenAI embeddings use 1536 dims, but Gemini uses 768
+  embedding VECTOR(768) NOT NULL,
+  
+  -- Source and importance metadata (optional, from knowledge_items)
+  source_type TEXT,
+  importance TEXT CHECK (importance IN ('primary', 'supporting', 'irrelevant') OR importance IS NULL),
+  
+  -- Timestamps
+  created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+-- =====================================================================
+-- Indexes for efficient querying
+-- =====================================================================
+
+-- Standard indexes for filtering by project and knowledge_item
+CREATE INDEX IF NOT EXISTS idx_knowledge_chunks_project_id 
+  ON knowledge_chunks (project_id);
+
+CREATE INDEX IF NOT EXISTS idx_knowledge_chunks_knowledge_item_id 
+  ON knowledge_chunks (knowledge_item_id);
+
+-- Composite index for project + knowledge_item queries
+CREATE INDEX IF NOT EXISTS idx_knowledge_chunks_project_knowledge 
+  ON knowledge_chunks (project_id, knowledge_item_id);
+
+-- Index for chunk ordering within a knowledge_item
+CREATE INDEX IF NOT EXISTS idx_knowledge_chunks_item_index 
+  ON knowledge_chunks (knowledge_item_id, chunk_index);
+
+-- Vector similarity index using IVFFlat (pgvector)
+-- This enables fast approximate nearest neighbor search
+-- The 'lists' parameter controls the number of clusters (tune based on data size)
+-- For < 100k rows, lists=100 is reasonable. Scale up for larger datasets.
+-- Using cosine distance (vector_cosine_ops) for semantic similarity
+CREATE INDEX IF NOT EXISTS idx_knowledge_chunks_embedding 
+  ON knowledge_chunks 
+  USING ivfflat (embedding vector_cosine_ops) 
+  WITH (lists = 100);
+
+-- Alternative: Use HNSW index for better recall at higher cost
+-- Uncomment if you prefer HNSW over IVFFlat:
+-- CREATE INDEX IF NOT EXISTS idx_knowledge_chunks_embedding_hnsw 
+--   ON knowledge_chunks 
+--   USING hnsw (embedding vector_cosine_ops) 
+--   WITH (m = 16, ef_construction = 64);
+
+-- =====================================================================
+-- Optional: Trigger to auto-update updated_at timestamp
+-- =====================================================================
+CREATE OR REPLACE FUNCTION update_updated_at_column()
+RETURNS TRIGGER AS $$
+BEGIN
+  NEW.updated_at = NOW();
+  RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER update_knowledge_chunks_updated_at
+  BEFORE UPDATE ON knowledge_chunks
+  FOR EACH ROW
+  EXECUTE FUNCTION update_updated_at_column();
+
+-- =====================================================================
+-- Helpful queries for monitoring and debugging
+-- =====================================================================
+
+-- Count chunks per project
+-- SELECT project_id, COUNT(*) as chunk_count FROM knowledge_chunks GROUP BY project_id;
+
+-- Count chunks per knowledge_item
+-- SELECT knowledge_item_id, COUNT(*) as chunk_count FROM knowledge_chunks GROUP BY knowledge_item_id;
+
+-- Find chunks similar to a query vector (example)
+-- SELECT id, content, 1 - (embedding <=> '[0.1, 0.2, ...]') AS similarity
+-- FROM knowledge_chunks
+-- WHERE project_id = 'your-project-id'
+-- ORDER BY embedding <=> '[0.1, 0.2, ...]'
+-- LIMIT 10;
+
+-- Check index usage
+-- SELECT schemaname, tablename, indexname, idx_scan, idx_tup_read, idx_tup_fetch
+-- FROM pg_stat_user_indexes
+-- WHERE tablename = 'knowledge_chunks';
+