[WIP] Evals

package.json

@@ -10,7 +10,8 @@
     "make": "electron-forge make",
     "publish": "electron-forge publish",
     "lint": "eslint --ext .ts,.tsx .",
-    "rebuild": "./node_modules/.bin/electron-rebuild"
+    "rebuild": "./node_modules/.bin/electron-rebuild",
+    "eval": "tsx src/bin/hide-eval.ts"
   },
   "devDependencies": {
     "@electron-forge/cli": "^7.6.0",
@@ -25,12 +26,12 @@
     "@electron/rebuild": "^3.7.1",
     "@tailwindcss/typography": "^0.5.15",
     "@types/better-sqlite3": "^7.6.12",
-    "@types/node": "^22.10.6",
+    "@types/node": "^20.17.10",
     "@types/react": "^19.0.1",
     "@types/react-dom": "^19.0.2",
     "@typescript-eslint/eslint-plugin": "^5.62.0",
     "@typescript-eslint/parser": "^5.62.0",
-    "@vercel/webpack-asset-relocator-loader": "^1.7.3",
+    "@vercel/webpack-asset-relocator-loader": "1.7.3",
     "autoprefixer": "^10.4.20",
     "copy-webpack-plugin": "^12.0.2",
     "css-loader": "^6.11.0",
@@ -46,16 +47,14 @@
     "ts-loader": "^9.5.1",
     "ts-node": "^10.9.2",
     "tsx": "^4.19.2",
-    "typescript": "~4.5.4"
+    "typescript": "5.3"
   },
   "keywords": [],
   "author": "artemm",
   "license": "MIT",
   "dependencies": {
-    "@ai-sdk/anthropic": "^1.0.8",
-    "@ai-sdk/openai": "^1.0.19",
     "@anthropic-ai/sdk": "^0.33.1",
-    "@modelcontextprotocol/sdk": "^1.0.4",
+    "@modelcontextprotocol/sdk": "^1.4.1",
     "@radix-ui/react-alert-dialog": "^1.1.4",
     "@radix-ui/react-avatar": "^1.1.2",
     "@radix-ui/react-dialog": "^1.1.4",
@@ -68,13 +67,14 @@
     "@radix-ui/react-tabs": "^1.1.2",
     "@radix-ui/react-tooltip": "^1.1.6",
     "@types/uuid": "^10.0.0",
-    "ai": "^4.0.36",
     "better-sqlite3": "^11.7.0",
     "class-variance-authority": "^0.7.1",
     "clsx": "^2.1.1",
+    "commander": "^12.1.0",
     "dotenv": "^16.4.7",
     "electron-squirrel-startup": "^1.0.1",
     "highlight.js": "^11.11.0",
+    "hyparquet": "^1.8.1",
     "lucide-react": "^0.468.0",
     "react": "^19.0.0",
     "react-dom": "^19.0.0",

src/bin/hide-eval.ts

@@ -0,0 +1,9 @@
+#!/usr/bin/env node
+import { runEvalCLI } from '../main/eval/cli.js';
+
+runEvalCLI(process.argv)
+  .then(() => process.exit(0))
+  .catch(err => {
+    console.error('Error:', err);
+    process.exit(1);
+  });

src/index.ts

@@ -131,7 +131,7 @@ const createWindow = (): BrowserWindow => {
 
   // and load the index.html of the app.
   mainWindow.loadURL(MAIN_WINDOW_WEBPACK_ENTRY);
-  
+
   return mainWindow;
 };
 
@@ -213,7 +213,8 @@ ipcMain.handle('dialog:showDirectoryPicker', async () => {
 
 app.whenReady().then(async () => {
   // Initialize the database and set up IPC handlers
-  initializeDatabase();
+  const dbPath = path.join(app.getPath('userData'), 'database.sqlite');
+  initializeDatabase(dbPath);
   setupDbHandlers();
 
   const { cmd, args } = await getMCPConfig();

src/lib/mcp/adapters.ts

@@ -1,6 +1,5 @@
 import type { Tool as AnthropicTool } from '@anthropic-ai/sdk/resources/messages';
 import type { Tool as MCPTool, CallToolResult as MCPToolResult } from '@modelcontextprotocol/sdk/types';
-import { CoreTool, tool as aiTool, jsonSchema } from 'ai';
 
 export function mcpToAnthropicTool(tool: MCPTool): AnthropicTool {
     const { name, description, inputSchema, ...rest } = tool;
@@ -15,40 +14,3 @@ export function mcpToAnthropicTool(tool: MCPTool): AnthropicTool {
         ...rest
     };
 }
-
-export function mcpToAiSdkTool(tool: MCPTool, callTool: (name: string, args: any) => Promise<MCPToolResult>): Record<string, CoreTool> {
-    const { name, description, inputSchema } = tool;
-    return {
-        [name]: aiTool({
-            description,
-            parameters: jsonSchema(inputSchema),
-            execute: async (args) => (callTool(name, args)),
-            // map to anthropic's tool result format:
-            experimental_toToolResultContent(result) {
-                if (typeof result === 'string') {
-                    return [{ type: 'text', text: result }];
-                }
-
-                if ('isError' in result && 'content' in result) {
-                    const r = result as MCPToolResult;
-                    return r.content.map(c => {
-                        if (c.type === 'text') {
-                            return { type: 'text', text: r.isError ? `Error: ${c.text}` : c.text };
-                        }
-
-                        if (c.type === 'image') {
-                            return { type: 'image', data: c.data, mimeType: c.mimeType };
-                        }
-
-                        if (c.type === 'resource') {
-                            console.error('Embedded resources are not supported');
-                        }
-                    });
-                }
-
-                console.error('Unexpected tool result:', result);
-                return [];
-            },
-        })
-    }
-}

src/main/db.ts

@@ -1,17 +1,15 @@
 import Database from 'better-sqlite3';
-import { app, ipcMain } from 'electron';
-import path from 'path';
+import { ipcMain } from 'electron';
 import { homedir } from 'os';
 import { v4 as uuidv4 } from 'uuid';
 import { Conversation, Project, Task } from '../types';
 import { UserSettings } from '../types/settings';
 
 let db: Database.Database;
 
-export const initializeDatabase = () => {
+export const initializeDatabase = (dbPath: string) => {
   if (db) return;
 
-  const dbPath = path.join(app.getPath('userData'), 'database.sqlite');
   db = new Database(dbPath);
 
   // Create projects table if it doesn't exist
@@ -95,6 +93,11 @@ export const getProjectById = (id: string): Project | undefined => {
   return stmt.get(id) as Project | undefined;
 };
 
+export const getProjectByName = (name: string): Project | undefined => {
+  const stmt = db.prepare('SELECT * FROM projects WHERE name = ?');
+  return stmt.get(name) as Project | undefined;
+};
+
 export const createProject = (project: Project): void => {
   const stmt = db.prepare('INSERT INTO projects (id, name, path, description) VALUES (?, ?, ?, ?)');
   stmt.run(project.id, project.name, project.path, project.description);

src/main/eval/cli.ts

@@ -0,0 +1,67 @@
+import * as fs from 'fs';
+import { program } from 'commander';
+import { EvalService } from './service';
+import { defaultEvalConfig } from './config';
+import { readJsonl, readParquet, writeJsonl } from './utils';
+import { SWEBenchInstance } from './types';
+import * as path from 'path';
+import { initializeDatabase } from '../db';
+
+export async function runEvalCLI(args: string[]) {
+  program
+    .name('eval')
+    .description('Evaluation tool for Hide')
+    .requiredOption('-d, --dataset <path>', 'Path to dataset file (required, supports .jsonl and .parquet)')
+    .option('-c, --config <path>', 'Path to config JSON file')
+    .option('-o, --output <path>', 'Path to output JSONL file')
+    .option('-b, --batch-size <number>', 'Batch size', '4')
+    .option('-l, --limit <number>', 'Limit', '4')
+    .option('-i, --instance-ids <string>', 'Comma-separated list of instance IDs to run')
+
+  program.parse(args);
+
+  const opts = program.opts();
+
+  // Load dataset
+  console.log('Loading dataset...');
+  const ext = path.extname(opts.dataset).toLowerCase();
+  let instances: SWEBenchInstance[];
+
+  if (ext === '.jsonl') {
+    instances = await readJsonl<SWEBenchInstance>(opts.dataset);
+  } else if (ext === '.parquet') {
+    instances = await readParquet<SWEBenchInstance>(opts.dataset);
+  } else {
+    throw new Error(`Unsupported file extension: ${ext}. Only .jsonl and .parquet files are supported.`);
+  }
+
+  console.log(`Loaded ${instances.length} instances`);
+
+  // Load config
+  const config = opts.config
+    ? JSON.parse(await fs.promises.readFile(opts.config, 'utf-8'))
+    : defaultEvalConfig;
+
+  if (opts.batchSize) {
+    config.batchSize = parseInt(opts.batchSize);
+  }
+
+  if (opts.instanceIds) {
+    config.instanceIDs = opts.instanceIds.split(',');
+  }
+
+  // Initialize services
+  console.log('Initializing services...');
+  // TODO: avoid hardcoding path
+  initializeDatabase("/Users/artemm/Library/Application Support/hide-app/database.sqlite");
+  const evalService = new EvalService(config);
+
+  // Run eval
+  console.log(`Starting evaluation of ${Math.min(instances.length, config.limit)} instances...`);
+  const results = await evalService.runEval(instances);
+
+  // Save results
+  const outputPath = opts.output || 'eval-results.jsonl';
+  await writeJsonl(outputPath, results);
+  console.log(`Evaluation complete. Results saved to ${outputPath}`);
+}

src/main/eval/config.ts

@@ -0,0 +1,33 @@
+import * as path from 'path';
+import * as os from 'os';
+import { EvalConfig } from './types';
+
+export const defaultEvalConfig: EvalConfig = {
+  projectName: 'eval',
+  batchSize: 4,
+  limit: 4,
+  workDir: path.join(os.homedir(), 'hide-eval'),
+  modelName: 'claude-3.5-sonnet',
+  promptTemplate: `<uploaded_files>
+{location}
+</uploaded_files>
+I've uploaded a python code repository in the directory {location} (not in /tmp/inputs). Consider the following PR description:
+
+<pr_description>
+{pr_description}
+</pr_description>
+
+Can you help me implement the necessary changes to the repository so that the requirements specified in the <pr_description> are met?
+I've already taken care of all changes to any of the test files described in the <pr_description>. This means you DON'T have to modify the testing logic or any of the tests in any way!
+
+Your task is to make the minimal changes to non-tests files in the {location} directory to ensure the <pr_description> is satisfied.
+
+Follow these steps to resolve the issue:
+1. As a first step, it might be a good idea to explore the repo to familiarize yourself with its structure.
+2. Create a script to reproduce the error and execute it with 'python <filename.py>' using the BashTool, to confirm the error
+3. Edit the sourcecode of the repo to resolve the issue
+4. Rerun your reproduce script and confirm that the error is fixed!
+5. Think about edgecases and make sure your fix handles them as well
+
+Your thinking should be thorough and so it's fine if it's very long.`
+};

src/main/eval/index.ts

@@ -0,0 +1,4 @@
+export { runEvalCLI } from './cli';
+export { EvalService } from './service';
+export { defaultEvalConfig } from './config';
+export * from './types';