diff --git a/package.json b/package.json index 14d734a..c88cd43 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,9 @@ "lint": "cross-env NODE_ENV=development eslint . --ext .js,.jsx,.ts,.tsx", "lint:fix": "cross-env NODE_ENV=development eslint . --ext .js,.jsx,.ts,.tsx --fix", "package": "ts-node ./.erb/scripts/clean.js dist && npm run build && electron-builder build --publish never && npm run build:dll", + "package-mac": "ts-node ./.erb/scripts/clean.js dist && npm run build && electron-builder build --mac --publish never && npm run build:dll", + "package-win": "ts-node ./.erb/scripts/clean.js dist && npm run build && electron-builder build --win --publish never && npm run build:dll", + "package-linux": "ts-node ./.erb/scripts/clean.js dist && npm run build && electron-builder build --linux --publish never && npm run build:dll", "rebuild": "electron-rebuild --parallel --types prod,dev,optional --module-dir release/app", "prestart": "cross-env NODE_ENV=development TS_NODE_TRANSPILE_ONLY=true NODE_OPTIONS=\"-r ts-node/register --no-warnings\" webpack --config ./.erb/configs/webpack.config.main.dev.ts", "start": "ts-node ./.erb/scripts/check-port-in-use.js && npm run prestart && npm run start:renderer", @@ -234,7 +237,12 @@ }, "win": { "target": [ - "nsis" + { + "target": "nsis", + "arch": [ + "x64" + ] + } ] }, "linux": { diff --git a/src/constant.ts b/src/constant.ts index 879669a..e965ef7 100644 --- a/src/constant.ts +++ b/src/constant.ts @@ -13,7 +13,7 @@ export const GEMINI_MODELS = { GEMINI_1_5_PRO: 'gemini-1.5-pro', GEMINI_2_0_FLASH: 'gemini-2.0-flash', GEMINI_2_0_FLASH_LITE: 'gemini-2.0-flash-lite', - GEMINI_2_5_PRO: 'gemini-2.5-pro-exp-03-25', + GEMINI_2_5_PRO: 'gemini-2.5-pro', } as const; export const RECOMMENDED_EXTRACTION_MODEL = [ diff --git a/src/main/helper/MainWindowHelper.ts b/src/main/helper/MainWindowHelper.ts index a851ff2..fe212ae 100644 --- a/src/main/helper/MainWindowHelper.ts +++ b/src/main/helper/MainWindowHelper.ts @@ -78,6 +78,9 @@ export class MainWindowHelper { ? path.join(__dirname, 'preload.js') : path.join(__dirname, '../../.erb/dll/preload.js'), scrollBounce: true, + // Disable hardware acceleration to fix screen sharing black screen + webSecurity: true, + backgroundThrottling: false, }, show: true, frame: false, @@ -120,19 +123,22 @@ export class MainWindowHelper { return { action: 'deny' }; }); - // Enhanced screen capture resistance - this.mainWindow.setContentProtection(true); + // Note: Commented out screen capture resistance features to fix black screen during screen sharing + // If you need these features, they may interfere with screen sharing functionality + + // Enhanced screen capture resistance - DISABLED for screen sharing compatibility + // this.mainWindow.setContentProtection(true); this.mainWindow.setVisibleOnAllWorkspaces(true, { visibleOnFullScreen: true, }); this.mainWindow.setAlwaysOnTop(true, 'screen-saver', 1); this.mainWindow.setIgnoreMouseEvents(true, { forward: true }); - // Additional screen capture resistance settings + // Additional screen capture resistance settings - MODIFIED for screen sharing compatibility if (process.platform === 'darwin') { - // Prevent window from being captured in screenshots + // Prevent window from being captured in screenshots - DISABLED this.mainWindow.setWindowButtonVisibility(false); - this.mainWindow.setHiddenInMissionControl(true); + // this.mainWindow.setHiddenInMissionControl(true); // DISABLED - causes issues with screen sharing this.mainWindow.setBackgroundColor('#00000000'); // Prevent window from being included in window switcher @@ -142,9 +148,9 @@ export class MainWindowHelper { this.mainWindow.setHasShadow(false); } - // Prevent the window from being captured by screen recording + // Prevent the window from being captured by screen recording - MODIFIED this.mainWindow.webContents.setBackgroundThrottling(false); - this.mainWindow.webContents.setFrameRate(60); + // this.mainWindow.webContents.setFrameRate(60); // This can cause issues with screen sharing stateManager.subscribe((state) => { if (this.mainWindow) { diff --git a/src/main/main.ts b/src/main/main.ts index efc57ca..5df744e 100644 --- a/src/main/main.ts +++ b/src/main/main.ts @@ -15,6 +15,12 @@ import { initializeIpcHandlers } from './ipcHandlers'; import { initializeStateManager } from './stateManager'; import { isDebug } from './constant'; +// Fix for black screen during screen sharing +// Disable hardware acceleration features that can interfere with screen capture +app.commandLine.appendSwitch('disable-features', 'VizDisplayCompositor,UseSkiaRenderer'); +app.commandLine.appendSwitch('disable-gpu-sandbox'); +app.commandLine.appendSwitch('disable-software-rasterizer'); + if (process.env.NODE_ENV === 'production') { const sourceMapSupport = require('source-map-support'); sourceMapSupport.install(); diff --git a/src/main/processor/GeminiHandler.ts b/src/main/processor/GeminiHandler.ts index 6141424..d3deefd 100644 --- a/src/main/processor/GeminiHandler.ts +++ b/src/main/processor/GeminiHandler.ts @@ -8,14 +8,14 @@ import { ToolConfig, } from '@google/generative-ai'; import { GeminiModel } from '../../types/models'; -import { ProblemSchema, SolutionSchema } from '../../types/ProblemInfo'; +import { ProblemSchema, SolutionSchema, UnifiedProblemSchema } from '../../types/ProblemInfo'; import stateManager from '../stateManager'; export async function extractProblemInfo( modelName: GeminiModel, // Use a Gemini model name imageDataList: string[], // signal: AbortSignal, // AbortSignal not directly supported by generateContent -): Promise { +): Promise { const { geminiApiKey } = stateManager.getState(); // Get Gemini key if (!geminiApiKey) { throw new Error('Gemini API key not set'); @@ -37,7 +37,16 @@ export async function extractProblemInfo( // 2. Prepare the text part (prompt) const textPart: Part = { text: - 'Extract the following information from this coding problem image:\n' + + 'Analyze this image and determine if it contains a Multiple Choice Question (MCQ) or a Coding Problem.\n\n' + + 'MCQ Format: Question on the left side of the screen with 4 options (A, B, C, D) on the right side.\n' + + 'Coding Problem Format: Programming challenge with problem statement, input/output format, constraints, and examples.\n\n' + + 'If it is an MCQ:\n' + + '- Extract the question text\n' + + '- Extract all 4 options (A, B, C, D)\n' + + '- Determine the correct answer based on your knowledge\n' + + '- Provide explanation for the correct answer\n\n' + + 'If it is a Coding Problem:\n' + + '- Extract the following information:\n' + '1. ENTIRE Problem statement (what needs to be solved)\n' + '2. Input/Output format\n' + '3. Constraints on the input\n' + @@ -58,148 +67,70 @@ export async function extractProblemInfo( { name: 'extract_problem_details', description: - 'Extract and structure the key components of a coding problem', + 'Extract and structure the key components of a problem - either MCQ or coding problem', parameters: { - type: SchemaType.OBJECT, // Use Enum for type safety + type: SchemaType.OBJECT, properties: { - problem_statement: { + type: { type: SchemaType.STRING, - description: - 'The ENTIRE main problem statement describing what needs to be solved', + format: 'enum', + enum: ['mcq', 'coding'], + description: 'Type of problem: MCQ or coding', }, - input_format: { + mcq_data: { type: SchemaType.OBJECT, properties: { - description: { + question: { type: SchemaType.STRING, - description: 'Description of the input format', + description: 'The MCQ question text', }, - parameters: { - type: SchemaType.ARRAY, - items: { - type: SchemaType.OBJECT, - properties: { - name: { - type: SchemaType.STRING, - description: 'Name of the parameter', - }, - type: { - type: SchemaType.STRING, - format: 'enum', - enum: [ - 'number', - 'string', - 'array', - 'array2d', - 'array3d', - 'matrix', - 'tree', - 'graph', - ], - description: 'Type of the parameter', - }, - subtype: { - type: SchemaType.STRING, - format: 'enum', - enum: [ - 'integer', - 'float', - 'string', - 'char', - 'boolean', - ], - description: - 'For arrays, specifies the type of elements', - }, - }, - required: ['name', 'type'], + options: { + type: SchemaType.OBJECT, + properties: { + A: { type: SchemaType.STRING, description: 'Option A' }, + B: { type: SchemaType.STRING, description: 'Option B' }, + C: { type: SchemaType.STRING, description: 'Option C' }, + D: { type: SchemaType.STRING, description: 'Option D' }, }, + required: ['A', 'B', 'C', 'D'], }, - }, - required: ['description', 'parameters'], - }, - output_format: { - type: SchemaType.OBJECT, - properties: { - description: { - type: SchemaType.STRING, - description: 'Description of the expected output format', - }, - type: { + correct_answer: { type: SchemaType.STRING, format: 'enum', - enum: [ - 'number', - 'string', - 'array', - 'array2d', - 'array3d', - 'matrix', - 'boolean', - ], - description: 'Type of the output', + enum: ['A', 'B', 'C', 'D'], + description: 'The correct answer option', }, - subtype: { + explanation: { type: SchemaType.STRING, - format: 'enum', - enum: ['integer', 'float', 'string', 'char', 'boolean'], - description: 'For arrays, specifies the type of elements', + description: 'Explanation for why this is the correct answer', }, }, - required: ['description', 'type'], + required: ['question', 'options', 'correct_answer', 'explanation'], }, - constraints: { - type: SchemaType.ARRAY, - items: { - type: SchemaType.OBJECT, - properties: { - description: { - type: SchemaType.STRING, - description: 'Description of the constraint', - }, - parameter: { - type: SchemaType.STRING, - description: 'The parameter this constraint applies to', - }, - range: { - type: SchemaType.OBJECT, - properties: { - min: { type: SchemaType.NUMBER }, - max: { type: SchemaType.NUMBER }, - }, - }, + coding_data: { + type: SchemaType.OBJECT, + properties: { + problem_statement: { + type: SchemaType.STRING, + description: + 'The ENTIRE main problem statement describing what needs to be solved', }, - required: ['description'], - }, - }, - test_cases: { - type: SchemaType.ARRAY, - minItems: 1, - items: { - type: SchemaType.OBJECT, - properties: { - input: { - type: SchemaType.OBJECT, - properties: { - args: { - type: SchemaType.ARRAY, - items: { - type: SchemaType.STRING, - description: - 'Arguments of the function call in order', - }, - description: - 'Array of arguments of mixed types (number, string, array, object, boolean, null)', - }, + input_format: { + type: SchemaType.OBJECT, + properties: { + description: { + type: SchemaType.STRING, + description: 'Description of the input format', }, - required: ['args'], - }, - output: { - type: SchemaType.OBJECT, - properties: { - result: { + parameters: { + type: SchemaType.ARRAY, + items: { type: SchemaType.OBJECT, properties: { + name: { + type: SchemaType.STRING, + description: 'Name of the parameter', + }, type: { type: SchemaType.STRING, format: 'enum', @@ -210,31 +141,151 @@ export async function extractProblemInfo( 'array2d', 'array3d', 'matrix', - 'boolean', + 'tree', + 'graph', ], - description: - 'Type of the expected result (number, string, array, object, boolean)', + description: 'Type of the parameter', }, subtype: { type: SchemaType.STRING, format: 'enum', - enum: ['integer', 'float', 'string', 'char'], + enum: [ + 'integer', + 'float', + 'string', + 'char', + 'boolean', + ], description: 'For arrays, specifies the type of elements', }, }, - description: - 'Expected result of mixed types (number, string, array, object, boolean, null)', + required: ['name', 'type'], + }, + }, + }, + required: ['description', 'parameters'], + }, + output_format: { + type: SchemaType.OBJECT, + properties: { + description: { + type: SchemaType.STRING, + description: 'Description of the expected output format', + }, + type: { + type: SchemaType.STRING, + format: 'enum', + enum: [ + 'number', + 'string', + 'array', + 'array2d', + 'array3d', + 'matrix', + 'boolean', + ], + description: 'Type of the output', + }, + subtype: { + type: SchemaType.STRING, + format: 'enum', + enum: ['integer', 'float', 'string', 'char', 'boolean'], + description: 'For arrays, specifies the type of elements', + }, + }, + required: ['description', 'type'], + }, + constraints: { + type: SchemaType.ARRAY, + items: { + type: SchemaType.OBJECT, + properties: { + description: { + type: SchemaType.STRING, + description: 'Description of the constraint', + }, + parameter: { + type: SchemaType.STRING, + description: 'The parameter this constraint applies to', + }, + range: { + type: SchemaType.OBJECT, + properties: { + min: { type: SchemaType.NUMBER }, + max: { type: SchemaType.NUMBER }, + }, }, }, - required: ['result'], + required: ['description'], + }, + }, + test_cases: { + type: SchemaType.ARRAY, + minItems: 1, + items: { + type: SchemaType.OBJECT, + properties: { + input: { + type: SchemaType.OBJECT, + properties: { + args: { + type: SchemaType.ARRAY, + items: { + type: SchemaType.STRING, + description: + 'Arguments of the function call in order', + }, + description: + 'Array of arguments of mixed types (number, string, array, object, boolean, null)', + }, + }, + required: ['args'], + }, + output: { + type: SchemaType.OBJECT, + properties: { + result: { + type: SchemaType.OBJECT, + properties: { + type: { + type: SchemaType.STRING, + format: 'enum', + enum: [ + 'number', + 'string', + 'array', + 'array2d', + 'array3d', + 'matrix', + 'boolean', + ], + description: + 'Type of the expected result (number, string, array, object, boolean)', + }, + subtype: { + type: SchemaType.STRING, + format: 'enum', + enum: ['integer', 'float', 'string', 'char'], + description: + 'For arrays, specifies the type of elements', + }, + }, + description: + 'Expected result of mixed types (number, string, array, object, boolean, null)', + }, + }, + required: ['result'], + }, + }, + required: ['input', 'output'], }, }, - required: ['input', 'output'], }, + required: ['problem_statement'], }, }, - required: ['problem_statement'], // Keep required fields + required: ['type'], }, }, ], @@ -279,7 +330,7 @@ export async function extractProblemInfo( // Gemini SDK often parses 'args' into an object directly // If it were a string (less common now), you'd use JSON.parse(args) // Validate the structure if necessary before casting - return args as ProblemSchema; // Assuming 'args' matches ProblemSchema + return args as UnifiedProblemSchema; // Assuming 'args' matches UnifiedProblemSchema } throw new Error( `Expected function call 'extract_problem_details' but got '${name}'`, @@ -323,13 +374,20 @@ export async function extractProblemInfo( export async function generateSolutionResponses( modelName: GeminiModel, - problemInfo: ProblemSchema, + problemInfo: UnifiedProblemSchema, ): Promise { const { geminiApiKey } = stateManager.getState(); // Get Gemini key if (!geminiApiKey) { throw new Error('Gemini API key not set'); } + // Only generate solutions for coding problems + if (problemInfo.type !== 'coding' || !problemInfo.coding_data) { + throw new Error('Solution generation is only available for coding problems'); + } + + const codingData = problemInfo.coding_data; + const genAI = new GoogleGenerativeAI(geminiApiKey); // Consider using a more powerful model like Pro for generation tasks if needed const model = genAI.getGenerativeModel({ @@ -343,28 +401,28 @@ export async function generateSolutionResponses( const promptContent = `Given the following coding problem: Problem Statement: -${problemInfo.problem_statement} +${codingData.problem_statement} Input Format: -${problemInfo.input_format.description ?? 'Input format not available'} +${codingData.input_format.description ?? 'Input format not available'} Parameters: ${ - problemInfo.input_format.parameters + codingData.input_format.parameters ?.map((p) => `- ${p.name}: ${p.type}${p.subtype ? ` of ${p.subtype}` : ''}`) .join('\n') ?? 'No parameters available' } Output Format: -${problemInfo.output_format.description ?? 'Output format not available'} -Returns: ${problemInfo.output_format.type ?? 'Type not specified'}${ - problemInfo.output_format?.subtype - ? ` of ${problemInfo.output_format.subtype}` +${codingData.output_format.description ?? 'Output format not available'} +Returns: ${codingData.output_format.type ?? 'Type not specified'}${ + codingData.output_format?.subtype + ? ` of ${codingData.output_format.subtype}` : '' } Constraints: ${ - problemInfo.constraints + codingData.constraints ?.map((c) => { let constraintStr = `- ${c.description}`; if (c.range) { @@ -376,7 +434,7 @@ ${ } Test Cases: -${JSON.stringify(problemInfo.test_cases ?? 'No test cases available', null, 2)} +${JSON.stringify(codingData.test_cases ?? 'No test cases available', null, 2)} Generate a solution strictly in this JSON format: { diff --git a/src/main/processor/OpenAIHandler.ts b/src/main/processor/OpenAIHandler.ts index e19850a..3e8c5a4 100644 --- a/src/main/processor/OpenAIHandler.ts +++ b/src/main/processor/OpenAIHandler.ts @@ -1,6 +1,6 @@ import axios, { AxiosError } from 'axios'; import { OpenAIModel } from '../../types/models'; -import { ProblemSchema, SolutionSchema } from '../../types/ProblemInfo'; +import { ProblemSchema, SolutionSchema, UnifiedProblemSchema } from '../../types/ProblemInfo'; import stateManager from '../stateManager'; const openAiAxios = axios.create({ @@ -27,7 +27,7 @@ export async function extractProblemInfo( model: OpenAIModel, imageDataList: string[], signal: AbortSignal, -): Promise { +): Promise { const { openAIApiKey } = stateManager.getState(); if (!openAIApiKey) { throw new Error('OpenAI API key not set'); @@ -264,8 +264,12 @@ export async function extractProblemInfo( const functionCallArguments = response.data.choices[0].message.function_call.arguments; - // Return the parsed function call arguments - return JSON.parse(functionCallArguments); + // Parse the result and wrap it in UnifiedProblemSchema format + const codingData = JSON.parse(functionCallArguments); + return { + type: 'coding' as const, + coding_data: codingData, + }; } catch (error) { if (error instanceof AxiosError) { if (error.response?.status === 401) { diff --git a/src/main/processor/index.ts b/src/main/processor/index.ts index 65d2610..5d08dbb 100644 --- a/src/main/processor/index.ts +++ b/src/main/processor/index.ts @@ -1,5 +1,5 @@ import { isGeminiModel, isOpenAIModel } from '../../types/models'; -import { ProblemSchema } from '../../types/ProblemInfo'; +import { ProblemSchema, UnifiedProblemSchema } from '../../types/ProblemInfo'; import stateManager from '../stateManager'; import * as OpenAIHandler from './OpenAIHandler'; import * as GeminiHandler from './GeminiHandler'; @@ -24,15 +24,20 @@ export const extractProblemInfo = async ( }; export const generateSolutionResponses = async ( - problemInfo: ProblemSchema, + problemInfo: UnifiedProblemSchema, signal: AbortSignal, ) => { const { solutionModel } = stateManager.getState(); + // Only generate solutions for coding problems + if (problemInfo.type !== 'coding' || !problemInfo.coding_data) { + throw new Error('Solution generation is only available for coding problems'); + } + if (isOpenAIModel(solutionModel)) return OpenAIHandler.generateSolutionResponses( solutionModel, - problemInfo, + problemInfo.coding_data, signal, ); diff --git a/src/renderer/features/solution/index.tsx b/src/renderer/features/solution/index.tsx index b0e3377..ae486f1 100644 --- a/src/renderer/features/solution/index.tsx +++ b/src/renderer/features/solution/index.tsx @@ -7,6 +7,48 @@ import { SolutionSection } from './components/SolutionSection'; function Solutions() { const { problemInfo, solutionData } = useSyncedStore(); + // Handle MCQ questions + if (problemInfo?.type === 'mcq' && problemInfo.mcq_data) { + return ( +
+ +
+
+
+ + + +
A. {problemInfo.mcq_data.options.A}
+
B. {problemInfo.mcq_data.options.B}
+
C. {problemInfo.mcq_data.options.C}
+
D. {problemInfo.mcq_data.options.D}
+
+ } + isLoading={!problemInfo} + /> + + +
+
+
+ + ); + } + + // Handle coding problems (existing logic) + const codingData = problemInfo?.type === 'coding' ? problemInfo.coding_data : null; + return (
{/* Navbar of commands with the SolutionsHelper */} @@ -18,7 +60,7 @@ function Solutions() {
diff --git a/src/types/ProblemInfo.ts b/src/types/ProblemInfo.ts index 95ab3d3..c350205 100644 --- a/src/types/ProblemInfo.ts +++ b/src/types/ProblemInfo.ts @@ -64,6 +64,24 @@ export interface ProblemSchema { test_cases: TestCase[]; } +export interface MCQSchema { + question: string; + options: { + A: string; + B: string; + C: string; + D: string; + }; + correct_answer: 'A' | 'B' | 'C' | 'D'; + explanation: string; +} + +export interface UnifiedProblemSchema { + type: 'mcq' | 'coding'; + mcq_data?: MCQSchema; + coding_data?: ProblemSchema; +} + export interface SolutionSchema { code: string; thoughts: string[]; diff --git a/src/types/index.ts b/src/types/index.ts index 5b6e125..36e3097 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -1,6 +1,6 @@ import { LANGUAGES, VIEW, SHORTCUTS } from '../constant'; import { ModelType } from './models'; -import { ProblemSchema, SolutionSchema } from './ProblemInfo'; +import { ProblemSchema, SolutionSchema, UnifiedProblemSchema } from './ProblemInfo'; export type ViewType = (typeof VIEW)[keyof typeof VIEW]; @@ -15,7 +15,7 @@ export interface Screenshot { export interface AppState { // Functional state view: ViewType; - problemInfo: ProblemSchema | null; + problemInfo: UnifiedProblemSchema | null; solutionData: SolutionSchema | null; screenshotQueue: Screenshot[]; extraScreenshotQueue: Screenshot[];