diff --git a/CHANGELOG.md b/CHANGELOG.md index 9aa07b9a..9ec7e854 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ - Fix tool loading bugs in static tool registration. - Fix xcodemake command argument corruption when project directory path appears as substring in non-path arguments. - Fix snapshot_ui warning state being isolated per UI automation tool, causing false warnings. +- Fixed screenshot tool capturing rotated images when simulator is in landscape orientation by detecting window dimensions and applying +90° rotation to correct the framebuffer capture. ([`#186`](https://github.com/cameroncooke/XcodeBuildMCP/pull/186) by [`@VincentStark`](https://github.com/VincentStark)) ## [1.14.0] - 2025-09-22 - Add video capture tool for simulators diff --git a/src/mcp/tools/simulator/__tests__/screenshot.test.ts b/src/mcp/tools/simulator/__tests__/screenshot.test.ts index 6817c171..ad6c75d4 100644 --- a/src/mcp/tools/simulator/__tests__/screenshot.test.ts +++ b/src/mcp/tools/simulator/__tests__/screenshot.test.ts @@ -10,6 +10,7 @@ import { createMockExecutor, createMockFileSystemExecutor, createCommandMatchingMockExecutor, + mockProcess, } from '../../../../test-utils/mock-executors.ts'; import type { CommandExecutor } from '../../../../utils/execution/index.ts'; import { SystemError } from '../../../../utils/responses/index.ts'; @@ -48,18 +49,34 @@ describe('screenshot plugin', () => { }); describe('Command Generation', () => { + // Mock device list JSON for proper device name lookup + const mockDeviceListJson = JSON.stringify({ + devices: { + 'com.apple.CoreSimulator.SimRuntime.iOS-17-2': [ + { udid: 'test-uuid', name: 'iPhone 15 Pro', state: 'Booted' }, + { udid: 'another-uuid', name: 'iPhone 15', state: 'Booted' }, + ], + }, + }); + it('should generate correct simctl and sips commands', async () => { const capturedCommands: string[][] = []; - const mockExecutor = createCommandMatchingMockExecutor({ - 'xcrun simctl': { success: true, output: 'Screenshot saved' }, - sips: { success: true, output: 'Image optimized' }, - }); - - // Wrap to capture both commands + // Wrap to capture commands and return appropriate mock responses const capturingExecutor = async (command: string[], ...args: any[]) => { capturedCommands.push(command); - return mockExecutor(command, ...args); + const cmdStr = command.join(' '); + // Return device list JSON for list command + if (cmdStr.includes('simctl list devices')) { + return { + success: true, + output: mockDeviceListJson, + error: undefined, + process: mockProcess, + }; + } + // Return success for all other commands + return { success: true, output: '', error: undefined, process: mockProcess }; }; const mockFileSystemExecutor = createMockFileSystemExecutor({ @@ -85,8 +102,8 @@ describe('screenshot plugin', () => { mockUuidDeps, ); - // Should execute both commands in sequence - expect(capturedCommands).toHaveLength(2); + // Should execute all commands in sequence: screenshot, list devices, orientation detection, optimization + expect(capturedCommands).toHaveLength(4); // First command: xcrun simctl screenshot expect(capturedCommands[0]).toEqual([ @@ -98,8 +115,17 @@ describe('screenshot plugin', () => { '/tmp/screenshot_mock-uuid-123.png', ]); - // Second command: sips optimization - expect(capturedCommands[1]).toEqual([ + // Second command: xcrun simctl list devices (to get device name) + expect(capturedCommands[1][0]).toBe('xcrun'); + expect(capturedCommands[1][1]).toBe('simctl'); + expect(capturedCommands[1][2]).toBe('list'); + + // Third command: swift orientation detection + expect(capturedCommands[2][0]).toBe('swift'); + expect(capturedCommands[2][1]).toBe('-e'); + + // Fourth command: sips optimization + expect(capturedCommands[3]).toEqual([ 'sips', '-Z', '800', @@ -118,15 +144,21 @@ describe('screenshot plugin', () => { it('should generate correct path with different uuid', async () => { const capturedCommands: string[][] = []; - const mockExecutor = createCommandMatchingMockExecutor({ - 'xcrun simctl': { success: true, output: 'Screenshot saved' }, - sips: { success: true, output: 'Image optimized' }, - }); - - // Wrap to capture both commands + // Wrap to capture commands and return appropriate mock responses const capturingExecutor = async (command: string[], ...args: any[]) => { capturedCommands.push(command); - return mockExecutor(command, ...args); + const cmdStr = command.join(' '); + // Return device list JSON for list command + if (cmdStr.includes('simctl list devices')) { + return { + success: true, + output: mockDeviceListJson, + error: undefined, + process: mockProcess, + }; + } + // Return success for all other commands + return { success: true, output: '', error: undefined, process: mockProcess }; }; const mockFileSystemExecutor = createMockFileSystemExecutor({ @@ -152,8 +184,8 @@ describe('screenshot plugin', () => { mockUuidDeps, ); - // Should execute both commands in sequence - expect(capturedCommands).toHaveLength(2); + // Should execute all commands in sequence: screenshot, list devices, orientation detection, optimization + expect(capturedCommands).toHaveLength(4); // First command: xcrun simctl screenshot expect(capturedCommands[0]).toEqual([ @@ -165,8 +197,17 @@ describe('screenshot plugin', () => { '/tmp/screenshot_different-uuid-456.png', ]); - // Second command: sips optimization - expect(capturedCommands[1]).toEqual([ + // Second command: xcrun simctl list devices (to get device name) + expect(capturedCommands[1][0]).toBe('xcrun'); + expect(capturedCommands[1][1]).toBe('simctl'); + expect(capturedCommands[1][2]).toBe('list'); + + // Third command: swift orientation detection + expect(capturedCommands[2][0]).toBe('swift'); + expect(capturedCommands[2][1]).toBe('-e'); + + // Fourth command: sips optimization + expect(capturedCommands[3]).toEqual([ 'sips', '-Z', '800', @@ -185,15 +226,21 @@ describe('screenshot plugin', () => { it('should use default dependencies when not provided', async () => { const capturedCommands: string[][] = []; - const mockExecutor = createCommandMatchingMockExecutor({ - 'xcrun simctl': { success: true, output: 'Screenshot saved' }, - sips: { success: true, output: 'Image optimized' }, - }); - - // Wrap to capture both commands + // Wrap to capture commands and return appropriate mock responses const capturingExecutor = async (command: string[], ...args: any[]) => { capturedCommands.push(command); - return mockExecutor(command, ...args); + const cmdStr = command.join(' '); + // Return device list JSON for list command + if (cmdStr.includes('simctl list devices')) { + return { + success: true, + output: mockDeviceListJson, + error: undefined, + process: mockProcess, + }; + } + // Return success for all other commands + return { success: true, output: '', error: undefined, process: mockProcess }; }; const mockFileSystemExecutor = createMockFileSystemExecutor({ @@ -208,8 +255,8 @@ describe('screenshot plugin', () => { mockFileSystemExecutor, ); - // Should execute both commands in sequence - expect(capturedCommands).toHaveLength(2); + // Should execute all commands in sequence: screenshot, list devices, orientation detection, optimization + expect(capturedCommands).toHaveLength(4); // First command should be generated with real os.tmpdir, path.join, and uuidv4 const firstCommand = capturedCommands[0]; @@ -221,14 +268,23 @@ describe('screenshot plugin', () => { expect(firstCommand[4]).toBe('screenshot'); expect(firstCommand[5]).toMatch(/\/.*\/screenshot_.*\.png/); - // Second command should be sips optimization - const secondCommand = capturedCommands[1]; - expect(secondCommand[0]).toBe('sips'); - expect(secondCommand[1]).toBe('-Z'); - expect(secondCommand[2]).toBe('800'); + // Second command should be xcrun simctl list devices + expect(capturedCommands[1][0]).toBe('xcrun'); + expect(capturedCommands[1][1]).toBe('simctl'); + expect(capturedCommands[1][2]).toBe('list'); + + // Third command should be swift orientation detection + expect(capturedCommands[2][0]).toBe('swift'); + expect(capturedCommands[2][1]).toBe('-e'); + + // Fourth command should be sips optimization + const thirdCommand = capturedCommands[3]; + expect(thirdCommand[0]).toBe('sips'); + expect(thirdCommand[1]).toBe('-Z'); + expect(thirdCommand[2]).toBe('800'); // Should have proper PNG input and JPG output paths - expect(secondCommand[secondCommand.length - 3]).toMatch(/\/.*\/screenshot_.*\.png/); - expect(secondCommand[secondCommand.length - 1]).toMatch(/\/.*\/screenshot_optimized_.*\.jpg/); + expect(thirdCommand[thirdCommand.length - 3]).toMatch(/\/.*\/screenshot_.*\.png/); + expect(thirdCommand[thirdCommand.length - 1]).toMatch(/\/.*\/screenshot_optimized_.*\.jpg/); }); }); @@ -370,15 +426,31 @@ describe('screenshot plugin', () => { it('should call correct command with direct execution', async () => { const capturedArgs: any[][] = []; - const mockExecutor = createCommandMatchingMockExecutor({ - 'xcrun simctl': { success: true, output: 'Screenshot saved' }, - sips: { success: true, output: 'Image optimized' }, + // Mock device list JSON for proper device name lookup + const mockDeviceListJson = JSON.stringify({ + devices: { + 'com.apple.CoreSimulator.SimRuntime.iOS-17-2': [ + { udid: 'test-uuid', name: 'iPhone 15 Pro', state: 'Booted' }, + ], + }, }); - // Wrap to capture both command executions + // Capture all command executions and return appropriate mock responses const capturingExecutor: CommandExecutor = async (...args) => { capturedArgs.push(args); - return mockExecutor(...args); + const command = args[0] as string[]; + const cmdStr = command.join(' '); + // Return device list JSON for list command + if (cmdStr.includes('simctl list devices')) { + return { + success: true, + output: mockDeviceListJson, + error: undefined, + process: mockProcess, + }; + } + // Return success for all other commands + return { success: true, output: '', error: undefined, process: mockProcess }; }; const mockFileSystemExecutor = createMockFileSystemExecutor({ @@ -404,8 +476,8 @@ describe('screenshot plugin', () => { mockUuidDeps, ); - // Should capture both command executions - expect(capturedArgs).toHaveLength(2); + // Should capture all command executions: screenshot, list devices, orientation detection, optimization + expect(capturedArgs).toHaveLength(4); // First call: xcrun simctl screenshot (3 args: command, logPrefix, useShell) expect(capturedArgs[0]).toEqual([ @@ -414,8 +486,21 @@ describe('screenshot plugin', () => { false, ]); - // Second call: sips optimization (3 args: command, logPrefix, useShell) - expect(capturedArgs[1]).toEqual([ + // Second call: xcrun simctl list devices (to get device name) + expect(capturedArgs[1][0][0]).toBe('xcrun'); + expect(capturedArgs[1][0][1]).toBe('simctl'); + expect(capturedArgs[1][0][2]).toBe('list'); + expect(capturedArgs[1][1]).toBe('[Screenshot]: list devices'); + expect(capturedArgs[1][2]).toBe(false); + + // Third call: swift orientation detection + expect(capturedArgs[2][0][0]).toBe('swift'); + expect(capturedArgs[2][0][1]).toBe('-e'); + expect(capturedArgs[2][1]).toBe('[Screenshot]: detect orientation'); + expect(capturedArgs[2][2]).toBe(false); + + // Fourth call: sips optimization (3 args: command, logPrefix, useShell) + expect(capturedArgs[3]).toEqual([ [ 'sips', '-Z', diff --git a/src/mcp/tools/ui-automation/__tests__/screenshot.test.ts b/src/mcp/tools/ui-automation/__tests__/screenshot.test.ts index d8b7fd22..8d1ac302 100644 --- a/src/mcp/tools/ui-automation/__tests__/screenshot.test.ts +++ b/src/mcp/tools/ui-automation/__tests__/screenshot.test.ts @@ -12,7 +12,11 @@ import { } from '../../../../test-utils/mock-executors.ts'; import { SystemError } from '../../../../utils/responses/index.ts'; import { sessionStore } from '../../../../utils/session-store.ts'; -import screenshotPlugin, { screenshotLogic } from '../screenshot.ts'; +import screenshotPlugin, { + screenshotLogic, + detectLandscapeMode, + rotateImage, +} from '../screenshot.ts'; describe('Screenshot Plugin', () => { beforeEach(() => { @@ -440,4 +444,447 @@ describe('Screenshot Plugin', () => { }); }); }); + + describe('Landscape Detection', () => { + it('should detect landscape mode when window width > height', async () => { + const mockExecutor = async () => ({ + success: true, + output: '844,390', + error: undefined, + process: mockProcess, + }); + + const result = await detectLandscapeMode(mockExecutor, 'iPhone 15 Pro'); + + expect(result).toBe(true); + }); + + it('should detect portrait mode when window height > width', async () => { + const mockExecutor = async () => ({ + success: true, + output: '390,844', + error: undefined, + process: mockProcess, + }); + + const result = await detectLandscapeMode(mockExecutor, 'iPhone 15 Pro'); + + expect(result).toBe(false); + }); + + it('should return false when swift command fails', async () => { + const mockExecutor = async () => ({ + success: false, + output: '', + error: 'Command failed', + process: mockProcess, + }); + + const result = await detectLandscapeMode(mockExecutor, 'iPhone 15 Pro'); + + expect(result).toBe(false); + }); + + it('should return false when output format is unexpected', async () => { + const mockExecutor = async () => ({ + success: true, + output: 'invalid output', + error: undefined, + process: mockProcess, + }); + + const result = await detectLandscapeMode(mockExecutor, 'iPhone 15 Pro'); + + expect(result).toBe(false); + }); + + it('should return false when executor throws an error', async () => { + const mockExecutor = async () => { + throw new Error('Execution failed'); + }; + + const result = await detectLandscapeMode(mockExecutor, 'iPhone 15 Pro'); + + expect(result).toBe(false); + }); + + it('should handle output with whitespace and newlines', async () => { + const mockExecutor = async () => ({ + success: true, + output: '\n 844,390 \n', + error: undefined, + process: mockProcess, + }); + + const result = await detectLandscapeMode(mockExecutor, 'iPhone 15 Pro'); + + expect(result).toBe(true); + }); + + it('should return false when no device name is provided', async () => { + const mockExecutor = async () => ({ + success: true, + output: '844,390', + error: undefined, + process: mockProcess, + }); + + // When no device name is provided, should skip orientation detection + const result = await detectLandscapeMode(mockExecutor); + + expect(result).toBe(false); + }); + }); + + describe('Image Rotation', () => { + it('should call sips with correct rotation arguments', async () => { + const capturedCommands: string[][] = []; + const mockExecutor = async (command: string[]) => { + capturedCommands.push(command); + return { + success: true, + output: '', + error: undefined, + process: mockProcess, + }; + }; + + await rotateImage('/tmp/test.png', 90, mockExecutor); + + expect(capturedCommands[0]).toEqual(['sips', '--rotate', '90', '/tmp/test.png']); + }); + + it('should return true on successful rotation', async () => { + const mockExecutor = async () => ({ + success: true, + output: '', + error: undefined, + process: mockProcess, + }); + + const result = await rotateImage('/tmp/test.png', 90, mockExecutor); + + expect(result).toBe(true); + }); + + it('should return false when rotation command fails', async () => { + const mockExecutor = async () => ({ + success: false, + output: '', + error: 'sips: error', + process: mockProcess, + }); + + const result = await rotateImage('/tmp/test.png', 90, mockExecutor); + + expect(result).toBe(false); + }); + + it('should return false when executor throws an error', async () => { + const mockExecutor = async () => { + throw new Error('Execution failed'); + }; + + const result = await rotateImage('/tmp/test.png', 90, mockExecutor); + + expect(result).toBe(false); + }); + + it('should handle different rotation angles', async () => { + const capturedCommands: string[][] = []; + const mockExecutor = async (command: string[]) => { + capturedCommands.push(command); + return { + success: true, + output: '', + error: undefined, + process: mockProcess, + }; + }; + + await rotateImage('/tmp/test.png', 270, mockExecutor); + + expect(capturedCommands[0]).toEqual(['sips', '--rotate', '270', '/tmp/test.png']); + }); + }); + + describe('Landscape Screenshot Integration', () => { + // Mock device list JSON response + const mockDeviceListJson = JSON.stringify({ + devices: { + 'com.apple.CoreSimulator.SimRuntime.iOS-17-2': [ + { + udid: '12345678-1234-4234-8234-123456789012', + name: 'iPhone 15 Pro', + state: 'Booted', + }, + ], + }, + }); + + it('should rotate screenshot when landscape mode is detected', async () => { + const capturedCommands: string[][] = []; + let commandIndex = 0; + const trackingExecutor = async (command: string[]) => { + capturedCommands.push(command); + const idx = commandIndex++; + + // First call: screenshot command + if (idx === 0) { + return { + success: true, + output: 'Screenshot saved', + error: undefined, + process: mockProcess, + }; + } + // Second call: list devices to get device name + if (idx === 1) { + return { + success: true, + output: mockDeviceListJson, + error: undefined, + process: mockProcess, + }; + } + // Third call: swift orientation detection (simulate landscape) + if (idx === 2) { + return { + success: true, + output: '844,390', + error: undefined, + process: mockProcess, + }; + } + // Fourth call: sips rotation + if (idx === 3) { + return { + success: true, + output: '', + error: undefined, + process: mockProcess, + }; + } + // Fifth call: sips optimization + return { + success: true, + output: '', + error: undefined, + process: mockProcess, + }; + }; + + const mockFileSystemExecutor = createMockFileSystemExecutor({ + readFile: async () => 'fake-image-data', + }); + + await screenshotLogic( + { simulatorId: '12345678-1234-4234-8234-123456789012' }, + trackingExecutor, + mockFileSystemExecutor, + { tmpdir: () => '/tmp', join: (...paths) => paths.join('/') }, + { v4: () => 'test-uuid' }, + ); + + // Verify rotation command was called with +90 degrees (index 3) + expect(capturedCommands[3]).toEqual([ + 'sips', + '--rotate', + '90', + '/tmp/screenshot_test-uuid.png', + ]); + }); + + it('should not rotate screenshot when portrait mode is detected', async () => { + const capturedCommands: string[][] = []; + let commandIndex = 0; + const trackingExecutor = async (command: string[]) => { + capturedCommands.push(command); + const idx = commandIndex++; + + // First call: screenshot command + if (idx === 0) { + return { + success: true, + output: 'Screenshot saved', + error: undefined, + process: mockProcess, + }; + } + // Second call: list devices to get device name + if (idx === 1) { + return { + success: true, + output: mockDeviceListJson, + error: undefined, + process: mockProcess, + }; + } + // Third call: swift orientation detection (simulate portrait) + if (idx === 2) { + return { + success: true, + output: '390,844', + error: undefined, + process: mockProcess, + }; + } + // Fourth call: sips optimization (no rotation in portrait) + return { + success: true, + output: '', + error: undefined, + process: mockProcess, + }; + }; + + const mockFileSystemExecutor = createMockFileSystemExecutor({ + readFile: async () => 'fake-image-data', + }); + + await screenshotLogic( + { simulatorId: '12345678-1234-4234-8234-123456789012' }, + trackingExecutor, + mockFileSystemExecutor, + { tmpdir: () => '/tmp', join: (...paths) => paths.join('/') }, + { v4: () => 'test-uuid' }, + ); + + // Should have: screenshot, list devices, orientation detection, optimization (no rotation) + expect(capturedCommands.length).toBe(4); + // Fourth command should be optimization, not rotation + expect(capturedCommands[3][0]).toBe('sips'); + expect(capturedCommands[3]).toContain('-Z'); + }); + + it('should continue without rotation if orientation detection fails', async () => { + const capturedCommands: string[][] = []; + let commandIndex = 0; + const trackingExecutor = async (command: string[]) => { + capturedCommands.push(command); + const idx = commandIndex++; + + // First call: screenshot command + if (idx === 0) { + return { + success: true, + output: 'Screenshot saved', + error: undefined, + process: mockProcess, + }; + } + // Second call: list devices to get device name + if (idx === 1) { + return { + success: true, + output: mockDeviceListJson, + error: undefined, + process: mockProcess, + }; + } + // Third call: swift orientation detection (fails) + if (idx === 2) { + return { + success: false, + output: '', + error: 'Swift not found', + process: mockProcess, + }; + } + // Fourth call: sips optimization + return { + success: true, + output: '', + error: undefined, + process: mockProcess, + }; + }; + + const mockFileSystemExecutor = createMockFileSystemExecutor({ + readFile: async () => 'fake-image-data', + }); + + const result = await screenshotLogic( + { simulatorId: '12345678-1234-4234-8234-123456789012' }, + trackingExecutor, + mockFileSystemExecutor, + { tmpdir: () => '/tmp', join: (...paths) => paths.join('/') }, + { v4: () => 'test-uuid' }, + ); + + // Should still succeed + expect(result.isError).toBe(false); + // Should have: screenshot, list devices, failed orientation detection, optimization + expect(capturedCommands.length).toBe(4); + }); + + it('should continue if rotation fails but still return image', async () => { + const capturedCommands: string[][] = []; + let commandIndex = 0; + const trackingExecutor = async (command: string[]) => { + capturedCommands.push(command); + const idx = commandIndex++; + + // First call: screenshot command + if (idx === 0) { + return { + success: true, + output: 'Screenshot saved', + error: undefined, + process: mockProcess, + }; + } + // Second call: list devices to get device name + if (idx === 1) { + return { + success: true, + output: mockDeviceListJson, + error: undefined, + process: mockProcess, + }; + } + // Third call: swift orientation detection (landscape) + if (idx === 2) { + return { + success: true, + output: '844,390', + error: undefined, + process: mockProcess, + }; + } + // Fourth call: sips rotation (fails) + if (idx === 3) { + return { + success: false, + output: '', + error: 'sips failed', + process: mockProcess, + }; + } + // Fifth call: sips optimization + return { + success: true, + output: '', + error: undefined, + process: mockProcess, + }; + }; + + const mockFileSystemExecutor = createMockFileSystemExecutor({ + readFile: async () => 'fake-image-data', + }); + + const result = await screenshotLogic( + { simulatorId: '12345678-1234-4234-8234-123456789012' }, + trackingExecutor, + mockFileSystemExecutor, + { tmpdir: () => '/tmp', join: (...paths) => paths.join('/') }, + { v4: () => 'test-uuid' }, + ); + + // Should still succeed even if rotation failed + expect(result.isError).toBe(false); + expect(result.content[0].type).toBe('image'); + }); + }); }); diff --git a/src/mcp/tools/ui-automation/screenshot.ts b/src/mcp/tools/ui-automation/screenshot.ts index 383544da..9455f575 100644 --- a/src/mcp/tools/ui-automation/screenshot.ts +++ b/src/mcp/tools/ui-automation/screenshot.ts @@ -1,5 +1,10 @@ /** * Screenshot tool plugin - Capture screenshots from iOS Simulator + * + * Note: The simctl screenshot command captures the raw framebuffer in portrait orientation + * regardless of the device's actual rotation. When the simulator is in landscape mode, + * this results in a rotated image. This plugin detects the simulator window orientation + * and applies a +90° rotation to correct landscape screenshots. */ import * as path from 'path'; import { tmpdir } from 'os'; @@ -20,6 +25,142 @@ import { const LOG_PREFIX = '[Screenshot]'; +/** + * Type for simctl device list response + */ +interface SimctlDevice { + udid: string; + name: string; + state?: string; +} + +interface SimctlDeviceList { + devices: Record; +} + +/** + * Generates Swift code to detect simulator window dimensions via CoreGraphics. + * Filters by device name to handle multiple open simulators correctly. + * Returns "width,height" of the matching simulator window. + */ +function getWindowDetectionSwiftCode(deviceName: string): string { + // Escape the device name for use in Swift string + const escapedDeviceName = deviceName.replace(/"/g, '\\"'); + // Use hasPrefix + boundary check to avoid matching "iPhone 15" when looking for "iPhone 15 Pro" + // Window titles are formatted like "iPhone 15 Pro – iOS 17.2" + return ` +import Cocoa +import CoreGraphics +let deviceName = "${escapedDeviceName}" +let opts = CGWindowListOption(arrayLiteral: .optionOnScreenOnly, .excludeDesktopElements) +if let wins = CGWindowListCopyWindowInfo(opts, kCGNullWindowID) as? [[String: Any]] { + for w in wins { + if let o = w[kCGWindowOwnerName as String] as? String, o == "Simulator", + let b = w[kCGWindowBounds as String] as? [String: Any], + let n = w[kCGWindowName as String] as? String { + // Check for exact match: name starts with deviceName followed by separator or end + let isMatch = n == deviceName || n.hasPrefix(deviceName + " ") + if isMatch { + print("\\(b["Width"] as? Int ?? 0),\\(b["Height"] as? Int ?? 0)") + break + } + } + } +}`.trim(); +} + +/** + * Gets the device name for a simulator ID using simctl. + * Returns the device name or null if not found. + */ +export async function getDeviceNameForSimulatorId( + simulatorId: string, + executor: CommandExecutor, +): Promise { + try { + const listCommand = ['xcrun', 'simctl', 'list', 'devices', '-j']; + const result = await executor(listCommand, `${LOG_PREFIX}: list devices`, false); + + if (result.success && result.output) { + const data = JSON.parse(result.output) as SimctlDeviceList; + const devices = data.devices; + + for (const runtime of Object.keys(devices)) { + for (const device of devices[runtime]) { + if (device.udid === simulatorId) { + log('info', `${LOG_PREFIX}: Found device name "${device.name}" for ${simulatorId}`); + return device.name; + } + } + } + } + log('warning', `${LOG_PREFIX}: Could not find device name for ${simulatorId}`); + return null; + } catch (error) { + log('warning', `${LOG_PREFIX}: Failed to get device name: ${error}`); + return null; + } +} + +/** + * Detects if the simulator window is in landscape orientation. + * Uses the device name to filter when multiple simulators are open. + * Returns true if width > height, indicating landscape mode. + */ +export async function detectLandscapeMode( + executor: CommandExecutor, + deviceName?: string, +): Promise { + try { + // If no device name available, skip orientation detection to avoid incorrect rotation + // This is safer than guessing, as we don't know if it's iPhone or iPad + if (!deviceName) { + log('warning', `${LOG_PREFIX}: No device name available, skipping orientation detection`); + return false; + } + const swiftCode = getWindowDetectionSwiftCode(deviceName); + const swiftCommand = ['swift', '-e', swiftCode]; + const result = await executor(swiftCommand, `${LOG_PREFIX}: detect orientation`, false); + + if (result.success && result.output) { + const match = result.output.trim().match(/(\d+),(\d+)/); + if (match) { + const width = parseInt(match[1], 10); + const height = parseInt(match[2], 10); + const isLandscape = width > height; + log( + 'info', + `${LOG_PREFIX}: Window dimensions ${width}x${height}, landscape=${isLandscape}`, + ); + return isLandscape; + } + } + log('warning', `${LOG_PREFIX}: Could not detect window orientation, assuming portrait`); + return false; + } catch (error) { + log('warning', `${LOG_PREFIX}: Orientation detection failed: ${error}`); + return false; + } +} + +/** + * Rotates an image by the specified degrees using sips. + */ +export async function rotateImage( + imagePath: string, + degrees: number, + executor: CommandExecutor, +): Promise { + try { + const rotateArgs = ['sips', '--rotate', degrees.toString(), imagePath]; + const result = await executor(rotateArgs, `${LOG_PREFIX}: rotate image`, false); + return result.success; + } catch (error) { + log('warning', `${LOG_PREFIX}: Image rotation failed: ${error}`); + return false; + } +} + // Define schema as ZodObject const screenshotSchema = z.object({ simulatorId: z.uuid({ message: 'Invalid Simulator UUID format' }), @@ -68,6 +209,19 @@ export async function screenshotLogic( log('info', `${LOG_PREFIX}/screenshot: Success for ${simulatorId}`); try { + // Fix landscape orientation: simctl captures in portrait orientation regardless of device rotation + // Get device name to identify the correct simulator window when multiple are open + const deviceName = await getDeviceNameForSimulatorId(simulatorId, executor); + // Detect if simulator window is landscape and rotate the image +90° to correct + const isLandscape = await detectLandscapeMode(executor, deviceName ?? undefined); + if (isLandscape) { + log('info', `${LOG_PREFIX}/screenshot: Landscape mode detected, rotating +90°`); + const rotated = await rotateImage(screenshotPath, 90, executor); + if (!rotated) { + log('warning', `${LOG_PREFIX}/screenshot: Rotation failed, continuing with original`); + } + } + // Optimize the image for LLM consumption: resize to max 800px width and convert to JPEG const optimizeArgs = [ 'sips',