diff --git a/deep-sea-stories/packages/backend/src/agent/api.ts b/deep-sea-stories/packages/backend/src/agent/api.ts index 6df97df..603b9bf 100644 --- a/deep-sea-stories/packages/backend/src/agent/api.ts +++ b/deep-sea-stories/packages/backend/src/agent/api.ts @@ -6,6 +6,7 @@ export interface AgentConfig { onEndGame: () => Promise; gameTimeLimitSeconds: number; onTranscription: (transcription: string) => void; + onReadyForPlayerInput?: () => void; } export interface VoiceAgentApi { diff --git a/deep-sea-stories/packages/backend/src/agent/gemini/session.ts b/deep-sea-stories/packages/backend/src/agent/gemini/session.ts index 9be0679..5823d82 100644 --- a/deep-sea-stories/packages/backend/src/agent/gemini/session.ts +++ b/deep-sea-stories/packages/backend/src/agent/gemini/session.ts @@ -25,6 +25,7 @@ export class GeminiSession implements VoiceAgentSession { private opening = false; private reconnecting = false; private ending = false; + private awaitingInitialGreeting = false; private talkingTimeLeft = 0; private talkingInterval: NodeJS.Timeout | null = null; @@ -87,6 +88,9 @@ export class GeminiSession implements VoiceAgentSession { this.opening = true; this.ending = false; + const shouldRequestIntro = + !this.previousHandle && !this.awaitingInitialGreeting; + const params: LiveConnectParameters = { model: GEMINI_MODEL, config: { @@ -130,7 +134,8 @@ export class GeminiSession implements VoiceAgentSession { this.session = await this.genai.live.connect(params); - if (!this.previousHandle) { + if (shouldRequestIntro) { + this.awaitingInitialGreeting = true; this.session.sendClientContent({ turns: [ { @@ -139,6 +144,8 @@ export class GeminiSession implements VoiceAgentSession { ], turnComplete: true, }); + } else if (!this.awaitingInitialGreeting) { + this.config.onReadyForPlayerInput?.(); } if (this.talkingInterval) clearInterval(this.talkingInterval); @@ -212,7 +219,13 @@ export class GeminiSession implements VoiceAgentSession { this.transcriptionParts = []; } - if (turnFinished) this.onTurnEnd?.(); + if (turnFinished) { + if (this.awaitingInitialGreeting) { + this.awaitingInitialGreeting = false; + this.config.onReadyForPlayerInput?.(); + } + this.onTurnEnd?.(); + } const base64 = message.data; if (base64) { diff --git a/deep-sea-stories/packages/backend/src/game/room.ts b/deep-sea-stories/packages/backend/src/game/room.ts index 4c41de9..c27dd29 100644 --- a/deep-sea-stories/packages/backend/src/game/room.ts +++ b/deep-sea-stories/packages/backend/src/game/room.ts @@ -146,6 +146,9 @@ export class GameRoom { timestamp: Date.now(), }); }, + onReadyForPlayerInput: () => { + this.gameSession?.setAiAgentMuted(false); + }, }); const { agent: fishjamAgent, agentId: fishjamAgentId } = @@ -166,6 +169,8 @@ export class GameRoom { this.notifierService, ); + this.gameSession.setAiAgentMuted(true); + console.log( `Starting game for ${this.players.size} players in room ${this.roomId}`, ); diff --git a/deep-sea-stories/packages/backend/src/game/session.ts b/deep-sea-stories/packages/backend/src/game/session.ts index 69d2407..9c5c503 100644 --- a/deep-sea-stories/packages/backend/src/game/session.ts +++ b/deep-sea-stories/packages/backend/src/game/session.ts @@ -35,6 +35,7 @@ export class GameSession { } setAiAgentMuted(muted: boolean) { + if (this.isAiAgentMuted === muted) return; this.isAiAgentMuted = muted; this.audioOrchestrator.setMuted(muted); diff --git a/deep-sea-stories/packages/backend/src/prompts/stories.json b/deep-sea-stories/packages/backend/src/prompts/stories.json index 38b1ef6..0649a61 100644 --- a/deep-sea-stories/packages/backend/src/prompts/stories.json +++ b/deep-sea-stories/packages/backend/src/prompts/stories.json @@ -3,18 +3,18 @@ "id": 1, "title": "The Diver in the Forest", "front": "A man is found dead in the middle of a burnt forest. He is wearing a full wet suit, flippers, and a scuba tank.", - "back": "The man was diving in the ocean near the coast. A massive forest fire was raging nearby. A firefighting plane scooped up a load of water from the ocean to dump on the fire. The diver was accidentally scooped up along with the water and dropped from a great height onto the burning forest. He died on impact." + "back": "The man was diving in the ocean near the coast. A massive forest fire was raging nearby. A firefighting plane scooped up a load of water from the ocean to dump on the fire. The diver was accidentally scooped up along with the water and dropped from a great height onto the burning forest." }, { "id": 2, "title": "The Light Switch", - "front": "A man turned off the light and went to sleep. The next morning, he woke up, looked out the window, saw the devastation, and immediately killed himself.", - "back": "The man was a lighthouse keeper. By turning off the light before going to sleep during a storm, he caused several ships to crash into the coast. When he saw the wreckage and bodies floating in the sea the next morning, he couldn't live with the guilt." + "front": "A man turned off the light and went to sleep. The next morning, he woke up, looked out the window, saw tremendous devastation.", + "back": "The man was a lighthouse keeper. By turning off the light before going to sleep during a storm, he caused several ships to crash into the coast. He saw the wreckage in the sea the next morning." }, { "id": 3, "title": "The High Card", - "front": "Four men sat around a table in a metal room. One man drew the highest card, smiled, and then immediately died.", + "front": "Four men sat around a table in a metal room. One man drew the highest card, froze, and passed away seconds later.", "back": "The four men were the crew of a military submarine that had lost power and was running out of oxygen. They realized there was only enough air left for three of them to survive until rescue arrived. They drew cards to decide who would sacrifice themselves to save the others. The man with the high card \"won\" the draw and shot himself to stop consuming oxygen." }, { diff --git a/deep-sea-stories/packages/web/src/components/PeerTile.tsx b/deep-sea-stories/packages/web/src/components/PeerTile.tsx index 5a79779..ab19d36 100644 --- a/deep-sea-stories/packages/web/src/components/PeerTile.tsx +++ b/deep-sea-stories/packages/web/src/components/PeerTile.tsx @@ -42,7 +42,7 @@ export const PeerTile: FC> = ({ }, [audioStream]); return ( -
+