From d570f9a91b47adaa9cc18e721d83662daed2947b Mon Sep 17 00:00:00 2001 From: Adrian Czerwiec Date: Wed, 11 Feb 2026 09:20:00 +0100 Subject: [PATCH 1/7] defer user input --- .../packages/backend/src/agent/api.ts | 1 + .../backend/src/agent/gemini/session.ts | 16 ++++++++++-- .../packages/backend/src/game/room.ts | 7 ++++++ .../packages/backend/src/game/session.ts | 25 ++++++++++++++++++- 4 files changed, 46 insertions(+), 3 deletions(-) diff --git a/deep-sea-stories/packages/backend/src/agent/api.ts b/deep-sea-stories/packages/backend/src/agent/api.ts index 6df97df..603b9bf 100644 --- a/deep-sea-stories/packages/backend/src/agent/api.ts +++ b/deep-sea-stories/packages/backend/src/agent/api.ts @@ -6,6 +6,7 @@ export interface AgentConfig { onEndGame: () => Promise; gameTimeLimitSeconds: number; onTranscription: (transcription: string) => void; + onReadyForPlayerInput?: () => void; } export interface VoiceAgentApi { diff --git a/deep-sea-stories/packages/backend/src/agent/gemini/session.ts b/deep-sea-stories/packages/backend/src/agent/gemini/session.ts index 9be0679..16478ba 100644 --- a/deep-sea-stories/packages/backend/src/agent/gemini/session.ts +++ b/deep-sea-stories/packages/backend/src/agent/gemini/session.ts @@ -25,6 +25,7 @@ export class GeminiSession implements VoiceAgentSession { private opening = false; private reconnecting = false; private ending = false; + private awaitingInitialGreeting = false; private talkingTimeLeft = 0; private talkingInterval: NodeJS.Timeout | null = null; @@ -87,6 +88,8 @@ export class GeminiSession implements VoiceAgentSession { this.opening = true; this.ending = false; + const shouldRequestIntro = !this.previousHandle && !this.awaitingInitialGreeting; + const params: LiveConnectParameters = { model: GEMINI_MODEL, config: { @@ -130,7 +133,8 @@ export class GeminiSession implements VoiceAgentSession { this.session = await this.genai.live.connect(params); - if (!this.previousHandle) { + if (shouldRequestIntro) { + this.awaitingInitialGreeting = true; this.session.sendClientContent({ turns: [ { @@ -139,6 +143,8 @@ export class GeminiSession implements VoiceAgentSession { ], turnComplete: true, }); + } else if (!this.awaitingInitialGreeting) { + this.config.onReadyForPlayerInput?.(); } if (this.talkingInterval) clearInterval(this.talkingInterval); @@ -212,7 +218,13 @@ export class GeminiSession implements VoiceAgentSession { this.transcriptionParts = []; } - if (turnFinished) this.onTurnEnd?.(); + if (turnFinished) { + if (this.awaitingInitialGreeting) { + this.awaitingInitialGreeting = false; + this.config.onReadyForPlayerInput?.(); + } + this.onTurnEnd?.(); + } const base64 = message.data; if (base64) { diff --git a/deep-sea-stories/packages/backend/src/game/room.ts b/deep-sea-stories/packages/backend/src/game/room.ts index 4c41de9..c8a9437 100644 --- a/deep-sea-stories/packages/backend/src/game/room.ts +++ b/deep-sea-stories/packages/backend/src/game/room.ts @@ -146,6 +146,11 @@ export class GameRoom { timestamp: Date.now(), }); }, + onReadyForPlayerInput: () => { + this.gameSession?.setAiAgentMuted(false, { + source: 'system', + }); + }, }); const { agent: fishjamAgent, agentId: fishjamAgentId } = @@ -166,6 +171,8 @@ export class GameRoom { this.notifierService, ); + this.gameSession.setAiAgentMuted(true, { source: 'system' }); + console.log( `Starting game for ${this.players.size} players in room ${this.roomId}`, ); diff --git a/deep-sea-stories/packages/backend/src/game/session.ts b/deep-sea-stories/packages/backend/src/game/session.ts index 69d2407..3363a6b 100644 --- a/deep-sea-stories/packages/backend/src/game/session.ts +++ b/deep-sea-stories/packages/backend/src/game/session.ts @@ -3,6 +3,8 @@ import type { AudioStreamingOrchestrator } from '../service/audio-streaming-orch import type { NotifierService } from '../service/notifier.js'; import type { Story } from '../types.js'; +type MuteSource = 'manual' | 'system'; + export class GameSession { readonly story: Story; readonly roomId: RoomId; @@ -11,6 +13,8 @@ export class GameSession { private readonly audioOrchestrator: AudioStreamingOrchestrator; private readonly notifierService: NotifierService; private isAiAgentMuted: boolean = false; + private manualMute: boolean = false; + private systemMute: boolean = false; constructor( roomId: RoomId, @@ -34,7 +38,26 @@ export class GameSession { this.audioOrchestrator.removePeer(peerId); } - setAiAgentMuted(muted: boolean) { + setAiAgentMuted(muted: boolean, options?: { source?: MuteSource }) { + const source = options?.source ?? 'manual'; + + if (source === 'manual') { + this.manualMute = muted; + this.systemMute = false; + } else { + this.systemMute = muted; + } + + this.updateMuteState(); + } + + private updateMuteState() { + const nextState = this.manualMute || this.systemMute; + this.applyMuteState(nextState); + } + + private applyMuteState(muted: boolean) { + if (this.isAiAgentMuted === muted) return; this.isAiAgentMuted = muted; this.audioOrchestrator.setMuted(muted); From eac8cc7b11e23486c9220e8734d8d6e136b8b3ee Mon Sep 17 00:00:00 2001 From: Adrian Czerwiec Date: Wed, 11 Feb 2026 14:33:53 +0100 Subject: [PATCH 2/7] adjust stories --- .../packages/backend/src/agent/gemini/session.ts | 3 ++- .../packages/backend/src/prompts/stories.json | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/deep-sea-stories/packages/backend/src/agent/gemini/session.ts b/deep-sea-stories/packages/backend/src/agent/gemini/session.ts index 16478ba..5823d82 100644 --- a/deep-sea-stories/packages/backend/src/agent/gemini/session.ts +++ b/deep-sea-stories/packages/backend/src/agent/gemini/session.ts @@ -88,7 +88,8 @@ export class GeminiSession implements VoiceAgentSession { this.opening = true; this.ending = false; - const shouldRequestIntro = !this.previousHandle && !this.awaitingInitialGreeting; + const shouldRequestIntro = + !this.previousHandle && !this.awaitingInitialGreeting; const params: LiveConnectParameters = { model: GEMINI_MODEL, diff --git a/deep-sea-stories/packages/backend/src/prompts/stories.json b/deep-sea-stories/packages/backend/src/prompts/stories.json index 38b1ef6..cad0f18 100644 --- a/deep-sea-stories/packages/backend/src/prompts/stories.json +++ b/deep-sea-stories/packages/backend/src/prompts/stories.json @@ -3,18 +3,18 @@ "id": 1, "title": "The Diver in the Forest", "front": "A man is found dead in the middle of a burnt forest. He is wearing a full wet suit, flippers, and a scuba tank.", - "back": "The man was diving in the ocean near the coast. A massive forest fire was raging nearby. A firefighting plane scooped up a load of water from the ocean to dump on the fire. The diver was accidentally scooped up along with the water and dropped from a great height onto the burning forest. He died on impact." + "back": "The man was diving in the ocean near the coast. A massive forest fire was raging nearby. A firefighting plane scooped up a load of water from the ocean to dump on the fire. The diver was accidentally scooped up along with the water and dropped from a great height onto the burning forest." }, { "id": 2, "title": "The Light Switch", - "front": "A man turned off the light and went to sleep. The next morning, he woke up, looked out the window, saw the devastation, and immediately killed himself.", - "back": "The man was a lighthouse keeper. By turning off the light before going to sleep during a storm, he caused several ships to crash into the coast. When he saw the wreckage and bodies floating in the sea the next morning, he couldn't live with the guilt." + "front": "A man turned off the light and went to sleep. The next morning, he woke up, looked out the window, saw a tremendous devastation.", + "back": "The man was a lighthouse keeper. By turning off the light before going to sleep during a storm, he caused several ships to crash into the coast. He saw the wreckage in the sea the next morning." }, { "id": 3, "title": "The High Card", - "front": "Four men sat around a table in a metal room. One man drew the highest card, smiled, and then immediately died.", + "front": "Four men sat around a table in a metal room. One man drew the highest card, froze, and passed away seconds later.", "back": "The four men were the crew of a military submarine that had lost power and was running out of oxygen. They realized there was only enough air left for three of them to survive until rescue arrived. They drew cards to decide who would sacrifice themselves to save the others. The man with the high card \"won\" the draw and shot himself to stop consuming oxygen." }, { From 331f75251c2b1d3a1d2eae8227ff9f00560852f5 Mon Sep 17 00:00:00 2001 From: Adrian Czerwiec Date: Thu, 12 Feb 2026 10:24:30 +0100 Subject: [PATCH 3/7] wider tiles --- deep-sea-stories/packages/web/src/components/PeerTile.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deep-sea-stories/packages/web/src/components/PeerTile.tsx b/deep-sea-stories/packages/web/src/components/PeerTile.tsx index 5a79779..ab19d36 100644 --- a/deep-sea-stories/packages/web/src/components/PeerTile.tsx +++ b/deep-sea-stories/packages/web/src/components/PeerTile.tsx @@ -42,7 +42,7 @@ export const PeerTile: FC> = ({ }, [audioStream]); return ( -
+
Date: Thu, 12 Feb 2026 11:11:58 +0100 Subject: [PATCH 4/7] Update deep-sea-stories/packages/backend/src/prompts/stories.json Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- deep-sea-stories/packages/backend/src/prompts/stories.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deep-sea-stories/packages/backend/src/prompts/stories.json b/deep-sea-stories/packages/backend/src/prompts/stories.json index cad0f18..0649a61 100644 --- a/deep-sea-stories/packages/backend/src/prompts/stories.json +++ b/deep-sea-stories/packages/backend/src/prompts/stories.json @@ -8,7 +8,7 @@ { "id": 2, "title": "The Light Switch", - "front": "A man turned off the light and went to sleep. The next morning, he woke up, looked out the window, saw a tremendous devastation.", + "front": "A man turned off the light and went to sleep. The next morning, he woke up, looked out the window, saw tremendous devastation.", "back": "The man was a lighthouse keeper. By turning off the light before going to sleep during a storm, he caused several ships to crash into the coast. He saw the wreckage in the sea the next morning." }, { From af5806d3e2595b4800d1ded1913da14f122ce067 Mon Sep 17 00:00:00 2001 From: Adrian Czerwiec Date: Thu, 12 Feb 2026 11:20:39 +0100 Subject: [PATCH 5/7] remove slop --- .../packages/backend/src/game/room.ts | 6 ++---- .../packages/backend/src/game/session.ts | 20 +------------------ 2 files changed, 3 insertions(+), 23 deletions(-) diff --git a/deep-sea-stories/packages/backend/src/game/room.ts b/deep-sea-stories/packages/backend/src/game/room.ts index c8a9437..c27dd29 100644 --- a/deep-sea-stories/packages/backend/src/game/room.ts +++ b/deep-sea-stories/packages/backend/src/game/room.ts @@ -147,9 +147,7 @@ export class GameRoom { }); }, onReadyForPlayerInput: () => { - this.gameSession?.setAiAgentMuted(false, { - source: 'system', - }); + this.gameSession?.setAiAgentMuted(false); }, }); @@ -171,7 +169,7 @@ export class GameRoom { this.notifierService, ); - this.gameSession.setAiAgentMuted(true, { source: 'system' }); + this.gameSession.setAiAgentMuted(true); console.log( `Starting game for ${this.players.size} players in room ${this.roomId}`, diff --git a/deep-sea-stories/packages/backend/src/game/session.ts b/deep-sea-stories/packages/backend/src/game/session.ts index 3363a6b..39ae733 100644 --- a/deep-sea-stories/packages/backend/src/game/session.ts +++ b/deep-sea-stories/packages/backend/src/game/session.ts @@ -38,25 +38,7 @@ export class GameSession { this.audioOrchestrator.removePeer(peerId); } - setAiAgentMuted(muted: boolean, options?: { source?: MuteSource }) { - const source = options?.source ?? 'manual'; - - if (source === 'manual') { - this.manualMute = muted; - this.systemMute = false; - } else { - this.systemMute = muted; - } - - this.updateMuteState(); - } - - private updateMuteState() { - const nextState = this.manualMute || this.systemMute; - this.applyMuteState(nextState); - } - - private applyMuteState(muted: boolean) { + setAiAgentMuted(muted: boolean) { if (this.isAiAgentMuted === muted) return; this.isAiAgentMuted = muted; this.audioOrchestrator.setMuted(muted); From fe3032bff1fe41acf9405dc1dc906abf01f98910 Mon Sep 17 00:00:00 2001 From: Adrian Czerwiec Date: Thu, 12 Feb 2026 11:22:13 +0100 Subject: [PATCH 6/7] remove slop2 --- .../packages/backend/src/game/session.ts | 112 +++++++++--------- 1 file changed, 54 insertions(+), 58 deletions(-) diff --git a/deep-sea-stories/packages/backend/src/game/session.ts b/deep-sea-stories/packages/backend/src/game/session.ts index 39ae733..75bc3ac 100644 --- a/deep-sea-stories/packages/backend/src/game/session.ts +++ b/deep-sea-stories/packages/backend/src/game/session.ts @@ -1,72 +1,68 @@ -import type { PeerId, RoomId } from '@fishjam-cloud/js-server-sdk'; -import type { AudioStreamingOrchestrator } from '../service/audio-streaming-orchestrator.js'; -import type { NotifierService } from '../service/notifier.js'; -import type { Story } from '../types.js'; - -type MuteSource = 'manual' | 'system'; +import type { PeerId, RoomId } from "@fishjam-cloud/js-server-sdk"; +import type { AudioStreamingOrchestrator } from "../service/audio-streaming-orchestrator.js"; +import type { NotifierService } from "../service/notifier.js"; +import type { Story } from "../types.js"; export class GameSession { - readonly story: Story; - readonly roomId: RoomId; - readonly agentId: PeerId; + readonly story: Story; + readonly roomId: RoomId; + readonly agentId: PeerId; - private readonly audioOrchestrator: AudioStreamingOrchestrator; - private readonly notifierService: NotifierService; - private isAiAgentMuted: boolean = false; - private manualMute: boolean = false; - private systemMute: boolean = false; + private readonly audioOrchestrator: AudioStreamingOrchestrator; + private readonly notifierService: NotifierService; + private isAiAgentMuted: boolean = false; - constructor( - roomId: RoomId, - agentId: PeerId, - story: Story, - audioOrchestrator: AudioStreamingOrchestrator, - notifierService: NotifierService, - ) { - this.roomId = roomId; - this.agentId = agentId; - this.story = story; - this.audioOrchestrator = audioOrchestrator; - this.notifierService = notifierService; - } + constructor( + roomId: RoomId, + agentId: PeerId, + story: Story, + audioOrchestrator: AudioStreamingOrchestrator, + notifierService: NotifierService, + ) { + this.roomId = roomId; + this.agentId = agentId; + this.story = story; + this.audioOrchestrator = audioOrchestrator; + this.notifierService = notifierService; + } - addPlayer(peerId: PeerId) { - this.audioOrchestrator.addPeer(peerId); - } + addPlayer(peerId: PeerId) { + this.audioOrchestrator.addPeer(peerId); + } - removePlayer(peerId: PeerId) { - this.audioOrchestrator.removePeer(peerId); - } + removePlayer(peerId: PeerId) { + this.audioOrchestrator.removePeer(peerId); + } - setAiAgentMuted(muted: boolean) { - if (this.isAiAgentMuted === muted) return; - this.isAiAgentMuted = muted; - this.audioOrchestrator.setMuted(muted); + setAiAgentMuted(muted: boolean) { + if (this.isAiAgentMuted === muted) return; + this.isAiAgentMuted = muted; + this.audioOrchestrator.setMuted(muted); - this.notifierService.emitNotification(this.roomId, { - type: 'aiAgentMutedStatusChanged' as const, - muted: muted, - timestamp: Date.now(), - }); + this.notifierService.emitNotification(this.roomId, { + type: "aiAgentMutedStatusChanged" as const, + muted: muted, + timestamp: Date.now(), + }); - console.log( - `AI agent in room ${this.roomId} is now ${muted ? 'muted' : 'unmuted'}`, - ); - } + console.log( + `AI agent in room ${this.roomId} is now ${muted ? "muted" : "unmuted"}`, + ); + } - isAiAgentMut(): boolean { - return this.isAiAgentMuted; - } + isAiAgentMut(): boolean { + return this.isAiAgentMuted; + } - async startGame() { - await this.audioOrchestrator.start(); - } + async startGame() { + await this.audioOrchestrator.start(); + } - async announceTimeExpired() { - await this.audioOrchestrator.voiceAgentSession.announceTimeExpired(); - } + async announceTimeExpired() { + await this.audioOrchestrator.voiceAgentSession.announceTimeExpired(); + } - async stopGame(wait: boolean = false) { - await this.audioOrchestrator.shutdown(wait); - } + async stopGame(wait: boolean = false) { + await this.audioOrchestrator.shutdown(wait); + } } From fb87a9ed2f94d3275929a23ce3f509942f8c5baa Mon Sep 17 00:00:00 2001 From: Adrian Czerwiec Date: Thu, 12 Feb 2026 11:49:41 +0100 Subject: [PATCH 7/7] formaty --- .../packages/backend/src/game/session.ts | 108 +++++++++--------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/deep-sea-stories/packages/backend/src/game/session.ts b/deep-sea-stories/packages/backend/src/game/session.ts index 75bc3ac..9c5c503 100644 --- a/deep-sea-stories/packages/backend/src/game/session.ts +++ b/deep-sea-stories/packages/backend/src/game/session.ts @@ -1,68 +1,68 @@ -import type { PeerId, RoomId } from "@fishjam-cloud/js-server-sdk"; -import type { AudioStreamingOrchestrator } from "../service/audio-streaming-orchestrator.js"; -import type { NotifierService } from "../service/notifier.js"; -import type { Story } from "../types.js"; +import type { PeerId, RoomId } from '@fishjam-cloud/js-server-sdk'; +import type { AudioStreamingOrchestrator } from '../service/audio-streaming-orchestrator.js'; +import type { NotifierService } from '../service/notifier.js'; +import type { Story } from '../types.js'; export class GameSession { - readonly story: Story; - readonly roomId: RoomId; - readonly agentId: PeerId; + readonly story: Story; + readonly roomId: RoomId; + readonly agentId: PeerId; - private readonly audioOrchestrator: AudioStreamingOrchestrator; - private readonly notifierService: NotifierService; - private isAiAgentMuted: boolean = false; + private readonly audioOrchestrator: AudioStreamingOrchestrator; + private readonly notifierService: NotifierService; + private isAiAgentMuted: boolean = false; - constructor( - roomId: RoomId, - agentId: PeerId, - story: Story, - audioOrchestrator: AudioStreamingOrchestrator, - notifierService: NotifierService, - ) { - this.roomId = roomId; - this.agentId = agentId; - this.story = story; - this.audioOrchestrator = audioOrchestrator; - this.notifierService = notifierService; - } + constructor( + roomId: RoomId, + agentId: PeerId, + story: Story, + audioOrchestrator: AudioStreamingOrchestrator, + notifierService: NotifierService, + ) { + this.roomId = roomId; + this.agentId = agentId; + this.story = story; + this.audioOrchestrator = audioOrchestrator; + this.notifierService = notifierService; + } - addPlayer(peerId: PeerId) { - this.audioOrchestrator.addPeer(peerId); - } + addPlayer(peerId: PeerId) { + this.audioOrchestrator.addPeer(peerId); + } - removePlayer(peerId: PeerId) { - this.audioOrchestrator.removePeer(peerId); - } + removePlayer(peerId: PeerId) { + this.audioOrchestrator.removePeer(peerId); + } - setAiAgentMuted(muted: boolean) { - if (this.isAiAgentMuted === muted) return; - this.isAiAgentMuted = muted; - this.audioOrchestrator.setMuted(muted); + setAiAgentMuted(muted: boolean) { + if (this.isAiAgentMuted === muted) return; + this.isAiAgentMuted = muted; + this.audioOrchestrator.setMuted(muted); - this.notifierService.emitNotification(this.roomId, { - type: "aiAgentMutedStatusChanged" as const, - muted: muted, - timestamp: Date.now(), - }); + this.notifierService.emitNotification(this.roomId, { + type: 'aiAgentMutedStatusChanged' as const, + muted: muted, + timestamp: Date.now(), + }); - console.log( - `AI agent in room ${this.roomId} is now ${muted ? "muted" : "unmuted"}`, - ); - } + console.log( + `AI agent in room ${this.roomId} is now ${muted ? 'muted' : 'unmuted'}`, + ); + } - isAiAgentMut(): boolean { - return this.isAiAgentMuted; - } + isAiAgentMut(): boolean { + return this.isAiAgentMuted; + } - async startGame() { - await this.audioOrchestrator.start(); - } + async startGame() { + await this.audioOrchestrator.start(); + } - async announceTimeExpired() { - await this.audioOrchestrator.voiceAgentSession.announceTimeExpired(); - } + async announceTimeExpired() { + await this.audioOrchestrator.voiceAgentSession.announceTimeExpired(); + } - async stopGame(wait: boolean = false) { - await this.audioOrchestrator.shutdown(wait); - } + async stopGame(wait: boolean = false) { + await this.audioOrchestrator.shutdown(wait); + } }