Add remastered OGV cutscene playback with MIDI music sync

CMakeLists.txt

@@ -53,7 +53,8 @@ include(GNUInstallDirs)
 option(ENABLE_TFE "Enable building “The Force Engine”" ON)
 option(ENABLE_SYSMIDI "Enable System-MIDI Output if RTMidi is available" ON)
 option(ENABLE_EDITOR "Enable TFE Editor" OFF)
-option(ENABLE_ADJUSTABLEHUD_MOD "Install the build‑in “AdjustableHud mod” with TFE" ON)
+option(ENABLE_OGV_CUTSCENES "Enable OGV (Ogg Theora) video cutscene support" OFF)
+option(ENABLE_ADJUSTABLEHUD_MOD "Install the build‑in "AdjustableHud mod" with TFE" ON)
 
 if(ENABLE_TFE)
 	add_executable(tfe)
@@ -119,6 +120,17 @@ if(ENABLE_TFE)
 	if(ENABLE_EDITOR)
 		set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DBUILD_EDITOR")
 	endif()
+	if(ENABLE_OGV_CUTSCENES)
+		if(UNIX)
+			pkg_check_modules(THEORA REQUIRED theoradec)
+			pkg_check_modules(OGG REQUIRED ogg)
+			pkg_check_modules(VORBIS REQUIRED vorbis vorbisfile)
+			target_include_directories(tfe PRIVATE ${THEORA_INCLUDE_DIRS} ${OGG_INCLUDE_DIRS} ${VORBIS_INCLUDE_DIRS})
+			target_link_libraries(tfe PRIVATE ${THEORA_LIBRARIES} ${OGG_LIBRARIES} ${VORBIS_LIBRARIES})
+			target_link_directories(tfe PRIVATE ${THEORA_LIBRARY_DIRS} ${OGG_LIBRARY_DIRS} ${VORBIS_LIBRARY_DIRS})
+		endif()
+		add_definitions("-DENABLE_OGV_CUTSCENES")
+	endif()
 	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DBUILD_FORCE_SCRIPT")
 
 

TheForceEngine/Shaders/yuv2rgb.frag

@@ -0,0 +1,24 @@
+// BT.601 YCbCr -> RGB for Theora video frames.
+uniform sampler2D TexY;
+uniform sampler2D TexCb;
+uniform sampler2D TexCr;
+
+in vec2 Frag_UV;
+out vec4 Out_Color;
+
+void main()
+{
+    float y  = texture(TexY,  Frag_UV).r;
+    float cb = texture(TexCb, Frag_UV).r - 0.5;
+    float cr = texture(TexCr, Frag_UV).r - 0.5;
+
+    // Rescale Y from studio range [16..235] to [0..1].
+    y = (y - 16.0 / 255.0) * (255.0 / 219.0);
+
+    vec3 rgb;
+    rgb.r = y + 1.596 * cr;
+    rgb.g = y - 0.391 * cb - 0.813 * cr;
+    rgb.b = y + 2.018 * cb;
+
+    Out_Color = vec4(clamp(rgb, 0.0, 1.0), 1.0);
+}

TheForceEngine/Shaders/yuv2rgb.vert

@@ -0,0 +1,11 @@
+uniform vec4 ScaleOffset;
+in vec2 vtx_pos;
+in vec2 vtx_uv;
+
+out vec2 Frag_UV;
+
+void main()
+{
+    Frag_UV = vec2(vtx_uv.x, 1.0 - vtx_uv.y);
+    gl_Position = vec4(vtx_pos.xy * ScaleOffset.xy + ScaleOffset.zw, 0, 1);
+}

TheForceEngine/TFE_Audio/audioSystem.cpp

@@ -74,6 +74,7 @@ namespace TFE_Audio
 
 	static AudioUpsampleFilter s_upsampleFilter = AUF_DEFAULT;
 	static AudioThreadCallback s_audioThreadCallback = nullptr;
+	static AudioDirectCallback s_directCallback = nullptr;
 
 	static void audioCallback(void*, unsigned char*, int);
 	void setSoundVolumeConsole(const ConsoleArgList& args);
@@ -227,6 +228,15 @@ namespace TFE_Audio
 		SDL_UnlockMutex(s_mutex);
 	}
 
+	void setDirectCallback(AudioDirectCallback callback)
+	{
+		if (s_nullDevice) { return; }
+
+		SDL_LockMutex(s_mutex);
+		s_directCallback = callback;
+		SDL_UnlockMutex(s_mutex);
+	}
+
 	const OutputDeviceInfo* getOutputDeviceList(s32& count, s32& curOutput)
 	{
 		return TFE_AudioDevice::getOutputDeviceList(count, curOutput);
@@ -467,9 +477,9 @@ namespace TFE_Audio
 
 		// First clear samples
 		memset(buffer, 0, bufferSize);
-			   
+
 		SDL_LockMutex(s_mutex);
-		// Then call the audio thread callback
+		// Call the audio thread callback (iMuse/game audio).
 		if (s_audioThreadCallback && !s_paused)
 		{
 			static f32 callbackBuffer[(AUDIO_CALLBACK_BUFFER_SIZE + 2)*AUDIO_CHANNEL_COUNT];	// 256 stereo + oversampling.
@@ -488,6 +498,11 @@ namespace TFE_Audio
 				}
 			}
 		}
+		// Direct callback adds at the full output rate (e.g. OGV video audio), mixing on top.
+		if (s_directCallback && !s_paused)
+		{
+			s_directCallback(buffer, frames, s_soundFxVolume * c_soundHeadroom);
+		}
 
 		// Then loop through the sources.
 		// Note: this is no longer used by Dark Forces. However I decided to keep direct sound support around

TheForceEngine/TFE_Audio/audioSystem.h

@@ -47,6 +47,9 @@ enum SoundType
 
 typedef void(*SoundFinishedCallback)(void* userData, s32 arg);
 typedef void(*AudioThreadCallback)(f32* buffer, u32 bufferSize, f32 systemVolume);
+// Direct callback writes stereo interleaved f32 at the full output rate (44100 Hz).
+// frameCount = number of stereo frames to fill.
+typedef void(*AudioDirectCallback)(f32* buffer, u32 frameCount, f32 systemVolume);
 
 namespace TFE_Audio
 {
@@ -76,6 +79,7 @@ namespace TFE_Audio
 	void bufferedAudioClear();
 
 	void setAudioThreadCallback(AudioThreadCallback callback = nullptr);
+	void setDirectCallback(AudioDirectCallback callback = nullptr);
 	const OutputDeviceInfo* getOutputDeviceList(s32& count, s32& curOutput);
 
 	// One shot, play and forget. Only do this if the client needs no control until stopAllSounds() is called.