From cc220bd0cd6ab9baaa27f75117294015c40002b4 Mon Sep 17 00:00:00 2001 From: ACrazyTown <47027981+ACrazyTown@users.noreply.github.com> Date: Wed, 9 Apr 2025 20:36:03 +0200 Subject: [PATCH] feat: Support analyzing streamed audio --- .vscode/settings.json | 3 + example/Project.xml | 8 +- example/source/PlayState.hx | 16 +++- src/funkin/vis/AudioBuffer.hx | 8 +- src/funkin/vis/AudioClip.hx | 1 + .../vis/audioclip/frontends/LimeAudioClip.hx | 56 ++++++++++--- src/funkin/vis/dsp/SpectralAnalyzer.hx | 79 ++++++++++++++++++- 7 files changed, 147 insertions(+), 24 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..0c9c33d --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "lime.projectFile": "example/Project.xml" +} diff --git a/example/Project.xml b/example/Project.xml index 0bb3a1b..3e36062 100644 --- a/example/Project.xml +++ b/example/Project.xml @@ -1,6 +1,6 @@ - + @@ -16,7 +16,7 @@ - + @@ -37,7 +37,7 @@ - + diff --git a/example/source/PlayState.hx b/example/source/PlayState.hx index 99ee2be..442d3f7 100644 --- a/example/source/PlayState.hx +++ b/example/source/PlayState.hx @@ -7,10 +7,15 @@ import haxe.io.BytesInput; import haxe.io.Input; import haxe.io.UInt16Array; import lime.media.AudioSource; -import lime.media.vorbis.VorbisFile; import lime.utils.Int16Array; import openfl.utils.Assets; +#if STREAM +import lime.media.vorbis.VorbisFile; +import lime.media.AudioBuffer; +import openfl.media.Sound; +#end + using StringTools; class PlayState extends FlxState @@ -27,10 +32,17 @@ class PlayState extends FlxState super.create(); // musicList = fillMusicList("assets/music/musicList.txt"); + + #if STREAM + var vorbis = VorbisFile.fromFile("assets/music/catStuck.ogg"); + var buffer = AudioBuffer.fromVorbisFile(vorbis); + FlxG.sound.playMusic(Sound.fromAudioBuffer(buffer)); + #else FlxG.sound.playMusic("assets/music/catStuck.ogg"); + #end @:privateAccess - musicSrc = cast FlxG.sound.music._channel.__source; + musicSrc = cast #if (openfl < "9.3.2") FlxG.sound.music._channel.__source #else FlxG.sound.music._channel.__audioSource #end; data = cast musicSrc.buffer.data; diff --git a/src/funkin/vis/AudioBuffer.hx b/src/funkin/vis/AudioBuffer.hx index fa3e261..dbaf752 100644 --- a/src/funkin/vis/AudioBuffer.hx +++ b/src/funkin/vis/AudioBuffer.hx @@ -6,10 +6,16 @@ class AudioBuffer { public var data(default, null):UInt16Array; public var sampleRate(default, null):Float; + public var length(default, null):Int; + public var bitsPerSample(default, null):Int; + public var channels(default, null):Int; - public function new(data:UInt16Array, sampleRate:Float) + public function new(data:UInt16Array, sampleRate:Float, length:Int, bitsPerSample:Int, channels:Int) { this.data = data; this.sampleRate = sampleRate; + this.length = length; + this.bitsPerSample = bitsPerSample; + this.channels = channels; } } \ No newline at end of file diff --git a/src/funkin/vis/AudioClip.hx b/src/funkin/vis/AudioClip.hx index 0b4e56b..163c4e9 100644 --- a/src/funkin/vis/AudioClip.hx +++ b/src/funkin/vis/AudioClip.hx @@ -8,4 +8,5 @@ interface AudioClip public var audioBuffer(default, null):AudioBuffer; public var currentFrame(get, never):Int; public var source:Dynamic; + public var streamed:Bool; } \ No newline at end of file diff --git a/src/funkin/vis/audioclip/frontends/LimeAudioClip.hx b/src/funkin/vis/audioclip/frontends/LimeAudioClip.hx index 46d584f..63da065 100644 --- a/src/funkin/vis/audioclip/frontends/LimeAudioClip.hx +++ b/src/funkin/vis/audioclip/frontends/LimeAudioClip.hx @@ -1,5 +1,6 @@ package funkin.vis.audioclip.frontends; +import haxe.Int64; import flixel.FlxG; import flixel.math.FlxMath; import funkin.vis.AudioBuffer; @@ -20,32 +21,61 @@ class LimeAudioClip implements funkin.vis.AudioClip public var audioBuffer(default, null):AudioBuffer; public var currentFrame(get, never):Int; public var source:Dynamic; + public var streamed:Bool; public function new(audioSource:AudioSource) { - var data:lime.utils.UInt16Array = cast audioSource.buffer.data; + var limeBuffer = audioSource.buffer; + var data:lime.utils.UInt16Array = cast limeBuffer.data; #if web - var sampleRate:Float = audioSource.buffer.src._sounds[0]._node.context.sampleRate; + streamed = false; + + var sampleRate:Float = limeBuffer.src._sounds[0]._node.context.sampleRate; + var length:Int = audioSource.length; + var bitsPerSample:Int = 32; + var channels:Int = 2; #else - var sampleRate = audioSource.buffer.sampleRate; + var sampleRate:Float = 0; + var length:Int = 0; + var bitsPerSample:Int = 0; + var channels:Int = 0; + + #if lime_vorbis + // If we have a ref to a VorbisFile it should be safe to assume + // this is a streamed sound! + @:privateAccess + if (limeBuffer.__srcVorbisFile != null) + { + streamed = true; + + var vorbisFile = limeBuffer.__srcVorbisFile; + var vorbisInfo = vorbisFile.info(); + + sampleRate = vorbisInfo.rate; + bitsPerSample = 16; + length = Std.int(Int64.toInt(vorbisFile.pcmTotal()) * vorbisInfo.channels * (bitsPerSample / 8)); + channels = vorbisInfo.channels; + } + else + #end + { + streamed = false; + + sampleRate = limeBuffer.sampleRate; + bitsPerSample = limeBuffer.bitsPerSample; + length = limeBuffer.data.length; + channels = limeBuffer.channels; + } #end - this.audioBuffer = new AudioBuffer(data, sampleRate); + this.audioBuffer = new AudioBuffer(data, sampleRate, length, bitsPerSample, channels); this.source = audioSource.buffer.src; } private function get_currentFrame():Int { - var dataLength:Int = 0; - - #if web - dataLength = source.length; - #else - dataLength = audioBuffer.data.length; - #end - - var value = Std.int(FlxMath.remapToRange(FlxG.sound.music.time, 0, FlxG.sound.music.length, 0, dataLength)); + var value = Std.int(FlxMath.remapToRange(FlxG.sound.music.time, 0, FlxG.sound.music.length, 0, audioBuffer.length)); if (value < 0) return -1; diff --git a/src/funkin/vis/dsp/SpectralAnalyzer.hx b/src/funkin/vis/dsp/SpectralAnalyzer.hx index 45c3c87..9e68463 100644 --- a/src/funkin/vis/dsp/SpectralAnalyzer.hx +++ b/src/funkin/vis/dsp/SpectralAnalyzer.hx @@ -1,5 +1,7 @@ package funkin.vis.dsp; +import haxe.io.Bytes; +import lime.utils.UInt8Array; import flixel.FlxG; import flixel.math.FlxMath; import funkin.vis._internal.html5.AnalyzerNode; @@ -8,6 +10,10 @@ import grig.audio.FFT; import grig.audio.FFTVisualization; import lime.media.AudioSource; +#if lime_vorbis +import lime.media.vorbis.VorbisFile; +#end + using grig.audio.lime.UInt8ArrayTools; typedef Bar = @@ -58,6 +64,9 @@ class SpectralAnalyzer #end private var _logGraphCache:Array = []; private var _mixedCache:Array = []; + #if lime_vorbis + private var vorbisBuffer:UInt8Array; + #end private function freqToBin(freq:Float, mathType:MathType = Round):Int { @@ -193,8 +202,8 @@ class SpectralAnalyzer return levels; #else - var numOctets = Std.int(audioSource.buffer.bitsPerSample / 8); - var wantedLength = fftN * numOctets * audioSource.buffer.channels; + var numOctets = Std.int(audioClip.audioBuffer.bitsPerSample / 8); + var wantedLength = fftN * numOctets * audioClip.audioBuffer.channels; var startFrame = audioClip.currentFrame; if (startFrame < 0) @@ -203,9 +212,34 @@ class SpectralAnalyzer } startFrame -= startFrame % numOctets; - var segment = audioSource.buffer.data.subarray(startFrame, min(startFrame + wantedLength, audioSource.buffer.data.length)); - getSignal(segment, audioSource.buffer.bitsPerSample); // Sets _buffer + var endFrame:Int = min(startFrame + wantedLength, audioClip.audioBuffer.length); + + #if lime_vorbis + if (audioClip.streamed) + { + @:privateAccess + var vorbisFile = audioSource.buffer.__srcVorbisFile; + + // reading from VorbisFile will automatically move the position + // which causes issues with playback, so we keep old time to go back to it + var prevPos = vorbisFile.pcmTell(); + + vorbisFile.pcmSeek(Std.int(startFrame / (numOctets + audioClip.audioBuffer.channels))); + + // calling this updates the vorbisBuffer array + readVorbisFileBuffer(vorbisFile, wantedLength); + + vorbisFile.pcmSeek(prevPos); + + getSignal(vorbisBuffer, audioClip.audioBuffer.bitsPerSample); + } + else + #end + { + var segment:UInt8Array = audioSource.buffer.data.subarray(startFrame, endFrame); + getSignal(segment, audioClip.audioBuffer.bitsPerSample); + } if (audioSource.buffer.channels > 1) { var wantedArrayLength = Std.int(_buffer.length / audioSource.buffer.channels); @@ -290,6 +324,43 @@ class SpectralAnalyzer return _buffer; } + #if lime_vorbis + // Pretty much copied from + // https://github.com/openfl/lime/blob/develop/src/lime/_internal/backend/native/NativeAudioSource.hx#L212 + function readVorbisFileBuffer(vorbisFile:VorbisFile, length:Int):UInt8Array + { + if (vorbisBuffer == null || vorbisBuffer.length != length) + vorbisBuffer = new UInt8Array(length); + + var read:Int = 0; + var total:Int = 0; + var readMax:Int = 4096; + + while (total < length) + { + readMax = 4096; + + if (readMax > length - total) + { + readMax = length - total; + } + + read = vorbisFile.read(vorbisBuffer.buffer, total, readMax); + + if (read > 0) + { + total += read; + } + else + { + break; + } + } + + return vorbisBuffer; + } + #end + @:generic static inline function clamp(val:T, min:T, max:T):T {