libobs: Implement asynchronous sample rate conversion for audio

docs/sphinx/reference-libobs-media-io.rst

@@ -514,6 +514,23 @@ FFmpeg wrapper to resample audio.
 
 ---------------------
 
+.. function:: void audio_resampler_set_compensation_error(audio_resampler_t *resampler, int error_ns)
+
+   Activate resampling compensation and set current error.
+
+   :param resampler: Audio resampler object
+   :param error_ns:  Current error in nanosecond
+
+---------------------
+
+.. function:: void audio_resampler_disable_compensation(audio_resampler_t *resampler)
+
+   Deactivate resampling compensation.
+
+   :param resampler: Audio resampler object
+
+---------------------
+
 .. function:: bool audio_resampler_resample(audio_resampler_t *resampler, uint8_t *output[], uint32_t *out_frames, uint64_t *ts_offset, const uint8_t *const input[], uint32_t in_frames)
 
    Resamples audio frames.

libobs/media-io/audio-resampler-ffmpeg.c

@@ -21,6 +21,9 @@
 #include <libavutil/avutil.h>
 #include <libavformat/avformat.h>
 #include <libswresample/swresample.h>
+#include "lag_lead_filter.h"
+
+// #define DEBUG_COMPENSATION
 
 struct audio_resampler {
 	struct SwrContext *context;
@@ -41,6 +44,12 @@ struct audio_resampler {
 	AVChannelLayout input_ch_layout;
 	AVChannelLayout output_ch_layout;
 #endif
+
+	struct lag_lead_filter compensation_filter;
+	bool compensation_filter_configured;
+
+	uint64_t total_input_samples;
+	uint64_t total_output_samples;
 };
 
 static inline enum AVSampleFormat convert_audio_format(enum audio_format format)
@@ -169,6 +178,14 @@ audio_resampler_t *audio_resampler_create(const struct resample_info *dst, const
 void audio_resampler_destroy(audio_resampler_t *rs)
 {
 	if (rs) {
+		uint64_t total_output_samples = rs->total_output_samples + swr_get_delay(rs->context, rs->output_freq);
+		if (rs->compensation_filter_configured)
+			blog(LOG_INFO,
+			     "audio_resampler (asynchronous compensation): input %" PRIu64 " samples, "
+			     "output %" PRIu64 " samples, estimated input frequency %f Hz",
+			     rs->total_input_samples, rs->total_output_samples,
+			     (double)rs->total_input_samples / total_output_samples * rs->output_freq);
+
 		if (rs->context)
 			swr_free(&rs->context);
 		if (rs->output_buffer[0])
@@ -178,6 +195,27 @@ void audio_resampler_destroy(audio_resampler_t *rs)
 	}
 }
 
+void audio_resampler_set_compensation_error(audio_resampler_t *rs, int error_ns)
+{
+	if (!rs->compensation_filter_configured) {
+		// Parameters are determined experimentally by SPICE simulation
+		// to have these characteristics.
+		// - Unity gain frequency: 1/180 Hz (inverse of 3 minutes)
+		// - Phase margin: 60 degrees
+		lag_lead_filter_set_parameters(&rs->compensation_filter, 3.756e-2, 8.250, 107.1);
+		lag_lead_filter_reset(&rs->compensation_filter);
+		rs->compensation_filter_configured = true;
+	}
+
+	lag_lead_filter_set_error_ns(&rs->compensation_filter, error_ns);
+}
+
+void audio_resampler_disable_compensation(audio_resampler_t *rs)
+{
+	rs->compensation_filter_configured = false;
+	swr_set_compensation(rs->context, 0, 0);
+}
+
 bool audio_resampler_resample(audio_resampler_t *rs, uint8_t *output[], uint32_t *out_frames, uint64_t *ts_offset,
 			      const uint8_t *const input[], uint32_t in_frames)
 {
@@ -187,6 +225,17 @@ bool audio_resampler_resample(audio_resampler_t *rs, uint8_t *output[], uint32_t
 	struct SwrContext *context = rs->context;
 	int ret;
 
+	if (rs->compensation_filter_configured) {
+		lag_lead_filter_tick(&rs->compensation_filter, rs->input_freq, in_frames);
+		double drift = lag_lead_filter_get_drift(&rs->compensation_filter);
+
+		ret = swr_set_compensation(rs->context, -(int)(drift * 65536 / rs->input_freq), 65536);
+		if (ret < 0) {
+			blog(LOG_ERROR, "swr_set_compensation failed: %d", ret);
+			return false;
+		}
+	}
+
 	int64_t delay = swr_get_delay(context, rs->input_freq);
 	int estimated = (int)av_rescale_rnd(delay + (int64_t)in_frames, (int64_t)rs->output_freq,
 					    (int64_t)rs->input_freq, AV_ROUND_UP);
@@ -210,9 +259,20 @@ bool audio_resampler_resample(audio_resampler_t *rs, uint8_t *output[], uint32_t
 		return false;
 	}
 
+#ifdef DEBUG_COMPENSATION
+	if (rs->compensation_filter_configured)
+		blog(LOG_INFO,
+		     "async-compensation in_frames=%d out_frames=%d error_ns=%" PRIi64 " internal-condition=(%f %f)",
+		     in_frames, ret, rs->compensation_filter.error_ns, rs->compensation_filter.vc1,
+		     rs->compensation_filter.vc2);
+#endif
+
 	for (uint32_t i = 0; i < rs->output_planes; i++)
 		output[i] = rs->output_buffer[i];
 
+	rs->total_input_samples += in_frames;
+	rs->total_output_samples += ret;
+
 	*out_frames = (uint32_t)ret;
 	return true;
 }

libobs/media-io/audio-resampler.h

@@ -36,6 +36,10 @@ struct resample_info {
 EXPORT audio_resampler_t *audio_resampler_create(const struct resample_info *dst, const struct resample_info *src);
 EXPORT void audio_resampler_destroy(audio_resampler_t *resampler);
 
+EXPORT void audio_resampler_set_compensation_error(audio_resampler_t *resampler, int delta_ns_per_s);
+
+EXPORT void audio_resampler_disable_compensation(audio_resampler_t *resampler);
+
 EXPORT bool audio_resampler_resample(audio_resampler_t *resampler, uint8_t *output[], uint32_t *out_frames,
 				     uint64_t *ts_offset, const uint8_t *const input[], uint32_t in_frames);
 

libobs/media-io/lag_lead_filter.h

@@ -0,0 +1,63 @@
+/*
+ *  Lag lead filter for asynchronous sample rate converter
+ *  Copyright (C) 2022-2026 Norihiro Kamae <norihiro@nagater.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, write to the Free Software Foundation, Inc.,
+ *  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#pragma once
+
+struct lag_lead_filter {
+	int64_t error_ns;
+	double vc1;
+	double vc2;
+
+	double gain;
+	double c1_inv;
+	double c2_inv;
+};
+
+static void lag_lead_filter_reset(struct lag_lead_filter *f)
+{
+	f->vc1 = 0.0;
+	f->vc2 = 0.0;
+}
+
+static void lag_lead_filter_set_parameters(struct lag_lead_filter *f, double gain, double c1, double c2)
+{
+	f->gain = gain;
+	f->c1_inv = 1.0 / c1;
+	f->c2_inv = 1.0 / c2;
+}
+
+static inline void lag_lead_filter_set_error_ns(struct lag_lead_filter *f, int32_t error_ns)
+{
+	f->error_ns = error_ns;
+}
+
+static void lag_lead_filter_tick(struct lag_lead_filter *f, uint32_t fs, uint32_t frames)
+{
+	const double icp = f->gain * f->error_ns * 1e-9;
+	const double ic2 = (f->vc1 - f->vc2) / fs;
+	const double ic1 = icp - ic2;
+
+	f->vc1 += ic1 * f->c1_inv * frames;
+	f->vc2 += ic2 * f->c2_inv * frames;
+}
+
+static inline double lag_lead_filter_get_drift(const struct lag_lead_filter *f)
+{
+	return f->vc1;
+}

libobs/obs-internal.h

@@ -861,6 +861,8 @@ struct obs_source {
 	bool audio_active;
 	bool user_muted;
 	bool muted;
+	volatile bool async_compensation;
+	bool last_async_compensation;
 	struct obs_source *next_audio_source;
 	struct obs_source **prev_next_audio_source;
 	uint64_t audio_ts;

libobs/obs-source.c

@@ -1635,6 +1635,16 @@ static void source_output_audio_data(obs_source_t *source, const struct audio_da
 		}
 	}
 
+	if (os_atomic_load_bool(&source->async_compensation) && source->resampler) {
+		uint64_t ts_buffered = in.timestamp;
+		uint64_t ts_data = data->timestamp - source->resample_offset;
+		int64_t error = (int64_t)(ts_buffered - ts_data);
+		if (push_back && INT32_MIN <= error && error <= INT32_MAX)
+			audio_resampler_set_compensation_error(source->resampler, (int)error);
+		else
+			audio_resampler_set_compensation_error(source->resampler, 0);
+	}
+
 	sync_offset = source->sync_offset;
 	in.timestamp += sync_offset;
 	in.timestamp -= source->resample_offset;
@@ -3894,16 +3904,20 @@ static inline void reset_resampler(obs_source_t *source, const struct obs_source
 	output_info.samples_per_sec = obs_info->samples_per_sec;
 	output_info.speakers = obs_info->speakers;
 
+	bool async_compensation = os_atomic_load_bool(&source->async_compensation);
+
 	source->sample_info.format = audio->format;
 	source->sample_info.samples_per_sec = audio->samples_per_sec;
 	source->sample_info.speakers = audio->speakers;
+	source->last_async_compensation = async_compensation;
 
 	audio_resampler_destroy(source->resampler);
 	source->resampler = NULL;
 	source->resample_offset = 0;
 
 	if (source->sample_info.samples_per_sec == obs_info->samples_per_sec &&
-	    source->sample_info.format == obs_info->format && source->sample_info.speakers == obs_info->speakers) {
+	    source->sample_info.format == obs_info->format && source->sample_info.speakers == obs_info->speakers &&
+	    !async_compensation) {
 		source->audio_failed = false;
 		return;
 	}
@@ -3996,7 +4010,8 @@ static void process_audio(obs_source_t *source, const struct obs_source_audio *a
 	bool mono_output;
 
 	if (source->sample_info.samples_per_sec != audio->samples_per_sec ||
-	    source->sample_info.format != audio->format || source->sample_info.speakers != audio->speakers)
+	    source->sample_info.format != audio->format || source->sample_info.speakers != audio->speakers ||
+	    source->last_async_compensation != os_atomic_load_bool(&source->async_compensation))
 		reset_resampler(source, audio);
 
 	if (source->audio_failed)
@@ -5608,6 +5623,21 @@ bool obs_source_async_unbuffered(const obs_source_t *source)
 	return obs_source_valid(source, "obs_source_async_unbuffered") ? source->async_unbuffered : false;
 }
 
+void obs_source_set_async_compensation(obs_source_t *source, bool compensate)
+{
+	if (!obs_source_valid(source, "obs_source_set_async_compensation"))
+		return;
+
+	os_atomic_store_bool(&source->async_compensation, compensate);
+}
+
+bool obs_source_async_compensation(const obs_source_t *source)
+{
+	return obs_source_valid(source, "obs_source_async_compensation")
+		       ? os_atomic_load_bool(&source->async_compensation)
+		       : false;
+}
+
 obs_data_t *obs_source_get_private_settings(obs_source_t *source)
 {
 	if (!obs_ptr_valid(source, "obs_source_get_private_settings"))

libobs/obs.h

@@ -1544,6 +1544,9 @@ EXPORT void obs_source_get_audio_mix(const obs_source_t *source, struct obs_sour
 EXPORT void obs_source_set_async_unbuffered(obs_source_t *source, bool unbuffered);
 EXPORT bool obs_source_async_unbuffered(const obs_source_t *source);
 
+EXPORT void obs_source_set_async_compensation(obs_source_t *source, bool compensate);
+EXPORT bool obs_source_async_compensation(const obs_source_t *source);
+
 /** Used to decouple audio from video so that audio doesn't attempt to sync up
  * with video.  I.E. Audio acts independently.  Only works when in unbuffered
  * mode. */

plugins/aja/aja-source.cpp

@@ -860,6 +860,8 @@ static void aja_source_update(void *data, obs_data_t *settings)
 		ajaSource->Deactivate();
 	}
 
+	obs_source_set_async_compensation(ajaSource->GetOBSSource(), obs_data_get_bool(settings, "async_compensation"));
+
 	auto &cardManager = aja::CardManager::Instance();
 	cardManager.EnumerateCards();
 	auto cardEntry = cardManager.GetCardEntry(wantCardID);
@@ -1015,6 +1017,8 @@ static obs_properties_t *aja_source_get_properties(void *data)
 	obs_property_set_modified_callback2(device_list, aja_source_device_changed, data);
 	obs_property_set_modified_callback2(io_select_list, aja_io_selection_changed, data);
 
+	obs_properties_add_bool(props, "async_compensation", obs_module_text("AsyncCompensation"));
+
 	return props;
 }
 
@@ -1029,6 +1033,7 @@ void aja_source_get_defaults(obs_data_t *settings)
 	obs_data_set_default_int(settings, kUIPropChannelFormat.id, kDefaultAudioCaptureChannels);
 	obs_data_set_default_bool(settings, kUIPropChannelSwap_FC_LFE.id, false);
 	obs_data_set_default_bool(settings, kUIPropDeactivateWhenNotShowing.id, false);
+	obs_data_set_default_bool(settings, "async_compensation", true);
 }
 
 static void aja_source_get_defaults_v1(obs_data_t *settings)

plugins/aja/data/locale/en-US.ini

@@ -25,3 +25,4 @@ ChannelFormat.5_1ch="5.1ch"
 ChannelFormat.7_1ch="7.1ch"
 SwapFC-LFE="Swap FC and LFE"
 SwapFC-LFE.Tooltip="Swap Front Center Channel and LFE Channel"
+AsyncCompensation="Enable Asynchronous Compensation"

plugins/decklink/const.h

@@ -15,6 +15,7 @@
 #define KEYER "keyer"
 #define SWAP "swap"
 #define ALLOW_10_BIT "allow_10_bit"
+#define ASYNC_COMPENSATION "async_compensation"
 
 #define TEXT_DEVICE obs_module_text("Device")
 #define TEXT_VIDEO_CONNECTION obs_module_text("VideoConnection")
@@ -43,3 +44,4 @@
 #define TEXT_SWAP obs_module_text("SwapFC-LFE")
 #define TEXT_SWAP_TOOLTIP obs_module_text("SwapFC-LFE.Tooltip")
 #define TEXT_ALLOW_10_BIT obs_module_text("Allow10Bit")
+#define TEXT_ASYNC_COMPENSATION obs_module_text("AsyncCompensation")

plugins/decklink/data/locale/en-US.ini

@@ -25,3 +25,4 @@ SwapFC-LFE.Tooltip="Swap Front Center Channel and LFE Channel"
 VideoConnection="Video Connection"
 AudioConnection="Audio Connection"
 Allow10Bit="Allow 10 Bit (Required for SDI captions, may cause performance overhead)"
+AsyncCompensation="Enable Asynchronous Compensation"

plugins/decklink/decklink-source.cpp

@@ -71,6 +71,8 @@ static void decklink_update(void *data, obs_data_t *settings)
 	decklink->swap = obs_data_get_bool(settings, SWAP);
 	decklink->allow10Bit = obs_data_get_bool(settings, ALLOW_10_BIT);
 	decklink->Activate(device, id, videoConnection, audioConnection);
+
+	obs_source_set_async_compensation(decklink->GetSource(), obs_data_get_bool(settings, ASYNC_COMPENSATION));
 }
 
 static void decklink_show(void *data)
@@ -99,6 +101,7 @@ static void decklink_get_defaults(obs_data_t *settings)
 	obs_data_set_default_int(settings, COLOR_RANGE, VIDEO_RANGE_DEFAULT);
 	obs_data_set_default_int(settings, CHANNEL_FORMAT, SPEAKERS_STEREO);
 	obs_data_set_default_bool(settings, SWAP, false);
+	obs_data_set_default_bool(settings, ASYNC_COMPENSATION, true);
 }
 
 static const char *decklink_get_name(void *)
@@ -261,6 +264,8 @@ static obs_properties_t *decklink_get_properties(void *data)
 
 	obs_properties_add_bool(props, ALLOW_10_BIT, TEXT_ALLOW_10_BIT);
 
+	obs_properties_add_bool(props, ASYNC_COMPENSATION, TEXT_ASYNC_COMPENSATION);
+
 	UNUSED_PARAMETER(data);
 	return props;
 }