From 9dcf747121f843901577cc9226f4bc25739f1807 Mon Sep 17 00:00:00 2001 From: Matin Lotfaliei Date: Mon, 30 Sep 2024 16:57:00 -0700 Subject: [PATCH 1/6] Add AudioIn lists to Mobu UI --- plugins/mobu/layout.cpp | 35 ++++++++++++++++++++++++++++++++++- plugins/mobu/layout.h | 4 ++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/plugins/mobu/layout.cpp b/plugins/mobu/layout.cpp index d512b9d9..336e63f7 100644 --- a/plugins/mobu/layout.cpp +++ b/plugins/mobu/layout.cpp @@ -3,6 +3,7 @@ #include "o3ds/o3ds.h" // for version# #include +#include #define OPEN3D_DEVICE__LAYOUT Open3D_Device_Layout @@ -195,6 +196,25 @@ void Open3D_Device_Layout::UICreate() mLayoutLeft.SetControl("JointsList", mMemoJoints); mMemoJoints.OnChange.Add(this, (FBCallback)&Open3D_Device_Layout::EventEditJoints); + // Microphone Source label - under JointsList + mLayoutLeft.AddRegion("LabelMicSource", "LabelMicSource", + lS, kFBAttachRight, "SourcesList", 1.00, + 2, kFBAttachBottom, "JointsList", 1.00, + 155, kFBAttachNone, "", 1.00, + lH, kFBAttachNone, NULL, 1.00); + mLayoutLeft.SetControl("LabelMicSource", mLabelMicSource); + mLabelJoints.Caption = "Microphone Source:"; + + // Microphones list - Under Microphone Source Label + mLayoutLeft.AddRegion("ListMicSource", "ListMicSource", + lS, kFBAttachRight, "SourcesList", 1.00, + 2, kFBAttachBottom, "LabelMicSource", 1.00, + 155, kFBAttachNone, "", 1.00, + lH, kFBAttachNone, NULL, 1.00); + mLayoutLeft.SetControl("ListMicSource", mListMicSource); + mListMicSource.OnChange.Add(this, (FBCallback)&Open3D_Device_Layout::EventSelectMicSource); + mListMicSource.Style = FBListStyle::kFBDropDownList; + // Version info - at bottom mLayoutLeft.AddRegion("VersionInfo", "VersionInfo", lS, kFBAttachRight, "SourcesList", 1.00, @@ -332,6 +352,7 @@ void Open3D_Device_Layout::UIConfigure() PopulateSubjectList(); PopulateSubjectFields(); + PopulateMicrophoneList(); mMemoLog.Enabled = false; @@ -480,6 +501,15 @@ void Open3D_Device_Layout::PopulateSubjectFields() } } +void Open3D_Device_Layout::PopulateMicrophoneList() { + mListMicSource.Items.Clear(); + auto micList = FBPropertyListAudioIn(); + FBAudioIn* mic; + for (auto i=0; (mic = micList[i]) != nullptr; i++) { + mListMicSource.Items.Add(mic->Name); + } +} + void Open3D_Device_Layout::EventSelectDevice(HISender pSender, HKEvent pEvent) { PopulateSubjectFields(); @@ -578,4 +608,7 @@ void Open3D_Device_Layout::EventEditJoints(HISender pSender, HKEvent pEvent) std::vector vstrings(begin, end); mDevice->Items[id]->mJoints.clear(); std::copy(vstrings.begin(), vstrings.end(), std::back_inserter(mDevice->Items[id]->mJoints)); -} \ No newline at end of file +} + +void Open3D_Device_Layout::EventSelectMicSource(HISender pSender, HKEvent pEvent) { +} \ No newline at end of file diff --git a/plugins/mobu/layout.h b/plugins/mobu/layout.h index a88d3e88..1e4d36d6 100644 --- a/plugins/mobu/layout.h +++ b/plugins/mobu/layout.h @@ -17,6 +17,7 @@ class Open3D_Device_Layout : public FBDeviceLayout void PopulateSubjectList(); void PopulateSubjectFields(); + void PopulateMicrophoneList(); // Events void EventDeviceStatusChange( HISender pSender, HKEvent pEvent ); @@ -26,6 +27,7 @@ class Open3D_Device_Layout : public FBDeviceLayout void EventSelectDevice(HISender pSender, HKEvent pEvent); void EventEditSubject(HISender pSender, HKEvent pEvent); void EventEditJoints(HISender pSender, HKEvent pEvent); + void EventSelectMicSource(HISender pSender, HKEvent pEvent); void EventEditIP(HISender pSender, HKEvent pEvent); void EventEditPort(HISender pSender, HKEvent pEvent); void EventEditProtocol(HISender pSender, HKEvent pEvent); @@ -57,6 +59,8 @@ class Open3D_Device_Layout : public FBDeviceLayout FBEdit mEditKey; FBLabel mLabelSamplingRate; FBEditNumber mEditSamplingRate; + FBLabel mLabelMicSource; + FBList mListMicSource; FBLabel mLabelPluginVersion; FBLabel mLabelJoints; FBEdit mMemoJoints; From cb361d844522a9470eb6e7c67dda6df54458ba52 Mon Sep 17 00:00:00 2001 From: Matin Lotfaliei Date: Sat, 5 Oct 2024 20:49:04 -0700 Subject: [PATCH 2/6] Correctly fetching Microphone list and set into the Open3D_Device --- plugins/mobu/device.cpp | 10 ++++++++++ plugins/mobu/device.h | 3 +++ plugins/mobu/layout.cpp | 16 ++++++++++++---- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/plugins/mobu/device.cpp b/plugins/mobu/device.cpp index 0575df6e..262398a2 100644 --- a/plugins/mobu/device.cpp +++ b/plugins/mobu/device.cpp @@ -257,6 +257,16 @@ bool Open3D_Device::Start() return true; } +void Open3D_Device::SetSelectedMicrophone(FBAudioIn *mic) { + if (mSelectedMicrophone != nullptr) { + if (mSelectedMicrophone->IsOnline()) { + mSelectedMicrophone->SetOnline(false); + mSelectedMicrophone->SetDestination(nullptr); + } + } + mSelectedMicrophone = mic; +} + bool Open3D_Device::IsActive() { if (mNetworkSocket != -1) return true; diff --git a/plugins/mobu/device.h b/plugins/mobu/device.h index 665f74d3..a8c2fa89 100644 --- a/plugins/mobu/device.h +++ b/plugins/mobu/device.h @@ -67,6 +67,8 @@ class Open3D_Device : public FBDevice FBDeviceSamplingMode GetSamplingType() { return mSamplingType; } void SetSamplingType( FBDeviceSamplingMode pType ) { mSamplingType = pType; } + void SetSelectedMicrophone(FBAudioIn* mic); + bool IsActive(); void InData(void *, size_t); @@ -92,6 +94,7 @@ class Open3D_Device : public FBDevice int mFrameCounter; uint32_t mIdSeq; + FBAudioIn* mSelectedMicrophone; }; #endif diff --git a/plugins/mobu/layout.cpp b/plugins/mobu/layout.cpp index 336e63f7..c8a6e09f 100644 --- a/plugins/mobu/layout.cpp +++ b/plugins/mobu/layout.cpp @@ -203,7 +203,7 @@ void Open3D_Device_Layout::UICreate() 155, kFBAttachNone, "", 1.00, lH, kFBAttachNone, NULL, 1.00); mLayoutLeft.SetControl("LabelMicSource", mLabelMicSource); - mLabelJoints.Caption = "Microphone Source:"; + mLabelMicSource.Caption = "Microphone Source:"; // Microphones list - Under Microphone Source Label mLayoutLeft.AddRegion("ListMicSource", "ListMicSource", @@ -503,9 +503,11 @@ void Open3D_Device_Layout::PopulateSubjectFields() void Open3D_Device_Layout::PopulateMicrophoneList() { mListMicSource.Items.Clear(); - auto micList = FBPropertyListAudioIn(); - FBAudioIn* mic; - for (auto i=0; (mic = micList[i]) != nullptr; i++) { + mListMicSource.Items.Add("None"); + + auto& micList = FBSystem::TheOne().AudioInputs; + for (auto i = 0; i < micList.GetCount(); ++i) { + const auto mic = micList[i]; mListMicSource.Items.Add(mic->Name); } } @@ -611,4 +613,10 @@ void Open3D_Device_Layout::EventEditJoints(HISender pSender, HKEvent pEvent) } void Open3D_Device_Layout::EventSelectMicSource(HISender pSender, HKEvent pEvent) { + if (mListMicSource.ItemIndex == 0) + mDevice->SetSelectedMicrophone(nullptr); + else { + auto& micList = FBSystem::TheOne().AudioInputs; + mDevice->SetSelectedMicrophone(micList[mListMicSource.ItemIndex - 1]); + } } \ No newline at end of file From 88e072f8225bd494964403dec74f19c8ddf030fa Mon Sep 17 00:00:00 2001 From: Matin Lotfaliei Date: Sun, 13 Oct 2024 17:29:14 -0700 Subject: [PATCH 3/6] Add audio .h/.cpp class and select microphone --- plugins/mobu/CMakeLists.txt | 10 ++-- plugins/mobu/audio.cpp | 100 ++++++++++++++++++++++++++++++++++++ plugins/mobu/audio.h | 34 ++++++++++++ plugins/mobu/device.cpp | 10 +--- plugins/mobu/device.h | 4 +- 5 files changed, 146 insertions(+), 12 deletions(-) create mode 100644 plugins/mobu/audio.cpp create mode 100644 plugins/mobu/audio.h diff --git a/plugins/mobu/CMakeLists.txt b/plugins/mobu/CMakeLists.txt index 83c1a073..3543c52a 100644 --- a/plugins/mobu/CMakeLists.txt +++ b/plugins/mobu/CMakeLists.txt @@ -3,7 +3,7 @@ project(MobuStreamDevice) find_package(nng CONFIG REQUIRED) -set(SRCS plugin.cpp layout.cpp device.cpp device.h layout.h mobuModel.h mobuModel.cpp) +set(SRCS plugin.cpp layout.cpp device.cpp device.h layout.h mobuModel.h mobuModel.cpp audio.cpp) function(build_mobu version path) if(NOT EXISTS ${path}) @@ -17,8 +17,12 @@ function(build_mobu version path) if(${version} GREATER_EQUAL "2024") target_compile_definitions(${mobu_target} PUBLIC ORSDKPRODUCT_VERSION=ORSDK${version} ) endif() - - target_link_libraries(${mobu_target} fbsdk.lib opengl32.lib glu32.lib Ws2_32.lib open3dstreamstatic ${NNG_LIBRARY}) + + set(audio_libraries + mmdevapi.lib + winmm.lib + ) + target_link_libraries(${mobu_target} fbsdk.lib opengl32.lib glu32.lib Ws2_32.lib open3dstreamstatic ${NNG_LIBRARY} ${audio_libraries}) target_include_directories(${mobu_target} PUBLIC "${path}/OpenRealitySDK/include" ${CMAKE_CURENT_LIST_DIR}../src) target_link_directories(${mobu_target} PUBLIC "${path}/OpenRealitySDK/lib/x64" "${path}/OpenRealitySDK/lib/x64/HIK2016") target_compile_definitions(${mobu_target} PUBLIC NNG_STATIC_LIB) diff --git a/plugins/mobu/audio.cpp b/plugins/mobu/audio.cpp new file mode 100644 index 00000000..188159bd --- /dev/null +++ b/plugins/mobu/audio.cpp @@ -0,0 +1,100 @@ +// +// Created by matin on 2024-10-13. +// + +#include "audio.h" + +#include + +#include +#include + +// Throw a std::system_error if the HRESULT indicates failure. +template +void ThrowIfFailed(HRESULT hr, T&& msg) { + if( FAILED( hr ) ) { + throw std::system_error{hr, std::system_category(), std::forward(msg)}; + } +} + +int WideCompare(const LPWSTR wstr, const char* str) +{ + wchar_t ws[255]; + swprintf(ws, 255, L"%hs", str); + return wcscmp(wstr, ws); +} + +Open3D_AudioInput::Open3D_AudioInput() { + //auto hr = CoInitializeEx(nullptr, COINIT_MULTITHREADED); + //ThrowIfFailed(hr, "Failed to initialize the MMDeviceAPI."); + + GUID IDevice_FriendlyName = { + 0xa45c254e, 0xdf1c, 0x4efd, + { 0x80, 0x20, 0x67, 0xd1, 0x46, 0xa8, 0x50, 0xe0 } + }; + mNameKey.pid = 14; + mNameKey.fmtid = IDevice_FriendlyName; +} + +void Open3D_AudioInput::recorder_release() { + if (recorder) { + recorder->Release(); + recorder = nullptr; + } +} + +Open3D_AudioInput::~Open3D_AudioInput() { + recorder_release(); + //CoUninitialize(); +} + +void Open3D_AudioInput::set_device(FBAudioIn *mic) { + recorder_release(); + if (mic == nullptr) { + return; + } + + IMMDeviceEnumerator* enumerator = nullptr; + IMMDeviceCollection* recorderCollection = nullptr; + IMMDevice* device = nullptr; + IPropertyStore* properties = nullptr; + UINT count; + + HRESULT hr = CoCreateInstance( + __uuidof(MMDeviceEnumerator), nullptr, CLSCTX_ALL, + __uuidof(IMMDeviceEnumerator), reinterpret_cast(&enumerator) + ); + ThrowIfFailed(hr, "Failed to create an instance of MMDeviceEnumerator."); + + hr = enumerator->EnumAudioEndpoints(eCapture, DEVICE_STATE_ACTIVE, &recorderCollection); + ThrowIfFailed(hr, "Failed to create an instance of MMDeviceEnumerator."); + + hr = recorderCollection->GetCount(&count); + ThrowIfFailed(hr, "Failed to get MMDevice counts."); + + for (UINT i=0; iItem(i, &device); + ThrowIfFailed(hr, "Failed to get MMDevice."); + + hr = device->OpenPropertyStore(STGM_READ, &properties); + ThrowIfFailed(hr, "Failed to open property store for the MMDevice."); + + PROPVARIANT varName; + PropVariantInit(&varName); + hr = properties->GetValue(mNameKey, &varName); + ThrowIfFailed(hr, "Failed to get property value for the MMDevice."); + + if (varName.vt != VT_EMPTY) { + if (WideCompare(varName.pwszVal, mic->Name.AsString()) == 0) { + recorder = device; + properties->Release(); + break; + } + } + + properties->Release(); + device->Release(); + } + recorderCollection->Release(); + enumerator->Release(); +} diff --git a/plugins/mobu/audio.h b/plugins/mobu/audio.h new file mode 100644 index 00000000..98c8b3aa --- /dev/null +++ b/plugins/mobu/audio.h @@ -0,0 +1,34 @@ +// +// Created by matin on 2024-10-13. +// + +#ifndef AUDIO_H +#define AUDIO_H + +#include +#include + +struct IMMDevice; +struct IAudioClient; +struct IAudioRenderClient; + +class Open3D_AudioInput +{ +public: + Open3D_AudioInput(); + ~Open3D_AudioInput(); + void set_device(FBAudioIn* mic); + +private: + IMMDevice* recorder = nullptr; + IAudioClient* recorderClient = nullptr; + IAudioRenderClient* renderService = nullptr; + WAVEFORMATEX* format = nullptr; + PROPERTYKEY mNameKey; + + void recorder_release(); +}; + + + +#endif //AUDIO_H diff --git a/plugins/mobu/device.cpp b/plugins/mobu/device.cpp index 262398a2..94f3b546 100644 --- a/plugins/mobu/device.cpp +++ b/plugins/mobu/device.cpp @@ -257,14 +257,8 @@ bool Open3D_Device::Start() return true; } -void Open3D_Device::SetSelectedMicrophone(FBAudioIn *mic) { - if (mSelectedMicrophone != nullptr) { - if (mSelectedMicrophone->IsOnline()) { - mSelectedMicrophone->SetOnline(false); - mSelectedMicrophone->SetDestination(nullptr); - } - } - mSelectedMicrophone = mic; +void Open3D_Device::SetSelectedMicrophone(FBAudioIn* mic) { + mAudioRecord.set_device(mic); } bool Open3D_Device::IsActive() diff --git a/plugins/mobu/device.h b/plugins/mobu/device.h index a8c2fa89..0b7f3bb4 100644 --- a/plugins/mobu/device.h +++ b/plugins/mobu/device.h @@ -3,6 +3,8 @@ #include #include + +#include "audio.h" #include "mobuModel.h" #include "o3ds/base_connector.h" #include "o3ds/tcp.h" @@ -94,7 +96,7 @@ class Open3D_Device : public FBDevice int mFrameCounter; uint32_t mIdSeq; - FBAudioIn* mSelectedMicrophone; + Open3D_AudioInput mAudioRecord; }; #endif From 1c32feacc5e2406b274b6db1b17d1ad2d58d26b3 Mon Sep 17 00:00:00 2001 From: Matin Lotfaliei Date: Sat, 19 Oct 2024 16:37:47 -0700 Subject: [PATCH 4/6] Add audio indicator using FBSlider --- plugins/mobu/CMakeLists.txt | 2 +- plugins/mobu/audio.cpp | 84 +++++++++++++++++++++++++++++++++---- plugins/mobu/audio.h | 41 ++++++++++++++++-- plugins/mobu/device.h | 6 +-- plugins/mobu/layout.cpp | 35 ++++++++++++++-- plugins/mobu/layout.h | 4 +- 6 files changed, 153 insertions(+), 19 deletions(-) diff --git a/plugins/mobu/CMakeLists.txt b/plugins/mobu/CMakeLists.txt index 3543c52a..68913381 100644 --- a/plugins/mobu/CMakeLists.txt +++ b/plugins/mobu/CMakeLists.txt @@ -3,7 +3,7 @@ project(MobuStreamDevice) find_package(nng CONFIG REQUIRED) -set(SRCS plugin.cpp layout.cpp device.cpp device.h layout.h mobuModel.h mobuModel.cpp audio.cpp) +set(SRCS plugin.cpp layout.cpp device.cpp device.h layout.h mobuModel.h mobuModel.cpp audio.h audio.cpp) function(build_mobu version path) if(NOT EXISTS ${path}) diff --git a/plugins/mobu/audio.cpp b/plugins/mobu/audio.cpp index 188159bd..904bc5c7 100644 --- a/plugins/mobu/audio.cpp +++ b/plugins/mobu/audio.cpp @@ -8,6 +8,9 @@ #include #include +#include + +constexpr int REFTIMES_PER_SEC = 10000000; // Throw a std::system_error if the HRESULT indicates failure. template @@ -17,7 +20,7 @@ void ThrowIfFailed(HRESULT hr, T&& msg) { } } -int WideCompare(const LPWSTR wstr, const char* str) +int WideCompare(LPWSTR wstr, const char* str) { wchar_t ws[255]; swprintf(ws, 255, L"%hs", str); @@ -28,28 +31,91 @@ Open3D_AudioInput::Open3D_AudioInput() { //auto hr = CoInitializeEx(nullptr, COINIT_MULTITHREADED); //ThrowIfFailed(hr, "Failed to initialize the MMDeviceAPI."); - GUID IDevice_FriendlyName = { + mNameKey.pid = 14; + mNameKey.fmtid = { 0xa45c254e, 0xdf1c, 0x4efd, { 0x80, 0x20, 0x67, 0xd1, 0x46, 0xa8, 0x50, 0xe0 } }; - mNameKey.pid = 14; - mNameKey.fmtid = IDevice_FriendlyName; } -void Open3D_AudioInput::recorder_release() { +void Open3D_AudioInput::publish_audio_captured() const { + for (const auto subscriber : subscribers) { + subscriber->audio_captured(captureBuffer, nFrames); + } +} + +void Open3D_AudioInput::activate() { + if (recorder == nullptr) { + return; + } + + auto hr = recorder->Activate(__uuidof(IAudioClient), CLSCTX_ALL, + nullptr, reinterpret_cast(&recorderClient)); + ThrowIfFailed(hr, "Failed to activate the recorder."); + + hr = recorderClient->GetMixFormat(&format); + ThrowIfFailed(hr, "Failed to get the recording format."); + + hr = recorderClient->Initialize(AUDCLNT_SHAREMODE_SHARED, 0, REFTIMES_PER_SEC, + 0, format, nullptr); + ThrowIfFailed(hr, "Failed to initialize the recording client with the recording format."); + + hr = recorderClient->GetService(__uuidof(IAudioCaptureClient), reinterpret_cast(&recorderService)); + ThrowIfFailed(hr, "Failed to get the capture service for the recorder client."); + + hr = recorderClient->Start(); + ThrowIfFailed(hr, "Failed to start the recorder."); + + recordThread = new std::thread(&Open3D_AudioInput::loop, this); +} + +void Open3D_AudioInput::release() { + if (recordThread) { + stopThread = true; + recordThread->join(); + delete recordThread; + recordThread = nullptr; + stopThread = false; + } + if (recorderClient) { + recorderClient->Stop(); + + if (recorderService) { + recorderService->Release(); + recorderService = nullptr; + } + recorderClient->Release(); + recorderClient = nullptr; + } if (recorder) { recorder->Release(); recorder = nullptr; } } +void Open3D_AudioInput::loop() { + HRESULT hr; + while (!stopThread) { + hr = recorderService->GetBuffer(&captureBuffer, &nFrames, &flags, nullptr, nullptr); + ThrowIfFailed(hr, "Failed to get recording buffer."); + + if (captureBuffer == nullptr) + continue; + + hr = recorderService->ReleaseBuffer(nFrames); + ThrowIfFailed(hr, "Failed to release recording buffer"); + + publish_audio_captured(); + } +} + Open3D_AudioInput::~Open3D_AudioInput() { - recorder_release(); + release(); //CoUninitialize(); } void Open3D_AudioInput::set_device(FBAudioIn *mic) { - recorder_release(); + release(); if (mic == nullptr) { return; } @@ -86,7 +152,7 @@ void Open3D_AudioInput::set_device(FBAudioIn *mic) { if (varName.vt != VT_EMPTY) { if (WideCompare(varName.pwszVal, mic->Name.AsString()) == 0) { - recorder = device; + recorder = device; properties->Release(); break; } @@ -97,4 +163,6 @@ void Open3D_AudioInput::set_device(FBAudioIn *mic) { } recorderCollection->Release(); enumerator->Release(); + + activate(); } diff --git a/plugins/mobu/audio.h b/plugins/mobu/audio.h index 98c8b3aa..85496e40 100644 --- a/plugins/mobu/audio.h +++ b/plugins/mobu/audio.h @@ -7,10 +7,20 @@ #include #include +#include struct IMMDevice; struct IAudioClient; struct IAudioRenderClient; +namespace std { + class thread; +} + +class IAudioSubscriber { +public: + virtual ~IAudioSubscriber() = default; + virtual void audio_captured(const BYTE* captureBuffer, UINT32 nFrames) = 0; +}; class Open3D_AudioInput { @@ -18,15 +28,40 @@ class Open3D_AudioInput Open3D_AudioInput(); ~Open3D_AudioInput(); void set_device(FBAudioIn* mic); + void subscribe(IAudioSubscriber* subscriber) { subscribers.push_back(subscriber); } + void unsubscribe(IAudioSubscriber* subscriber) { subscribers.erase(std::remove(subscribers.begin(), subscribers.end(), subscriber), subscribers.end()); } private: + // Audio Device Activation Section + // =============================== + IMMDevice* recorder = nullptr; IAudioClient* recorderClient = nullptr; - IAudioRenderClient* renderService = nullptr; + IAudioCaptureClient* recorderService = nullptr; + WAVEFORMATEX* format = nullptr; - PROPERTYKEY mNameKey; + PROPERTYKEY mNameKey = {}; + + void activate(); + void release(); + + // Audio Capture Loop Section + // ========================== + + BYTE* captureBuffer = nullptr; + UINT32 nFrames = 0; + DWORD flags = 0; + + std::thread* recordThread = nullptr; + bool stopThread = false; + + void loop(); + + // Audio Capture Publishing to Subscribers Section + // =============================================== - void recorder_release(); + std::vector subscribers; + void publish_audio_captured() const; }; diff --git a/plugins/mobu/device.h b/plugins/mobu/device.h index 0b7f3bb4..268472cf 100644 --- a/plugins/mobu/device.h +++ b/plugins/mobu/device.h @@ -83,6 +83,8 @@ class Open3D_Device : public FBDevice SOCKET mNetworkSocket; std::vector mClients; + Open3D_AudioInput mAudioRecord; + private: double mSamplingRate; FBDeviceSamplingMode mSamplingType; @@ -94,9 +96,7 @@ class Open3D_Device : public FBDevice int mNetworkPort; bool mStreaming; int mFrameCounter; - uint32_t mIdSeq; - - Open3D_AudioInput mAudioRecord; + uint32_t mIdSeq; }; #endif diff --git a/plugins/mobu/layout.cpp b/plugins/mobu/layout.cpp index c8a6e09f..ada7b15d 100644 --- a/plugins/mobu/layout.cpp +++ b/plugins/mobu/layout.cpp @@ -20,6 +20,7 @@ bool Open3D_Device_Layout::FBCreate() // Add device & system callbacks mDevice->OnStatusChange.Add ( this,(FBCallback)&Open3D_Device_Layout::EventDeviceStatusChange ); OnIdle.Add ( this,(FBCallback)&Open3D_Device_Layout::EventUIIdle ); + mDevice->mAudioRecord.subscribe(this); return true; } @@ -29,6 +30,7 @@ void Open3D_Device_Layout::FBDestroy() // Remove device & system callbacks OnIdle.Remove ( this,(FBCallback)&Open3D_Device_Layout::EventUIIdle ); mDevice->OnStatusChange.Remove ( this,(FBCallback)&Open3D_Device_Layout::EventDeviceStatusChange ); + mDevice->mAudioRecord.unsubscribe(this); } void Open3D_Device_Layout::UICreate() @@ -213,7 +215,24 @@ void Open3D_Device_Layout::UICreate() lH, kFBAttachNone, NULL, 1.00); mLayoutLeft.SetControl("ListMicSource", mListMicSource); mListMicSource.OnChange.Add(this, (FBCallback)&Open3D_Device_Layout::EventSelectMicSource); - mListMicSource.Style = FBListStyle::kFBDropDownList; + mListMicSource.Style = kFBDropDownList; + + // Microphone Capture Indicator - Under Microphones List + mLayoutLeft.AddRegion("MicCaptureIndicator", "MicCaptureIndicator", + lS, kFBAttachRight, "SourcesList", 1.00, + 5, kFBAttachBottom, "ListMicSource", 1.00, + 155, kFBAttachNone, "", 1.00, + lH, kFBAttachNone, NULL, 1.00); + mLayoutLeft.SetControl("MicCaptureIndicator", mMicCaptureIndicator); + mMicCaptureIndicator.Height = 10; + mMicCaptureIndicator.Max = 0; + mMicCaptureIndicator.Max = 1; + mMicCaptureIndicator.Value = 0; + mMicCaptureIndicator.LargeStep = 0.01; + mMicCaptureIndicator.SmallStep = 0.01; + mMicCaptureIndicator.Visible = false; + mMicCaptureIndicator.Orientation = kFBHorizontal; + mMicCaptureIndicator.ReadOnly = true; // Version info - at bottom mLayoutLeft.AddRegion("VersionInfo", "VersionInfo", @@ -416,6 +435,12 @@ void Open3D_Device_Layout::EventEditDelta(HISender pSender, HKEvent pEvent) mDevice->SetDeltaThreshold(value); } +void Open3D_Device_Layout::audio_captured(const BYTE* captureBuffer, const UINT32 nFrames) +{ + const auto captureBufferFloat = reinterpret_cast(captureBuffer); + mMicCaptureIndicator.Value = fabsf(captureBufferFloat[0]); +} + void Open3D_Device_Layout::EventDeviceStatusChange( HISender pSender, HKEvent pEvent ) { UIReset(); @@ -613,10 +638,14 @@ void Open3D_Device_Layout::EventEditJoints(HISender pSender, HKEvent pEvent) } void Open3D_Device_Layout::EventSelectMicSource(HISender pSender, HKEvent pEvent) { - if (mListMicSource.ItemIndex == 0) + if (mListMicSource.ItemIndex == 0) { mDevice->SetSelectedMicrophone(nullptr); + mMicCaptureIndicator.Value = 0; + mMicCaptureIndicator.Visible = false; + } else { auto& micList = FBSystem::TheOne().AudioInputs; mDevice->SetSelectedMicrophone(micList[mListMicSource.ItemIndex - 1]); + mMicCaptureIndicator.Visible = true; } -} \ No newline at end of file +} diff --git a/plugins/mobu/layout.h b/plugins/mobu/layout.h index 1e4d36d6..fad7ed08 100644 --- a/plugins/mobu/layout.h +++ b/plugins/mobu/layout.h @@ -3,7 +3,7 @@ #include "device.h" -class Open3D_Device_Layout : public FBDeviceLayout +class Open3D_Device_Layout : public FBDeviceLayout, IAudioSubscriber { FBDeviceLayoutDeclare( Open3D_Device_Layout, FBDeviceLayout ); public: @@ -34,6 +34,7 @@ class Open3D_Device_Layout : public FBDeviceLayout void EventEditSampleRate(HISender pSender, HKEvent pEvent); void EventEditKey(HISender pSender, HKEvent pEvent); void EventEditDelta(HISender pSender, HKEvent pEvent); + void audio_captured(const BYTE* captureBuffer, UINT32 nFrames) override; private: FBLayout mLayoutLeft; @@ -61,6 +62,7 @@ class Open3D_Device_Layout : public FBDeviceLayout FBEditNumber mEditSamplingRate; FBLabel mLabelMicSource; FBList mListMicSource; + FBSlider mMicCaptureIndicator; FBLabel mLabelPluginVersion; FBLabel mLabelJoints; FBEdit mMemoJoints; From ec1624378c576b649e52319d24adfb212a5e89ff Mon Sep 17 00:00:00 2001 From: Matin Lotfaliei Date: Sun, 20 Oct 2024 15:53:35 -0700 Subject: [PATCH 5/6] Add and send audio to SubjectUpdate --- plugins/mobu/device.cpp | 93 +++++++++++++++++++++++++++-------------- plugins/mobu/device.h | 6 ++- plugins/mobu/layout.cpp | 64 +++++++++++++++++----------- src/o3ds.fbs | 1 + src/o3ds/model.cpp | 3 +- src/o3ds/model.h | 13 ++++++ 6 files changed, 122 insertions(+), 58 deletions(-) diff --git a/plugins/mobu/device.cpp b/plugins/mobu/device.cpp index 94f3b546..d3563c25 100644 --- a/plugins/mobu/device.cpp +++ b/plugins/mobu/device.cpp @@ -84,6 +84,8 @@ bool Open3D_Device::FBCreate() mFrameCounter = 0; mIdSeq = 0; + + mAudioRecord.subscribe(this); return true; } @@ -259,6 +261,15 @@ bool Open3D_Device::Start() void Open3D_Device::SetSelectedMicrophone(FBAudioIn* mic) { mAudioRecord.set_device(mic); + + // check if there should be a subject for audio. + if (mic == nullptr) { + Items.removeSubject(audioSubjectName); + } else { + if (Items.findSubject(audioSubjectName) == nullptr) { + Items.addSubject(audioSubjectName); + } + } } bool Open3D_Device::IsActive() @@ -338,6 +349,18 @@ uint32_t Open3D_Device::WriteTcp(O3DS::TcpSocket &socket, void *data, uint32_t b } +void Open3D_Device::audio_captured(const BYTE *captureBuffer, UINT32 nFrames) { + const auto audioSubject = Items.findSubject(audioSubjectName); + if (Online == true) { + audioSubject->mAudioBuffer.insert(audioSubject->mAudioBuffer.end(), + captureBuffer, captureBuffer + nFrames); + } + else + { + audioSubject->mAudioBuffer.assign(captureBuffer, captureBuffer + nFrames); + } +} + void Open3D_Device::DeviceIONotify(kDeviceIOs pAction, FBDeviceNotifyInfo &pDeviceNotifyInfo) { std::vector buf; @@ -354,7 +377,7 @@ void Open3D_Device::DeviceIONotify(kDeviceIOs pAction, FBDeviceNotifyInfo &pDev case kIOPlayModeWrite: case kIOStopModeWrite: { - if (mNetworkSocket != -1) + if (Online == true) { Items.update(); @@ -368,6 +391,10 @@ void Open3D_Device::DeviceIONotify(kDeviceIOs pAction, FBDeviceNotifyInfo &pDev else { bucketSize = Items.SerializeUpdate(buf, count, MobuTime.GetSecondDouble()); + + // Clear the sent pieces of audio buffers. + auto audioSubject = Items.findSubject(audioSubjectName); + audioSubject->mAudioBuffer.clear(); } if (mFrameCounter > 100) { @@ -387,23 +414,27 @@ void Open3D_Device::DeviceIONotify(kDeviceIOs pAction, FBDeviceNotifyInfo &pDev if (mTcpIp.Accept(newSocket)) mClients.push_back(newSocket); - for (std::vector::iterator client = mClients.begin(); client != mClients.end();) - { - O3DS::TcpSocket s(*client); - if (WriteTcp(s, &buf[0], bucketSize)) { - client++; - } - else { - client = mClients.erase(client); + for (std::vector::iterator client = mClients.begin(); client != mClients.end();) + { + O3DS::TcpSocket s(*client); + if (WriteTcp(s, &buf[0], bucketSize)) { + client++; + } + else { + client = mClients.erase(client); + } } + + std::ostringstream oss; + oss << bucketSize << " bytes"; + Status = oss.str().c_str(); } - } if (mProtocol == Open3D_Device::kTCPClient) { - if(mTcpIp.Valid()) - { - if (WriteTcp(mTcpIp, &buf[0], bucketSize) == 0) + if(mTcpIp.Valid()) { + if (WriteTcp(mTcpIp, &buf[0], bucketSize) == 0) + { Status = "TCP Disconnected"; } } @@ -414,28 +445,28 @@ void Open3D_Device::DeviceIONotify(kDeviceIOs pAction, FBDeviceNotifyInfo &pDev struct in_addr ret; if (!inet_pton_alt(AF_INET, mNetworkAddress.operator char *(), &ret)) { - Status = "UDP ERROR"; - return; - } + Status = "UDP ERROR"; + return; + } - struct sockaddr_in dest_addr; - dest_addr.sin_family = AF_INET; - dest_addr.sin_addr.s_addr = ret.S_un.S_addr; - dest_addr.sin_port = htons(mNetworkPort); + struct sockaddr_in dest_addr; + dest_addr.sin_family = AF_INET; + dest_addr.sin_addr.s_addr = ret.S_un.S_addr; + dest_addr.sin_port = htons(mNetworkPort); - size_t sentSz = 0; + size_t sentSz = 0; - mIdSeq++; + mIdSeq++; - std::vector fragData; - UdpFragmenter frag(buf.data(), bucketSize, 512); - for (int i = 0; i < frag.mFrames; i++) - { - frag.makeFragment(mIdSeq, i, fragData); - sentSz+= sendto(mNetworkSocket, fragData.data(), fragData.size(), 0, - (struct sockaddr*)&dest_addr, - (int)sizeof(struct sockaddr_in)); - } + std::vector fragData; + UdpFragmenter frag(buf.data(), bucketSize, 512); + for (int i = 0; i < frag.mFrames; i++) + { + frag.makeFragment(mIdSeq, i, fragData); + sentSz+= sendto(mNetworkSocket, fragData.data(), fragData.size(), 0, + (struct sockaddr*)&dest_addr, + (int)sizeof(struct sockaddr_in)); + } std::ostringstream oss; oss << frag.mFrames << "/" << sentSz << " bytes"; diff --git a/plugins/mobu/device.h b/plugins/mobu/device.h index 268472cf..908962f2 100644 --- a/plugins/mobu/device.h +++ b/plugins/mobu/device.h @@ -12,8 +12,10 @@ #define OPEN3D_DEVICE__CLASSNAME Open3D_Device #define OPEN3D_DEVICE__CLASSSTR "Open3D_Device" +constexpr auto audioSubjectName = "Audio"; -class Open3D_Device : public FBDevice + +class Open3D_Device : public FBDevice, IAudioSubscriber { FBDeviceDeclare( Open3D_Device, FBDevice ); public: @@ -86,6 +88,8 @@ class Open3D_Device : public FBDevice Open3D_AudioInput mAudioRecord; private: + void audio_captured(const BYTE *captureBuffer, UINT32 nFrames) override; + double mSamplingRate; FBDeviceSamplingMode mSamplingType; FBPlayerControl mPlayerControl; diff --git a/plugins/mobu/layout.cpp b/plugins/mobu/layout.cpp index ada7b15d..9641e52f 100644 --- a/plugins/mobu/layout.cpp +++ b/plugins/mobu/layout.cpp @@ -420,7 +420,13 @@ void Open3D_Device_Layout::EventEditSubject(HISender pSender, HKEvent pEvent) int id = mSourcesList.ItemIndex; if (id < 0) return; - FBString newValue = mEditSubject.Text.AsString(); + const FBString newValue = mEditSubject.Text.AsString(); + + // Don't let changing the name of the Audio subject. + if (mDevice->Items[id]->mName == audioSubjectName && newValue != audioSubjectName) { + mEditSubject.Text.SetString(audioSubjectName); + return; + } mDevice->Items[id]->mName = newValue.operator char *(); mSourcesList.Items.RemoveAt(id); @@ -487,32 +493,36 @@ void Open3D_Device_Layout::PopulateSubjectFields() { if (id < mDevice->Items.size()) { + std::ostringstream oss; O3DS::Subject* subject = mDevice->Items[id]; - FBModel *model = static_cast(subject->mReference); + const auto model = static_cast(subject->mReference); + + mEditSubject.Text = subject->mName.c_str(); - mEditSubject.Text = mDevice->Items[id]->mName.c_str(); - mEditSource.Text = model->GetFullName(); + if (model != nullptr) { + mEditSource.Text = model->GetFullName(); - std::ostringstream imploded, oss; - std::copy(mDevice->Items[id]->mJoints.begin(), mDevice->Items[id]->mJoints.end(), - std::ostream_iterator(imploded, " ")); + std::ostringstream imploded; + std::copy(subject->mJoints.begin(), subject->mJoints.end(), + std::ostream_iterator(imploded, " ")); - mMemoJoints.Text = imploded.str().c_str(); + mMemoJoints.Text = imploded.str().c_str(); - O3DS::Mobu::TraverseSubject(subject, model); + O3DS::Mobu::TraverseSubject(subject, model); - if (model->Is(FBModelNull::TypeInfo)) - oss << "Null" << std::endl; - if (model->Is(FBModelRoot::TypeInfo)) - oss << "Root" << std::endl; - if (model->Is(FBCamera::TypeInfo)) - oss << "Camera" << std::endl; - if (model->Is(FBModelSkeleton::TypeInfo)) - oss << "Joint" << std::endl; + if (model->Is(FBModelNull::TypeInfo)) + oss << "Null" << std::endl; + if (model->Is(FBModelRoot::TypeInfo)) + oss << "Root" << std::endl; + if (model->Is(FBCamera::TypeInfo)) + oss << "Camera" << std::endl; + if (model->Is(FBModelSkeleton::TypeInfo)) + oss << "Joint" << std::endl; + } - oss << "Items: " << mDevice->Items[id]->mTransforms.size() << std::endl; + oss << "Items: " << subject->mTransforms.size() << std::endl; - oss << "Joints: " << mDevice->Items[id]->mJoints.size() << std::endl; + oss << "Joints: " << subject->mJoints.size() << std::endl; std::vector buf; mDevice->Items.Serialize(buf, count); @@ -575,12 +585,15 @@ void Open3D_Device_Layout::EventAdd(HISender pSender, HKEvent pEvent) void Open3D_Device_Layout::EventDel(HISender pSender, HKEvent pEvent) { - int id = mSourcesList.ItemIndex; - if (id >= 0) - { - mDevice->Items.mItems.erase(mDevice->Items.begin() + id); - PopulateSubjectList(); - } + const int id = mSourcesList.ItemIndex; + if (id < 0) return; + + // Don't let changing the name of the Audio subject. + if (mDevice->Items[id]->mName == audioSubjectName) + return; + + mDevice->Items.mItems.erase(mDevice->Items.begin() + id); + PopulateSubjectList(); } void Open3D_Device_Layout::EventEditIP(HISender pSender, HKEvent pEvent) @@ -648,4 +661,5 @@ void Open3D_Device_Layout::EventSelectMicSource(HISender pSender, HKEvent pEvent mDevice->SetSelectedMicrophone(micList[mListMicSource.ItemIndex - 1]); mMicCaptureIndicator.Visible = true; } + PopulateSubjectList(); } diff --git a/src/o3ds.fbs b/src/o3ds.fbs index 2c15d0f4..ce9a8ed3 100644 --- a/src/o3ds.fbs +++ b/src/o3ds.fbs @@ -86,6 +86,7 @@ table Subject { table SubjectUpdate { name:string; + audio:[byte]; translations:[TranslationUpdate]; rotation:[RotationUpdate]; scale:[ScaleUpdate]; diff --git a/src/o3ds/model.cpp b/src/o3ds/model.cpp index 8a9d1035..9bcb5a85 100644 --- a/src/o3ds/model.cpp +++ b/src/o3ds/model.cpp @@ -421,10 +421,11 @@ namespace O3DS transformId++; } + auto audio = builder.CreateVector(mAudioBuffer); auto tr = builder.CreateVectorOfStructs(translations); auto ro = builder.CreateVectorOfStructs(rotations); auto sc = builder.CreateVectorOfStructs(scales); - return CreateSubjectUpdate(builder, oSubjectName, tr, ro, sc); + return CreateSubjectUpdate(builder, oSubjectName, audio, tr, ro, sc); } int Subject::Serialize(std::vector &outbuf, double timestamp) diff --git a/src/o3ds/model.h b/src/o3ds/model.h index 23d33381..eded2cd3 100644 --- a/src/o3ds/model.h +++ b/src/o3ds/model.h @@ -150,6 +150,7 @@ namespace O3DS std::string mName; std::vector mJoints; + std::vector mAudioBuffer; TransformList mTransforms; void* mReference; @@ -238,6 +239,18 @@ namespace O3DS return nullptr; } + void removeSubject(const std::string &name) + { + const auto item = findSubject(name); + if (item == nullptr) { + return; + } + // move item to the end + const auto iter = std::remove(mItems.begin(), mItems.end(), item); + // remove item from the end + mItems.erase(iter, mItems.end()); + } + void update() { for (auto i : mItems) From 0da5c06f6f1696aec01eb268f1274200a6d0d475 Mon Sep 17 00:00:00 2001 From: Matin Lotfaliei Date: Sun, 20 Oct 2024 23:01:12 -0700 Subject: [PATCH 6/6] Check if audio subject is present --- plugins/mobu/device.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/mobu/device.cpp b/plugins/mobu/device.cpp index d3563c25..f31b43ab 100644 --- a/plugins/mobu/device.cpp +++ b/plugins/mobu/device.cpp @@ -377,7 +377,7 @@ void Open3D_Device::DeviceIONotify(kDeviceIOs pAction, FBDeviceNotifyInfo &pDev case kIOPlayModeWrite: case kIOStopModeWrite: { - if (Online == true) + if (mNetworkSocket != -1) { Items.update(); @@ -393,8 +393,8 @@ void Open3D_Device::DeviceIONotify(kDeviceIOs pAction, FBDeviceNotifyInfo &pDev bucketSize = Items.SerializeUpdate(buf, count, MobuTime.GetSecondDouble()); // Clear the sent pieces of audio buffers. - auto audioSubject = Items.findSubject(audioSubjectName); - audioSubject->mAudioBuffer.clear(); + if (auto audioSubject = Items.findSubject(audioSubjectName); audioSubject != nullptr) + audioSubject->mAudioBuffer.clear(); } if (mFrameCounter > 100) {