diff --git a/plugins/mobu/CMakeLists.txt b/plugins/mobu/CMakeLists.txt index 83c1a073..68913381 100644 --- a/plugins/mobu/CMakeLists.txt +++ b/plugins/mobu/CMakeLists.txt @@ -3,7 +3,7 @@ project(MobuStreamDevice) find_package(nng CONFIG REQUIRED) -set(SRCS plugin.cpp layout.cpp device.cpp device.h layout.h mobuModel.h mobuModel.cpp) +set(SRCS plugin.cpp layout.cpp device.cpp device.h layout.h mobuModel.h mobuModel.cpp audio.h audio.cpp) function(build_mobu version path) if(NOT EXISTS ${path}) @@ -17,8 +17,12 @@ function(build_mobu version path) if(${version} GREATER_EQUAL "2024") target_compile_definitions(${mobu_target} PUBLIC ORSDKPRODUCT_VERSION=ORSDK${version} ) endif() - - target_link_libraries(${mobu_target} fbsdk.lib opengl32.lib glu32.lib Ws2_32.lib open3dstreamstatic ${NNG_LIBRARY}) + + set(audio_libraries + mmdevapi.lib + winmm.lib + ) + target_link_libraries(${mobu_target} fbsdk.lib opengl32.lib glu32.lib Ws2_32.lib open3dstreamstatic ${NNG_LIBRARY} ${audio_libraries}) target_include_directories(${mobu_target} PUBLIC "${path}/OpenRealitySDK/include" ${CMAKE_CURENT_LIST_DIR}../src) target_link_directories(${mobu_target} PUBLIC "${path}/OpenRealitySDK/lib/x64" "${path}/OpenRealitySDK/lib/x64/HIK2016") target_compile_definitions(${mobu_target} PUBLIC NNG_STATIC_LIB) diff --git a/plugins/mobu/audio.cpp b/plugins/mobu/audio.cpp new file mode 100644 index 00000000..904bc5c7 --- /dev/null +++ b/plugins/mobu/audio.cpp @@ -0,0 +1,168 @@ +// +// Created by matin on 2024-10-13. +// + +#include "audio.h" + +#include + +#include +#include +#include + +constexpr int REFTIMES_PER_SEC = 10000000; + +// Throw a std::system_error if the HRESULT indicates failure. +template +void ThrowIfFailed(HRESULT hr, T&& msg) { + if( FAILED( hr ) ) { + throw std::system_error{hr, std::system_category(), std::forward(msg)}; + } +} + +int WideCompare(LPWSTR wstr, const char* str) +{ + wchar_t ws[255]; + swprintf(ws, 255, L"%hs", str); + return wcscmp(wstr, ws); +} + +Open3D_AudioInput::Open3D_AudioInput() { + //auto hr = CoInitializeEx(nullptr, COINIT_MULTITHREADED); + //ThrowIfFailed(hr, "Failed to initialize the MMDeviceAPI."); + + mNameKey.pid = 14; + mNameKey.fmtid = { + 0xa45c254e, 0xdf1c, 0x4efd, + { 0x80, 0x20, 0x67, 0xd1, 0x46, 0xa8, 0x50, 0xe0 } + }; +} + +void Open3D_AudioInput::publish_audio_captured() const { + for (const auto subscriber : subscribers) { + subscriber->audio_captured(captureBuffer, nFrames); + } +} + +void Open3D_AudioInput::activate() { + if (recorder == nullptr) { + return; + } + + auto hr = recorder->Activate(__uuidof(IAudioClient), CLSCTX_ALL, + nullptr, reinterpret_cast(&recorderClient)); + ThrowIfFailed(hr, "Failed to activate the recorder."); + + hr = recorderClient->GetMixFormat(&format); + ThrowIfFailed(hr, "Failed to get the recording format."); + + hr = recorderClient->Initialize(AUDCLNT_SHAREMODE_SHARED, 0, REFTIMES_PER_SEC, + 0, format, nullptr); + ThrowIfFailed(hr, "Failed to initialize the recording client with the recording format."); + + hr = recorderClient->GetService(__uuidof(IAudioCaptureClient), reinterpret_cast(&recorderService)); + ThrowIfFailed(hr, "Failed to get the capture service for the recorder client."); + + hr = recorderClient->Start(); + ThrowIfFailed(hr, "Failed to start the recorder."); + + recordThread = new std::thread(&Open3D_AudioInput::loop, this); +} + +void Open3D_AudioInput::release() { + if (recordThread) { + stopThread = true; + recordThread->join(); + delete recordThread; + recordThread = nullptr; + stopThread = false; + } + if (recorderClient) { + recorderClient->Stop(); + + if (recorderService) { + recorderService->Release(); + recorderService = nullptr; + } + recorderClient->Release(); + recorderClient = nullptr; + } + if (recorder) { + recorder->Release(); + recorder = nullptr; + } +} + +void Open3D_AudioInput::loop() { + HRESULT hr; + while (!stopThread) { + hr = recorderService->GetBuffer(&captureBuffer, &nFrames, &flags, nullptr, nullptr); + ThrowIfFailed(hr, "Failed to get recording buffer."); + + if (captureBuffer == nullptr) + continue; + + hr = recorderService->ReleaseBuffer(nFrames); + ThrowIfFailed(hr, "Failed to release recording buffer"); + + publish_audio_captured(); + } +} + +Open3D_AudioInput::~Open3D_AudioInput() { + release(); + //CoUninitialize(); +} + +void Open3D_AudioInput::set_device(FBAudioIn *mic) { + release(); + if (mic == nullptr) { + return; + } + + IMMDeviceEnumerator* enumerator = nullptr; + IMMDeviceCollection* recorderCollection = nullptr; + IMMDevice* device = nullptr; + IPropertyStore* properties = nullptr; + UINT count; + + HRESULT hr = CoCreateInstance( + __uuidof(MMDeviceEnumerator), nullptr, CLSCTX_ALL, + __uuidof(IMMDeviceEnumerator), reinterpret_cast(&enumerator) + ); + ThrowIfFailed(hr, "Failed to create an instance of MMDeviceEnumerator."); + + hr = enumerator->EnumAudioEndpoints(eCapture, DEVICE_STATE_ACTIVE, &recorderCollection); + ThrowIfFailed(hr, "Failed to create an instance of MMDeviceEnumerator."); + + hr = recorderCollection->GetCount(&count); + ThrowIfFailed(hr, "Failed to get MMDevice counts."); + + for (UINT i=0; iItem(i, &device); + ThrowIfFailed(hr, "Failed to get MMDevice."); + + hr = device->OpenPropertyStore(STGM_READ, &properties); + ThrowIfFailed(hr, "Failed to open property store for the MMDevice."); + + PROPVARIANT varName; + PropVariantInit(&varName); + hr = properties->GetValue(mNameKey, &varName); + ThrowIfFailed(hr, "Failed to get property value for the MMDevice."); + + if (varName.vt != VT_EMPTY) { + if (WideCompare(varName.pwszVal, mic->Name.AsString()) == 0) { + recorder = device; + properties->Release(); + break; + } + } + + properties->Release(); + device->Release(); + } + recorderCollection->Release(); + enumerator->Release(); + + activate(); +} diff --git a/plugins/mobu/audio.h b/plugins/mobu/audio.h new file mode 100644 index 00000000..85496e40 --- /dev/null +++ b/plugins/mobu/audio.h @@ -0,0 +1,69 @@ +// +// Created by matin on 2024-10-13. +// + +#ifndef AUDIO_H +#define AUDIO_H + +#include +#include +#include + +struct IMMDevice; +struct IAudioClient; +struct IAudioRenderClient; +namespace std { + class thread; +} + +class IAudioSubscriber { +public: + virtual ~IAudioSubscriber() = default; + virtual void audio_captured(const BYTE* captureBuffer, UINT32 nFrames) = 0; +}; + +class Open3D_AudioInput +{ +public: + Open3D_AudioInput(); + ~Open3D_AudioInput(); + void set_device(FBAudioIn* mic); + void subscribe(IAudioSubscriber* subscriber) { subscribers.push_back(subscriber); } + void unsubscribe(IAudioSubscriber* subscriber) { subscribers.erase(std::remove(subscribers.begin(), subscribers.end(), subscriber), subscribers.end()); } + +private: + // Audio Device Activation Section + // =============================== + + IMMDevice* recorder = nullptr; + IAudioClient* recorderClient = nullptr; + IAudioCaptureClient* recorderService = nullptr; + + WAVEFORMATEX* format = nullptr; + PROPERTYKEY mNameKey = {}; + + void activate(); + void release(); + + // Audio Capture Loop Section + // ========================== + + BYTE* captureBuffer = nullptr; + UINT32 nFrames = 0; + DWORD flags = 0; + + std::thread* recordThread = nullptr; + bool stopThread = false; + + void loop(); + + // Audio Capture Publishing to Subscribers Section + // =============================================== + + std::vector subscribers; + void publish_audio_captured() const; +}; + + + +#endif //AUDIO_H diff --git a/plugins/mobu/device.cpp b/plugins/mobu/device.cpp index 0575df6e..f31b43ab 100644 --- a/plugins/mobu/device.cpp +++ b/plugins/mobu/device.cpp @@ -84,6 +84,8 @@ bool Open3D_Device::FBCreate() mFrameCounter = 0; mIdSeq = 0; + + mAudioRecord.subscribe(this); return true; } @@ -257,6 +259,19 @@ bool Open3D_Device::Start() return true; } +void Open3D_Device::SetSelectedMicrophone(FBAudioIn* mic) { + mAudioRecord.set_device(mic); + + // check if there should be a subject for audio. + if (mic == nullptr) { + Items.removeSubject(audioSubjectName); + } else { + if (Items.findSubject(audioSubjectName) == nullptr) { + Items.addSubject(audioSubjectName); + } + } +} + bool Open3D_Device::IsActive() { if (mNetworkSocket != -1) return true; @@ -334,6 +349,18 @@ uint32_t Open3D_Device::WriteTcp(O3DS::TcpSocket &socket, void *data, uint32_t b } +void Open3D_Device::audio_captured(const BYTE *captureBuffer, UINT32 nFrames) { + const auto audioSubject = Items.findSubject(audioSubjectName); + if (Online == true) { + audioSubject->mAudioBuffer.insert(audioSubject->mAudioBuffer.end(), + captureBuffer, captureBuffer + nFrames); + } + else + { + audioSubject->mAudioBuffer.assign(captureBuffer, captureBuffer + nFrames); + } +} + void Open3D_Device::DeviceIONotify(kDeviceIOs pAction, FBDeviceNotifyInfo &pDeviceNotifyInfo) { std::vector buf; @@ -364,6 +391,10 @@ void Open3D_Device::DeviceIONotify(kDeviceIOs pAction, FBDeviceNotifyInfo &pDev else { bucketSize = Items.SerializeUpdate(buf, count, MobuTime.GetSecondDouble()); + + // Clear the sent pieces of audio buffers. + if (auto audioSubject = Items.findSubject(audioSubjectName); audioSubject != nullptr) + audioSubject->mAudioBuffer.clear(); } if (mFrameCounter > 100) { @@ -383,23 +414,27 @@ void Open3D_Device::DeviceIONotify(kDeviceIOs pAction, FBDeviceNotifyInfo &pDev if (mTcpIp.Accept(newSocket)) mClients.push_back(newSocket); - for (std::vector::iterator client = mClients.begin(); client != mClients.end();) - { - O3DS::TcpSocket s(*client); - if (WriteTcp(s, &buf[0], bucketSize)) { - client++; - } - else { - client = mClients.erase(client); + for (std::vector::iterator client = mClients.begin(); client != mClients.end();) + { + O3DS::TcpSocket s(*client); + if (WriteTcp(s, &buf[0], bucketSize)) { + client++; + } + else { + client = mClients.erase(client); + } } + + std::ostringstream oss; + oss << bucketSize << " bytes"; + Status = oss.str().c_str(); } - } if (mProtocol == Open3D_Device::kTCPClient) { - if(mTcpIp.Valid()) - { - if (WriteTcp(mTcpIp, &buf[0], bucketSize) == 0) + if(mTcpIp.Valid()) { + if (WriteTcp(mTcpIp, &buf[0], bucketSize) == 0) + { Status = "TCP Disconnected"; } } @@ -410,28 +445,28 @@ void Open3D_Device::DeviceIONotify(kDeviceIOs pAction, FBDeviceNotifyInfo &pDev struct in_addr ret; if (!inet_pton_alt(AF_INET, mNetworkAddress.operator char *(), &ret)) { - Status = "UDP ERROR"; - return; - } + Status = "UDP ERROR"; + return; + } - struct sockaddr_in dest_addr; - dest_addr.sin_family = AF_INET; - dest_addr.sin_addr.s_addr = ret.S_un.S_addr; - dest_addr.sin_port = htons(mNetworkPort); + struct sockaddr_in dest_addr; + dest_addr.sin_family = AF_INET; + dest_addr.sin_addr.s_addr = ret.S_un.S_addr; + dest_addr.sin_port = htons(mNetworkPort); - size_t sentSz = 0; + size_t sentSz = 0; - mIdSeq++; + mIdSeq++; - std::vector fragData; - UdpFragmenter frag(buf.data(), bucketSize, 512); - for (int i = 0; i < frag.mFrames; i++) - { - frag.makeFragment(mIdSeq, i, fragData); - sentSz+= sendto(mNetworkSocket, fragData.data(), fragData.size(), 0, - (struct sockaddr*)&dest_addr, - (int)sizeof(struct sockaddr_in)); - } + std::vector fragData; + UdpFragmenter frag(buf.data(), bucketSize, 512); + for (int i = 0; i < frag.mFrames; i++) + { + frag.makeFragment(mIdSeq, i, fragData); + sentSz+= sendto(mNetworkSocket, fragData.data(), fragData.size(), 0, + (struct sockaddr*)&dest_addr, + (int)sizeof(struct sockaddr_in)); + } std::ostringstream oss; oss << frag.mFrames << "/" << sentSz << " bytes"; diff --git a/plugins/mobu/device.h b/plugins/mobu/device.h index 665f74d3..908962f2 100644 --- a/plugins/mobu/device.h +++ b/plugins/mobu/device.h @@ -3,6 +3,8 @@ #include #include + +#include "audio.h" #include "mobuModel.h" #include "o3ds/base_connector.h" #include "o3ds/tcp.h" @@ -10,8 +12,10 @@ #define OPEN3D_DEVICE__CLASSNAME Open3D_Device #define OPEN3D_DEVICE__CLASSSTR "Open3D_Device" +constexpr auto audioSubjectName = "Audio"; + -class Open3D_Device : public FBDevice +class Open3D_Device : public FBDevice, IAudioSubscriber { FBDeviceDeclare( Open3D_Device, FBDevice ); public: @@ -67,6 +71,8 @@ class Open3D_Device : public FBDevice FBDeviceSamplingMode GetSamplingType() { return mSamplingType; } void SetSamplingType( FBDeviceSamplingMode pType ) { mSamplingType = pType; } + void SetSelectedMicrophone(FBAudioIn* mic); + bool IsActive(); void InData(void *, size_t); @@ -79,7 +85,11 @@ class Open3D_Device : public FBDevice SOCKET mNetworkSocket; std::vector mClients; + Open3D_AudioInput mAudioRecord; + private: + void audio_captured(const BYTE *captureBuffer, UINT32 nFrames) override; + double mSamplingRate; FBDeviceSamplingMode mSamplingType; FBPlayerControl mPlayerControl; @@ -90,8 +100,7 @@ class Open3D_Device : public FBDevice int mNetworkPort; bool mStreaming; int mFrameCounter; - uint32_t mIdSeq; - + uint32_t mIdSeq; }; #endif diff --git a/plugins/mobu/layout.cpp b/plugins/mobu/layout.cpp index d512b9d9..9641e52f 100644 --- a/plugins/mobu/layout.cpp +++ b/plugins/mobu/layout.cpp @@ -3,6 +3,7 @@ #include "o3ds/o3ds.h" // for version# #include +#include #define OPEN3D_DEVICE__LAYOUT Open3D_Device_Layout @@ -19,6 +20,7 @@ bool Open3D_Device_Layout::FBCreate() // Add device & system callbacks mDevice->OnStatusChange.Add ( this,(FBCallback)&Open3D_Device_Layout::EventDeviceStatusChange ); OnIdle.Add ( this,(FBCallback)&Open3D_Device_Layout::EventUIIdle ); + mDevice->mAudioRecord.subscribe(this); return true; } @@ -28,6 +30,7 @@ void Open3D_Device_Layout::FBDestroy() // Remove device & system callbacks OnIdle.Remove ( this,(FBCallback)&Open3D_Device_Layout::EventUIIdle ); mDevice->OnStatusChange.Remove ( this,(FBCallback)&Open3D_Device_Layout::EventDeviceStatusChange ); + mDevice->mAudioRecord.unsubscribe(this); } void Open3D_Device_Layout::UICreate() @@ -195,6 +198,42 @@ void Open3D_Device_Layout::UICreate() mLayoutLeft.SetControl("JointsList", mMemoJoints); mMemoJoints.OnChange.Add(this, (FBCallback)&Open3D_Device_Layout::EventEditJoints); + // Microphone Source label - under JointsList + mLayoutLeft.AddRegion("LabelMicSource", "LabelMicSource", + lS, kFBAttachRight, "SourcesList", 1.00, + 2, kFBAttachBottom, "JointsList", 1.00, + 155, kFBAttachNone, "", 1.00, + lH, kFBAttachNone, NULL, 1.00); + mLayoutLeft.SetControl("LabelMicSource", mLabelMicSource); + mLabelMicSource.Caption = "Microphone Source:"; + + // Microphones list - Under Microphone Source Label + mLayoutLeft.AddRegion("ListMicSource", "ListMicSource", + lS, kFBAttachRight, "SourcesList", 1.00, + 2, kFBAttachBottom, "LabelMicSource", 1.00, + 155, kFBAttachNone, "", 1.00, + lH, kFBAttachNone, NULL, 1.00); + mLayoutLeft.SetControl("ListMicSource", mListMicSource); + mListMicSource.OnChange.Add(this, (FBCallback)&Open3D_Device_Layout::EventSelectMicSource); + mListMicSource.Style = kFBDropDownList; + + // Microphone Capture Indicator - Under Microphones List + mLayoutLeft.AddRegion("MicCaptureIndicator", "MicCaptureIndicator", + lS, kFBAttachRight, "SourcesList", 1.00, + 5, kFBAttachBottom, "ListMicSource", 1.00, + 155, kFBAttachNone, "", 1.00, + lH, kFBAttachNone, NULL, 1.00); + mLayoutLeft.SetControl("MicCaptureIndicator", mMicCaptureIndicator); + mMicCaptureIndicator.Height = 10; + mMicCaptureIndicator.Max = 0; + mMicCaptureIndicator.Max = 1; + mMicCaptureIndicator.Value = 0; + mMicCaptureIndicator.LargeStep = 0.01; + mMicCaptureIndicator.SmallStep = 0.01; + mMicCaptureIndicator.Visible = false; + mMicCaptureIndicator.Orientation = kFBHorizontal; + mMicCaptureIndicator.ReadOnly = true; + // Version info - at bottom mLayoutLeft.AddRegion("VersionInfo", "VersionInfo", lS, kFBAttachRight, "SourcesList", 1.00, @@ -332,6 +371,7 @@ void Open3D_Device_Layout::UIConfigure() PopulateSubjectList(); PopulateSubjectFields(); + PopulateMicrophoneList(); mMemoLog.Enabled = false; @@ -380,7 +420,13 @@ void Open3D_Device_Layout::EventEditSubject(HISender pSender, HKEvent pEvent) int id = mSourcesList.ItemIndex; if (id < 0) return; - FBString newValue = mEditSubject.Text.AsString(); + const FBString newValue = mEditSubject.Text.AsString(); + + // Don't let changing the name of the Audio subject. + if (mDevice->Items[id]->mName == audioSubjectName && newValue != audioSubjectName) { + mEditSubject.Text.SetString(audioSubjectName); + return; + } mDevice->Items[id]->mName = newValue.operator char *(); mSourcesList.Items.RemoveAt(id); @@ -395,6 +441,12 @@ void Open3D_Device_Layout::EventEditDelta(HISender pSender, HKEvent pEvent) mDevice->SetDeltaThreshold(value); } +void Open3D_Device_Layout::audio_captured(const BYTE* captureBuffer, const UINT32 nFrames) +{ + const auto captureBufferFloat = reinterpret_cast(captureBuffer); + mMicCaptureIndicator.Value = fabsf(captureBufferFloat[0]); +} + void Open3D_Device_Layout::EventDeviceStatusChange( HISender pSender, HKEvent pEvent ) { UIReset(); @@ -441,32 +493,36 @@ void Open3D_Device_Layout::PopulateSubjectFields() { if (id < mDevice->Items.size()) { + std::ostringstream oss; O3DS::Subject* subject = mDevice->Items[id]; - FBModel *model = static_cast(subject->mReference); + const auto model = static_cast(subject->mReference); + + mEditSubject.Text = subject->mName.c_str(); - mEditSubject.Text = mDevice->Items[id]->mName.c_str(); - mEditSource.Text = model->GetFullName(); + if (model != nullptr) { + mEditSource.Text = model->GetFullName(); - std::ostringstream imploded, oss; - std::copy(mDevice->Items[id]->mJoints.begin(), mDevice->Items[id]->mJoints.end(), - std::ostream_iterator(imploded, " ")); + std::ostringstream imploded; + std::copy(subject->mJoints.begin(), subject->mJoints.end(), + std::ostream_iterator(imploded, " ")); - mMemoJoints.Text = imploded.str().c_str(); + mMemoJoints.Text = imploded.str().c_str(); - O3DS::Mobu::TraverseSubject(subject, model); + O3DS::Mobu::TraverseSubject(subject, model); - if (model->Is(FBModelNull::TypeInfo)) - oss << "Null" << std::endl; - if (model->Is(FBModelRoot::TypeInfo)) - oss << "Root" << std::endl; - if (model->Is(FBCamera::TypeInfo)) - oss << "Camera" << std::endl; - if (model->Is(FBModelSkeleton::TypeInfo)) - oss << "Joint" << std::endl; + if (model->Is(FBModelNull::TypeInfo)) + oss << "Null" << std::endl; + if (model->Is(FBModelRoot::TypeInfo)) + oss << "Root" << std::endl; + if (model->Is(FBCamera::TypeInfo)) + oss << "Camera" << std::endl; + if (model->Is(FBModelSkeleton::TypeInfo)) + oss << "Joint" << std::endl; + } - oss << "Items: " << mDevice->Items[id]->mTransforms.size() << std::endl; + oss << "Items: " << subject->mTransforms.size() << std::endl; - oss << "Joints: " << mDevice->Items[id]->mJoints.size() << std::endl; + oss << "Joints: " << subject->mJoints.size() << std::endl; std::vector buf; mDevice->Items.Serialize(buf, count); @@ -480,6 +536,17 @@ void Open3D_Device_Layout::PopulateSubjectFields() } } +void Open3D_Device_Layout::PopulateMicrophoneList() { + mListMicSource.Items.Clear(); + mListMicSource.Items.Add("None"); + + auto& micList = FBSystem::TheOne().AudioInputs; + for (auto i = 0; i < micList.GetCount(); ++i) { + const auto mic = micList[i]; + mListMicSource.Items.Add(mic->Name); + } +} + void Open3D_Device_Layout::EventSelectDevice(HISender pSender, HKEvent pEvent) { PopulateSubjectFields(); @@ -518,12 +585,15 @@ void Open3D_Device_Layout::EventAdd(HISender pSender, HKEvent pEvent) void Open3D_Device_Layout::EventDel(HISender pSender, HKEvent pEvent) { - int id = mSourcesList.ItemIndex; - if (id >= 0) - { - mDevice->Items.mItems.erase(mDevice->Items.begin() + id); - PopulateSubjectList(); - } + const int id = mSourcesList.ItemIndex; + if (id < 0) return; + + // Don't let changing the name of the Audio subject. + if (mDevice->Items[id]->mName == audioSubjectName) + return; + + mDevice->Items.mItems.erase(mDevice->Items.begin() + id); + PopulateSubjectList(); } void Open3D_Device_Layout::EventEditIP(HISender pSender, HKEvent pEvent) @@ -578,4 +648,18 @@ void Open3D_Device_Layout::EventEditJoints(HISender pSender, HKEvent pEvent) std::vector vstrings(begin, end); mDevice->Items[id]->mJoints.clear(); std::copy(vstrings.begin(), vstrings.end(), std::back_inserter(mDevice->Items[id]->mJoints)); -} \ No newline at end of file +} + +void Open3D_Device_Layout::EventSelectMicSource(HISender pSender, HKEvent pEvent) { + if (mListMicSource.ItemIndex == 0) { + mDevice->SetSelectedMicrophone(nullptr); + mMicCaptureIndicator.Value = 0; + mMicCaptureIndicator.Visible = false; + } + else { + auto& micList = FBSystem::TheOne().AudioInputs; + mDevice->SetSelectedMicrophone(micList[mListMicSource.ItemIndex - 1]); + mMicCaptureIndicator.Visible = true; + } + PopulateSubjectList(); +} diff --git a/plugins/mobu/layout.h b/plugins/mobu/layout.h index a88d3e88..fad7ed08 100644 --- a/plugins/mobu/layout.h +++ b/plugins/mobu/layout.h @@ -3,7 +3,7 @@ #include "device.h" -class Open3D_Device_Layout : public FBDeviceLayout +class Open3D_Device_Layout : public FBDeviceLayout, IAudioSubscriber { FBDeviceLayoutDeclare( Open3D_Device_Layout, FBDeviceLayout ); public: @@ -17,6 +17,7 @@ class Open3D_Device_Layout : public FBDeviceLayout void PopulateSubjectList(); void PopulateSubjectFields(); + void PopulateMicrophoneList(); // Events void EventDeviceStatusChange( HISender pSender, HKEvent pEvent ); @@ -26,12 +27,14 @@ class Open3D_Device_Layout : public FBDeviceLayout void EventSelectDevice(HISender pSender, HKEvent pEvent); void EventEditSubject(HISender pSender, HKEvent pEvent); void EventEditJoints(HISender pSender, HKEvent pEvent); + void EventSelectMicSource(HISender pSender, HKEvent pEvent); void EventEditIP(HISender pSender, HKEvent pEvent); void EventEditPort(HISender pSender, HKEvent pEvent); void EventEditProtocol(HISender pSender, HKEvent pEvent); void EventEditSampleRate(HISender pSender, HKEvent pEvent); void EventEditKey(HISender pSender, HKEvent pEvent); void EventEditDelta(HISender pSender, HKEvent pEvent); + void audio_captured(const BYTE* captureBuffer, UINT32 nFrames) override; private: FBLayout mLayoutLeft; @@ -57,6 +60,9 @@ class Open3D_Device_Layout : public FBDeviceLayout FBEdit mEditKey; FBLabel mLabelSamplingRate; FBEditNumber mEditSamplingRate; + FBLabel mLabelMicSource; + FBList mListMicSource; + FBSlider mMicCaptureIndicator; FBLabel mLabelPluginVersion; FBLabel mLabelJoints; FBEdit mMemoJoints; diff --git a/src/o3ds.fbs b/src/o3ds.fbs index 2c15d0f4..ce9a8ed3 100644 --- a/src/o3ds.fbs +++ b/src/o3ds.fbs @@ -86,6 +86,7 @@ table Subject { table SubjectUpdate { name:string; + audio:[byte]; translations:[TranslationUpdate]; rotation:[RotationUpdate]; scale:[ScaleUpdate]; diff --git a/src/o3ds/model.cpp b/src/o3ds/model.cpp index 8a9d1035..9bcb5a85 100644 --- a/src/o3ds/model.cpp +++ b/src/o3ds/model.cpp @@ -421,10 +421,11 @@ namespace O3DS transformId++; } + auto audio = builder.CreateVector(mAudioBuffer); auto tr = builder.CreateVectorOfStructs(translations); auto ro = builder.CreateVectorOfStructs(rotations); auto sc = builder.CreateVectorOfStructs(scales); - return CreateSubjectUpdate(builder, oSubjectName, tr, ro, sc); + return CreateSubjectUpdate(builder, oSubjectName, audio, tr, ro, sc); } int Subject::Serialize(std::vector &outbuf, double timestamp) diff --git a/src/o3ds/model.h b/src/o3ds/model.h index 23d33381..eded2cd3 100644 --- a/src/o3ds/model.h +++ b/src/o3ds/model.h @@ -150,6 +150,7 @@ namespace O3DS std::string mName; std::vector mJoints; + std::vector mAudioBuffer; TransformList mTransforms; void* mReference; @@ -238,6 +239,18 @@ namespace O3DS return nullptr; } + void removeSubject(const std::string &name) + { + const auto item = findSubject(name); + if (item == nullptr) { + return; + } + // move item to the end + const auto iter = std::remove(mItems.begin(), mItems.end(), item); + // remove item from the end + mItems.erase(iter, mItems.end()); + } + void update() { for (auto i : mItems)