From 2415edd8e5071540ad5848ba6122f1bdb9b4f615 Mon Sep 17 00:00:00 2001 From: Sean DuBois Date: Fri, 30 Jun 2023 15:47:33 -0400 Subject: [PATCH 1/3] Convert plugins/obs-webrtc to C++ --- .github/workflows/main.yml | 2 +- buildspec.json | 22 +-- plugins/obs-webrtc/whip-output.cpp | 237 +++++++++++++++-------------- plugins/obs-webrtc/whip-output.h | 17 ++- 4 files changed, 140 insertions(+), 138 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2b7a31dd379479..eb5f89342270fe 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,7 +19,7 @@ env: CEF_BUILD_VERSION_WIN: '5060' QT_VERSION_MAC: '6.4.3' QT_VERSION_WIN: '6.4.3' - DEPS_VERSION_WIN: '2023-06-22' + DEPS_VERSION_WIN: '2023-06-28' VLC_VERSION_WIN: '3.0.0-git' TWITCH_CLIENTID: ${{ secrets.TWITCH_CLIENT_ID }} TWITCH_HASH: ${{ secrets.TWITCH_HASH }} diff --git a/buildspec.json b/buildspec.json index 835ad700f29482..547c29652bf1e6 100644 --- a/buildspec.json +++ b/buildspec.json @@ -1,26 +1,26 @@ { "dependencies": { "prebuilt": { - "version": "2023-06-22", - "baseUrl": "https://github.com/obsproject/obs-deps/releases/download", + "version": "2023-06-28", + "baseUrl": "https://github.com/Sean-Der/obs-deps/releases/download", "label": "Pre-Built obs-deps", "hashes": { - "macos-universal": "a0d2e03f0ea79681634c31627430a220d9b62113d6ff58174d0bdab6fafdd32b", - "windows-x64": "1b12e86e2d62a97a889866d66b95fe47ddc6f7fa9b13e88aedfab4ea9e298ea2", - "windows-x86": "8c6bfda01ec7d38a50a61ac4954dc00d6fcb171830a7f32593c5bd20a44be3a9", - "linux-x86_64": "1b82ab4247bc2730ae84caa5a009e76637eac77b05bfb4b4a2bae610e7a75262" + "macos-universal": "3118c8f99cd6a862ed1316b0098f03c5293957c28b1137bf4260723d0eb7acc5", + "windows-x64": "58d73e33da1eaf2bf9855f630ff123c595cd759dc815bc7b29c50500a0fa26b0", + "windows-x86": "bf22ccd72a5d1537ab056d5a986daca0e4a6d4444a5e32a18314220bdfcdae65", + "linux-x86_64": "a1a4a42ee35010e492cf4431bfeb9c1df7b12692456bf12516158efa59dbae73" } }, "qt6": { - "version": "2023-06-22", - "baseUrl": "https://github.com/obsproject/obs-deps/releases/download", + "version": "2023-06-28", + "baseUrl": "https://github.com/Sean-Der/obs-deps/releases/download", "label": "Pre-Built Qt6", "hashes": { - "macos-universal": "f890d258a1afa7ba409b79c8ee55d53155e5c72105b8b18a3f52047ee70fc0aa", - "windows-x64": "1907fbcbcef69527154b29316c425b0885afb77ad69a9a2af7a1471d79512195" + "macos-universal": "11f4da10dc98effce8012be31dd790ef508ba2780906946ed52eb835ed5530dd", + "windows-x64": "96e5c91c1febf092bbb938c825425c13b513da6e1f006eff7e278402e51254d8" }, "debugSymbols": { - "windows-x64": "b461a7ade0c099505baea857fa5b98c4f8e9b702681be019ea354735d062e065" + "windows-x64": "af1b4958ad8e005f9ae507d35f61dd72fc7b2278fc6958afb4118fbd4769a9e4" } }, "cef": { diff --git a/plugins/obs-webrtc/whip-output.cpp b/plugins/obs-webrtc/whip-output.cpp index 3879e65b71e392..1b4c6add177c1e 100644 --- a/plugins/obs-webrtc/whip-output.cpp +++ b/plugins/obs-webrtc/whip-output.cpp @@ -7,12 +7,10 @@ const char signaling_media_id_valid_char[] = "0123456789" const uint32_t audio_ssrc = 5002; const char *audio_mid = "0"; -const uint32_t audio_clockrate = 48000; const uint8_t audio_payload_type = 111; const uint32_t video_ssrc = 5000; const char *video_mid = "1"; -const uint32_t video_clockrate = 90000; const uint8_t video_payload_type = 96; WHIPOutput::WHIPOutput(obs_data_t *, obs_output_t *output) @@ -23,9 +21,9 @@ WHIPOutput::WHIPOutput(obs_data_t *, obs_output_t *output) running(false), start_stop_mutex(), start_stop_thread(), - peer_connection(-1), - audio_track(-1), - video_track(-1), + peer_connection(nullptr), + audio_track(nullptr), + video_track(nullptr), total_bytes_sent(0), connect_time_ms(0), start_time_ns(0), @@ -78,11 +76,14 @@ void WHIPOutput::Data(struct encoder_packet *packet) if (packet->type == OBS_ENCODER_AUDIO) { int64_t duration = packet->dts_usec - last_audio_timestamp; - Send(packet->data, packet->size, duration, audio_track); + Send(packet->data, packet->size, duration, audio_track, + audio_sr_reporter); last_audio_timestamp = packet->dts_usec; } else if (packet->type == OBS_ENCODER_VIDEO) { + auto height = obs_encoder_get_width(packet->encoder); int64_t duration = packet->dts_usec - last_video_timestamp; - Send(packet->data, packet->size, duration, video_track); + Send(packet->data, packet->size, duration, video_track, + video_sr_reporter); last_video_timestamp = packet->dts_usec; } } @@ -92,30 +93,25 @@ void WHIPOutput::ConfigureAudioTrack(std::string media_stream_id, { auto media_stream_track_id = std::string(media_stream_id + "-audio"); - rtcTrackInit track_init = { - RTC_DIRECTION_SENDONLY, - RTC_CODEC_OPUS, - audio_payload_type, - audio_ssrc, - audio_mid, - cname.c_str(), - media_stream_id.c_str(), - media_stream_track_id.c_str(), - }; - - rtcPacketizationHandlerInit packetizer_init = {audio_ssrc, - cname.c_str(), - audio_payload_type, - audio_clockrate, - 0, - 0, - RTC_NAL_SEPARATOR_LENGTH, - 0}; - - audio_track = rtcAddTrackEx(peer_connection, &track_init); - rtcSetOpusPacketizationHandler(audio_track, &packetizer_init); - rtcChainRtcpSrReporter(audio_track); - rtcChainRtcpNackResponder(audio_track, 1000); + rtc::Description::Audio audio_description( + audio_mid, rtc::Description::Direction::SendOnly); + audio_description.addOpusCodec(audio_payload_type); + audio_description.addSSRC(audio_ssrc, cname, media_stream_id, + media_stream_track_id); + audio_track = peer_connection->addTrack(audio_description); + + auto rtp_config = std::make_shared( + audio_ssrc, cname, audio_payload_type, + rtc::OpusRtpPacketizer::defaultClockRate); + auto packetizer = std::make_shared(rtp_config); + audio_sr_reporter = std::make_shared(rtp_config); + auto nack_responder = std::make_shared(); + + auto opus_handler = + std::make_shared(packetizer); + opus_handler->addToChain(audio_sr_reporter); + opus_handler->addToChain(nack_responder); + audio_track->setMediaHandler(opus_handler); } void WHIPOutput::ConfigureVideoTrack(std::string media_stream_id, @@ -123,31 +119,26 @@ void WHIPOutput::ConfigureVideoTrack(std::string media_stream_id, { auto media_stream_track_id = std::string(media_stream_id + "-video"); - rtcTrackInit track_init = { - RTC_DIRECTION_SENDONLY, - RTC_CODEC_H264, - video_payload_type, - video_ssrc, - video_mid, - cname.c_str(), - media_stream_id.c_str(), - media_stream_track_id.c_str(), - }; - - rtcPacketizationHandlerInit packetizer_init = { - video_ssrc, - cname.c_str(), - video_payload_type, - video_clockrate, - 0, - 0, - RTC_NAL_SEPARATOR_START_SEQUENCE, - 0}; - - video_track = rtcAddTrackEx(peer_connection, &track_init); - rtcSetH264PacketizationHandler(video_track, &packetizer_init); - rtcChainRtcpSrReporter(video_track); - rtcChainRtcpNackResponder(video_track, 1000); + rtc::Description::Video video_description( + video_mid, rtc::Description::Direction::SendOnly); + video_description.addH264Codec(video_payload_type); + video_description.addSSRC(video_ssrc, cname, media_stream_id, + media_stream_track_id); + video_track = peer_connection->addTrack(video_description); + + auto rtp_config = std::make_shared( + video_ssrc, cname, video_payload_type, + rtc::H264RtpPacketizer::defaultClockRate); + auto packetizer = std::make_shared( + rtc::H264RtpPacketizer::Separator::StartSequence, rtp_config); + video_sr_reporter = std::make_shared(rtp_config); + auto nack_responder = std::make_shared(); + + auto h264_handler = + std::make_shared(packetizer); + h264_handler->addToChain(video_sr_reporter); + h264_handler->addToChain(nack_responder); + video_track->setMediaHandler(h264_handler); } bool WHIPOutput::Setup() @@ -167,51 +158,40 @@ bool WHIPOutput::Setup() bearer_token = obs_service_get_connect_info( service, OBS_SERVICE_CONNECT_INFO_BEARER_TOKEN); - rtcConfiguration config; - memset(&config, 0, sizeof(config)); - - peer_connection = rtcCreatePeerConnection(&config); - rtcSetUserPointer(peer_connection, this); + peer_connection = std::make_shared(); - rtcSetStateChangeCallback(peer_connection, [](int, rtcState state, - void *ptr) { - auto whipOutput = static_cast(ptr); + peer_connection->onStateChange([this](rtc::PeerConnection::State state) { switch (state) { - case RTC_NEW: - do_log_s(LOG_INFO, "PeerConnection state is now: New"); + case rtc::PeerConnection::State::New: + do_log(LOG_INFO, "PeerConnection state is now: New"); break; - case RTC_CONNECTING: - do_log_s(LOG_INFO, - "PeerConnection state is now: Connecting"); - whipOutput->start_time_ns = os_gettime_ns(); + case rtc::PeerConnection::State::Connecting: + do_log(LOG_INFO, + "PeerConnection state is now: Connecting"); + start_time_ns = os_gettime_ns(); break; - case RTC_CONNECTED: - do_log_s(LOG_INFO, - "PeerConnection state is now: Connected"); - whipOutput->connect_time_ms = - (int)((os_gettime_ns() - - whipOutput->start_time_ns) / + case rtc::PeerConnection::State::Connected: + do_log(LOG_INFO, + "PeerConnection state is now: Connected"); + connect_time_ms = + (int)((os_gettime_ns() - start_time_ns) / 1000000.0); - do_log_s(LOG_INFO, "Connect time: %dms", - whipOutput->connect_time_ms.load()); + do_log(LOG_INFO, "Connect time: %dms", + connect_time_ms.load()); break; - case RTC_DISCONNECTED: - do_log_s(LOG_INFO, - "PeerConnection state is now: Disconnected"); - whipOutput->Stop(false); - obs_output_signal_stop(whipOutput->output, - OBS_OUTPUT_DISCONNECTED); + case rtc::PeerConnection::State::Disconnected: + do_log(LOG_INFO, + "PeerConnection state is now: Disconnected"); + Stop(false); + obs_output_signal_stop(output, OBS_OUTPUT_DISCONNECTED); break; - case RTC_FAILED: - do_log_s(LOG_INFO, - "PeerConnection state is now: Failed"); - whipOutput->Stop(false); - obs_output_signal_stop(whipOutput->output, - OBS_OUTPUT_ERROR); + case rtc::PeerConnection::State::Failed: + do_log(LOG_INFO, "PeerConnection state is now: Failed"); + Stop(false); + obs_output_signal_stop(output, OBS_OUTPUT_ERROR); break; - case RTC_CLOSED: - do_log_s(LOG_INFO, - "PeerConnection state is now: Closed"); + case rtc::PeerConnection::State::Closed: + do_log(LOG_INFO, "PeerConnection state is now: Closed"); break; } }); @@ -231,7 +211,7 @@ bool WHIPOutput::Setup() ConfigureAudioTrack(media_stream_id, cname); ConfigureVideoTrack(media_stream_id, cname); - rtcSetLocalDescription(peer_connection, "offer"); + peer_connection->setLocalDescription(); return true; } @@ -249,8 +229,8 @@ bool WHIPOutput::Connect() std::string read_buffer; std::string location_header; - char offer_sdp[4096] = {0}; - rtcGetLocalDescription(peer_connection, offer_sdp, sizeof(offer_sdp)); + auto offer_sdp = + std::string(peer_connection->localDescription().value()); CURL *c = curl_easy_init(); curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, curl_writefunction); @@ -260,7 +240,7 @@ bool WHIPOutput::Connect() curl_easy_setopt(c, CURLOPT_HTTPHEADER, headers); curl_easy_setopt(c, CURLOPT_URL, endpoint_url.c_str()); curl_easy_setopt(c, CURLOPT_POST, 1L); - curl_easy_setopt(c, CURLOPT_COPYPOSTFIELDS, offer_sdp); + curl_easy_setopt(c, CURLOPT_COPYPOSTFIELDS, offer_sdp.c_str()); curl_easy_setopt(c, CURLOPT_TIMEOUT, 8L); auto cleanup = [&]() { @@ -318,7 +298,11 @@ bool WHIPOutput::Connect() curl_url_cleanup(h); } - rtcSetRemoteDescription(peer_connection, read_buffer.c_str(), "answer"); + auto response = std::string(read_buffer); + response.erase(0, response.find("v=0")); + + rtc::Description answer(response, "answer"); + peer_connection->setRemoteDescription(answer); cleanup(); return true; } @@ -329,10 +313,10 @@ void WHIPOutput::StartThread() return; if (!Connect()) { - rtcDeletePeerConnection(peer_connection); - peer_connection = -1; - audio_track = -1; - video_track = -1; + peer_connection->close(); + peer_connection = nullptr; + audio_track = nullptr; + video_track = nullptr; return; } @@ -394,11 +378,11 @@ void WHIPOutput::SendDelete() void WHIPOutput::StopThread(bool signal) { - if (peer_connection != -1) { - rtcDeletePeerConnection(peer_connection); - peer_connection = -1; - audio_track = -1; - video_track = -1; + if (peer_connection != nullptr) { + peer_connection->close(); + peer_connection = nullptr; + audio_track = nullptr; + video_track = nullptr; } SendDelete(); @@ -421,26 +405,43 @@ void WHIPOutput::StopThread(bool signal) last_video_timestamp = 0; } -void WHIPOutput::Send(void *data, uintptr_t size, uint64_t duration, int track) +void WHIPOutput::Send(void *data, uintptr_t size, uint64_t duration, + std::shared_ptr track, + std::shared_ptr rtcp_sr_reporter) { - if (!running) + if (!track->isOpen()) return; + std::vector sample{(rtc::byte *)data, + (rtc::byte *)data + size}; + + auto rtp_config = rtcp_sr_reporter->rtpConfig; + // sample time is in us, we need to convert it to seconds auto elapsed_seconds = double(duration) / (1000.0 * 1000.0); // get elapsed time in clock rate - uint32_t elapsed_timestamp = 0; - rtcTransformSecondsToTimestamp(track, elapsed_seconds, - &elapsed_timestamp); + uint32_t elapsed_timestamp = + rtp_config->secondsToTimestamp(elapsed_seconds); // set new timestamp - uint32_t current_timestamp = 0; - rtcGetCurrentTrackTimestamp(track, ¤t_timestamp); - rtcSetTrackRtpTimestamp(track, current_timestamp + elapsed_timestamp); - - total_bytes_sent += size; - rtcSendMessage(track, reinterpret_cast(data), (int)size); + rtp_config->timestamp = rtp_config->timestamp + elapsed_timestamp; + + // get elapsed time in clock rate from last RTCP sender report + auto report_elapsed_timestamp = + rtp_config->timestamp - + rtcp_sr_reporter->lastReportedTimestamp(); + + // check if last report was at least 1 second ago + if (rtp_config->timestampToSeconds(report_elapsed_timestamp) > 1) + rtcp_sr_reporter->setNeedsToReport(); + + try { + track->send(sample); + total_bytes_sent += sample.size(); + } catch (const std::exception &e) { + do_log(LOG_ERROR, "error: %s ", e.what()); + } } void register_whip_output() diff --git a/plugins/obs-webrtc/whip-output.h b/plugins/obs-webrtc/whip-output.h index b8b10b5323e1ab..568b1f4ee392bd 100644 --- a/plugins/obs-webrtc/whip-output.h +++ b/plugins/obs-webrtc/whip-output.h @@ -10,14 +10,11 @@ #include #include -#include +#include #define do_log(level, format, ...) \ blog(level, "[obs-webrtc] [whip_output: '%s'] " format, \ obs_output_get_name(output), ##__VA_ARGS__) -#define do_log_s(level, format, ...) \ - blog(level, "[obs-webrtc] [whip_output: '%s'] " format, \ - obs_output_get_name(whipOutput->output), ##__VA_ARGS__) class WHIPOutput { public: @@ -44,7 +41,9 @@ class WHIPOutput { void SendDelete(); void StopThread(bool signal); - void Send(void *data, uintptr_t size, uint64_t duration, int track); + void Send(void *data, uintptr_t size, uint64_t duration, + std::shared_ptr track, + std::shared_ptr rtcp_sr_reporter); obs_output_t *output; @@ -57,9 +56,11 @@ class WHIPOutput { std::mutex start_stop_mutex; std::thread start_stop_thread; - int peer_connection; - int audio_track; - int video_track; + std::shared_ptr peer_connection; + std::shared_ptr audio_track; + std::shared_ptr video_track; + std::shared_ptr audio_sr_reporter; + std::shared_ptr video_sr_reporter; std::atomic total_bytes_sent; std::atomic connect_time_ms; From 74e0e65f2b16e12400384f8a2f2b58d18801a201 Mon Sep 17 00:00:00 2001 From: Sean DuBois Date: Fri, 30 Jun 2023 15:53:42 -0400 Subject: [PATCH 2/3] foo --- plugins/obs-webrtc/whip-output.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/obs-webrtc/whip-output.cpp b/plugins/obs-webrtc/whip-output.cpp index 1b4c6add177c1e..e19129006a74a0 100644 --- a/plugins/obs-webrtc/whip-output.cpp +++ b/plugins/obs-webrtc/whip-output.cpp @@ -80,7 +80,6 @@ void WHIPOutput::Data(struct encoder_packet *packet) audio_sr_reporter); last_audio_timestamp = packet->dts_usec; } else if (packet->type == OBS_ENCODER_VIDEO) { - auto height = obs_encoder_get_width(packet->encoder); int64_t duration = packet->dts_usec - last_video_timestamp; Send(packet->data, packet->size, duration, video_track, video_sr_reporter); From 3a19b254fd50c6c8eac89a70b4c53610675a8521 Mon Sep 17 00:00:00 2001 From: Sean DuBois Date: Fri, 30 Jun 2023 15:48:10 -0400 Subject: [PATCH 3/3] Add WebRTC Simulcast Support --- UI/window-basic-main-outputs.cpp | 56 ++++++++++++++----- plugins/obs-webrtc/whip-output.cpp | 89 +++++++++++++++++++++++++++--- plugins/obs-webrtc/whip-output.h | 13 ++++- 3 files changed, 133 insertions(+), 25 deletions(-) diff --git a/UI/window-basic-main-outputs.cpp b/UI/window-basic-main-outputs.cpp index 22256c9ca6c389..07b8eaafa1ecc5 100644 --- a/UI/window-basic-main-outputs.cpp +++ b/UI/window-basic-main-outputs.cpp @@ -1456,7 +1456,7 @@ struct AdvancedOutput : BasicOutputHandler { OBSEncoder streamAudioEnc; OBSEncoder streamArchiveEnc; OBSEncoder recordTrack[MAX_AUDIO_MIXES]; - OBSEncoder videoStreaming; + OBSEncoder videoStreaming[3]; OBSEncoder videoRecording; bool ffmpegOutput; @@ -1619,13 +1619,25 @@ AdvancedOutput::AdvancedOutput(OBSBasic *main_) : BasicOutputHandler(main_) } } - videoStreaming = obs_video_encoder_create(streamEncoder, - "advanced_video_stream", - streamEncSettings, nullptr); - if (!videoStreaming) + videoStreaming[0] = obs_video_encoder_create(streamEncoder, + "advanced_video_stream_1", + streamEncSettings, + nullptr); + + videoStreaming[1] = obs_video_encoder_create(streamEncoder, + "advanced_video_stream_2", + streamEncSettings, + nullptr); + + videoStreaming[2] = obs_video_encoder_create(streamEncoder, + "advanced_video_stream_3", + streamEncSettings, + nullptr); + + if (!videoStreaming[0]) throw "Failed to create streaming video encoder " "(advanced output)"; - obs_encoder_release(videoStreaming); + obs_encoder_release(videoStreaming[0]); const char *rate_control = obs_data_get_string( useStreamEncoder ? streamEncSettings : recordEncSettings, @@ -1697,7 +1709,7 @@ void AdvancedOutput::UpdateStreamSettings() config_get_string(main->Config(), "AdvOut", "Encoder"); OBSData settings = GetDataFromJsonFile("streamEncoder.json"); - ApplyEncoderDefaults(settings, videoStreaming); + ApplyEncoderDefaults(settings, videoStreaming[0]); if (applyServiceSettings) { int bitrate = (int)obs_data_get_int(settings, "bitrate"); @@ -1726,11 +1738,11 @@ void AdvancedOutput::UpdateStreamSettings() case VIDEO_FORMAT_P010: break; default: - obs_encoder_set_preferred_video_format(videoStreaming, + obs_encoder_set_preferred_video_format(videoStreaming[0], VIDEO_FORMAT_NV12); } - obs_encoder_update(videoStreaming, settings); + obs_encoder_update(videoStreaming[0], settings); } inline void AdvancedOutput::UpdateRecordingSettings() @@ -1775,14 +1787,14 @@ inline void AdvancedOutput::SetupStreaming() } obs_output_set_audio_encoder(streamOutput, streamAudioEnc, 0); - obs_encoder_set_scaled_size(videoStreaming, cx, cy); + obs_encoder_set_scaled_size(videoStreaming[0], cx, cy); const char *id = obs_service_get_id(main->GetService()); if (strcmp(id, "rtmp_custom") == 0) { OBSDataAutoRelease settings = obs_data_create(); obs_service_apply_encoder_settings(main->GetService(), settings, nullptr); - obs_encoder_update(videoStreaming, settings); + obs_encoder_update(videoStreaming[0], settings); } } @@ -1821,10 +1833,18 @@ inline void AdvancedOutput::SetupRecording() tracks = config_get_int(main->Config(), "AdvOut", "TrackIndex"); if (useStreamEncoder) { - obs_output_set_video_encoder(fileOutput, videoStreaming); + // Sean-Der + obs_encoder_set_scaled_size(videoStreaming[0], 640, 480); + obs_encoder_set_scaled_size(videoStreaming[1], 800, 600); + obs_encoder_set_scaled_size(videoStreaming[2], 1024, 768); + + obs_output_set_video_encoder2(fileOutput, videoStreaming[0], 0); + obs_output_set_video_encoder2(fileOutput, videoStreaming[1], 1); + obs_output_set_video_encoder2(fileOutput, videoStreaming[2], 2); + if (replayBuffer) obs_output_set_video_encoder(replayBuffer, - videoStreaming); + videoStreaming[0]); } else { if (rescale && rescaleRes && *rescaleRes) { if (sscanf(rescaleRes, "%ux%u", &cx, &cy) != 2) { @@ -2013,7 +2033,10 @@ inline void AdvancedOutput::UpdateAudioSettings() void AdvancedOutput::SetupOutputs() { - obs_encoder_set_video(videoStreaming, obs_get_video()); + obs_encoder_set_video(videoStreaming[0], obs_get_video()); + obs_encoder_set_video(videoStreaming[1], obs_get_video()); + obs_encoder_set_video(videoStreaming[2], obs_get_video()); + if (videoRecording) obs_encoder_set_video(videoRecording, obs_get_video()); for (size_t i = 0; i < MAX_AUDIO_MIXES; i++) @@ -2123,7 +2146,10 @@ bool AdvancedOutput::SetupStreaming(obs_service_t *service) outputType = type; } - obs_output_set_video_encoder(streamOutput, videoStreaming); + obs_output_set_video_encoder2(streamOutput, videoStreaming[0], 0); + obs_output_set_video_encoder2(streamOutput, videoStreaming[1], 1); + obs_output_set_video_encoder2(streamOutput, videoStreaming[2], 2); + obs_output_set_audio_encoder(streamOutput, streamAudioEnc, 0); return true; diff --git a/plugins/obs-webrtc/whip-output.cpp b/plugins/obs-webrtc/whip-output.cpp index e19129006a74a0..2d0026a962f04b 100644 --- a/plugins/obs-webrtc/whip-output.cpp +++ b/plugins/obs-webrtc/whip-output.cpp @@ -9,10 +9,21 @@ const uint32_t audio_ssrc = 5002; const char *audio_mid = "0"; const uint8_t audio_payload_type = 111; -const uint32_t video_ssrc = 5000; +const uint32_t video_l_ssrc = 5000; +const uint32_t video_m_ssrc = 5004; +const uint32_t video_h_ssrc = 5006; + const char *video_mid = "1"; const uint8_t video_payload_type = 96; +const std::string rtpHeaderExtUriMid = "urn:ietf:params:rtp-hdrext:sdes:mid"; +const std::string rtpHeaderExtUriRid = + "urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id"; + +const std::string highRid = "h"; +const std::string medRid = "m"; +const std::string lowRid = "l"; + WHIPOutput::WHIPOutput(obs_data_t *, obs_output_t *output) : output(output), endpoint_url(), @@ -24,11 +35,19 @@ WHIPOutput::WHIPOutput(obs_data_t *, obs_output_t *output) peer_connection(nullptr), audio_track(nullptr), video_track(nullptr), + hSequenceNumber(0), + hRTPTimestamp(0), + hLastVideoTimestamp(0), + mSequenceNumber(0), + mRTPTimestamp(0), + mLastVideoTimestamp(0), + lSequenceNumber(0), + lRTPTimestamp(0), + lLastVideoTimestamp(0), total_bytes_sent(0), connect_time_ms(0), start_time_ns(0), - last_audio_timestamp(0), - last_video_timestamp(0) + last_audio_timestamp(0) { } @@ -80,10 +99,46 @@ void WHIPOutput::Data(struct encoder_packet *packet) audio_sr_reporter); last_audio_timestamp = packet->dts_usec; } else if (packet->type == OBS_ENCODER_VIDEO) { - int64_t duration = packet->dts_usec - last_video_timestamp; + auto rtp_config = video_sr_reporter->rtpConfig; + + auto height = obs_encoder_get_width(packet->encoder); + int64_t duration = 0; + if (height == 640) { + rtp_config->sequenceNumber = lSequenceNumber; + rtp_config->ssrc = video_l_ssrc; + rtp_config->rid = lowRid; + rtp_config->timestamp = lRTPTimestamp; + duration = packet->dts_usec - lLastVideoTimestamp; + } else if (height == 800) { + rtp_config->sequenceNumber = mSequenceNumber; + rtp_config->ssrc = video_m_ssrc; + rtp_config->rid = medRid; + rtp_config->timestamp = mRTPTimestamp; + duration = packet->dts_usec - mLastVideoTimestamp; + } else if (height == 1024) { + rtp_config->sequenceNumber = hSequenceNumber; + rtp_config->ssrc = video_h_ssrc; + rtp_config->rid = highRid; + rtp_config->timestamp = hRTPTimestamp; + duration = packet->dts_usec - hLastVideoTimestamp; + } + Send(packet->data, packet->size, duration, video_track, video_sr_reporter); - last_video_timestamp = packet->dts_usec; + + if (height == 640) { + lSequenceNumber = rtp_config->sequenceNumber; + lLastVideoTimestamp = packet->dts_usec; + lRTPTimestamp = rtp_config->timestamp; + } else if (height == 800) { + mSequenceNumber = rtp_config->sequenceNumber; + mLastVideoTimestamp = packet->dts_usec; + mRTPTimestamp = rtp_config->timestamp; + } else if (height == 1024) { + hSequenceNumber = rtp_config->sequenceNumber; + hLastVideoTimestamp = packet->dts_usec; + hRTPTimestamp = rtp_config->timestamp; + } } } @@ -121,13 +176,26 @@ void WHIPOutput::ConfigureVideoTrack(std::string media_stream_id, rtc::Description::Video video_description( video_mid, rtc::Description::Direction::SendOnly); video_description.addH264Codec(video_payload_type); - video_description.addSSRC(video_ssrc, cname, media_stream_id, + video_description.addSSRC(video_l_ssrc, cname, media_stream_id, media_stream_track_id); + + video_description.addExtMap( + rtc::Description::Entry::ExtMap(1, rtpHeaderExtUriMid)); + video_description.addExtMap( + rtc::Description::Entry::ExtMap(2, rtpHeaderExtUriRid)); + video_description.addRid(highRid); + video_description.addRid(medRid); + video_description.addRid(lowRid); + video_track = peer_connection->addTrack(video_description); auto rtp_config = std::make_shared( - video_ssrc, cname, video_payload_type, + video_l_ssrc, cname, video_payload_type, rtc::H264RtpPacketizer::defaultClockRate); + rtp_config->midId = 1; + rtp_config->ridId = 2; + rtp_config->mid = video_mid; + auto packetizer = std::make_shared( rtc::H264RtpPacketizer::Separator::StartSequence, rtp_config); video_sr_reporter = std::make_shared(rtp_config); @@ -401,7 +469,9 @@ void WHIPOutput::StopThread(bool signal) connect_time_ms = 0; start_time_ns = 0; last_audio_timestamp = 0; - last_video_timestamp = 0; + hLastVideoTimestamp = 0; + mLastVideoTimestamp = 0; + lLastVideoTimestamp = 0; } void WHIPOutput::Send(void *data, uintptr_t size, uint64_t duration, @@ -448,7 +518,8 @@ void register_whip_output() struct obs_output_info info = {}; info.id = "whip_output"; - info.flags = OBS_OUTPUT_AV | OBS_OUTPUT_ENCODED | OBS_OUTPUT_SERVICE; + info.flags = OBS_OUTPUT_AV | OBS_OUTPUT_ENCODED | OBS_OUTPUT_SERVICE | + OBS_OUTPUT_MULTI_TRACK_AV; info.get_name = [](void *) -> const char * { return obs_module_text("Output.Name"); }; diff --git a/plugins/obs-webrtc/whip-output.h b/plugins/obs-webrtc/whip-output.h index 568b1f4ee392bd..8c6b577511e9a3 100644 --- a/plugins/obs-webrtc/whip-output.h +++ b/plugins/obs-webrtc/whip-output.h @@ -62,11 +62,22 @@ class WHIPOutput { std::shared_ptr audio_sr_reporter; std::shared_ptr video_sr_reporter; + uint16_t hSequenceNumber; + uint32_t hRTPTimestamp; + int64_t hLastVideoTimestamp; + + uint16_t mSequenceNumber; + uint32_t mRTPTimestamp; + int64_t mLastVideoTimestamp; + + uint16_t lSequenceNumber; + uint32_t lRTPTimestamp; + int64_t lLastVideoTimestamp; + std::atomic total_bytes_sent; std::atomic connect_time_ms; int64_t start_time_ns; int64_t last_audio_timestamp; - int64_t last_video_timestamp; }; void register_whip_output();