Skip to content

Commit

Permalink
WHIP をサイマルキャストマルチコーデックに対応する
Browse files Browse the repository at this point in the history
  • Loading branch information
@melpon
melpon committed Jan 24, 2025
1 parent 45213b4 commit f834e43
Show file tree
Hide file tree
Showing 4 changed files with 174 additions and 12 deletions.
18 changes: 17 additions & 1 deletion .vscode/launch.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
//"preLaunchTask": "C/C++: cl.exe アクティブなファイルのビルド"
},
{
"name": "macOS リリースビルドのデバッグ",
"name": "[hello] macOS リリースビルドのデバッグ",
"type": "cppdbg",
"request": "launch",
"program": "${workspaceFolder}/_build/macos_x86_64/release/test/hello",
Expand All @@ -52,6 +52,22 @@
"externalConsole": false,
"MIMode": "lldb"
},
{
"name": "[whip][debug] Ubuntu 24.04 x86_64",
"type": "lldb-dap",
"request": "launch",
"program": "${workspaceFolder}/_build/ubuntu-24.04_x86_64/debug/test/whip",
// "args": ["${workspaceFolder}/test/.testparam.json"],
"args": [],
"initCommands": [
"settings set target.debug-file-search-paths ${workspaceRoot}/../webrtc-build/_build/ubuntu-24.04_x86_64/debug/webrtc",
],
"sourceMap": [
["../../../../_source/ubuntu-24.04_x86_64/webrtc", "${workspaceRoot}/../webrtc-build/_source/ubuntu-24.04_x86_64/webrtc"],
],
"cwd": "${workspaceRoot}",
"runInTerminal": true
},

]
}
1 change: 1 addition & 0 deletions include/sora/sora_signaling_whip.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ struct SoraSignalingWhipConfig {

std::string signaling_url;
std::string channel_id;
std::optional<std::vector<webrtc::RtpEncodingParameters>> send_encodings;
rtc::scoped_refptr<webrtc::VideoTrackSourceInterface> video_source;
};

Expand Down
115 changes: 106 additions & 9 deletions src/sora_signaling_whip.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#include <boost/algorithm/string/split.hpp>

// WebRTC
#include <media/base/codec_comparators.h>
#include <p2p/client/basic_port_allocator.h>
#include <pc/rtp_media_utils.h>
#include <pc/session_description.h>
Expand Down Expand Up @@ -84,12 +85,17 @@ void SoraSignalingWhip::Connect() {
}
transceiver.value()->SetCodecPreferences(codecs);
}
webrtc::RtpTransceiverInit video_init;
if (self->config_.video_source != nullptr) {
std::string video_track_id = rtc::CreateRandomString(16);
auto video_track = self->config_.pc_factory->CreateVideoTrack(
self->config_.video_source, video_track_id);
webrtc::RtpTransceiverInit init;
auto& init = video_init;
init.direction = webrtc::RtpTransceiverDirection::kSendOnly;
init.stream_ids = {rtc::CreateRandomString(16)};
if (self->config_.send_encodings) {
init.send_encodings = *self->config_.send_encodings;
}
auto transceiver = pc->AddTransceiver(video_track, init);
if (!transceiver.ok()) {
RTC_LOG(LS_ERROR) << "Failed to AddTransceiver(video): error="
Expand All @@ -100,8 +106,54 @@ void SoraSignalingWhip::Connect() {
auto cap = self->config_.pc_factory->GetRtpSenderCapabilities(
cricket::MediaType::MEDIA_TYPE_VIDEO);
std::vector<webrtc::RtpCodecCapability> codecs;
for (const auto& send_encoding : init.send_encodings) {
for (const webrtc::RtpCodecCapability& codec : cap.codecs) {
auto codec_format =
webrtc::SdpVideoFormat(codec.name, codec.parameters);
if (send_encoding.codec) {
auto encoding_format = webrtc::SdpVideoFormat(
send_encoding.codec->name, send_encoding.codec->parameters);
if (codec_format == encoding_format) {
auto it = std::find_if(
codecs.begin(), codecs.end(),
[&codec_format](const webrtc::RtpCodecCapability& c) {
auto format = webrtc::SdpVideoFormat(c.name, c.parameters);
return codec_format == format;
});
if (it == codecs.end()) {
codecs.push_back(codec);
}
break;
}
}
}
}
//for (const webrtc::RtpCodecCapability& codec : cap.codecs) {
// if (codec.name == "H264") {
// codecs.push_back(codec);
// break;
// }
//}
//for (const webrtc::RtpCodecCapability& codec : cap.codecs) {
// if (codec.name == "H265") {
// codecs.push_back(codec);
// break;
// }
//}
//for (const webrtc::RtpCodecCapability& codec : cap.codecs) {
// if (codec.name == "VP9") {
// codecs.push_back(codec);
// break;
// }
//}
//for (const webrtc::RtpCodecCapability& codec : cap.codecs) {
// if (codec.name == "AV1") {
// codecs.push_back(codec);
// break;
// }
//}
for (const webrtc::RtpCodecCapability& codec : cap.codecs) {
if (codec.name == "AV1") {
if (codec.name == "rtx") {
codecs.push_back(codec);
break;
}
Expand All @@ -111,17 +163,49 @@ void SoraSignalingWhip::Connect() {

pc->CreateOffer(
CreateSessionDescriptionThunk::Create(
[self](webrtc::SessionDescriptionInterface* description) {
[self,
video_init](webrtc::SessionDescriptionInterface* description) {
auto offer = std::unique_ptr<webrtc::SessionDescriptionInterface>(
description);

// 各 RtpEncodingParameters の利用するコーデックと payload_type を関連付ける
std::map<std::string, int> rid_payload_type_map;
auto& content = offer->description()->contents()[1];
auto media_desc = content.media_description();
for (auto& send_encoding : video_init.send_encodings) {
RTC_LOG(LS_WARNING)
<< "send_encoding: " << send_encoding.codec->name;
for (auto& codec : media_desc->codecs()) {
RTC_LOG(LS_WARNING) << "codec: " << codec.name;
if (send_encoding.codec &&
webrtc::IsSameRtpCodec(codec, *send_encoding.codec)) {
RTC_LOG(LS_WARNING) << "rid=" << send_encoding.rid
<< " codec=" << codec.name
<< " payload_type=" << codec.id;
rid_payload_type_map[send_encoding.rid] = codec.id;
}
}
}
auto& track = media_desc->mutable_streams()[0];
auto rids = track.rids();
for (auto& rid : rids) {
auto it = rid_payload_type_map.find(rid.rid);
if (it == rid_payload_type_map.end()) {
continue;
}
rid.payload_types.push_back(it->second);
}
track.set_rids(rids);

std::string offer_sdp;
if (!offer->ToString(&offer_sdp)) {
RTC_LOG(LS_ERROR) << "Failed to get SDP";
return;
}
RTC_LOG(LS_INFO) << "Offer SDP: " << offer_sdp;

boost::asio::post(*self->config_.io_context, [self, offer_sdp]() {
boost::asio::post(*self->config_.io_context, [self, offer_sdp,
video_init]() {
URLParts parts;
if (!URLParts::Parse(self->config_.signaling_url, parts)) {
RTC_LOG(LS_ERROR)
Expand All @@ -130,7 +214,8 @@ void SoraSignalingWhip::Connect() {
}

self->req_.target(parts.path_query_fragment + "/" +
self->config_.channel_id);
self->config_.channel_id +
"?video_bit_rate=6000");
self->req_.method(boost::beast::http::verb::post);
// self->req_.set(boost::beast::http::field::authorization, "Bearer " + self->config_.secret_key);
self->req_.set(boost::beast::http::field::content_type,
Expand All @@ -140,8 +225,8 @@ void SoraSignalingWhip::Connect() {
self->req_.body() = offer_sdp;
self->req_.prepare_payload();
RTC_LOG(LS_INFO) << "Send request to: " << self->req_.target();
self->SendRequest([self,
offer_sdp](boost::beast::error_code ec) {
self->SendRequest([self, offer_sdp,
video_init](boost::beast::error_code ec) {
if (ec) {
return;
}
Expand Down Expand Up @@ -197,15 +282,27 @@ void SoraSignalingWhip::Connect() {
webrtc::SdpType::kOffer, offer_sdp);
self->pc_->SetLocalDescription(
SetSessionDescriptionThunk::Create(
[self]() {
[self, video_init]() {
auto answer = webrtc::CreateSessionDescription(
webrtc::SdpType::kAnswer, self->res_.body());
self->pc_->SetRemoteDescription(
SetSessionDescriptionThunk::Create(
[self]() {
[self, video_init]() {
RTC_LOG(LS_INFO)
<< "Succeeded to "
"SetRemoteDescription";
auto p = self->pc_->GetSenders()[1]
->GetParameters();
for (int i = 0; i < p.encodings.size();
i++) {
p.encodings[i].codec =
video_init.send_encodings[i].codec;
p.encodings[i].scalability_mode =
video_init.send_encodings[i]
.scalability_mode;
}
self->pc_->GetSenders()[1]->SetParameters(
p);
},
[](webrtc::RTCError error) {
RTC_LOG(LS_ERROR)
Expand Down
52 changes: 50 additions & 2 deletions test/whip.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,14 @@ class SoraClient : public std::enable_shared_from_this<SoraClient>,
sora::SoraClientContextConfig context_config;
context_config.use_audio_device = false;
context_config.use_hardware_encoder = false;
context_config.openh264 = "hoge-";
context_ = sora::SoraClientContext::Create(context_config);

ioc_.reset(new boost::asio::io_context(1));

FakeVideoCapturerConfig fake_config;
fake_config.width = 640;
fake_config.height = 480;
fake_config.width = 1920;
fake_config.height = 1080;
fake_config.fps = 30;
auto video_source = CreateFakeVideoCapturer(fake_config);

Expand All @@ -46,6 +47,53 @@ class SoraClient : public std::enable_shared_from_this<SoraClient>,
config.signaling_url = "hoge-";
config.channel_id = "sora";
config.video_source = video_source;

auto& send_encodings = config.send_encodings.emplace();
webrtc::RtpCodecCapability vp9_codec;
webrtc::RtpCodecCapability av1_codec;
webrtc::RtpCodecCapability h264_codec;
webrtc::RtpCodecCapability h265_codec;
vp9_codec.kind = cricket::MEDIA_TYPE_VIDEO;
vp9_codec.name = "VP9";
vp9_codec.parameters["profile-id"] = "0";
vp9_codec.clock_rate = 90000;
av1_codec.kind = cricket::MEDIA_TYPE_VIDEO;
av1_codec.name = "AV1";
av1_codec.clock_rate = 90000;
av1_codec.parameters["level-idx"] = "5";
av1_codec.parameters["profile"] = "0";
av1_codec.parameters["tier"] = "0";
h264_codec.kind = cricket::MEDIA_TYPE_VIDEO;
h264_codec.name = "H264";
h264_codec.clock_rate = 90000;
h264_codec.parameters["profile-level-id"] = "42001f";
h264_codec.parameters["level-asymmetry-allowed"] = "1";
h264_codec.parameters["packetization-mode"] = "1";
h265_codec.kind = cricket::MEDIA_TYPE_VIDEO;
h265_codec.name = "H265";
h265_codec.clock_rate = 90000;
send_encodings.resize(3);
send_encodings[0].rid = "r0";
send_encodings[0].scale_resolution_down_by = 4.0;
send_encodings[1].rid = "r1";
send_encodings[1].scale_resolution_down_by = 2.0;
send_encodings[2].rid = "r2";
send_encodings[2].scale_resolution_down_by = 1.0;

send_encodings[0].codec = av1_codec;
send_encodings[0].scalability_mode = "L1T2";
//send_encodings[0].codec = h264_codec;

send_encodings[1].codec = av1_codec;
send_encodings[1].scalability_mode = "L1T2";
//send_encodings[1].codec = h264_codec;

//send_encodings[2].codec = av1_codec;
//send_encodings[2].scalability_mode = "L1T2";
//send_encodings[2].codec = h264_codec;
//send_encodings[2].codec = h265_codec;
send_encodings[2].codec = vp9_codec;

conn_ = sora::SoraSignalingWhip::Create(config);

boost::asio::executor_work_guard<boost::asio::io_context::executor_type>
Expand Down

0 comments on commit f834e43

Please sign in to comment.