Enable IC background blur feature

build_overrides/owt.gni

@@ -0,0 +1,9 @@
+# Copyright (C) <2021> Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+declare_args() {
+  owt_msdk_lib_root = ""
+  owt_msdk_header_root = ""
+  owt_build_ic = false
+}

scripts/build-win.py

@@ -11,6 +11,7 @@
 '''
 
 import os
+import re
 import sys
 import subprocess
 import argparse
@@ -35,7 +36,7 @@
     ]
 
 
-def gngen(arch, ssl_root, msdk_root, quic_root, scheme, tests):
+def gngen(arch, ssl_root, msdk_root, quic_root, openvino_root, scheme, tests):
     gn_args = list(GN_ARGS)
     gn_args.append('target_cpu="%s"' % arch)
     using_llvm = False
@@ -49,9 +50,15 @@ def gngen(arch, ssl_root, msdk_root, quic_root, scheme, tests):
         gn_args.append('use_custom_libcxx=false')
     if scheme == 'release':
         gn_args.append('is_debug=false')
+        if openvino_root:
+            gn_args.append('lib_suffix=".lib"')
+            gn_args.append('dll_suffix=".dll"')
     else:
         gn_args.append('is_debug=true')
         gn_args.append('enable_iterator_debugging=true')
+        if openvino_root:
+            gn_args.append('lib_suffix="d.lib"')
+            gn_args.append('dll_suffix="d.dll"')
     if ssl_root:
         gn_args.append('owt_use_openssl=true')
         gn_args.append('owt_openssl_header_root="%s"' % (ssl_root + r'\include'))
@@ -76,6 +83,17 @@ def gngen(arch, ssl_root, msdk_root, quic_root, scheme, tests):
         else:
             return False
         gn_args.append('owt_use_quic=true')
+    if openvino_root:
+        gn_args.append('owt_build_ic=true')
+        gn_args.append('owt_openvino_root="{}"'.format(openvino_root))
+        opencv_version_bin = os.path.join(openvino_root, 'opencv', 'bin', 'opencv_version.exe')
+        try:
+            output = subprocess.check_output([opencv_version_bin])
+            opencv_version = ''.join(re.findall('\d', output.decode()))
+            gn_args.append('owt_openvino_opencv_version={}'.format(opencv_version))
+        except FileNotFoundError as e:
+            print('File not found: {}'.format(opencv_version_bin))
+            return False
     if tests:
         gn_args.append('rtc_include_tests=true')
         gn_args.append('owt_include_tests=true')
@@ -183,6 +201,7 @@ def main():
     parser.add_argument('--ssl_root', help='Path for OpenSSL.')
     parser.add_argument('--msdk_root', help='Path for MSDK.')
     parser.add_argument('--quic_root', help='Path to QUIC library')
+    parser.add_argument('--openvino_root', help='Path for OpenVINO.')
     parser.add_argument('--scheme', default='debug', choices=('debug', 'release'),
                         help='Schemes for building. Supported value: debug, release')
     parser.add_argument('--gn_gen', default=False, action='store_true',
@@ -195,19 +214,16 @@ def main():
                         help='To generate the API document.')
     parser.add_argument('--output_path', help='Path to copy sdk.')
     opts = parser.parse_args()
-    if opts.ssl_root and not os.path.exists(os.path.expanduser(opts.ssl_root)):
-        print('Invalid ssl_root.')
-        return 1
-    if opts.msdk_root and not os.path.exists(os.path.expanduser(opts.msdk_root)):
-        print('Invalid msdk_root')
-        return 1
-    if opts.quic_root and not os.path.exists(os.path.expanduser(opts.quic_root)):
-        print('Invalid quic_root')
-        return 1
+    for name in ['ssl', 'msdk', 'quic', 'openvino']:
+        attr = name + '_root'
+        path = getattr(opts, attr)
+        if path is not None and not os.path.exists(os.path.expanduser(path)):
+            print('Invalid {}.'.format(attr))
+            return 1
     print(opts)
     if opts.gn_gen:
         if not gngen(opts.arch, opts.ssl_root, opts.msdk_root, opts.quic_root,
-                     opts.scheme, opts.tests):
+                     opts.openvino_root, opts.scheme, opts.tests):
             return 1
     if opts.sdk:
         if not ninjabuild(opts.arch, opts.scheme):

talk/owt/BUILD.gn

@@ -3,12 +3,11 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import("//build_overrides/webrtc.gni")
+import("//build_overrides/owt.gni")
 import("//testing/test.gni")
 
 declare_args() {
   include_internal_audio_device = true
-  owt_msdk_lib_root = ""
-  owt_msdk_header_root = ""
 }
 
 # Introduced for using libvpx config files. We only enable libvpx rate
@@ -99,6 +98,7 @@ static_library("owt_sdk_base") {
     "sdk/base/functionalobserver.cc",
     "sdk/base/functionalobserver.h",
     "sdk/base/globalconfiguration.cc",
+    "sdk/base/intelligentcollaborationparameters.cc",
     "sdk/base/localcamerastreamparameters.cc",
     "sdk/base/logging.cc",
     "sdk/base/logsinks.cc",
@@ -109,6 +109,7 @@ static_library("owt_sdk_base") {
     "sdk/base/peerconnectionchannel.h",
     "sdk/base/peerconnectiondependencyfactory.cc",
     "sdk/base/peerconnectiondependencyfactory.h",
+    "sdk/base/pluginmanager.cc",
     "sdk/base/sdputils.cc",
     "sdk/base/sdputils.h",
     "sdk/base/stream.cc",
@@ -128,8 +129,11 @@ static_library("owt_sdk_base") {
     "sdk/include/cpp/owt/base/framegeneratorinterface.h",
     "sdk/include/cpp/owt/base/localcamerastreamparameters.h",
     "sdk/include/cpp/owt/base/logging.h",
+    "sdk/include/cpp/owt/base/pluginmanager.h",
     "sdk/include/cpp/owt/base/stream.h",
+    "sdk/include/cpp/owt/base/videoframepostprocessor.h",
     "sdk/include/cpp/owt/base/videorendererinterface.h",
+    "sdk/include/cpp/owt/ic/intelligentcollaborationparameters.h",
   ]
   if (is_win || is_linux) {
     sources += [
@@ -141,10 +145,13 @@ static_library("owt_sdk_base") {
       "sdk/base/customizedvideosource.h",
       "sdk/base/encodedvideoencoderfactory.cc",
       "sdk/base/encodedvideoencoderfactory.h",
+      "sdk/base/sharedobjectloader.h",
+      "sdk/base/sharedobjectpointer.h",
       "sdk/base/webrtcvideorendererimpl.cc",
       "sdk/base/webrtcvideorendererimpl.h",
       "sdk/base/windowcapturer.cc",
       "sdk/include/cpp/owt/base/videodecoderinterface.h",
+      "sdk/include/cpp/owt/ic/icmanagerinterface.h",
     ]
   }
   public_deps = [
@@ -261,6 +268,7 @@ static_library("owt_sdk_base") {
       "sdk/base/win/device_info_mf.cc",
       "sdk/base/win/video_capture_mf.h",
       "sdk/base/win/video_capture_mf.cc",
+      "sdk/base/win/sharedobjectloader.cc",
     ]
     public_deps += [ "//third_party/webrtc/modules/audio_device:audio_device_module_from_input_and_output" ]
 
@@ -286,6 +294,7 @@ static_library("owt_sdk_base") {
       ]
     }
     sources += [
+        "sdk/base/linux/sharedobjectloader.cc",
         "sdk/base/linux/xwindownativeframe.h",
         "sdk/base/linux/videorenderlinux.cc",
         "sdk/base/linux/videorenderlinux.h",
@@ -316,6 +325,10 @@ static_library("owt_sdk_base") {
       "-Wno-reorder",
     ]
   }
+  if (owt_build_ic) {
+    defines += [ "OWT_BUILD_IC" ]
+    data_deps = [ "sdk/ic:owt_ic" ]
+  }
 }
 static_library("owt_sdk_p2p") {
   deps = [
@@ -498,7 +511,10 @@ if (owt_include_tests) {
 
     # Only the root target should depend on this.
     visibility = [ "//:default" ]
-    deps = [ ":owt_unittests" ]
+    deps = [
+      ":owt_unittests",
+      "//talk/owt/sdk/sample",
+    ]
   }
   test("owt_unittests") {
     testonly = true

talk/owt/sdk/base/cameravideocapturer.cc

@@ -36,14 +36,25 @@ void CameraVideoCapturer::OnFrame(const webrtc::VideoFrame& frame) {
     return;
   }
 
+  rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer =
+      frame.video_frame_buffer();
+
   if (out_height != frame.height() || out_width != frame.width()) {
     // Video adapter has requested a down-scale. Allocate a new buffer and
     // return scaled version.
     rtc::scoped_refptr<webrtc::I420Buffer> scaled_buffer =
         webrtc::I420Buffer::Create(out_width, out_height);
     scaled_buffer->ScaleFrom(*frame.video_frame_buffer()->ToI420());
+    buffer = scaled_buffer;
+  }
+
+  for (auto& post_processor : video_frame_post_processors_) {
+    buffer = post_processor->Process(buffer);
+  }
+
+  if (buffer != frame.video_frame_buffer()) {
     broadcaster_.OnFrame(webrtc::VideoFrame::Builder()
-                             .set_video_frame_buffer(scaled_buffer)
+                             .set_video_frame_buffer(buffer)
                              .set_rotation(webrtc::kVideoRotation_0)
                              .set_timestamp_us(frame.timestamp_us())
                              .set_id(frame.id())
@@ -71,6 +82,11 @@ void CameraVideoCapturer::RemoveSink(
   UpdateVideoAdapter();
 }
 
+void CameraVideoCapturer::AddVideoFramePostProcessor(
+    std::shared_ptr<VideoFramePostProcessor> post_processor) {
+  video_frame_post_processors_.emplace_back(post_processor);
+}
+
 void CameraVideoCapturer::UpdateVideoAdapter() {
   video_adapter_.OnSinkWants(broadcaster_.wants());
 }

talk/owt/sdk/base/cameravideocapturer.h

@@ -13,12 +13,15 @@
 #include <stddef.h>
 
 #include <memory>
+#include <vector>
 
 #include "api/video/video_frame.h"
 #include "api/video/video_source_interface.h"
 #include "media/base/video_adapter.h"
 #include "media/base/video_broadcaster.h"
 
+#include "talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h"
+
 // This file is borrowed from webrtc project
 namespace owt {
 namespace base {
@@ -32,6 +35,9 @@ class CameraVideoCapturer : public rtc::VideoSourceInterface<webrtc::VideoFrame>
                        const rtc::VideoSinkWants& wants) override;
   void RemoveSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override;
 
+  void AddVideoFramePostProcessor(
+      const std::shared_ptr<VideoFramePostProcessor> post_processor);
+
  protected:
   void OnFrame(const webrtc::VideoFrame& frame);
   rtc::VideoSinkWants GetSinkWants();
@@ -41,6 +47,8 @@ class CameraVideoCapturer : public rtc::VideoSourceInterface<webrtc::VideoFrame>
 
   rtc::VideoBroadcaster broadcaster_;
   cricket::VideoAdapter video_adapter_;
+  std::vector<std::shared_ptr<owt::base::VideoFramePostProcessor>>
+      video_frame_post_processors_;
 };
 }  // namespace base
 }  // namespace owt

talk/owt/sdk/base/intelligentcollaborationparameters.cc

@@ -0,0 +1,21 @@
+// Copyright (C) <2021> Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+#include "talk/owt/sdk/include/cpp/owt/ic/intelligentcollaborationparameters.h"
+
+namespace owt {
+namespace base {
+
+std::vector<std::shared_ptr<owt::base::VideoFramePostProcessor>>&
+IntelligentCollaborationParameters::PostProcessors() {
+  return post_processors_;
+}
+
+const std::vector<std::shared_ptr<owt::base::VideoFramePostProcessor>>&
+IntelligentCollaborationParameters::PostProcessors() const {
+  return post_processors_;
+}
+
+}  // namespace base
+}  // namespace owt

talk/owt/sdk/base/linux/sharedobjectloader.cc

@@ -0,0 +1,26 @@
+// Copyright (C) <2021> Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+#include "talk/owt/sdk/base/sharedobjectloader.h"
+
+#include <dlfcn.h>
+
+namespace owt {
+namespace base {
+
+SharedObjectLoader::SharedObjectLoader(const char* path)
+    : shared_object(dlopen(path, RTLD_NOW), dlclose) {}
+
+SharedObjectLoader::~SharedObjectLoader() {}
+
+bool SharedObjectLoader::IsLoaded() const {
+  return shared_object_.get();
+}
+
+void* SharedObjectLoader::get_symbol(const char* name) const {
+  return shared_object_ ? dlsym(shared_object.get(), name) : nullptr;
+}
+
+}  // namespace base
+}  // namespace owt

talk/owt/sdk/base/localcamerastreamparameters.cc

@@ -19,6 +19,14 @@ LocalCameraStreamParameters::LocalCameraStreamParameters(bool audio_enabled,
 void LocalCameraStreamParameters::Fps(int fps) {
   fps_ = fps;
 }
+IntelligentCollaborationParameters&
+LocalCameraStreamParameters::ICParams() {
+  return ic_params_;
+}
+const IntelligentCollaborationParameters&
+LocalCameraStreamParameters::ICParams() const {
+  return ic_params_;
+}
 void LocalCameraStreamParameters::CameraId(const std::string& camera_id) {
   camera_id_ = camera_id;
 }

talk/owt/sdk/base/peerconnectiondependencyfactory.cc

@@ -47,6 +47,7 @@
 #pragma comment(lib, "wmcodecdspuuid.lib")
 #pragma comment(lib, "amstrmid.lib")
 #pragma comment(lib, "strmiids.lib")
+#pragma comment(lib, "dcomp.lib")
 #ifdef OWT_USE_MSDK
 #pragma comment(lib, "mf.lib")
 #pragma comment(lib, "mfplat.lib")

talk/owt/sdk/base/pluginmanager.cc

@@ -0,0 +1,32 @@
+// Copyright (C) <2021> Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+#include "talk/owt/sdk/include/cpp/owt/base/pluginmanager.h"
+
+#include "talk/owt/sdk/base/sharedobjectpointer.h"
+
+#if defined(WEBRTC_WIN)
+#define DLL_SUFFIX ".dll"
+#elif defined(WEBRTC_LINUX)
+#define DLL_SUFFIX ".so"
+#endif
+
+namespace owt {
+namespace base {
+
+#if defined(WEBRTC_WIN) || defined(WEBRTC_LINUX)
+template <>
+struct SOTrait<owt::ic::ICManagerInterface> {
+  static constexpr auto name = "ICManager";
+};
+
+owt::ic::ICManagerInterface* PluginManager::ICPlugin() {
+  static owt::base::SharedObjectPointer<owt::ic::ICManagerInterface> ic_plugin(
+      "owt_ic" DLL_SUFFIX);
+  return ic_plugin.Get();
+}
+#endif
+
+}  // namespace base
+}  // namespace owt