[iOS] Audio is not captured in videos played with hls player.

[HyundongHwang]

Unity version

2022.3.37f1

Unity editor platform

macOS

AVPro Movie Capture edition

Full

AVPro Movie Capture Version

5.3.3

Xcode version being used to build your Unity project.

16.2

Which iOS version(s) are you using?

18.1.1

Hardware

ipad air 5

Which capture component are you using?

Capture From Texture

Capture mode

Realtime

Which output mode are you using?

Video file

Video codecs

H264

Audio source

Unity

Audio codecs

AAC

Any other component configuration

No response

The issue

Hello.
Audio is not captured from the video played with hls player.
I set the capture settings as follows.

Video: MediaPlayer -> ResolveToRenderTexture -> ExternalTexture -> CaptureFromTexture
Audio: MediaPlayer -> audioOutput -> mainCam's CaptureAudioFromAudioListener -> CaptureFromTexture

With the same test code, when the source is a local mp4 file, it succeeded, but when it is an hls url, only the video is captured and the audio is entered as a blank value.

    private async void _HLS_REC()
    {
        ZUiUtil.HideAllChildren(_divTop.go);
        _rimg0.go.SetActive(true);

        const string HLS_URL = "https://dwgayteeh8hg5.cloudfront.net/class/class000046_NewJeans_Supernatural_hoois127/class_front_hls/master.m3u8";
        var mp0 = ZUiUtil.GetOrAddChildComponent<MediaPlayer>("mp0");
        var rr0 = ZUiUtil.GetOrAddChildComponent<ResolveToRenderTexture>("rr0");
        var capTx = ZUiUtil.GetOrAddChildComponent<CaptureFromTexture>("capTx");
        var audioOutput = ZUiUtil.GetOrAddChildComponent<AudioOutput>($"audioOutput_{GetHashCode()}");
        var mainCam = ZUiUtil.GetOrAddChildComponent<Camera>("mainCam");
        var captureAudioFromAudioListener = mainCam.GetComponent<CaptureAudioFromAudioListener>();

        if (captureAudioFromAudioListener == null)
            captureAudioFromAudioListener = mainCam.AddComponent<CaptureAudioFromAudioListener>();

        mp0.PlatformOptionsAndroid.videoApi = Android.VideoApi.ExoPlayer;
        mp0.PlatformOptionsWindows._audioMode = Windows.AudioOutput.Unity;
        mp0.PlatformOptionsAndroid.audioMode = MediaPlayer.PlatformOptions.AudioMode.Unity;
        mp0.PlatformOptions_iOS.audioMode = MediaPlayer.PlatformOptions.AudioMode.Unity;
        mp0.PlatformOptions_macOS.audioMode = MediaPlayer.PlatformOptions.AudioMode.Unity;

        audioOutput.Player = mp0;
        mp0.Loop = true;
        rr0.MediaPlayer = mp0;

        capTx.AudioCaptureSource = AudioCaptureSource.Unity;
        capTx.UnityAudioCapture = captureAudioFromAudioListener;

        mp0.OpenMedia(MediaPathType.AbsolutePathOrURL, HLS_URL, false);

        await _MpWaitForOpen(mp0);

        var rt = new RenderTexture(mp0.Info.GetVideoWidth(), mp0.Info.GetVideoHeight(), 0, RenderTextureFormat.ARGB32);
        rr0.ExternalTexture = rt;
        _rimg0.rawImage.color = Color.white;
        _rimg0.rawImage.texture = rt;
        capTx.SetSourceTexture(rt);
        mp0.Play();
        capTx.StartCapture();

        await Task.Delay(10_000);

        capTx.StopCapture();
        LogSloth.d($"capTx.LastFilePath:{capTx.LastFilePath}");
        ZAssert.IsTrue(File.Exists(capTx.LastFilePath));
        mp0.ForceDispose();
        Destroy(mp0.gameObject);
        Destroy(rr0.gameObject);
        Destroy(rt);
        Destroy(capTx);
        Destroy(audioOutput);
    }

    private async void _MP4_REC()
    {
        ZUiUtil.HideAllChildren(_divTop.go);
        _rimg0.go.SetActive(true);

        await _SyncClassContent_1();
        var LOCAL_MP4_PATH = ZStreamContentUtil.GetAvailPath_ClassFront1080p(ZConst.DBG_CLASS_ID);
        LogSloth.d($"LOCAL_MP4_PATH:{LOCAL_MP4_PATH}");
        var mp0 = ZUiUtil.GetOrAddChildComponent<MediaPlayer>("mp0");
        var rr0 = ZUiUtil.GetOrAddChildComponent<ResolveToRenderTexture>("rr0");
        var capTx = ZUiUtil.GetOrAddChildComponent<CaptureFromTexture>("capTx");
        var audioOutput = ZUiUtil.GetOrAddChildComponent<AudioOutput>($"audioOutput_{GetHashCode()}");
        var mainCam = ZUiUtil.GetOrAddChildComponent<Camera>("mainCam");
        var captureAudioFromAudioListener = mainCam.GetComponent<CaptureAudioFromAudioListener>();

        if (captureAudioFromAudioListener == null)
            captureAudioFromAudioListener = mainCam.AddComponent<CaptureAudioFromAudioListener>();

        mp0.PlatformOptionsAndroid.videoApi = Android.VideoApi.ExoPlayer;
        mp0.PlatformOptionsWindows._audioMode = Windows.AudioOutput.Unity;
        mp0.PlatformOptionsAndroid.audioMode = MediaPlayer.PlatformOptions.AudioMode.Unity;
        mp0.PlatformOptions_iOS.audioMode = MediaPlayer.PlatformOptions.AudioMode.Unity;
        mp0.PlatformOptions_macOS.audioMode = MediaPlayer.PlatformOptions.AudioMode.Unity;

        audioOutput.Player = mp0;
        mp0.Loop = true;
        rr0.MediaPlayer = mp0;

        capTx.AudioCaptureSource = AudioCaptureSource.Unity;
        capTx.UnityAudioCapture = captureAudioFromAudioListener;

        mp0.OpenMedia(MediaPathType.AbsolutePathOrURL, LOCAL_MP4_PATH, false);

        await _MpWaitForOpen(mp0);

        var rt = new RenderTexture(mp0.Info.GetVideoWidth(), mp0.Info.GetVideoHeight(), 0, RenderTextureFormat.ARGB32);
        rr0.ExternalTexture = rt;
        _rimg0.rawImage.color = Color.white;
        _rimg0.rawImage.texture = rt;
        capTx.SetSourceTexture(rt);
        mp0.Play();
        capTx.StartCapture();

        await Task.Delay(10_000);

        capTx.StopCapture();
        LogSloth.d($"capTx.LastFilePath:{capTx.LastFilePath}");
        ZAssert.IsTrue(File.Exists(capTx.LastFilePath));

        mp0.ForceDispose();
        Destroy(mp0.gameObject);
        Destroy(rr0.gameObject);
        Destroy(rt);
        Destroy(capTx);
        Destroy(audioOutput);
    }

    private static async Task _MpWaitForOpen(MediaPlayer mp)
    {
        while (true)
        {
            await Task.Delay(100);

            if (mp.Info == null)
                continue;

            if (mp.Info.GetVideoWidth() <= 0 && mp.Info.GetVideoHeight() <= 0)
                continue;

            if (mp.Control == null)
                continue;

            if (mp.Cache == null)
                continue;

            break;
        }
    }

Log output

No response

[MorrisRH]

Unfortunately we're unable to route audio from HLS media through to Unity and as a result the capture component will not be able to capture it. This is a limitation of Apple's framework and not something we can fix. More information is available here: RenderHeads/UnityPlugin-AVProVideo#1242

[HyundongHwang]

Thank you for your reply.
#445 Does Android have the same issue?