Could You provide example of decoding audio file to buffer compatible with SFML::

[baziorek]

I have example how to do something with FFMPEG and SFML - how to play almost any audio/video. I have code which is decoding audio/video file with ffmpeg to array then the code is sending the array to SFML which is playing the music.

I want to show to students really simple code, which does not require knowledge about audio in programming, that is why I want to change my code from using ffmpeg directly to use AVcpp.

I found example: https://github.com/h4tr3d/avcpp/blob/master/example/api2-samples/api2-decode-audio.cpp but I can't adapt its to work with my code.

Here is code which is playing music with ffmpeg + SFML:

#include <iostream>
#include <vector>
#include <thread>
#include <chrono>
#include <SFML/Audio.hpp>

extern "C" {
    #include <libavformat/avformat.h>
    #include <libavcodec/avcodec.h>
    #include <libswresample/swresample.h>
    #include <libavutil/channel_layout.h>
    #include <libavutil/samplefmt.h>
    #include <libavutil/mem.h>
}

struct AudioFileData
{
    int channels;
    int sampleRate;
   
    std::vector < sf::Int16 > pcmAudioBuffer;
   
    bool successReading;
};

AudioFileData decodeAudioWithFFmpeg( const std::string & filePath );

void playSound( const AudioFileData & audioFileData );

int main( int argc, char * * argv ) {
    if( argc < 2 ) {
        std::cerr << "Usage: " << argv[ 0 ] << " <audio file>" << std::endl;
        return 1;
    }
   
    const std::string filePath = argv[ 1 ];
   
    AudioFileData audioData = decodeAudioWithFFmpeg( filePath );
    if( audioData.pcmAudioBuffer.empty() )
         std::cerr << "Failed to decode or play audio from file " << filePath << std::endl;
    else
         playSound( audioData );
   
}

AudioFileData decodeAudioWithFFmpeg( const std::string & filePath ) {
    avformat_network_init();
   
    AVFormatContext * formatContext = nullptr;
    if( avformat_open_input( & formatContext, filePath.c_str(), nullptr, nullptr ) != 0 ) {
        std::cerr << "Error: Could not open audio file with FFmpeg: " << filePath << std::endl;
        return { };
    }
   
    if( avformat_find_stream_info( formatContext, nullptr ) < 0 ) {
        std::cerr << "Error: Could not retrieve stream info." << std::endl;
        avformat_close_input( & formatContext );
        return { };
    }
   
    const AVCodec * codec = nullptr;
    AVCodecContext * codecContext = nullptr;
    int streamIndex = - 1;
   
    for( unsigned int i = 0; i < formatContext->nb_streams; ++i ) {
        if( formatContext->streams[ i ]->codecpar->codec_type == AVMEDIA_TYPE_AUDIO ) {
            codec = avcodec_find_decoder( formatContext->streams[ i ]->codecpar->codec_id );
            if( !codec ) {
                std::cerr << "Error: Unsupported codec for audio stream." << std::endl;
                avformat_close_input( & formatContext );
                return { };
            }
           
            codecContext = avcodec_alloc_context3( codec );
            avcodec_parameters_to_context( codecContext, formatContext->streams[ i ]->codecpar );
            if( avcodec_open2( codecContext, codec, nullptr ) < 0 ) {
                std::cerr << "Error: Could not open codec." << std::endl;
                avcodec_free_context( & codecContext );
                avformat_close_input( & formatContext );
                return { };
            }
            streamIndex = i;
            break;
        }
    }
   
    if( streamIndex == - 1 ) {
        std::cerr << "Error: No audio stream found in file." << std::endl;
        avcodec_free_context( & codecContext );
        avformat_close_input( & formatContext );
        return { };
    }
   
    SwrContext * swrContext = swr_alloc();
    if( !swrContext ) {
        std::cerr << "Error: Could not allocate SwrContext." << std::endl;
        avcodec_free_context( & codecContext );
        avformat_close_input( & formatContext );
        return { };
    }
   
    swr_alloc_set_opts2( & swrContext,
    & codecContext->ch_layout, AV_SAMPLE_FMT_S16, codecContext->sample_rate,
    & codecContext->ch_layout, codecContext->sample_fmt, codecContext->sample_rate,
    0, nullptr );
   
    if( swr_init( swrContext ) < 0 ) {
        std::cerr << "Error: Could not initialize SwrContext." << std::endl;
        swr_free( & swrContext );
        avcodec_free_context( & codecContext );
        avformat_close_input( & formatContext );
        return { };
    }
   
    AVPacket * packet = av_packet_alloc();
    AVFrame * frame = av_frame_alloc();
   
    AudioFileData audioFileData;
    audioFileData.sampleRate = codecContext->sample_rate;
    audioFileData.channels = codecContext->ch_layout.nb_channels;
   
    while( av_read_frame( formatContext, packet ) >= 0 ) {
        if( packet->stream_index == streamIndex ) {
            if( avcodec_send_packet( codecContext, packet ) == 0 ) {
                while( avcodec_receive_frame( codecContext, frame ) == 0 ) {
                    int bufferSize = av_samples_get_buffer_size( nullptr, audioFileData.channels, frame->nb_samples, AV_SAMPLE_FMT_S16, 1 );
                    std::vector < uint8_t > tempBuffer( bufferSize );
                   
                    uint8_t * tempBufferPtr = tempBuffer.data();
                    swr_convert( swrContext, & tempBufferPtr, frame->nb_samples, frame->data, frame->nb_samples );
                   
                    audioFileData.pcmAudioBuffer.insert( end( audioFileData.pcmAudioBuffer ),
                    ( sf::Int16 * ) tempBuffer.data(),
                    ( sf::Int16 * ) tempBuffer.data() + bufferSize / sizeof( sf::Int16 ) );
                }
            }
        }
        av_packet_unref( packet );
    }
   
    av_frame_free( & frame );
    av_packet_free( & packet );
    swr_free( & swrContext );
    avcodec_free_context( & codecContext );
    avformat_close_input( & formatContext );
   
    return audioFileData;
}

void playSound( const AudioFileData & audioFileData ) {
    sf::SoundBuffer soundBuffer;
    if( !soundBuffer.loadFromSamples( audioFileData.pcmAudioBuffer.data(), audioFileData.pcmAudioBuffer.size(), audioFileData.channels, audioFileData.sampleRate ) ) {
        std::cerr << "Error: Could not load buffer for playback" << std::endl;
        return;
    }
   
    sf::Sound sound;
    sound.setBuffer( soundBuffer );
    sound.play();
   
    std::cout << "Playing file, duration: " << soundBuffer.getDuration().asSeconds() << " seconds" << std::endl;
    while( sound.getStatus() == sf::Sound::Playing ) {
        std::this_thread::sleep_for( std::chrono::milliseconds( 500 ) );
    }
   
    std::cout << "Playback finished." << std::endl;
}

I know that SFML can play few audio formats, but I want to play various formats. Just SFML is IMO easiest code to play audio file:

#include <iostream>
#include <thread>
#include <chrono>
#include <SFML/Audio.hpp>

int main() {
    const std::string filePath = "audio.wav";

    sf::Music music;
    if (!music.openFromFile(filePath)) {
        std::cerr << "Error: Could not load file " << filePath << std::endl;
        return 1;
    }

    std::cout << "Now playing: " << filePath << ", duration: " << music.getDuration().asSeconds() << " seconds" << std::endl;

    music.play();

    while (sf::Music::Playing == music.getStatus()) {
        std::cout << "\rCurrent position: " << music.getPlayingOffset().asSeconds()
        << " / " << music.getDuration().asSeconds() << " seconds" << std::flush;
        std::this_thread::sleep_for(std::chrono::milliseconds(500));
    }

    std::cout << "\nPlayback finished." << std::endl;
}

---

Fastest way to install with vcpkg

git clone https://github.com/Microsoft/vcpkg.git --depth=1
./vcpkg/bootstrap-vcpkg.sh --disable-metrics

./vcpkg/vcpkg install 'sfml[audio]'

./vcpkg/vcpkg install 'ffmpeg[all]'

Then sample CMakeLists.txt:

cmake_minimum_required(VERSION 3.31)

project(AudioTerminal LANGUAGES CXX)

set(CMAKE_CXX_STANDARD 23)
set(CMAKE_CXX_STANDARD_REQUIRED ON)


# binary1 (just SFML)
find_package(SFML COMPONENTS system window graphics audio CONFIG REQUIRED)

add_executable(SFML_AUDIO sfml1.cpp)
target_link_libraries(SFML_AUDIO PRIVATE sfml-system sfml-network sfml-graphics sfml-window sfml-audio)

# binary2 (SFML + FFMPEG)
find_package(PkgConfig REQUIRED)
pkg_check_modules(FFMPEG REQUIRED IMPORTED_TARGET
    libavformat
    libavcodec
    libswresample
    libavutil
)

add_executable(SFML_FFMPEG sfml_ffmpeg.cpp)
target_include_directories(SFML_FFMPEG PRIVATE ${PROJECT_SOURCE_DIR} PkgConfig::FFMPEG)
target_link_libraries(SFML_FFMPEG PRIVATE PkgConfig::FFMPEG sfml-audio sfml-system)

As I know to install avcpp with vcpkg we need just:
vcpkg install avcpp
but probably You have better way to install the library:D.

---

BTW. Really good job with the library avcpp. I'd love to use its, but I need help.

[h4tr3d]

Hi, will try to prepare in near two days.

Installing avcpp via vcpkg is a good case :)

[h4tr3d]

Look into attached sample: sfml-audio-pcm16.zip

[baziorek]

Look into attached sample: sfml-audio-pcm16.zip

Thanks a lot for the example!
It works, only few things I've noticed to consider before adding to main branch:

1. adding header #include <optional> will make code more portable, because when I used g++ directly there were compilation error

#if 0
    AudioFileData audioData = decodeAudioWithFFmpeg( filePath );
#else
    AudioFileData audioData = decodeAudioWithAvcpp( filePath );
#endif

---

BTW. About vcpkg: when we install:

./vcpkg/vcpkg install avcpp
Then it is installing * ffmpeg[avcodec,avdevice,avfilter,avformat,core,gpl,postproc,swresample,swscale]:[email protected]#2, and one of them is ffmpeg[gpl]. So avcpp when installing with VCPKG is having licence GPL (as I understand licenses).
So I'm suggesting to have more options of avcpp in vcpkg eg. avcpp[with-gpl] and avcpp[no-gpl]

---

PS: I'm creating article for site cpp0x.pl (article in Polish) where I'm trying to find IMO best way how to play music from C++ for programmers who don't know much about playing musics, codecs, etc. (article mostly for students). Can I paste Your code in the article?

[h4tr3d]

So I'm suggesting to have more options of avcpp in vcpkg eg. avcpp[with-gpl] and avcpp[no-gpl]

vcpkg support was not introduced by me. I have no time to dive deeply into details of the vcpkg work and ports Features supports (and its transitivity).

Can I paste Your code in the article?

Sure! I suggest only remove debug output just to reduce lines of code.

[baziorek]

So I'm suggesting to have more options of avcpp in vcpkg eg. avcpp[with-gpl] and avcpp[no-gpl]

vcpkg support was not introduced by me. I have no time to dive deeply into details of the vcpkg work and ports Features supports (and its transitivity).

I've created issue for VCPKG microsoft/vcpkg#43345 - as I remember community is adding this fast.

---

Can I paste Your code in the article?

Sure! I suggest only remove debug output just to reduce lines of code.
Thanks a lot!
Here is modified example from You:

#include <iostream>
#include <vector>
#include <thread>
#include <chrono>
#include <filesystem>
#include <optional>

#include <SFML/Audio.hpp>

#include "avcpp/av.h"
#include "avcpp/format.h"
#include "avcpp/formatcontext.h"
#include "avcpp/codec.h"
#include "avcpp/codeccontext.h"
#include "avcpp/audioresampler.h"

struct AudioFileData
{
    int channels;
    int sampleRate;

    std::vector < sf::Int16 > pcmAudioBuffer;

    bool successReading;
};

AudioFileData decodeAudioWithAvcpp(const std::filesystem::path& filePath);
void          playSound( const AudioFileData & audioFileData );

int main( int argc, char * * argv ) {
    if( argc < 2 ) {
        std::cerr << "Usage: " << argv[ 0 ] << " <audio file>" << std::endl;
        return 1;
    }

    const std::string filePath = argv[ 1 ];

    AudioFileData audioData = decodeAudioWithAvcpp( filePath );

    if( audioData.pcmAudioBuffer.empty() )
        std::cerr << "Failed to decode or play audio from file " << filePath << std::endl;
    else
        playSound( audioData );
}

AudioFileData decodeAudioWithAvcpp(const std::filesystem::path& filePath)
{
    /// Reference to https://github.com/h4tr3d/avcpp/blob/master/example/api2-samples/api2-decode-audio.cpp

    std::error_code ec;
    std::optional<size_t> audioStream;

    av::init();
    av::setFFmpegLoggingLevel(AV_LOG_DEBUG);

    av::FormatContext ictx;
    av::Stream ast;
    av::AudioDecoderContext adec;

    ictx.openInput(filePath); // exception on fail
    ictx.findStreamInfo();    // process input and collect streams information

    /// Lookup first audio stream
    for (auto i = 0u; i < ictx.streamsCount(); ++i) {
        auto st = ictx.stream(i);
        if (st.isAudio()) {
            audioStream = i;
            ast = st;
            break;
        }
    }

    if (ast.isNull()) {
        std::cerr << "Audio stream not found\n";
        exit(1);
    }

    if (ast.isValid()) {
        adec = av::AudioDecoderContext(ast);
        adec.open(av::Codec()); // throw on error
    }

    /// Resample: required to convert into proper SFML format. SFML supports only PCM_S16 samples
    av::AudioResampler resampler(adec.channelLayout(), adec.sampleRate(), AV_SAMPLE_FMT_S16,
                                 adec.channelLayout(), adec.sampleRate(), adec.sampleFormat());


    /// Output data
    AudioFileData audioFileData;
    audioFileData.sampleRate = adec.sampleRate();
    audioFileData.channels = adec.channels();

    /// Process input
    while (true) {
        auto pkt = ictx.readPacket(ec);
        // handle in proper way, like EGAIN
        if (ec) {
            std::clog << "Packet reading error: " << ec << ", " << ec.message() << std::endl;
            break;
        }

        // EOF
        if (!pkt) {
            break;
        }

        // Skip non-audio packets
        if (pkt.streamIndex() != *audioStream)
            continue;

        std::clog << "Read packet: " << pkt.pts() << " / " << pkt.pts().seconds() << " / " << pkt.timeBase() << " / st: " << pkt.streamIndex() << std::endl;

        auto samples = adec.decode(pkt);

        std::clog << "  Samples [in]: " << samples.samplesCount()
                  << ", ch: "   << samples.channelsCount()
                  << ", freq: " << samples.sampleRate()
                  << ", name: " << samples.channelsLayoutString()
                  << ", pts: "  << samples.pts().seconds()
                  << ", ref="   << samples.isReferenced() << ":" << samples.refCount()
                  << std::endl;

        // skip empty frames if any
        if (!samples)
            continue;

        // Resample it
        resampler.push(samples);

        /// Pop resampler data: in common way, single input samples frame can produce multiple output. Or empty, if single output frame
        /// requires more input one.
        /// Note, we can only push data here and process it at once on the Flushing. But too much more memory is needed.

        auto ouSamples = av::AudioSamples::null();
        while ((ouSamples = resampler.pop(samples.samplesCount(), ec))) {
            std::clog << "  Samples [ou]: " << ouSamples.samplesCount()
                      << ", ch: " << ouSamples.channelsCount()
                      << ", freq: " << ouSamples.sampleRate()
                      << ", name: " << ouSamples.channelsLayoutString()
                      << ", pts: " << ouSamples.pts().seconds()
                      << ", ref=" << ouSamples.isReferenced() << ":" << ouSamples.refCount()
                      << ", size=" << ouSamples.size()
                      << std::endl;

            // UB due to strict aliasing...
            audioFileData.pcmAudioBuffer.insert(std::end(audioFileData.pcmAudioBuffer),
                    (sf::Int16 *) ouSamples.data(),
                    (sf::Int16 *) ouSamples.data() + ouSamples.size() / sizeof( sf::Int16 ) );
        }
    }

    /// Flush resampler
    {
        while (resampler.delay()) {
            auto ouSamples = resampler.pop(0, ec); // request all remain data at once

            if (ec) {
                std::clog << "Resampling status: " << ec << ", text: " << ec.message() << std::endl;
                break;
            } else if (!ouSamples) {
                break;
            } else {
                std::clog << "  Samples [ou]: " << ouSamples.samplesCount()
                          << ", ch: " << ouSamples.channelsCount()
                          << ", freq: " << ouSamples.sampleRate()
                          << ", name: " << ouSamples.channelsLayoutString()
                          << ", pts: " << ouSamples.pts().seconds()
                          << ", ref=" << ouSamples.isReferenced() << ":" << ouSamples.refCount()
                          << ", size=" << ouSamples.size()
                          << std::endl;

            // UB due to strict aliasing...
            audioFileData.pcmAudioBuffer.insert(std::end(audioFileData.pcmAudioBuffer),
                    (sf::Int16 *) ouSamples.data(),
                    (sf::Int16 *) ouSamples.data() + ouSamples.size() / sizeof( sf::Int16 ) );
            }
        }
    }

    return audioFileData;
}

void playSound( const AudioFileData & audioFileData ) {
    sf::SoundBuffer soundBuffer;
    if( !soundBuffer.loadFromSamples( audioFileData.pcmAudioBuffer.data(), audioFileData.pcmAudioBuffer.size(), audioFileData.channels, audioFileData.sampleRate ) ) {
        std::cerr << "Error: Could not load buffer for playback" << std::endl;
        return;
    }

    sf::Sound sound;
    sound.setBuffer( soundBuffer );
    sound.play();

    std::cout << "Playing file, duration: " << soundBuffer.getDuration().asSeconds() << " seconds" << std::endl;
    while( sound.getStatus() == sf::Sound::Playing ) {
        std::this_thread::sleep_for( std::chrono::milliseconds( 500 ) );
    }

    std::cout << "Playback finished." << std::endl;
}

I'll let You know when my article will be published here: https://cpp0x.pl/ - it will be in Polish, but hopefully many people will found out about You great work!