Everywhere: Hoist the Libraries folder to the top-level

Author: https://github.com/trflynn89
Commit: 93712b24bf
Pull-request: https://github.com/LadybirdBrowser/ladybird/pull/2256
Reviewed-by: https://github.com/sideshowbarker

Libraries/LibMedia/Audio/FFmpegLoader.cpp

@@ -0,0 +1,349 @@
+/*
+ * Copyright (c) 2024, Jelle Raaijmakers <jelle@ladybird.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include "FFmpegLoader.h"
+#include <AK/BitStream.h>
+#include <AK/NumericLimits.h>
+#include <LibCore/System.h>
+
+#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(59, 24, 100)
+#    define USE_FFMPEG_CH_LAYOUT
+#endif
+#if LIBAVFORMAT_VERSION_INT >= AV_VERSION_INT(59, 0, 100)
+#    define USE_CONSTIFIED_POINTERS
+#endif
+
+namespace Audio {
+
+static constexpr int BUFFER_MAX_PROBE_SIZE = 64 * KiB;
+
+FFmpegIOContext::FFmpegIOContext(AVIOContext* avio_context)
+    : m_avio_context(avio_context)
+{
+}
+
+FFmpegIOContext::~FFmpegIOContext()
+{
+    // NOTE: free the buffer inside the AVIO context, since it might be changed since its initial allocation
+    av_free(m_avio_context->buffer);
+    avio_context_free(&m_avio_context);
+}
+
+ErrorOr<NonnullOwnPtr<FFmpegIOContext>, LoaderError> FFmpegIOContext::create(AK::SeekableStream& stream)
+{
+    auto* avio_buffer = av_malloc(PAGE_SIZE);
+    if (avio_buffer == nullptr)
+        return LoaderError { LoaderError::Category::IO, "Failed to allocate AVIO buffer" };
+
+    // This AVIOContext explains to avformat how to interact with our stream
+    auto* avio_context = avio_alloc_context(
+        static_cast<unsigned char*>(avio_buffer),
+        PAGE_SIZE,
+        0,
+        &stream,
+        [](void* opaque, u8* buffer, int size) -> int {
+            auto& stream = *static_cast<SeekableStream*>(opaque);
+            AK::Bytes buffer_bytes { buffer, AK::min<size_t>(size, PAGE_SIZE) };
+            auto read_bytes_or_error = stream.read_some(buffer_bytes);
+            if (read_bytes_or_error.is_error()) {
+                if (read_bytes_or_error.error().code() == EOF)
+                    return AVERROR_EOF;
+                return AVERROR_UNKNOWN;
+            }
+            int number_of_bytes_read = read_bytes_or_error.value().size();
+            if (number_of_bytes_read == 0)
+                return AVERROR_EOF;
+            return number_of_bytes_read;
+        },
+        nullptr,
+        [](void* opaque, int64_t offset, int whence) -> int64_t {
+            whence &= ~AVSEEK_FORCE;
+
+            auto& stream = *static_cast<SeekableStream*>(opaque);
+            if (whence == AVSEEK_SIZE)
+                return static_cast<int64_t>(stream.size().value());
+
+            auto seek_mode_from_whence = [](int origin) -> SeekMode {
+                if (origin == SEEK_CUR)
+                    return SeekMode::FromCurrentPosition;
+                if (origin == SEEK_END)
+                    return SeekMode::FromEndPosition;
+                return SeekMode::SetPosition;
+            };
+            auto offset_or_error = stream.seek(offset, seek_mode_from_whence(whence));
+            if (offset_or_error.is_error())
+                return -EIO;
+            return 0;
+        });
+    if (avio_context == nullptr) {
+        av_free(avio_buffer);
+        return LoaderError { LoaderError::Category::IO, "Failed to allocate AVIO context" };
+    }
+
+    return make<FFmpegIOContext>(avio_context);
+}
+
+FFmpegLoaderPlugin::FFmpegLoaderPlugin(NonnullOwnPtr<SeekableStream> stream, NonnullOwnPtr<FFmpegIOContext> io_context)
+    : LoaderPlugin(move(stream))
+    , m_io_context(move(io_context))
+{
+}
+
+FFmpegLoaderPlugin::~FFmpegLoaderPlugin()
+{
+    if (m_frame != nullptr)
+        av_frame_free(&m_frame);
+    if (m_packet != nullptr)
+        av_packet_free(&m_packet);
+    if (m_codec_context != nullptr)
+        avcodec_free_context(&m_codec_context);
+    if (m_format_context != nullptr)
+        avformat_close_input(&m_format_context);
+}
+
+ErrorOr<NonnullOwnPtr<LoaderPlugin>, LoaderError> FFmpegLoaderPlugin::create(NonnullOwnPtr<SeekableStream> stream)
+{
+    auto io_context = TRY(FFmpegIOContext::create(*stream));
+    auto loader = make<FFmpegLoaderPlugin>(move(stream), move(io_context));
+    TRY(loader->initialize());
+    return loader;
+}
+
+MaybeLoaderError FFmpegLoaderPlugin::initialize()
+{
+    // Open the container
+    m_format_context = avformat_alloc_context();
+    if (m_format_context == nullptr)
+        return LoaderError { LoaderError::Category::IO, "Failed to allocate format context" };
+    m_format_context->pb = m_io_context->avio_context();
+    if (avformat_open_input(&m_format_context, nullptr, nullptr, nullptr) < 0)
+        return LoaderError { LoaderError::Category::IO, "Failed to open input for format parsing" };
+
+    // Read stream info; doing this is required for headerless formats like MPEG
+    if (avformat_find_stream_info(m_format_context, nullptr) < 0)
+        return LoaderError { LoaderError::Category::IO, "Failed to find stream info" };
+
+#ifdef USE_CONSTIFIED_POINTERS
+    AVCodec const* codec {};
+#else
+    AVCodec* codec {};
+#endif
+    // Find the best stream to play within the container
+    int best_stream_index = av_find_best_stream(m_format_context, AVMediaType::AVMEDIA_TYPE_AUDIO, -1, -1, &codec, 0);
+    if (best_stream_index == AVERROR_STREAM_NOT_FOUND)
+        return LoaderError { LoaderError::Category::Format, "No audio stream found in container" };
+    if (best_stream_index == AVERROR_DECODER_NOT_FOUND)
+        return LoaderError { LoaderError::Category::Format, "No suitable decoder found for stream" };
+    if (best_stream_index < 0)
+        return LoaderError { LoaderError::Category::Format, "Failed to find an audio stream" };
+    m_audio_stream = m_format_context->streams[best_stream_index];
+
+    // Set up the context to decode the audio stream
+    m_codec_context = avcodec_alloc_context3(codec);
+    if (m_codec_context == nullptr)
+        return LoaderError { LoaderError::Category::IO, "Failed to allocate the codec context" };
+
+    if (avcodec_parameters_to_context(m_codec_context, m_audio_stream->codecpar) < 0)
+        return LoaderError { LoaderError::Category::IO, "Failed to copy codec parameters" };
+
+    m_codec_context->pkt_timebase = m_audio_stream->time_base;
+    m_codec_context->thread_count = AK::min(static_cast<int>(Core::System::hardware_concurrency()), 4);
+
+    if (avcodec_open2(m_codec_context, codec, nullptr) < 0)
+        return LoaderError { LoaderError::Category::IO, "Failed to open input for decoding" };
+
+    // This is an initial estimate of the total number of samples in the stream.
+    // During decoding, we might need to increase the number as more frames come in.
+    double duration_in_seconds = static_cast<double>(m_audio_stream->duration) * time_base();
+    if (duration_in_seconds < 0)
+        return LoaderError { LoaderError::Category::Format, "Negative stream duration" };
+    m_total_samples = AK::round_to<decltype(m_total_samples)>(sample_rate() * duration_in_seconds);
+
+    // Allocate packet (logical chunk of data) and frame (video / audio frame) buffers
+    m_packet = av_packet_alloc();
+    if (m_packet == nullptr)
+        return LoaderError { LoaderError::Category::IO, "Failed to allocate packet" };
+
+    m_frame = av_frame_alloc();
+    if (m_frame == nullptr)
+        return LoaderError { LoaderError::Category::IO, "Failed to allocate frame" };
+
+    return {};
+}
+
+double FFmpegLoaderPlugin::time_base() const
+{
+    return av_q2d(m_audio_stream->time_base);
+}
+
+bool FFmpegLoaderPlugin::sniff(SeekableStream& stream)
+{
+    auto io_context = MUST(FFmpegIOContext::create(stream));
+#ifdef USE_CONSTIFIED_POINTERS
+    AVInputFormat const* detected_format {};
+#else
+    AVInputFormat* detected_format {};
+#endif
+    auto score = av_probe_input_buffer2(io_context->avio_context(), &detected_format, nullptr, nullptr, 0, BUFFER_MAX_PROBE_SIZE);
+    return score > 0;
+}
+
+static ErrorOr<FixedArray<Sample>> extract_samples_from_frame(AVFrame& frame)
+{
+    size_t number_of_samples = frame.nb_samples;
+    VERIFY(number_of_samples > 0);
+
+#ifdef USE_FFMPEG_CH_LAYOUT
+    size_t number_of_channels = frame.ch_layout.nb_channels;
+#else
+    size_t number_of_channels = frame.channels;
+#endif
+    auto format = static_cast<AVSampleFormat>(frame.format);
+    auto packed_format = av_get_packed_sample_fmt(format);
+    auto is_planar = av_sample_fmt_is_planar(format) == 1;
+
+    // FIXME: handle number_of_channels > 2
+    if (number_of_channels != 1 && number_of_channels != 2)
+        return Error::from_string_view("Unsupported number of channels"sv);
+
+    switch (format) {
+    case AV_SAMPLE_FMT_FLTP:
+    case AV_SAMPLE_FMT_S16:
+    case AV_SAMPLE_FMT_S32:
+        break;
+    default:
+        // FIXME: handle other formats
+        return Error::from_string_view("Unsupported sample format"sv);
+    }
+
+    auto get_plane_pointer = [&](size_t channel_index) -> uint8_t* {
+        return is_planar ? frame.extended_data[channel_index] : frame.extended_data[0];
+    };
+    auto index_in_plane = [&](size_t sample_index, size_t channel_index) {
+        if (is_planar)
+            return sample_index;
+        return sample_index * number_of_channels + channel_index;
+    };
+    auto read_sample = [&](uint8_t* data, size_t index) -> float {
+        switch (packed_format) {
+        case AV_SAMPLE_FMT_FLT:
+            return reinterpret_cast<float*>(data)[index];
+        case AV_SAMPLE_FMT_S16:
+            return reinterpret_cast<i16*>(data)[index] / static_cast<float>(NumericLimits<i16>::max());
+        case AV_SAMPLE_FMT_S32:
+            return reinterpret_cast<i32*>(data)[index] / static_cast<float>(NumericLimits<i32>::max());
+        default:
+            VERIFY_NOT_REACHED();
+        }
+    };
+
+    auto samples = TRY(FixedArray<Sample>::create(number_of_samples));
+    for (size_t sample = 0; sample < number_of_samples; ++sample) {
+        if (number_of_channels == 1) {
+            samples.unchecked_at(sample) = Sample { read_sample(get_plane_pointer(0), index_in_plane(sample, 0)) };
+        } else {
+            samples.unchecked_at(sample) = Sample {
+                read_sample(get_plane_pointer(0), index_in_plane(sample, 0)),
+                read_sample(get_plane_pointer(1), index_in_plane(sample, 1)),
+            };
+        }
+    }
+    return samples;
+}
+
+ErrorOr<Vector<FixedArray<Sample>>, LoaderError> FFmpegLoaderPlugin::load_chunks(size_t samples_to_read_from_input)
+{
+    Vector<FixedArray<Sample>> chunks {};
+
+    do {
+        // Obtain a packet
+        auto read_frame_error = av_read_frame(m_format_context, m_packet);
+        if (read_frame_error < 0) {
+            if (read_frame_error == AVERROR_EOF)
+                break;
+            return LoaderError { LoaderError::Category::IO, "Failed to read frame" };
+        }
+        if (m_packet->stream_index != m_audio_stream->index) {
+            av_packet_unref(m_packet);
+            continue;
+        }
+
+        // Send the packet to the decoder
+        if (avcodec_send_packet(m_codec_context, m_packet) < 0)
+            return LoaderError { LoaderError::Category::IO, "Failed to send packet" };
+        av_packet_unref(m_packet);
+
+        // Ask the decoder for a new frame. We might not have sent enough data yet
+        auto receive_frame_error = avcodec_receive_frame(m_codec_context, m_frame);
+        if (receive_frame_error != 0) {
+            if (receive_frame_error == AVERROR(EAGAIN))
+                continue;
+            if (receive_frame_error == AVERROR_EOF)
+                break;
+            return LoaderError { LoaderError::Category::IO, "Failed to receive frame" };
+        }
+
+        chunks.append(TRY(extract_samples_from_frame(*m_frame)));
+
+        // Use the frame's presentation timestamp to set the number of loaded samples
+        m_loaded_samples = static_cast<int>(m_frame->pts * sample_rate() * time_base());
+        if (m_loaded_samples > m_total_samples) [[unlikely]]
+            m_total_samples = m_loaded_samples;
+
+        samples_to_read_from_input -= AK::min(samples_to_read_from_input, m_frame->nb_samples);
+    } while (samples_to_read_from_input > 0);
+
+    return chunks;
+}
+
+MaybeLoaderError FFmpegLoaderPlugin::reset()
+{
+    return seek(0);
+}
+
+MaybeLoaderError FFmpegLoaderPlugin::seek(int sample_index)
+{
+    auto sample_position_in_seconds = static_cast<double>(sample_index) / sample_rate();
+    auto sample_timestamp = AK::round_to<int64_t>(sample_position_in_seconds / time_base());
+
+    if (av_seek_frame(m_format_context, m_audio_stream->index, sample_timestamp, AVSEEK_FLAG_ANY) < 0)
+        return LoaderError { LoaderError::Category::IO, "Failed to seek" };
+    avcodec_flush_buffers(m_codec_context);
+
+    m_loaded_samples = sample_index;
+    return {};
+}
+
+u32 FFmpegLoaderPlugin::sample_rate()
+{
+    VERIFY(m_codec_context != nullptr);
+    return m_codec_context->sample_rate;
+}
+
+u16 FFmpegLoaderPlugin::num_channels()
+{
+    VERIFY(m_codec_context != nullptr);
+#ifdef USE_FFMPEG_CH_LAYOUT
+    return m_codec_context->ch_layout.nb_channels;
+#else
+    return m_codec_context->channels;
+#endif
+}
+
+PcmSampleFormat FFmpegLoaderPlugin::pcm_format()
+{
+    // FIXME: pcm_format() is unused, always return Float for now
+    return PcmSampleFormat::Float32;
+}
+
+ByteString FFmpegLoaderPlugin::format_name()
+{
+    if (!m_format_context)
+        return "unknown";
+    return m_format_context->iformat->name;
+}
+
+}

Libraries/LibMedia/Audio/FFmpegLoader.h

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2024, Jelle Raaijmakers <jelle@ladybird.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include "Loader.h"
+#include <AK/Error.h>
+#include <AK/NonnullOwnPtr.h>
+
+extern "C" {
+#include <libavcodec/avcodec.h>
+#include <libavformat/avformat.h>
+#include <libavutil/samplefmt.h>
+}
+
+namespace Audio {
+
+class FFmpegIOContext {
+public:
+    explicit FFmpegIOContext(AVIOContext*);
+    ~FFmpegIOContext();
+
+    static ErrorOr<NonnullOwnPtr<FFmpegIOContext>, LoaderError> create(AK::SeekableStream& stream);
+
+    AVIOContext* avio_context() const { return m_avio_context; }
+
+private:
+    AVIOContext* m_avio_context { nullptr };
+};
+
+class FFmpegLoaderPlugin : public LoaderPlugin {
+public:
+    explicit FFmpegLoaderPlugin(NonnullOwnPtr<SeekableStream>, NonnullOwnPtr<FFmpegIOContext>);
+    virtual ~FFmpegLoaderPlugin();
+
+    static bool sniff(SeekableStream& stream);
+    static ErrorOr<NonnullOwnPtr<LoaderPlugin>, LoaderError> create(NonnullOwnPtr<SeekableStream>);
+
+    virtual ErrorOr<Vector<FixedArray<Sample>>, LoaderError> load_chunks(size_t samples_to_read_from_input) override;
+
+    virtual MaybeLoaderError reset() override;
+    virtual MaybeLoaderError seek(int sample_index) override;
+
+    virtual int loaded_samples() override { return m_loaded_samples; }
+    virtual int total_samples() override { return m_total_samples; }
+    virtual u32 sample_rate() override;
+    virtual u16 num_channels() override;
+    virtual PcmSampleFormat pcm_format() override;
+    virtual ByteString format_name() override;
+
+private:
+    MaybeLoaderError initialize();
+    double time_base() const;
+
+    AVStream* m_audio_stream;
+    AVCodecContext* m_codec_context { nullptr };
+    AVFormatContext* m_format_context { nullptr };
+    AVFrame* m_frame { nullptr };
+    NonnullOwnPtr<FFmpegIOContext> m_io_context;
+    int m_loaded_samples { 0 };
+    AVPacket* m_packet { nullptr };
+    int m_total_samples { 0 };
+};
+
+}

Libraries/LibMedia/Audio/Forward.h

@@ -0,0 +1,16 @@
+/*
+ * Copyright (c) 2023, Tim Flynn <trflynn89@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+namespace Audio {
+
+class ConnectionToServer;
+class Loader;
+class PlaybackStream;
+struct Sample;
+
+}

Libraries/LibMedia/Audio/Loader.cpp

@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2018-2023, the SerenityOS developers.
+ * Copyright (c) 2024, Jelle Raaijmakers <jelle@ladybird.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include "Loader.h"
+#include "FFmpegLoader.h"
+#include <AK/TypedTransfer.h>
+#include <LibCore/MappedFile.h>
+
+namespace Audio {
+
+LoaderPlugin::LoaderPlugin(NonnullOwnPtr<SeekableStream> stream)
+    : m_stream(move(stream))
+{
+}
+
+Loader::Loader(NonnullOwnPtr<LoaderPlugin> plugin)
+    : m_plugin(move(plugin))
+{
+}
+
+struct LoaderPluginInitializer {
+    bool (*sniff)(SeekableStream&);
+    ErrorOr<NonnullOwnPtr<LoaderPlugin>, LoaderError> (*create)(NonnullOwnPtr<SeekableStream>);
+};
+
+static constexpr LoaderPluginInitializer s_initializers[] = {
+    { FFmpegLoaderPlugin::sniff, FFmpegLoaderPlugin::create },
+};
+
+ErrorOr<NonnullRefPtr<Loader>, LoaderError> Loader::create(StringView path)
+{
+    auto stream = TRY(Core::MappedFile::map(path, Core::MappedFile::Mode::ReadOnly));
+    auto plugin = TRY(Loader::create_plugin(move(stream)));
+    return adopt_ref(*new (nothrow) Loader(move(plugin)));
+}
+
+ErrorOr<NonnullRefPtr<Loader>, LoaderError> Loader::create(ReadonlyBytes buffer)
+{
+    auto stream = TRY(try_make<FixedMemoryStream>(buffer));
+    auto plugin = TRY(Loader::create_plugin(move(stream)));
+    return adopt_ref(*new (nothrow) Loader(move(plugin)));
+}
+
+ErrorOr<NonnullOwnPtr<LoaderPlugin>, LoaderError> Loader::create_plugin(NonnullOwnPtr<SeekableStream> stream)
+{
+    for (auto const& loader : s_initializers) {
+        if (loader.sniff(*stream)) {
+            TRY(stream->seek(0, SeekMode::SetPosition));
+            return loader.create(move(stream));
+        }
+        TRY(stream->seek(0, SeekMode::SetPosition));
+    }
+
+    return LoaderError { "No loader plugin available" };
+}
+
+LoaderSamples Loader::get_more_samples(size_t samples_to_read_from_input)
+{
+    if (m_plugin_at_end_of_stream && m_buffer.is_empty())
+        return FixedArray<Sample> {};
+
+    size_t remaining_samples = total_samples() - loaded_samples();
+    size_t samples_to_read = min(remaining_samples, samples_to_read_from_input);
+    auto samples = TRY(FixedArray<Sample>::create(samples_to_read));
+
+    size_t sample_index = 0;
+
+    if (m_buffer.size() > 0) {
+        size_t to_transfer = min(m_buffer.size(), samples_to_read);
+        AK::TypedTransfer<Sample>::move(samples.data(), m_buffer.data(), to_transfer);
+        if (to_transfer < m_buffer.size())
+            m_buffer.remove(0, to_transfer);
+        else
+            m_buffer.clear_with_capacity();
+
+        sample_index += to_transfer;
+    }
+
+    while (sample_index < samples_to_read) {
+        auto chunk_data = TRY(m_plugin->load_chunks(samples_to_read - sample_index));
+        chunk_data.remove_all_matching([](auto& chunk) { return chunk.is_empty(); });
+        if (chunk_data.is_empty()) {
+            m_plugin_at_end_of_stream = true;
+            break;
+        }
+        for (auto& chunk : chunk_data) {
+            if (sample_index < samples_to_read) {
+                auto count = min(samples_to_read - sample_index, chunk.size());
+                AK::TypedTransfer<Sample>::move(samples.span().offset(sample_index), chunk.data(), count);
+                // We didn't read all of the chunk; transfer the rest into the buffer.
+                if (count < chunk.size()) {
+                    auto remaining_samples_count = chunk.size() - count;
+                    // We will always have an empty buffer at this point!
+                    TRY(m_buffer.try_append(chunk.span().offset(count), remaining_samples_count));
+                }
+            } else {
+                // We're now past what the user requested. Transfer the entirety of the data into the buffer.
+                TRY(m_buffer.try_append(chunk.data(), chunk.size()));
+            }
+            sample_index += chunk.size();
+        }
+    }
+
+    return samples;
+}
+
+}

Libraries/LibMedia/Audio/Loader.h

@@ -0,0 +1,115 @@
+/*
+ * Copyright (c) 2018-2022, the SerenityOS developers.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include "LoaderError.h"
+#include "Sample.h"
+#include "SampleFormats.h"
+#include <AK/Error.h>
+#include <AK/FixedArray.h>
+#include <AK/NonnullOwnPtr.h>
+#include <AK/NonnullRefPtr.h>
+#include <AK/RefCounted.h>
+#include <AK/Stream.h>
+#include <AK/StringView.h>
+#include <AK/Vector.h>
+
+namespace Audio {
+
+// Experimentally determined to be a decent buffer size on i686:
+// 4K (the default) is slightly worse, and 64K is much worse.
+// At sufficiently large buffer sizes, the advantage of infrequent read() calls is outweighed by the memmove() overhead.
+// There was no intensive fine-tuning done to determine this value, so improvements may definitely be possible.
+constexpr size_t const loader_buffer_size = 8 * KiB;
+
+// Two seek points should ideally not be farther apart than this.
+// This variable is a heuristic for seek table-constructing loaders.
+constexpr u64 const maximum_seekpoint_distance_ms = 1000;
+// Seeking should be at least as precise as this.
+// That means: The actual achieved seek position must not be more than this amount of time before the requested seek position.
+constexpr u64 const seek_tolerance_ms = 5000;
+
+using LoaderSamples = ErrorOr<FixedArray<Sample>, LoaderError>;
+using MaybeLoaderError = ErrorOr<void, LoaderError>;
+
+class LoaderPlugin {
+public:
+    explicit LoaderPlugin(NonnullOwnPtr<SeekableStream> stream);
+    virtual ~LoaderPlugin() = default;
+
+    // Load as many audio chunks as necessary to get up to the required samples.
+    // A chunk can be anything that is convenient for the plugin to load in one go without requiring to move samples around different buffers.
+    // For example: A FLAC, MP3 or QOA frame.
+    // The chunks are returned in a vector, so the loader can simply add chunks until the requested sample amount is reached.
+    // The sample count MAY be surpassed, but only as little as possible. It CAN be undershot when the end of the stream is reached.
+    // If the loader has no chunking limitations (e.g. WAV), it may return a single exact-sized chunk.
+    virtual ErrorOr<Vector<FixedArray<Sample>>, LoaderError> load_chunks(size_t samples_to_read_from_input) = 0;
+
+    virtual MaybeLoaderError reset() = 0;
+
+    virtual MaybeLoaderError seek(int const sample_index) = 0;
+
+    // total_samples() and loaded_samples() should be independent
+    // of the number of channels.
+    //
+    // For example, with a three-second-long, stereo, 44.1KHz audio file:
+    //    num_channels() should return 2
+    //    sample_rate() should return 44100 (each channel is sampled at this rate)
+    //    total_samples() should return 132300 (sample_rate * three seconds)
+    virtual int loaded_samples() = 0;
+    virtual int total_samples() = 0;
+    virtual u32 sample_rate() = 0;
+    virtual u16 num_channels() = 0;
+
+    // Human-readable name of the file format, of the form <full abbreviation> (.<ending>)
+    virtual ByteString format_name() = 0;
+    virtual PcmSampleFormat pcm_format() = 0;
+
+protected:
+    NonnullOwnPtr<SeekableStream> m_stream;
+};
+
+class Loader : public RefCounted<Loader> {
+public:
+    static ErrorOr<NonnullRefPtr<Loader>, LoaderError> create(StringView path);
+    static ErrorOr<NonnullRefPtr<Loader>, LoaderError> create(ReadonlyBytes buffer);
+
+    // Will only read less samples if we're at the end of the stream.
+    LoaderSamples get_more_samples(size_t samples_to_read_from_input = 128 * KiB);
+
+    MaybeLoaderError reset() const
+    {
+        m_plugin_at_end_of_stream = false;
+        return m_plugin->reset();
+    }
+    MaybeLoaderError seek(int const position) const
+    {
+        m_buffer.clear_with_capacity();
+        m_plugin_at_end_of_stream = false;
+        return m_plugin->seek(position);
+    }
+
+    int loaded_samples() const { return m_plugin->loaded_samples() - (int)m_buffer.size(); }
+    int total_samples() const { return m_plugin->total_samples(); }
+    u32 sample_rate() const { return m_plugin->sample_rate(); }
+    u16 num_channels() const { return m_plugin->num_channels(); }
+    ByteString format_name() const { return m_plugin->format_name(); }
+    u16 bits_per_sample() const { return pcm_bits_per_sample(m_plugin->pcm_format()); }
+    PcmSampleFormat pcm_format() const { return m_plugin->pcm_format(); }
+
+private:
+    static ErrorOr<NonnullOwnPtr<LoaderPlugin>, LoaderError> create_plugin(NonnullOwnPtr<SeekableStream> stream);
+
+    explicit Loader(NonnullOwnPtr<LoaderPlugin>);
+
+    mutable NonnullOwnPtr<LoaderPlugin> m_plugin;
+    // The plugin can signal an end of stream by returning no (or only empty) chunks.
+    mutable bool m_plugin_at_end_of_stream { false };
+    mutable Vector<Sample, loader_buffer_size> m_buffer;
+};
+
+}

Libraries/LibMedia/Audio/LoaderError.h

@@ -0,0 +1,96 @@
+/*
+ * Copyright (c) 2021, kleines Filmröllchen <filmroellchen@serenityos.org>.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/DeprecatedFlyString.h>
+#include <AK/Error.h>
+#include <errno.h>
+
+namespace Audio {
+
+struct LoaderError {
+
+    enum class Category : u32 {
+        // The error category is unknown.
+        Unknown = 0,
+        IO,
+        // The read file doesn't follow the file format.
+        Format,
+        // Equivalent to an ASSERT(), except non-crashing.
+        Internal,
+        // The loader encountered something in the format that is not yet implemented.
+        Unimplemented,
+    };
+    Category category { Category::Unknown };
+    // Binary index: where in the file the error occurred.
+    size_t index { 0 };
+    DeprecatedFlyString description { ByteString::empty() };
+
+    constexpr LoaderError() = default;
+    LoaderError(Category category, size_t index, DeprecatedFlyString description)
+        : category(category)
+        , index(index)
+        , description(move(description))
+    {
+    }
+    LoaderError(DeprecatedFlyString description)
+        : description(move(description))
+    {
+    }
+    LoaderError(Category category, DeprecatedFlyString description)
+        : category(category)
+        , description(move(description))
+    {
+    }
+
+    LoaderError(LoaderError&) = default;
+    LoaderError(LoaderError&&) = default;
+
+    LoaderError(Error&& error)
+    {
+        if (error.is_errno()) {
+            auto code = error.code();
+            description = ByteString::formatted("{} ({})", strerror(code), code);
+            if (code == EBADF || code == EBUSY || code == EEXIST || code == EIO || code == EISDIR || code == ENOENT || code == ENOMEM || code == EPIPE)
+                category = Category::IO;
+        } else {
+            description = error.string_literal();
+        }
+    }
+};
+
+}
+
+namespace AK {
+
+template<>
+struct Formatter<Audio::LoaderError> : Formatter<FormatString> {
+    ErrorOr<void> format(FormatBuilder& builder, Audio::LoaderError const& error)
+    {
+        StringView category;
+        switch (error.category) {
+        case Audio::LoaderError::Category::Unknown:
+            category = "Unknown"sv;
+            break;
+        case Audio::LoaderError::Category::IO:
+            category = "I/O"sv;
+            break;
+        case Audio::LoaderError::Category::Format:
+            category = "Format"sv;
+            break;
+        case Audio::LoaderError::Category::Internal:
+            category = "Internal"sv;
+            break;
+        case Audio::LoaderError::Category::Unimplemented:
+            category = "Unimplemented"sv;
+            break;
+        }
+        return Formatter<FormatString>::format(builder, "{} error: {} (at {})"sv, category, error.description, error.index);
+    }
+};
+
+}

Libraries/LibMedia/Audio/PlaybackStream.cpp

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2023, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include "PlaybackStream.h"
+
+#include <AK/Platform.h>
+#include <LibCore/ThreadedPromise.h>
+
+#if defined(HAVE_PULSEAUDIO)
+#    include "PlaybackStreamPulseAudio.h"
+#elif defined(AK_OS_MACOS)
+#    include "PlaybackStreamAudioUnit.h"
+#elif defined(AK_OS_ANDROID)
+#    include "PlaybackStreamOboe.h"
+#endif
+
+namespace Audio {
+
+ErrorOr<NonnullRefPtr<PlaybackStream>> PlaybackStream::create(OutputState initial_output_state, u32 sample_rate, u8 channels, u32 target_latency_ms, AudioDataRequestCallback&& data_request_callback)
+{
+    VERIFY(data_request_callback);
+    // Create the platform-specific implementation for this stream.
+#if defined(HAVE_PULSEAUDIO)
+    return PlaybackStreamPulseAudio::create(initial_output_state, sample_rate, channels, target_latency_ms, move(data_request_callback));
+#elif defined(AK_OS_MACOS)
+    return PlaybackStreamAudioUnit::create(initial_output_state, sample_rate, channels, target_latency_ms, move(data_request_callback));
+#elif defined(AK_OS_ANDROID)
+    return PlaybackStreamOboe::create(initial_output_state, sample_rate, channels, target_latency_ms, move(data_request_callback));
+#else
+    (void)initial_output_state, (void)sample_rate, (void)channels, (void)target_latency_ms;
+    return Error::from_string_literal("Audio output is not available for this platform");
+#endif
+}
+
+}

Libraries/LibMedia/Audio/PlaybackStream.h

@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2023, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include "SampleFormats.h"
+#include <AK/AtomicRefCounted.h>
+#include <AK/Function.h>
+#include <AK/Queue.h>
+#include <AK/Time.h>
+#include <LibCore/Forward.h>
+#include <LibThreading/ConditionVariable.h>
+#include <LibThreading/MutexProtected.h>
+#include <LibThreading/Thread.h>
+
+namespace Audio {
+
+enum class OutputState {
+    Playing,
+    Suspended,
+};
+
+// This class implements high-level audio playback behavior. It is primarily intended as an abstract cross-platform
+// interface to be used by Ladybird (and its dependent libraries) for playback.
+//
+// The interface is designed to be simple and robust. All control functions can be called safely from any thread.
+// Timing information provided by the class should allow audio timestamps to be tracked with the best accuracy possible.
+class PlaybackStream : public AtomicRefCounted<PlaybackStream> {
+public:
+    using AudioDataRequestCallback = Function<ReadonlyBytes(Bytes buffer, PcmSampleFormat format, size_t sample_count)>;
+
+    // Creates a new audio Output class.
+    //
+    // The initial_output_state parameter determines whether it will begin playback immediately.
+    //
+    // The AudioDataRequestCallback will be called when the Output needs more audio data to fill
+    // its buffers and continue playback.
+    static ErrorOr<NonnullRefPtr<PlaybackStream>> create(OutputState initial_output_state, u32 sample_rate, u8 channels, u32 target_latency_ms, AudioDataRequestCallback&&);
+
+    virtual ~PlaybackStream() = default;
+
+    // Sets the callback function that will be fired whenever the server consumes more data than is made available
+    // by the data request callback. It will fire when either the data request runs too long, or the data request
+    // returns no data. If all the input data has been exhausted and this event fires, that means that playback
+    // has ended.
+    virtual void set_underrun_callback(Function<void()>) = 0;
+
+    // Resume playback from the suspended state, requesting new data for audio buffers as soon as possible.
+    //
+    // The value provided to the promise resolution will match the `total_time_played()` at the exact moment that
+    // the stream was resumed.
+    virtual NonnullRefPtr<Core::ThreadedPromise<AK::Duration>> resume() = 0;
+    // Completes playback of any buffered audio data and then suspends playback and buffering.
+    virtual NonnullRefPtr<Core::ThreadedPromise<void>> drain_buffer_and_suspend() = 0;
+    // Drops any buffered audio data and then suspends playback and buffering. This can used be to stop playback
+    // as soon as possible instead of waiting for remaining audio to play.
+    virtual NonnullRefPtr<Core::ThreadedPromise<void>> discard_buffer_and_suspend() = 0;
+
+    // Returns a accurate monotonically-increasing time duration that is based on the number of samples that have
+    // been played by the output device. The value is interpolated and takes into account latency to the speakers
+    // whenever possible.
+    //
+    // This function should be able to run from any thread safely.
+    virtual ErrorOr<AK::Duration> total_time_played() = 0;
+
+    virtual NonnullRefPtr<Core::ThreadedPromise<void>> set_volume(double volume) = 0;
+};
+
+}

Libraries/LibMedia/Audio/PlaybackStreamAudioUnit.cpp

@@ -0,0 +1,391 @@
+/*
+ * Copyright (c) 2023, Andrew Kaster <akaster@serenityos.org>
+ * Copyright (c) 2023, Tim Flynn <trflynn89@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include "PlaybackStreamAudioUnit.h"
+#include <AK/Atomic.h>
+#include <AK/SourceLocation.h>
+#include <LibCore/SharedCircularQueue.h>
+#include <LibCore/ThreadedPromise.h>
+
+#include <AudioUnit/AudioUnit.h>
+
+namespace Audio {
+
+static constexpr AudioUnitElement AUDIO_UNIT_OUTPUT_BUS = 0;
+
+static void log_os_error_code(OSStatus error_code, SourceLocation location = SourceLocation::current());
+
+#define AU_TRY(expression)                                                         \
+    ({                                                                             \
+        /* Ignore -Wshadow to allow nesting the macro. */                          \
+        AK_IGNORE_DIAGNOSTIC("-Wshadow", auto&& _temporary_result = (expression)); \
+        if (_temporary_result != noErr) [[unlikely]] {                             \
+            log_os_error_code(_temporary_result);                                  \
+            return Error::from_errno(_temporary_result);                           \
+        }                                                                          \
+    })
+
+struct AudioTask {
+    enum class Type {
+        Play,
+        Pause,
+        PauseAndDiscard,
+        Volume,
+    };
+
+    void resolve(AK::Duration time)
+    {
+        promise.visit(
+            [](Empty) { VERIFY_NOT_REACHED(); },
+            [&](NonnullRefPtr<Core::ThreadedPromise<void>>& promise) {
+                promise->resolve();
+            },
+            [&](NonnullRefPtr<Core::ThreadedPromise<AK::Duration>>& promise) {
+                promise->resolve(move(time));
+            });
+    }
+
+    void reject(OSStatus error)
+    {
+        log_os_error_code(error);
+
+        promise.visit(
+            [](Empty) { VERIFY_NOT_REACHED(); },
+            [error](auto& promise) {
+                promise->reject(Error::from_errno(error));
+            });
+    }
+
+    Type type;
+    Variant<Empty, NonnullRefPtr<Core::ThreadedPromise<void>>, NonnullRefPtr<Core::ThreadedPromise<AK::Duration>>> promise;
+    Optional<double> data {};
+};
+
+class AudioState : public RefCounted<AudioState> {
+public:
+    using AudioTaskQueue = Core::SharedSingleProducerCircularQueue<AudioTask>;
+
+    static ErrorOr<NonnullRefPtr<AudioState>> create(AudioStreamBasicDescription description, PlaybackStream::AudioDataRequestCallback data_request_callback, OutputState initial_output_state)
+    {
+        auto task_queue = TRY(AudioTaskQueue::create());
+        auto state = TRY(adopt_nonnull_ref_or_enomem(new (nothrow) AudioState(description, move(task_queue), move(data_request_callback), initial_output_state)));
+
+        AudioComponentDescription component_description;
+        component_description.componentType = kAudioUnitType_Output;
+        component_description.componentSubType = kAudioUnitSubType_DefaultOutput;
+        component_description.componentManufacturer = kAudioUnitManufacturer_Apple;
+        component_description.componentFlags = 0;
+        component_description.componentFlagsMask = 0;
+
+        auto* component = AudioComponentFindNext(NULL, &component_description);
+        AU_TRY(AudioComponentInstanceNew(component, &state->m_audio_unit));
+
+        AU_TRY(AudioUnitSetProperty(
+            state->m_audio_unit,
+            kAudioUnitProperty_StreamFormat,
+            kAudioUnitScope_Input,
+            AUDIO_UNIT_OUTPUT_BUS,
+            &description,
+            sizeof(description)));
+
+        AURenderCallbackStruct callbackStruct;
+        callbackStruct.inputProc = &AudioState::on_audio_unit_buffer_request;
+        callbackStruct.inputProcRefCon = state.ptr();
+
+        AU_TRY(AudioUnitSetProperty(
+            state->m_audio_unit,
+            kAudioUnitProperty_SetRenderCallback,
+            kAudioUnitScope_Global,
+            AUDIO_UNIT_OUTPUT_BUS,
+            &callbackStruct,
+            sizeof(callbackStruct)));
+
+        AU_TRY(AudioUnitInitialize(state->m_audio_unit));
+        AU_TRY(AudioOutputUnitStart(state->m_audio_unit));
+
+        return state;
+    }
+
+    ~AudioState()
+    {
+        if (m_audio_unit != nullptr)
+            AudioOutputUnitStop(m_audio_unit);
+    }
+
+    ErrorOr<void> queue_task(AudioTask task)
+    {
+        return m_task_queue.blocking_enqueue(move(task), []() {
+            usleep(10'000);
+        });
+    }
+
+    AK::Duration last_sample_time() const
+    {
+        return AK::Duration::from_milliseconds(m_last_sample_time.load());
+    }
+
+private:
+    AudioState(AudioStreamBasicDescription description, AudioTaskQueue task_queue, PlaybackStream::AudioDataRequestCallback data_request_callback, OutputState initial_output_state)
+        : m_description(description)
+        , m_task_queue(move(task_queue))
+        , m_paused(initial_output_state == OutputState::Playing ? Paused::No : Paused::Yes)
+        , m_data_request_callback(move(data_request_callback))
+    {
+    }
+
+    static OSStatus on_audio_unit_buffer_request(void* user_data, AudioUnitRenderActionFlags*, AudioTimeStamp const* time_stamp, UInt32 element, UInt32 frames_to_render, AudioBufferList* output_buffer_list)
+    {
+        VERIFY(element == AUDIO_UNIT_OUTPUT_BUS);
+        VERIFY(output_buffer_list->mNumberBuffers == 1);
+
+        auto& state = *static_cast<AudioState*>(user_data);
+
+        VERIFY(time_stamp->mFlags & kAudioTimeStampSampleTimeValid);
+        auto sample_time_seconds = time_stamp->mSampleTime / state.m_description.mSampleRate;
+
+        auto last_sample_time = static_cast<i64>(sample_time_seconds * 1000.0);
+        state.m_last_sample_time.store(last_sample_time);
+
+        if (auto result = state.m_task_queue.dequeue(); result.is_error()) {
+            VERIFY(result.error() == AudioTaskQueue::QueueStatus::Empty);
+        } else {
+            auto task = result.release_value();
+            OSStatus error = noErr;
+
+            switch (task.type) {
+            case AudioTask::Type::Play:
+                state.m_paused = Paused::No;
+                break;
+
+            case AudioTask::Type::Pause:
+                state.m_paused = Paused::Yes;
+                break;
+
+            case AudioTask::Type::PauseAndDiscard:
+                error = AudioUnitReset(state.m_audio_unit, kAudioUnitScope_Global, AUDIO_UNIT_OUTPUT_BUS);
+                state.m_paused = Paused::Yes;
+                break;
+
+            case AudioTask::Type::Volume:
+                VERIFY(task.data.has_value());
+                error = AudioUnitSetParameter(state.m_audio_unit, kHALOutputParam_Volume, kAudioUnitScope_Global, 0, static_cast<float>(*task.data), 0);
+                break;
+            }
+
+            if (error == noErr)
+                task.resolve(AK::Duration::from_milliseconds(last_sample_time));
+            else
+                task.reject(error);
+        }
+
+        Bytes output_buffer {
+            reinterpret_cast<u8*>(output_buffer_list->mBuffers[0].mData),
+            output_buffer_list->mBuffers[0].mDataByteSize
+        };
+
+        if (state.m_paused == Paused::No) {
+            auto written_bytes = state.m_data_request_callback(output_buffer, PcmSampleFormat::Float32, frames_to_render);
+
+            if (written_bytes.is_empty())
+                state.m_paused = Paused::Yes;
+        }
+
+        if (state.m_paused == Paused::Yes)
+            output_buffer.fill(0);
+
+        return noErr;
+    }
+
+    AudioComponentInstance m_audio_unit { nullptr };
+    AudioStreamBasicDescription m_description {};
+
+    AudioTaskQueue m_task_queue;
+
+    enum class Paused {
+        Yes,
+        No,
+    };
+    Paused m_paused { Paused::Yes };
+
+    PlaybackStream::AudioDataRequestCallback m_data_request_callback;
+    Atomic<i64> m_last_sample_time { 0 };
+};
+
+ErrorOr<NonnullRefPtr<PlaybackStream>> PlaybackStreamAudioUnit::create(OutputState initial_output_state, u32 sample_rate, u8 channels, u32, AudioDataRequestCallback&& data_request_callback)
+{
+    AudioStreamBasicDescription description {};
+    description.mFormatID = kAudioFormatLinearPCM;
+    description.mFormatFlags = kLinearPCMFormatFlagIsFloat | kLinearPCMFormatFlagIsPacked;
+    description.mSampleRate = sample_rate;
+    description.mChannelsPerFrame = channels;
+    description.mBitsPerChannel = sizeof(float) * 8;
+    description.mBytesPerFrame = sizeof(float) * channels;
+    description.mBytesPerPacket = description.mBytesPerFrame;
+    description.mFramesPerPacket = 1;
+
+    auto state = TRY(AudioState::create(description, move(data_request_callback), initial_output_state));
+    return TRY(adopt_nonnull_ref_or_enomem(new (nothrow) PlaybackStreamAudioUnit(move(state))));
+}
+
+PlaybackStreamAudioUnit::PlaybackStreamAudioUnit(NonnullRefPtr<AudioState> impl)
+    : m_state(move(impl))
+{
+}
+
+PlaybackStreamAudioUnit::~PlaybackStreamAudioUnit() = default;
+
+void PlaybackStreamAudioUnit::set_underrun_callback(Function<void()>)
+{
+    // FIXME: Implement this.
+}
+
+NonnullRefPtr<Core::ThreadedPromise<AK::Duration>> PlaybackStreamAudioUnit::resume()
+{
+    auto promise = Core::ThreadedPromise<AK::Duration>::create();
+    AudioTask task { AudioTask::Type::Play, promise };
+
+    if (auto result = m_state->queue_task(move(task)); result.is_error())
+        promise->reject(result.release_error());
+
+    return promise;
+}
+
+NonnullRefPtr<Core::ThreadedPromise<void>> PlaybackStreamAudioUnit::drain_buffer_and_suspend()
+{
+    auto promise = Core::ThreadedPromise<void>::create();
+    AudioTask task { AudioTask::Type::Pause, promise };
+
+    if (auto result = m_state->queue_task(move(task)); result.is_error())
+        promise->reject(result.release_error());
+
+    return promise;
+}
+
+NonnullRefPtr<Core::ThreadedPromise<void>> PlaybackStreamAudioUnit::discard_buffer_and_suspend()
+{
+    auto promise = Core::ThreadedPromise<void>::create();
+    AudioTask task { AudioTask::Type::PauseAndDiscard, promise };
+
+    if (auto result = m_state->queue_task(move(task)); result.is_error())
+        promise->reject(result.release_error());
+
+    return promise;
+}
+
+ErrorOr<AK::Duration> PlaybackStreamAudioUnit::total_time_played()
+{
+    return m_state->last_sample_time();
+}
+
+NonnullRefPtr<Core::ThreadedPromise<void>> PlaybackStreamAudioUnit::set_volume(double volume)
+{
+    auto promise = Core::ThreadedPromise<void>::create();
+    AudioTask task { AudioTask::Type::Volume, promise, volume };
+
+    if (auto result = m_state->queue_task(move(task)); result.is_error())
+        promise->reject(result.release_error());
+
+    return promise;
+}
+
+void log_os_error_code([[maybe_unused]] OSStatus error_code, [[maybe_unused]] SourceLocation location)
+{
+#if AUDIO_DEBUG
+    auto error_string = "Unknown error"sv;
+
+    // Errors listed in AUComponent.h
+    switch (error_code) {
+    case kAudioUnitErr_InvalidProperty:
+        error_string = "InvalidProperty"sv;
+        break;
+    case kAudioUnitErr_InvalidParameter:
+        error_string = "InvalidParameter"sv;
+        break;
+    case kAudioUnitErr_InvalidElement:
+        error_string = "InvalidElement"sv;
+        break;
+    case kAudioUnitErr_NoConnection:
+        error_string = "NoConnection"sv;
+        break;
+    case kAudioUnitErr_FailedInitialization:
+        error_string = "FailedInitialization"sv;
+        break;
+    case kAudioUnitErr_TooManyFramesToProcess:
+        error_string = "TooManyFramesToProcess"sv;
+        break;
+    case kAudioUnitErr_InvalidFile:
+        error_string = "InvalidFile"sv;
+        break;
+    case kAudioUnitErr_UnknownFileType:
+        error_string = "UnknownFileType"sv;
+        break;
+    case kAudioUnitErr_FileNotSpecified:
+        error_string = "FileNotSpecified"sv;
+        break;
+    case kAudioUnitErr_FormatNotSupported:
+        error_string = "FormatNotSupported"sv;
+        break;
+    case kAudioUnitErr_Uninitialized:
+        error_string = "Uninitialized"sv;
+        break;
+    case kAudioUnitErr_InvalidScope:
+        error_string = "InvalidScope"sv;
+        break;
+    case kAudioUnitErr_PropertyNotWritable:
+        error_string = "PropertyNotWritable"sv;
+        break;
+    case kAudioUnitErr_CannotDoInCurrentContext:
+        error_string = "CannotDoInCurrentContext"sv;
+        break;
+    case kAudioUnitErr_InvalidPropertyValue:
+        error_string = "InvalidPropertyValue"sv;
+        break;
+    case kAudioUnitErr_PropertyNotInUse:
+        error_string = "PropertyNotInUse"sv;
+        break;
+    case kAudioUnitErr_Initialized:
+        error_string = "Initialized"sv;
+        break;
+    case kAudioUnitErr_InvalidOfflineRender:
+        error_string = "InvalidOfflineRender"sv;
+        break;
+    case kAudioUnitErr_Unauthorized:
+        error_string = "Unauthorized"sv;
+        break;
+    case kAudioUnitErr_MIDIOutputBufferFull:
+        error_string = "MIDIOutputBufferFull"sv;
+        break;
+    case kAudioComponentErr_InstanceTimedOut:
+        error_string = "InstanceTimedOut"sv;
+        break;
+    case kAudioComponentErr_InstanceInvalidated:
+        error_string = "InstanceInvalidated"sv;
+        break;
+    case kAudioUnitErr_RenderTimeout:
+        error_string = "RenderTimeout"sv;
+        break;
+    case kAudioUnitErr_ExtensionNotFound:
+        error_string = "ExtensionNotFound"sv;
+        break;
+    case kAudioUnitErr_InvalidParameterValue:
+        error_string = "InvalidParameterValue"sv;
+        break;
+    case kAudioUnitErr_InvalidFilePath:
+        error_string = "InvalidFilePath"sv;
+        break;
+    case kAudioUnitErr_MissingKey:
+        error_string = "MissingKey"sv;
+        break;
+    default:
+        break;
+    }
+
+    warnln("{}: Audio Unit error {}: {}", location, error_code, error_string);
+#endif
+}
+
+}

Libraries/LibMedia/Audio/PlaybackStreamAudioUnit.h

@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2023, Andrew Kaster <akaster@serenityos.org>
+ * Copyright (c) 2023, Tim Flynn <trflynn89@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include "PlaybackStream.h"
+#include <AK/Error.h>
+#include <AK/NonnullRefPtr.h>
+
+namespace Audio {
+
+class AudioState;
+
+class PlaybackStreamAudioUnit final : public PlaybackStream {
+public:
+    static ErrorOr<NonnullRefPtr<PlaybackStream>> create(OutputState initial_output_state, u32 sample_rate, u8 channels, u32 target_latency_ms, AudioDataRequestCallback&& data_request_callback);
+
+    virtual void set_underrun_callback(Function<void()>) override;
+
+    virtual NonnullRefPtr<Core::ThreadedPromise<AK::Duration>> resume() override;
+    virtual NonnullRefPtr<Core::ThreadedPromise<void>> drain_buffer_and_suspend() override;
+    virtual NonnullRefPtr<Core::ThreadedPromise<void>> discard_buffer_and_suspend() override;
+
+    virtual ErrorOr<AK::Duration> total_time_played() override;
+
+    virtual NonnullRefPtr<Core::ThreadedPromise<void>> set_volume(double) override;
+
+private:
+    explicit PlaybackStreamAudioUnit(NonnullRefPtr<AudioState>);
+    ~PlaybackStreamAudioUnit();
+
+    NonnullRefPtr<AudioState> m_state;
+};
+
+}

Libraries/LibMedia/Audio/PlaybackStreamOboe.cpp

@@ -0,0 +1,157 @@
+/*
+ * Copyright (c) 2024, Olekoop <mlglol360xd@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include "PlaybackStreamOboe.h"
+#include <AK/Atomic.h>
+#include <AK/SourceLocation.h>
+#include <LibCore/SharedCircularQueue.h>
+#include <LibCore/ThreadedPromise.h>
+#include <memory>
+
+#include <oboe/Oboe.h>
+
+namespace Audio {
+
+class OboeCallback : public oboe::AudioStreamDataCallback {
+public:
+    virtual oboe::DataCallbackResult onAudioReady(oboe::AudioStream* oboeStream, void* audioData, int32_t numFrames) override
+    {
+        Bytes output_buffer {
+            reinterpret_cast<u8*>(audioData),
+            static_cast<size_t>(numFrames * oboeStream->getChannelCount() * sizeof(float))
+        };
+        auto written_bytes = m_data_request_callback(output_buffer, PcmSampleFormat::Float32, numFrames);
+        if (written_bytes.is_empty())
+            return oboe::DataCallbackResult::Stop;
+
+        auto timestamp = oboeStream->getTimestamp(CLOCK_MONOTONIC);
+        if (timestamp == oboe::Result::OK) {
+            m_number_of_samples_enqueued = timestamp.value().position;
+        } else {
+            // Fallback for OpenSLES
+            m_number_of_samples_enqueued += numFrames;
+        }
+        auto last_sample_time = static_cast<i64>(m_number_of_samples_enqueued / oboeStream->getSampleRate());
+        m_last_sample_time.store(last_sample_time);
+
+        float* output = (float*)audioData;
+        for (int frames = 0; frames < numFrames; frames++) {
+            for (int channels = 0; channels < oboeStream->getChannelCount(); channels++) {
+                *output++ *= m_volume.load();
+            }
+        }
+        return oboe::DataCallbackResult::Continue;
+    }
+    OboeCallback(PlaybackStream::AudioDataRequestCallback data_request_callback)
+        : m_data_request_callback(move(data_request_callback))
+    {
+    }
+    AK::Duration last_sample_time() const
+    {
+        return AK::Duration::from_seconds(m_last_sample_time.load());
+    }
+    void set_volume(float volume)
+    {
+        m_volume.store(volume);
+    }
+
+private:
+    PlaybackStream::AudioDataRequestCallback m_data_request_callback;
+    Atomic<i64> m_last_sample_time { 0 };
+    size_t m_number_of_samples_enqueued { 0 };
+    Atomic<float> m_volume { 1.0 };
+};
+
+class PlaybackStreamOboe::Storage : public RefCounted<PlaybackStreamOboe::Storage> {
+public:
+    Storage(std::shared_ptr<oboe::AudioStream> stream, std::shared_ptr<OboeCallback> oboe_callback)
+        : m_stream(move(stream))
+        , m_oboe_callback(move(oboe_callback))
+    {
+    }
+    std::shared_ptr<oboe::AudioStream> stream() const { return m_stream; }
+    std::shared_ptr<OboeCallback> oboe_callback() const { return m_oboe_callback; }
+
+private:
+    std::shared_ptr<oboe::AudioStream> m_stream;
+    std::shared_ptr<OboeCallback> m_oboe_callback;
+};
+
+PlaybackStreamOboe::PlaybackStreamOboe(NonnullRefPtr<Storage> storage)
+    : m_storage(move(storage))
+{
+}
+
+ErrorOr<NonnullRefPtr<PlaybackStream>> PlaybackStreamOboe::create(OutputState initial_output_state, u32 sample_rate, u8 channels, u32, AudioDataRequestCallback&& data_request_callback)
+{
+    std::shared_ptr<oboe::AudioStream> stream;
+    auto oboe_callback = std::make_shared<OboeCallback>(move(data_request_callback));
+    oboe::AudioStreamBuilder builder;
+    auto result = builder.setSharingMode(oboe::SharingMode::Shared)
+                      ->setPerformanceMode(oboe::PerformanceMode::LowLatency)
+                      ->setFormat(oboe::AudioFormat::Float)
+                      ->setDataCallback(oboe_callback)
+                      ->setChannelCount(channels)
+                      ->setSampleRate(sample_rate)
+                      ->openStream(stream);
+
+    if (result != oboe::Result::OK)
+        return Error::from_string_literal("Oboe failed to start");
+
+    if (initial_output_state == OutputState::Playing)
+        stream->requestStart();
+
+    auto storage = TRY(adopt_nonnull_ref_or_enomem(new PlaybackStreamOboe::Storage(move(stream), move(oboe_callback))));
+    return TRY(adopt_nonnull_ref_or_enomem(new (nothrow) PlaybackStreamOboe(move(storage))));
+}
+
+PlaybackStreamOboe::~PlaybackStreamOboe() = default;
+
+void PlaybackStreamOboe::set_underrun_callback(Function<void()>)
+{
+    // FIXME: Implement this.
+}
+
+NonnullRefPtr<Core::ThreadedPromise<AK::Duration>> PlaybackStreamOboe::resume()
+{
+    auto promise = Core::ThreadedPromise<AK::Duration>::create();
+    auto time = MUST(total_time_played());
+    m_storage->stream()->start();
+    promise->resolve(move(time));
+    return promise;
+}
+
+NonnullRefPtr<Core::ThreadedPromise<void>> PlaybackStreamOboe::drain_buffer_and_suspend()
+{
+    auto promise = Core::ThreadedPromise<void>::create();
+    m_storage->stream()->stop();
+    promise->resolve();
+    return promise;
+}
+
+NonnullRefPtr<Core::ThreadedPromise<void>> PlaybackStreamOboe::discard_buffer_and_suspend()
+{
+    auto promise = Core::ThreadedPromise<void>::create();
+    m_storage->stream()->pause();
+    m_storage->stream()->flush();
+    promise->resolve();
+    return promise;
+}
+
+ErrorOr<AK::Duration> PlaybackStreamOboe::total_time_played()
+{
+    return m_storage->oboe_callback()->last_sample_time();
+}
+
+NonnullRefPtr<Core::ThreadedPromise<void>> PlaybackStreamOboe::set_volume(double volume)
+{
+    auto promise = Core::ThreadedPromise<void>::create();
+    m_storage->oboe_callback()->set_volume(volume);
+    promise->resolve();
+    return promise;
+}
+
+}

Libraries/LibMedia/Audio/PlaybackStreamOboe.h

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2024, Olekoop <mlglol360xd@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include "PlaybackStream.h"
+#include <AK/Error.h>
+#include <AK/NonnullRefPtr.h>
+
+namespace Audio {
+
+class PlaybackStreamOboe final : public PlaybackStream {
+public:
+    static ErrorOr<NonnullRefPtr<PlaybackStream>> create(OutputState initial_output_state, u32 sample_rate, u8 channels, u32 target_latency_ms, AudioDataRequestCallback&& data_request_callback);
+
+    virtual void set_underrun_callback(Function<void()>) override;
+
+    virtual NonnullRefPtr<Core::ThreadedPromise<AK::Duration>> resume() override;
+    virtual NonnullRefPtr<Core::ThreadedPromise<void>> drain_buffer_and_suspend() override;
+    virtual NonnullRefPtr<Core::ThreadedPromise<void>> discard_buffer_and_suspend() override;
+
+    virtual ErrorOr<AK::Duration> total_time_played() override;
+
+    virtual NonnullRefPtr<Core::ThreadedPromise<void>> set_volume(double) override;
+
+private:
+    class Storage;
+    explicit PlaybackStreamOboe(NonnullRefPtr<Storage>);
+    ~PlaybackStreamOboe();
+    RefPtr<Storage> m_storage;
+};
+
+}

Libraries/LibMedia/Audio/PlaybackStreamPulseAudio.cpp

@@ -0,0 +1,181 @@
+/*
+ * Copyright (c) 2023, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include "PlaybackStreamPulseAudio.h"
+
+#include <LibCore/ThreadedPromise.h>
+
+namespace Audio {
+
+#define TRY_OR_EXIT_THREAD(expression)                                                                       \
+    ({                                                                                                       \
+        auto&& __temporary_result = (expression);                                                            \
+        if (__temporary_result.is_error()) [[unlikely]] {                                                    \
+            warnln("Failure in PulseAudio control thread: {}", __temporary_result.error().string_literal()); \
+            internal_state->exit();                                                                          \
+            return 1;                                                                                        \
+        }                                                                                                    \
+        __temporary_result.release_value();                                                                  \
+    })
+
+ErrorOr<NonnullRefPtr<PlaybackStream>> PlaybackStreamPulseAudio::create(OutputState initial_state, u32 sample_rate, u8 channels, u32 target_latency_ms, AudioDataRequestCallback&& data_request_callback)
+{
+    VERIFY(data_request_callback);
+
+    // Create an internal state for the control thread to hold on to.
+    auto internal_state = TRY(adopt_nonnull_ref_or_enomem(new (nothrow) InternalState()));
+    auto playback_stream = TRY(adopt_nonnull_ref_or_enomem(new (nothrow) PlaybackStreamPulseAudio(internal_state)));
+
+    // Create the control thread and start it.
+    auto thread = TRY(Threading::Thread::try_create([=, data_request_callback = move(data_request_callback)]() mutable {
+        auto context = TRY_OR_EXIT_THREAD(PulseAudioContext::instance());
+        internal_state->set_stream(TRY_OR_EXIT_THREAD(context->create_stream(initial_state, sample_rate, channels, target_latency_ms, [data_request_callback = move(data_request_callback)](PulseAudioStream&, Bytes buffer, size_t sample_count) {
+            return data_request_callback(buffer, PcmSampleFormat::Float32, sample_count);
+        })));
+
+        // PulseAudio retains the last volume it sets for an application. We want to consistently
+        // start at 100% volume instead.
+        TRY_OR_EXIT_THREAD(internal_state->stream()->set_volume(1.0));
+
+        internal_state->thread_loop();
+        return 0;
+    },
+        "Audio::PlaybackStream"sv));
+
+    thread->start();
+    thread->detach();
+    return playback_stream;
+}
+
+PlaybackStreamPulseAudio::PlaybackStreamPulseAudio(NonnullRefPtr<InternalState> state)
+    : m_state(move(state))
+{
+}
+
+PlaybackStreamPulseAudio::~PlaybackStreamPulseAudio()
+{
+    m_state->exit();
+}
+
+#define TRY_OR_REJECT(expression, ...)                           \
+    ({                                                           \
+        auto&& __temporary_result = (expression);                \
+        if (__temporary_result.is_error()) [[unlikely]] {        \
+            promise->reject(__temporary_result.release_error()); \
+            return __VA_ARGS__;                                  \
+        }                                                        \
+        __temporary_result.release_value();                      \
+    })
+
+void PlaybackStreamPulseAudio::set_underrun_callback(Function<void()> callback)
+{
+    m_state->enqueue([this, callback = move(callback)]() mutable {
+        m_state->stream()->set_underrun_callback(move(callback));
+    });
+}
+
+NonnullRefPtr<Core::ThreadedPromise<AK::Duration>> PlaybackStreamPulseAudio::resume()
+{
+    auto promise = Core::ThreadedPromise<AK::Duration>::create();
+    TRY_OR_REJECT(m_state->check_is_running(), promise);
+    m_state->enqueue([this, promise]() {
+        TRY_OR_REJECT(m_state->stream()->resume());
+        promise->resolve(TRY_OR_REJECT(m_state->stream()->total_time_played()));
+    });
+    return promise;
+}
+
+NonnullRefPtr<Core::ThreadedPromise<void>> PlaybackStreamPulseAudio::drain_buffer_and_suspend()
+{
+    auto promise = Core::ThreadedPromise<void>::create();
+    TRY_OR_REJECT(m_state->check_is_running(), promise);
+    m_state->enqueue([this, promise]() {
+        TRY_OR_REJECT(m_state->stream()->drain_and_suspend());
+        promise->resolve();
+    });
+    return promise;
+}
+
+NonnullRefPtr<Core::ThreadedPromise<void>> PlaybackStreamPulseAudio::discard_buffer_and_suspend()
+{
+    auto promise = Core::ThreadedPromise<void>::create();
+    TRY_OR_REJECT(m_state->check_is_running(), promise);
+    m_state->enqueue([this, promise]() {
+        TRY_OR_REJECT(m_state->stream()->flush_and_suspend());
+        promise->resolve();
+    });
+    return promise;
+}
+
+ErrorOr<AK::Duration> PlaybackStreamPulseAudio::total_time_played()
+{
+    if (m_state->stream() != nullptr)
+        return m_state->stream()->total_time_played();
+    return AK::Duration::zero();
+}
+
+NonnullRefPtr<Core::ThreadedPromise<void>> PlaybackStreamPulseAudio::set_volume(double volume)
+{
+    auto promise = Core::ThreadedPromise<void>::create();
+    TRY_OR_REJECT(m_state->check_is_running(), promise);
+    m_state->enqueue([this, promise, volume]() {
+        TRY_OR_REJECT(m_state->stream()->set_volume(volume));
+        promise->resolve();
+    });
+    return promise;
+}
+
+ErrorOr<void> PlaybackStreamPulseAudio::InternalState::check_is_running()
+{
+    if (m_exit)
+        return Error::from_string_literal("PulseAudio control thread loop is not running");
+    return {};
+}
+
+void PlaybackStreamPulseAudio::InternalState::set_stream(NonnullRefPtr<PulseAudioStream> const& stream)
+{
+    m_stream = stream;
+}
+
+RefPtr<PulseAudioStream> PlaybackStreamPulseAudio::InternalState::stream()
+{
+    return m_stream;
+}
+
+void PlaybackStreamPulseAudio::InternalState::enqueue(Function<void()>&& task)
+{
+    Threading::MutexLocker locker { m_mutex };
+    m_tasks.enqueue(forward<Function<void()>>(task));
+    m_wake_condition.signal();
+}
+
+void PlaybackStreamPulseAudio::InternalState::thread_loop()
+{
+    while (true) {
+        auto task = [this]() -> Function<void()> {
+            Threading::MutexLocker locker { m_mutex };
+
+            while (m_tasks.is_empty() && !m_exit)
+                m_wake_condition.wait();
+            if (m_exit)
+                return nullptr;
+            return m_tasks.dequeue();
+        }();
+        if (!task) {
+            VERIFY(m_exit);
+            break;
+        }
+        task();
+    }
+}
+
+void PlaybackStreamPulseAudio::InternalState::exit()
+{
+    m_exit = true;
+    m_wake_condition.signal();
+}
+
+}

Libraries/LibMedia/Audio/PlaybackStreamPulseAudio.h

@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2023, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include "PlaybackStream.h"
+#include "PulseAudioWrappers.h"
+
+namespace Audio {
+
+class PlaybackStreamPulseAudio final
+    : public PlaybackStream {
+public:
+    static ErrorOr<NonnullRefPtr<PlaybackStream>> create(OutputState initial_state, u32 sample_rate, u8 channels, u32 target_latency_ms, AudioDataRequestCallback&& data_request_callback);
+
+    virtual void set_underrun_callback(Function<void()>) override;
+
+    virtual NonnullRefPtr<Core::ThreadedPromise<AK::Duration>> resume() override;
+    virtual NonnullRefPtr<Core::ThreadedPromise<void>> drain_buffer_and_suspend() override;
+    virtual NonnullRefPtr<Core::ThreadedPromise<void>> discard_buffer_and_suspend() override;
+
+    virtual ErrorOr<AK::Duration> total_time_played() override;
+
+    virtual NonnullRefPtr<Core::ThreadedPromise<void>> set_volume(double) override;
+
+private:
+    // This struct is kept alive until the control thread exits to prevent a use-after-free without blocking on
+    // the UI thread.
+    class InternalState : public AtomicRefCounted<InternalState> {
+    public:
+        void set_stream(NonnullRefPtr<PulseAudioStream> const&);
+        RefPtr<PulseAudioStream> stream();
+
+        void enqueue(Function<void()>&&);
+        void thread_loop();
+        ErrorOr<void> check_is_running();
+        void exit();
+
+    private:
+        RefPtr<PulseAudioStream> m_stream { nullptr };
+
+        Queue<Function<void()>> m_tasks;
+        Threading::Mutex m_mutex;
+        Threading::ConditionVariable m_wake_condition { m_mutex };
+
+        Atomic<bool> m_exit { false };
+    };
+
+    PlaybackStreamPulseAudio(NonnullRefPtr<InternalState>);
+    ~PlaybackStreamPulseAudio();
+
+    RefPtr<InternalState> m_state;
+};
+
+}

Libraries/LibMedia/Audio/PulseAudioWrappers.cpp

@@ -0,0 +1,499 @@
+/*
+ * Copyright (c) 2023, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include "PulseAudioWrappers.h"
+
+#include <AK/WeakPtr.h>
+#include <LibThreading/Mutex.h>
+
+namespace Audio {
+
+WeakPtr<PulseAudioContext> PulseAudioContext::weak_instance()
+{
+    // Use a weak pointer to allow the context to be shut down if we stop outputting audio.
+    static WeakPtr<PulseAudioContext> the_instance;
+    return the_instance;
+}
+
+ErrorOr<NonnullRefPtr<PulseAudioContext>> PulseAudioContext::instance()
+{
+    static Threading::Mutex instantiation_mutex;
+    // Lock and unlock the mutex to ensure that the mutex is fully unlocked at application
+    // exit.
+    atexit([]() {
+        instantiation_mutex.lock();
+        instantiation_mutex.unlock();
+    });
+
+    auto instantiation_locker = Threading::MutexLocker(instantiation_mutex);
+
+    auto the_instance = weak_instance();
+    RefPtr<PulseAudioContext> strong_instance_pointer = the_instance.strong_ref();
+
+    if (strong_instance_pointer == nullptr) {
+        auto* main_loop = pa_threaded_mainloop_new();
+        if (main_loop == nullptr)
+            return Error::from_string_literal("Failed to create PulseAudio main loop");
+
+        auto* api = pa_threaded_mainloop_get_api(main_loop);
+        if (api == nullptr)
+            return Error::from_string_literal("Failed to get PulseAudio API");
+
+        auto* context = pa_context_new(api, "Ladybird");
+        if (context == nullptr)
+            return Error::from_string_literal("Failed to get PulseAudio connection context");
+
+        strong_instance_pointer = make_ref_counted<PulseAudioContext>(main_loop, api, context);
+
+        // Set a callback to signal ourselves to wake when the state changes, so that we can
+        // synchronously wait for the connection.
+        pa_context_set_state_callback(
+            context, [](pa_context*, void* user_data) {
+                static_cast<PulseAudioContext*>(user_data)->signal_to_wake();
+            },
+            strong_instance_pointer.ptr());
+
+        if (auto error = pa_context_connect(context, nullptr, PA_CONTEXT_NOFLAGS, nullptr); error < 0) {
+            warnln("Starting PulseAudio context connection failed with error: {}", pulse_audio_error_to_string(static_cast<PulseAudioErrorCode>(-error)));
+            return Error::from_string_literal("Error while starting PulseAudio daemon connection");
+        }
+
+        if (auto error = pa_threaded_mainloop_start(main_loop); error < 0) {
+            warnln("Starting PulseAudio main loop failed with error: {}", pulse_audio_error_to_string(static_cast<PulseAudioErrorCode>(-error)));
+            return Error::from_string_literal("Failed to start PulseAudio main loop");
+        }
+
+        {
+            auto locker = strong_instance_pointer->main_loop_locker();
+            while (true) {
+                bool is_ready = false;
+                switch (strong_instance_pointer->get_connection_state()) {
+                case PulseAudioContextState::Connecting:
+                case PulseAudioContextState::Authorizing:
+                case PulseAudioContextState::SettingName:
+                    break;
+                case PulseAudioContextState::Ready:
+                    is_ready = true;
+                    break;
+                case PulseAudioContextState::Failed:
+                    warnln("PulseAudio server connection failed with error: {}", pulse_audio_error_to_string(strong_instance_pointer->get_last_error()));
+                    return Error::from_string_literal("Failed to connect to PulseAudio server");
+                case PulseAudioContextState::Unconnected:
+                case PulseAudioContextState::Terminated:
+                    VERIFY_NOT_REACHED();
+                    break;
+                }
+
+                if (is_ready)
+                    break;
+
+                strong_instance_pointer->wait_for_signal();
+            }
+
+            pa_context_set_state_callback(context, nullptr, nullptr);
+        }
+
+        the_instance = strong_instance_pointer;
+    }
+
+    return strong_instance_pointer.release_nonnull();
+}
+
+PulseAudioContext::PulseAudioContext(pa_threaded_mainloop* main_loop, pa_mainloop_api* api, pa_context* context)
+    : m_main_loop(main_loop)
+    , m_api(api)
+    , m_context(context)
+{
+}
+
+PulseAudioContext::~PulseAudioContext()
+{
+    {
+        auto locker = main_loop_locker();
+        pa_context_disconnect(m_context);
+        pa_context_unref(m_context);
+    }
+    pa_threaded_mainloop_stop(m_main_loop);
+    pa_threaded_mainloop_free(m_main_loop);
+}
+
+bool PulseAudioContext::current_thread_is_main_loop_thread()
+{
+    return static_cast<bool>(pa_threaded_mainloop_in_thread(m_main_loop));
+}
+
+void PulseAudioContext::lock_main_loop()
+{
+    if (!current_thread_is_main_loop_thread())
+        pa_threaded_mainloop_lock(m_main_loop);
+}
+
+void PulseAudioContext::unlock_main_loop()
+{
+    if (!current_thread_is_main_loop_thread())
+        pa_threaded_mainloop_unlock(m_main_loop);
+}
+
+void PulseAudioContext::wait_for_signal()
+{
+    pa_threaded_mainloop_wait(m_main_loop);
+}
+
+void PulseAudioContext::signal_to_wake()
+{
+    pa_threaded_mainloop_signal(m_main_loop, 0);
+}
+
+PulseAudioContextState PulseAudioContext::get_connection_state()
+{
+    return static_cast<PulseAudioContextState>(pa_context_get_state(m_context));
+}
+
+bool PulseAudioContext::connection_is_good()
+{
+    return PA_CONTEXT_IS_GOOD(pa_context_get_state(m_context));
+}
+
+PulseAudioErrorCode PulseAudioContext::get_last_error()
+{
+    return static_cast<PulseAudioErrorCode>(pa_context_errno(m_context));
+}
+
+#define STREAM_SIGNAL_CALLBACK(stream)                                          \
+    [](auto*, int, void* user_data) {                                           \
+        static_cast<PulseAudioStream*>(user_data)->m_context->signal_to_wake(); \
+    },                                                                          \
+        (stream)
+
+ErrorOr<NonnullRefPtr<PulseAudioStream>> PulseAudioContext::create_stream(OutputState initial_state, u32 sample_rate, u8 channels, u32 target_latency_ms, PulseAudioDataRequestCallback write_callback)
+{
+    auto locker = main_loop_locker();
+
+    VERIFY(get_connection_state() == PulseAudioContextState::Ready);
+    pa_sample_spec sample_specification {
+        // FIXME: Support more audio sample types.
+        __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ ? PA_SAMPLE_FLOAT32LE : PA_SAMPLE_FLOAT32BE,
+        sample_rate,
+        channels,
+    };
+
+    // Check the sample specification and channel map here. These are also checked by stream_new(),
+    // but we can return a more accurate error if we check beforehand.
+    if (pa_sample_spec_valid(&sample_specification) == 0)
+        return Error::from_string_literal("PulseAudio sample specification is invalid");
+    pa_channel_map channel_map;
+    if (pa_channel_map_init_auto(&channel_map, sample_specification.channels, PA_CHANNEL_MAP_DEFAULT) == 0) {
+        warnln("Getting default PulseAudio channel map failed with error: {}", pulse_audio_error_to_string(get_last_error()));
+        return Error::from_string_literal("Failed to get default PulseAudio channel map");
+    }
+
+    // Create the stream object and set a callback to signal ourselves to wake when the stream changes states,
+    // allowing us to wait synchronously for it to become Ready or Failed.
+    auto* stream = pa_stream_new_with_proplist(m_context, "Audio Stream", &sample_specification, &channel_map, nullptr);
+    if (stream == nullptr) {
+        warnln("Instantiating PulseAudio stream failed with error: {}", pulse_audio_error_to_string(get_last_error()));
+        return Error::from_string_literal("Failed to create PulseAudio stream");
+    }
+    pa_stream_set_state_callback(
+        stream, [](pa_stream*, void* user_data) {
+            static_cast<PulseAudioContext*>(user_data)->signal_to_wake();
+        },
+        this);
+
+    auto stream_wrapper = TRY(adopt_nonnull_ref_or_enomem(new (nothrow) PulseAudioStream(NonnullRefPtr(*this), stream)));
+
+    stream_wrapper->m_write_callback = move(write_callback);
+    pa_stream_set_write_callback(
+        stream, [](pa_stream* stream, size_t bytes_to_write, void* user_data) {
+            auto& stream_wrapper = *static_cast<PulseAudioStream*>(user_data);
+            VERIFY(stream_wrapper.m_stream == stream);
+            stream_wrapper.on_write_requested(bytes_to_write);
+        },
+        stream_wrapper.ptr());
+
+    // Borrowing logic from cubeb to set reasonable buffer sizes for a target latency:
+    // https://searchfox.org/mozilla-central/rev/3b707c8fd7e978eebf24279ee51ccf07895cfbcb/third_party/rust/cubeb-sys/libcubeb/src/cubeb_pulse.c#910-927
+    pa_buffer_attr buffer_attributes;
+    buffer_attributes.maxlength = -1;
+    buffer_attributes.prebuf = -1;
+    buffer_attributes.tlength = target_latency_ms * sample_rate / 1000;
+    buffer_attributes.minreq = buffer_attributes.tlength / 4;
+    buffer_attributes.fragsize = buffer_attributes.minreq;
+    auto flags = static_cast<pa_stream_flags>(PA_STREAM_AUTO_TIMING_UPDATE | PA_STREAM_INTERPOLATE_TIMING | PA_STREAM_ADJUST_LATENCY | PA_STREAM_RELATIVE_VOLUME);
+
+    if (initial_state == OutputState::Suspended) {
+        stream_wrapper->m_suspended = true;
+        flags = static_cast<pa_stream_flags>(static_cast<u32>(flags) | PA_STREAM_START_CORKED);
+    }
+
+    // This is a workaround for an issue with starting the stream corked, see PulseAudioPlaybackStream::total_time_played().
+    pa_stream_set_started_callback(
+        stream, [](pa_stream* stream, void* user_data) {
+            static_cast<PulseAudioStream*>(user_data)->m_started_playback = true;
+            pa_stream_set_started_callback(stream, nullptr, nullptr);
+        },
+        stream_wrapper.ptr());
+
+    pa_stream_set_underflow_callback(
+        stream, [](pa_stream*, void* user_data) {
+            auto& stream = *static_cast<PulseAudioStream*>(user_data);
+            if (stream.m_underrun_callback)
+                stream.m_underrun_callback();
+        },
+        stream_wrapper.ptr());
+
+    if (auto error = pa_stream_connect_playback(stream, nullptr, &buffer_attributes, flags, nullptr, nullptr); error != 0) {
+        warnln("Failed to start PulseAudio stream connection with error: {}", pulse_audio_error_to_string(static_cast<PulseAudioErrorCode>(error)));
+        return Error::from_string_literal("Error while connecting the PulseAudio stream");
+    }
+
+    while (true) {
+        bool is_ready = false;
+        switch (stream_wrapper->get_connection_state()) {
+        case PulseAudioStreamState::Creating:
+            break;
+        case PulseAudioStreamState::Ready:
+            is_ready = true;
+            break;
+        case PulseAudioStreamState::Failed:
+            warnln("PulseAudio stream connection failed with error: {}", pulse_audio_error_to_string(get_last_error()));
+            return Error::from_string_literal("Failed to connect to PulseAudio daemon");
+        case PulseAudioStreamState::Unconnected:
+        case PulseAudioStreamState::Terminated:
+            VERIFY_NOT_REACHED();
+            break;
+        }
+        if (is_ready)
+            break;
+
+        wait_for_signal();
+    }
+
+    pa_stream_set_state_callback(stream, nullptr, nullptr);
+
+    return stream_wrapper;
+}
+
+PulseAudioStream::~PulseAudioStream()
+{
+    auto locker = m_context->main_loop_locker();
+    pa_stream_set_write_callback(m_stream, nullptr, nullptr);
+    pa_stream_set_underflow_callback(m_stream, nullptr, nullptr);
+    pa_stream_set_started_callback(m_stream, nullptr, nullptr);
+    pa_stream_disconnect(m_stream);
+    pa_stream_unref(m_stream);
+}
+
+PulseAudioStreamState PulseAudioStream::get_connection_state()
+{
+    return static_cast<PulseAudioStreamState>(pa_stream_get_state(m_stream));
+}
+
+bool PulseAudioStream::connection_is_good()
+{
+    return PA_STREAM_IS_GOOD(pa_stream_get_state(m_stream));
+}
+
+void PulseAudioStream::set_underrun_callback(Function<void()> callback)
+{
+    auto locker = m_context->main_loop_locker();
+    m_underrun_callback = move(callback);
+}
+
+u32 PulseAudioStream::sample_rate()
+{
+    return pa_stream_get_sample_spec(m_stream)->rate;
+}
+
+size_t PulseAudioStream::sample_size()
+{
+    return pa_sample_size(pa_stream_get_sample_spec(m_stream));
+}
+
+size_t PulseAudioStream::frame_size()
+{
+    return pa_frame_size(pa_stream_get_sample_spec(m_stream));
+}
+
+u8 PulseAudioStream::channel_count()
+{
+    return pa_stream_get_sample_spec(m_stream)->channels;
+}
+
+void PulseAudioStream::on_write_requested(size_t bytes_to_write)
+{
+    VERIFY(m_write_callback);
+    if (m_suspended)
+        return;
+    while (bytes_to_write > 0) {
+        auto buffer = begin_write(bytes_to_write).release_value_but_fixme_should_propagate_errors();
+        auto frame_size = this->frame_size();
+        VERIFY(buffer.size() % frame_size == 0);
+        auto written_buffer = m_write_callback(*this, buffer, buffer.size() / frame_size);
+        if (written_buffer.size() == 0) {
+            cancel_write().release_value_but_fixme_should_propagate_errors();
+            break;
+        }
+        bytes_to_write -= written_buffer.size();
+        write(written_buffer).release_value_but_fixme_should_propagate_errors();
+    }
+}
+
+ErrorOr<Bytes> PulseAudioStream::begin_write(size_t bytes_to_write)
+{
+    void* data_pointer;
+    size_t data_size = bytes_to_write;
+    if (pa_stream_begin_write(m_stream, &data_pointer, &data_size) != 0 || data_pointer == nullptr)
+        return Error::from_string_literal("Failed to get the playback stream's write buffer from PulseAudio");
+    return Bytes { data_pointer, data_size };
+}
+
+ErrorOr<void> PulseAudioStream::write(ReadonlyBytes data)
+{
+    if (pa_stream_write(m_stream, data.data(), data.size(), nullptr, 0, PA_SEEK_RELATIVE) != 0)
+        return Error::from_string_literal("Failed to write data to PulseAudio playback stream");
+    return {};
+}
+
+ErrorOr<void> PulseAudioStream::cancel_write()
+{
+    if (pa_stream_cancel_write(m_stream) != 0)
+        return Error::from_string_literal("Failed to get the playback stream's write buffer from PulseAudio");
+    return {};
+}
+
+bool PulseAudioStream::is_suspended() const
+{
+    return m_suspended;
+}
+
+StringView pulse_audio_error_to_string(PulseAudioErrorCode code)
+{
+    if (code < PulseAudioErrorCode::OK || code >= PulseAudioErrorCode::Sentinel)
+        return "Unknown error code"sv;
+
+    char const* string = pa_strerror(static_cast<int>(code));
+    return StringView { string, strlen(string) };
+}
+
+ErrorOr<void> PulseAudioStream::wait_for_operation(pa_operation* operation, StringView error_message)
+{
+    while (pa_operation_get_state(operation) == PA_OPERATION_RUNNING)
+        m_context->wait_for_signal();
+    if (!m_context->connection_is_good() || !this->connection_is_good()) {
+        auto pulse_audio_error_name = pulse_audio_error_to_string(m_context->get_last_error());
+        warnln("Encountered stream error: {}", pulse_audio_error_name);
+        return Error::from_string_view(error_message);
+    }
+    pa_operation_unref(operation);
+    return {};
+}
+
+ErrorOr<void> PulseAudioStream::drain_and_suspend()
+{
+    auto locker = m_context->main_loop_locker();
+
+    if (m_suspended)
+        return {};
+    m_suspended = true;
+
+    if (pa_stream_is_corked(m_stream) > 0)
+        return {};
+
+    TRY(wait_for_operation(pa_stream_drain(m_stream, STREAM_SIGNAL_CALLBACK(this)), "Draining PulseAudio stream failed"sv));
+    TRY(wait_for_operation(pa_stream_cork(m_stream, 1, STREAM_SIGNAL_CALLBACK(this)), "Corking PulseAudio stream after drain failed"sv));
+    return {};
+}
+
+ErrorOr<void> PulseAudioStream::flush_and_suspend()
+{
+    auto locker = m_context->main_loop_locker();
+
+    if (m_suspended)
+        return {};
+    m_suspended = true;
+
+    if (pa_stream_is_corked(m_stream) > 0)
+        return {};
+
+    TRY(wait_for_operation(pa_stream_flush(m_stream, STREAM_SIGNAL_CALLBACK(this)), "Flushing PulseAudio stream failed"sv));
+    TRY(wait_for_operation(pa_stream_cork(m_stream, 1, STREAM_SIGNAL_CALLBACK(this)), "Corking PulseAudio stream after flush failed"sv));
+    return {};
+}
+
+ErrorOr<void> PulseAudioStream::resume()
+{
+    auto locker = m_context->main_loop_locker();
+
+    if (!m_suspended)
+        return {};
+    m_suspended = false;
+
+    TRY(wait_for_operation(pa_stream_cork(m_stream, 0, STREAM_SIGNAL_CALLBACK(this)), "Uncorking PulseAudio stream failed"sv));
+
+    // Defer a write to the playback buffer on the PulseAudio main loop. Otherwise, playback will not
+    // begin again, despite the fact that we uncorked.
+    // NOTE: We ref here and then unref in the callback so that this stream will not be deleted until
+    //       it finishes.
+    ref();
+    pa_mainloop_api_once(
+        m_context->m_api, [](pa_mainloop_api*, void* user_data) {
+            auto& stream = *static_cast<PulseAudioStream*>(user_data);
+            // NOTE: writable_size() returns -1 in case of an error. However, the value is still safe
+            //       since begin_write() will interpret -1 as a default parameter and choose a good size.
+            auto bytes_to_write = pa_stream_writable_size(stream.m_stream);
+            stream.on_write_requested(bytes_to_write);
+            stream.unref();
+        },
+        this);
+    return {};
+}
+
+ErrorOr<AK::Duration> PulseAudioStream::total_time_played()
+{
+    auto locker = m_context->main_loop_locker();
+
+    // NOTE: This is a workaround for a PulseAudio issue. When a stream is started corked,
+    //       the time smoother doesn't seem to be aware of it, so it will return the time
+    //       since the stream was connected. Once the playback actually starts, the time
+    //       resets back to zero. However, since we request monotonically-increasing time,
+    //       this means that the smoother will register that it had a larger time before,
+    //       and return that time instead, until we reach a timestamp greater than the
+    //       last-returned time. If we never call pa_stream_get_time() until after giving
+    //       the stream its first samples, the issue never occurs.
+    if (!m_started_playback)
+        return AK::Duration::zero();
+
+    pa_usec_t time = 0;
+    auto error = pa_stream_get_time(m_stream, &time);
+    if (error == -PA_ERR_NODATA)
+        return AK::Duration::zero();
+    if (error != 0)
+        return Error::from_string_literal("Failed to get time from PulseAudio stream");
+    if (time > NumericLimits<i64>::max()) {
+        warnln("WARNING: Audio time is too large!");
+        time -= NumericLimits<i64>::max();
+    }
+    return AK::Duration::from_microseconds(static_cast<i64>(time));
+}
+
+ErrorOr<void> PulseAudioStream::set_volume(double volume)
+{
+    auto locker = m_context->main_loop_locker();
+
+    auto index = pa_stream_get_index(m_stream);
+    if (index == PA_INVALID_INDEX)
+        return Error::from_string_literal("Failed to get PulseAudio stream index while setting volume");
+
+    auto pulse_volume = pa_sw_volume_from_linear(volume);
+    pa_cvolume per_channel_volumes;
+    pa_cvolume_set(&per_channel_volumes, channel_count(), pulse_volume);
+
+    auto* operation = pa_context_set_sink_input_volume(m_context->m_context, index, &per_channel_volumes, STREAM_SIGNAL_CALLBACK(this));
+    return wait_for_operation(operation, "Failed to set PulseAudio stream volume"sv);
+}
+
+}

Libraries/LibMedia/Audio/PulseAudioWrappers.h

@@ -0,0 +1,185 @@
+/*
+ * Copyright (c) 2023, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include "Forward.h"
+#include "PlaybackStream.h"
+#include "SampleFormats.h"
+#include <AK/AtomicRefCounted.h>
+#include <AK/Error.h>
+#include <AK/NonnullRefPtr.h>
+#include <AK/Time.h>
+#include <LibThreading/Thread.h>
+#include <pulse/pulseaudio.h>
+
+namespace Audio {
+
+class PulseAudioStream;
+
+enum class PulseAudioContextState {
+    Unconnected = PA_CONTEXT_UNCONNECTED,
+    Connecting = PA_CONTEXT_CONNECTING,
+    Authorizing = PA_CONTEXT_AUTHORIZING,
+    SettingName = PA_CONTEXT_SETTING_NAME,
+    Ready = PA_CONTEXT_READY,
+    Failed = PA_CONTEXT_FAILED,
+    Terminated = PA_CONTEXT_TERMINATED,
+};
+
+enum class PulseAudioErrorCode;
+
+using PulseAudioDataRequestCallback = Function<ReadonlyBytes(PulseAudioStream&, Bytes buffer, size_t sample_count)>;
+
+// A wrapper around the PulseAudio main loop and context structs.
+// Generally, only one instance of this should be needed for a single process.
+class PulseAudioContext
+    : public AtomicRefCounted<PulseAudioContext>
+    , public Weakable<PulseAudioContext> {
+public:
+    static AK::WeakPtr<PulseAudioContext> weak_instance();
+    static ErrorOr<NonnullRefPtr<PulseAudioContext>> instance();
+
+    explicit PulseAudioContext(pa_threaded_mainloop*, pa_mainloop_api*, pa_context*);
+    PulseAudioContext(PulseAudioContext const& other) = delete;
+    ~PulseAudioContext();
+
+    bool current_thread_is_main_loop_thread();
+    void lock_main_loop();
+    void unlock_main_loop();
+    [[nodiscard]] auto main_loop_locker()
+    {
+        lock_main_loop();
+        return ScopeGuard([this]() { unlock_main_loop(); });
+    }
+    // Waits for signal_to_wake() to be called.
+    // This must be called with the main loop locked.
+    void wait_for_signal();
+    // Signals to wake all threads from calls to signal_to_wake()
+    void signal_to_wake();
+
+    PulseAudioContextState get_connection_state();
+    bool connection_is_good();
+    PulseAudioErrorCode get_last_error();
+
+    ErrorOr<NonnullRefPtr<PulseAudioStream>> create_stream(OutputState initial_state, u32 sample_rate, u8 channels, u32 target_latency_ms, PulseAudioDataRequestCallback write_callback);
+
+private:
+    friend class PulseAudioStream;
+
+    pa_threaded_mainloop* m_main_loop { nullptr };
+    pa_mainloop_api* m_api { nullptr };
+    pa_context* m_context;
+};
+
+enum class PulseAudioStreamState {
+    Unconnected = PA_STREAM_UNCONNECTED,
+    Creating = PA_STREAM_CREATING,
+    Ready = PA_STREAM_READY,
+    Failed = PA_STREAM_FAILED,
+    Terminated = PA_STREAM_TERMINATED,
+};
+
+class PulseAudioStream : public AtomicRefCounted<PulseAudioStream> {
+public:
+    static constexpr bool start_corked = true;
+
+    ~PulseAudioStream();
+
+    PulseAudioStreamState get_connection_state();
+    bool connection_is_good();
+
+    // Sets the callback to be run when the server consumes more of the buffer than
+    // has been written yet.
+    void set_underrun_callback(Function<void()>);
+
+    u32 sample_rate();
+    size_t sample_size();
+    size_t frame_size();
+    u8 channel_count();
+    // Gets a data buffer that can be written to and then passed back to PulseAudio through
+    // the write() function. This avoids a copy vs directly calling write().
+    ErrorOr<Bytes> begin_write(size_t bytes_to_write = NumericLimits<size_t>::max());
+    // Writes a data buffer to the playback stream.
+    ErrorOr<void> write(ReadonlyBytes data);
+    // Cancels the previous begin_write() call.
+    ErrorOr<void> cancel_write();
+
+    bool is_suspended() const;
+    // Plays back all buffered data and corks the stream. Until resume() is called, no data
+    // will be written to the stream.
+    ErrorOr<void> drain_and_suspend();
+    // Drops all buffered data and corks the stream. Until resume() is called, no data will
+    // be written to the stream.
+    ErrorOr<void> flush_and_suspend();
+    // Uncorks the stream and forces data to be written to the buffers to force playback to
+    // resume as soon as possible.
+    ErrorOr<void> resume();
+    ErrorOr<AK::Duration> total_time_played();
+
+    ErrorOr<void> set_volume(double volume);
+
+    PulseAudioContext& context() { return *m_context; }
+
+private:
+    friend class PulseAudioContext;
+
+    explicit PulseAudioStream(NonnullRefPtr<PulseAudioContext>&& context, pa_stream* stream)
+        : m_context(context)
+        , m_stream(stream)
+    {
+    }
+    PulseAudioStream(PulseAudioStream const& other) = delete;
+
+    ErrorOr<void> wait_for_operation(pa_operation*, StringView error_message);
+
+    void on_write_requested(size_t bytes_to_write);
+
+    NonnullRefPtr<PulseAudioContext> m_context;
+    pa_stream* m_stream { nullptr };
+    bool m_started_playback { false };
+    PulseAudioDataRequestCallback m_write_callback { nullptr };
+    // Determines whether we will allow the write callback to run. This should only be true
+    // if the stream is becoming or is already corked.
+    bool m_suspended { false };
+
+    Function<void()> m_underrun_callback;
+};
+
+enum class PulseAudioErrorCode {
+    OK = 0,
+    AccessFailure,
+    UnknownCommand,
+    InvalidArgument,
+    EntityExists,
+    NoSuchEntity,
+    ConnectionRefused,
+    ProtocolError,
+    Timeout,
+    NoAuthenticationKey,
+    InternalError,
+    ConnectionTerminated,
+    EntityKilled,
+    InvalidServer,
+    NoduleInitFailed,
+    BadState,
+    NoData,
+    IncompatibleProtocolVersion,
+    DataTooLarge,
+    NotSupported,
+    Unknown,
+    NoExtension,
+    Obsolete,
+    NotImplemented,
+    CalledFromFork,
+    IOError,
+    Busy,
+    Sentinel
+};
+
+StringView pulse_audio_error_to_string(PulseAudioErrorCode code);
+
+}

Libraries/LibMedia/Audio/Sample.h

@@ -0,0 +1,174 @@
+/*
+ * Copyright (c) 2018-2020, Andreas Kling <andreas@ladybird.org>
+ * Copyright (c) 2021, kleines Filmröllchen <filmroellchen@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/Format.h>
+#include <AK/Math.h>
+
+namespace Audio {
+using AK::Exponentials::exp;
+using AK::Exponentials::log;
+// Constants for logarithmic volume. See Sample::linear_to_log
+// Corresponds to 60dB
+constexpr float DYNAMIC_RANGE = 1000;
+constexpr float VOLUME_A = 1 / DYNAMIC_RANGE;
+float const VOLUME_B = log(DYNAMIC_RANGE);
+
+// A single sample in an audio buffer.
+// Values are floating point, and should range from -1.0 to +1.0
+struct Sample {
+    constexpr Sample() = default;
+
+    // For mono
+    constexpr explicit Sample(float left)
+        : left(left)
+        , right(left)
+    {
+    }
+
+    // For stereo
+    constexpr Sample(float left, float right)
+        : left(left)
+        , right(right)
+    {
+    }
+
+    // Returns the absolute maximum range (separate per channel) of the given sample buffer.
+    // For example { 0.8, 0 } means that samples on the left channel occupy the range { -0.8, 0.8 },
+    // while all samples on the right channel are 0.
+    static Sample max_range(ReadonlySpan<Sample> span)
+    {
+        Sample result { NumericLimits<float>::min_normal(), NumericLimits<float>::min_normal() };
+        for (Sample sample : span) {
+            result.left = max(result.left, AK::fabs(sample.left));
+            result.right = max(result.right, AK::fabs(sample.right));
+        }
+        return result;
+    }
+
+    void clip()
+    {
+        if (left > 1)
+            left = 1;
+        else if (left < -1)
+            left = -1;
+
+        if (right > 1)
+            right = 1;
+        else if (right < -1)
+            right = -1;
+    }
+
+    // Logarithmic scaling, as audio should ALWAYS do.
+    // Reference: https://www.dr-lex.be/info-stuff/volumecontrols.html
+    // We use the curve `factor = a * exp(b * change)`,
+    // where change is the input fraction we want to change by,
+    // a = 1/1000, b = ln(1000) = 6.908 and factor is the multiplier used.
+    // The value 1000 represents the dynamic range in sound pressure, which corresponds to 60 dB(A).
+    // This is a good dynamic range because it can represent all loudness values from
+    // 30 dB(A) (barely hearable with background noise)
+    // to 90 dB(A) (almost too loud to hear and about the reasonable limit of actual sound equipment).
+    //
+    // Format ranges:
+    // - Linear:        0.0 to 1.0
+    // - Logarithmic:   0.0 to 1.0
+
+    ALWAYS_INLINE float linear_to_log(float const change) const
+    {
+        // TODO: Add linear slope around 0
+        return VOLUME_A * exp(VOLUME_B * change);
+    }
+
+    ALWAYS_INLINE float log_to_linear(float const val) const
+    {
+        // TODO: Add linear slope around 0
+        return log(val / VOLUME_A) / VOLUME_B;
+    }
+
+    ALWAYS_INLINE Sample& log_multiply(float const change)
+    {
+        float factor = linear_to_log(change);
+        left *= factor;
+        right *= factor;
+        return *this;
+    }
+
+    ALWAYS_INLINE Sample log_multiplied(float const volume_change) const
+    {
+        Sample new_frame { left, right };
+        new_frame.log_multiply(volume_change);
+        return new_frame;
+    }
+
+    // Constant power panning
+    ALWAYS_INLINE Sample& pan(float const position)
+    {
+        float const pi_over_2 = AK::Pi<float> * 0.5f;
+        float const root_over_2 = AK::sqrt<float>(2.0) * 0.5f;
+        float const angle = position * pi_over_2 * 0.5f;
+        float s, c;
+        AK::sincos<float>(angle, s, c);
+        left *= root_over_2 * (c - s);
+        right *= root_over_2 * (c + s);
+        return *this;
+    }
+
+    ALWAYS_INLINE Sample panned(float const position) const
+    {
+        Sample new_sample { left, right };
+        new_sample.pan(position);
+        return new_sample;
+    }
+
+    constexpr Sample& operator*=(float const mult)
+    {
+        left *= mult;
+        right *= mult;
+        return *this;
+    }
+
+    constexpr Sample operator*(float const mult) const
+    {
+        return { left * mult, right * mult };
+    }
+
+    constexpr Sample& operator+=(Sample const& other)
+    {
+        left += other.left;
+        right += other.right;
+        return *this;
+    }
+    constexpr Sample& operator+=(float other)
+    {
+        left += other;
+        right += other;
+        return *this;
+    }
+
+    constexpr Sample operator+(Sample const& other) const
+    {
+        return { left + other.left, right + other.right };
+    }
+
+    float left { 0 };
+    float right { 0 };
+};
+
+}
+
+namespace AK {
+
+template<>
+struct Formatter<Audio::Sample> : Formatter<FormatString> {
+    ErrorOr<void> format(FormatBuilder& builder, Audio::Sample const& value)
+    {
+        return Formatter<FormatString>::format(builder, "[{}, {}]"sv, value.left, value.right);
+    }
+};
+
+}

Libraries/LibMedia/Audio/SampleFormats.cpp

@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2022, kleines Filmröllchen <filmroellchen@serenityos.org>.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include "SampleFormats.h"
+
+namespace Audio {
+
+u16 pcm_bits_per_sample(PcmSampleFormat format)
+{
+    switch (format) {
+    case PcmSampleFormat::Uint8:
+        return 8;
+    case PcmSampleFormat::Int16:
+        return 16;
+    case PcmSampleFormat::Int24:
+        return 24;
+    case PcmSampleFormat::Int32:
+    case PcmSampleFormat::Float32:
+        return 32;
+    case PcmSampleFormat::Float64:
+        return 64;
+    default:
+        VERIFY_NOT_REACHED();
+    }
+}
+
+}

Libraries/LibMedia/Audio/SampleFormats.h

@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2022, kleines Filmröllchen <filmroellchen@serenityos.org>.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/ByteString.h>
+#include <AK/Types.h>
+
+namespace Audio {
+
+// Supported PCM sample formats.
+enum class PcmSampleFormat : u8 {
+    Uint8,
+    Int16,
+    Int24,
+    Int32,
+    Float32,
+    Float64,
+};
+
+// Most of the read code only cares about how many bits to read or write
+u16 pcm_bits_per_sample(PcmSampleFormat format);
+
+}

Libraries/LibMedia/CMakeLists.txt

@@ -0,0 +1,52 @@
+if (NOT ANDROID AND NOT WIN32)
+    include(ffmpeg)
+    include(pulseaudio)
+endif()
+
+set(SOURCES
+    Audio/Loader.cpp
+    Audio/PlaybackStream.cpp
+    Audio/SampleFormats.cpp
+    Color/ColorConverter.cpp
+    Color/ColorPrimaries.cpp
+    Color/TransferCharacteristics.cpp
+    Containers/Matroska/MatroskaDemuxer.cpp
+    Containers/Matroska/Reader.cpp
+    PlaybackManager.cpp
+    VideoFrame.cpp
+)
+
+serenity_lib(LibMedia media)
+target_link_libraries(LibMedia PRIVATE LibCore LibCrypto LibRIFF LibIPC LibGfx LibThreading LibUnicode)
+
+if (NOT ANDROID AND NOT WIN32)
+    target_sources(LibMedia PRIVATE
+            Audio/FFmpegLoader.cpp
+            FFmpeg/FFmpegVideoDecoder.cpp
+    )
+    target_link_libraries(LibMedia PRIVATE PkgConfig::AVCODEC PkgConfig::AVFORMAT PkgConfig::AVUTIL)
+else()
+    # FIXME: Need to figure out how to build or replace ffmpeg libs on Android and Windows
+    target_sources(LibMedia PRIVATE FFmpeg/FFmpegVideoDecoderStub.cpp)
+endif()
+
+# Audio backend -- how we output audio to the speakers
+if (HAVE_PULSEAUDIO)
+    target_sources(LibMedia PRIVATE
+        Audio/PlaybackStreamPulseAudio.cpp
+        Audio/PulseAudioWrappers.cpp
+    )
+    target_link_libraries(LibMedia PRIVATE PkgConfig::PULSEAUDIO)
+    target_compile_definitions(LibMedia PUBLIC HAVE_PULSEAUDIO=1)
+elseif (APPLE AND NOT IOS)
+    target_sources(LibMedia PRIVATE Audio/PlaybackStreamAudioUnit.cpp)
+
+    find_library(AUDIO_UNIT AudioUnit REQUIRED)
+    target_link_libraries(LibMedia PRIVATE ${AUDIO_UNIT})
+elseif (ANDROID)
+    target_sources(LibMedia PRIVATE Audio/PlaybackStreamOboe.cpp)
+    find_package(oboe REQUIRED CONFIG)
+    target_link_libraries(LibMedia PRIVATE log oboe::oboe)
+else()
+    message(WARNING "No audio backend available")
+endif()

Libraries/LibMedia/CodecID.h

@@ -0,0 +1,85 @@
+/*
+ * Copyright (c) 2023, Stephan Vedder <stephan.vedder@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/Format.h>
+
+namespace Media {
+
+enum class CodecID : u32 {
+    Unknown,
+    // On2 / Google
+    VP8,
+    VP9,
+    // MPEG
+    H261,
+    MPEG1,
+    H262,
+    H263,
+    H264,
+    H265,
+    // AOMedia
+    AV1,
+    // Xiph
+    Theora,
+    Vorbis,
+    Opus,
+};
+
+}
+
+namespace AK {
+template<>
+struct Formatter<Media::CodecID> : Formatter<StringView> {
+    ErrorOr<void> format(FormatBuilder& builder, Media::CodecID value)
+    {
+        StringView codec;
+        switch (value) {
+        case Media::CodecID::Unknown:
+            codec = "Unknown"sv;
+            break;
+        case Media::CodecID::VP8:
+            codec = "VP8"sv;
+            break;
+        case Media::CodecID::VP9:
+            codec = "VP9"sv;
+            break;
+        case Media::CodecID::H261:
+            codec = "H.261"sv;
+            break;
+        case Media::CodecID::H262:
+            codec = "H.262"sv;
+            break;
+        case Media::CodecID::H263:
+            codec = "H.263"sv;
+            break;
+        case Media::CodecID::H264:
+            codec = "H.264"sv;
+            break;
+        case Media::CodecID::H265:
+            codec = "H.265"sv;
+            break;
+        case Media::CodecID::MPEG1:
+            codec = "MPEG1"sv;
+            break;
+        case Media::CodecID::AV1:
+            codec = "AV1"sv;
+            break;
+        case Media::CodecID::Theora:
+            codec = "Theora"sv;
+            break;
+        case Media::CodecID::Vorbis:
+            codec = "Vorbis"sv;
+            break;
+        case Media::CodecID::Opus:
+            codec = "Opus"sv;
+            break;
+        }
+        return builder.put_string(codec);
+    }
+};
+}

Libraries/LibMedia/Color/CodingIndependentCodePoints.h

@@ -0,0 +1,303 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/Format.h>
+#include <AK/StringView.h>
+
+namespace Media {
+
+// CICP is defined by H.273:
+// https://www.itu.int/rec/T-REC-H.273/en
+// See the Section 8.
+// Current edition is from 07/21.
+
+enum class ColorPrimaries : u8 {
+    Reserved = 0,
+    BT709 = 1,
+    Unspecified = 2, // Used by codecs to indicate that an alternative value may be used
+    BT470M = 4,
+    BT470BG = 5,
+    BT601 = 6,
+    SMPTE240 = 7,
+    GenericFilm = 8,
+    BT2020 = 9,
+    XYZ = 10,
+    SMPTE431 = 11,
+    SMPTE432 = 12,
+    EBU3213 = 22,
+    // All other values are also Reserved for later use.
+};
+
+enum class TransferCharacteristics : u8 {
+    Reserved = 0,
+    BT709 = 1,
+    Unspecified = 2, // Used by codecs to indicate that an alternative value may be used
+    BT470M = 4,
+    BT470BG = 5,
+    BT601 = 6, // BT.601 or Rec. 601
+    SMPTE240 = 7,
+    Linear = 8,
+    Log100 = 9,
+    Log100Sqrt10 = 10,
+    IEC61966 = 11,
+    BT1361 = 12,
+    SRGB = 13,
+    BT2020BitDepth10 = 14,
+    BT2020BitDepth12 = 15,
+    SMPTE2084 = 16, // Also known as PQ
+    SMPTE428 = 17,
+    HLG = 18,
+    // All other values are also Reserved for later use.
+};
+
+enum class MatrixCoefficients : u8 {
+    Identity = 0, // Applies no transformation to input values
+    BT709 = 1,
+    Unspecified = 2, // Used by codecs to indicate that an alternative value may be used
+    FCC = 4,
+    BT470BG = 5,
+    BT601 = 6,
+    SMPTE240 = 7,
+    YCgCo = 8,
+    BT2020NonConstantLuminance = 9,
+    BT2020ConstantLuminance = 10,
+    SMPTE2085 = 11,
+    ChromaticityDerivedNonConstantLuminance = 12,
+    ChromaticityDerivedConstantLuminance = 13,
+    ICtCp = 14,
+    // All other values are Reserved for later use.
+};
+
+enum class VideoFullRangeFlag : u8 {
+    Studio = 0,      // Y range 16..235, UV range 16..240
+    Full = 1,        // 0..255
+    Unspecified = 2, // Not part of the spec, serenity-specific addition for convenience.
+};
+
+// https://en.wikipedia.org/wiki/Coding-independent_code_points
+struct CodingIndependentCodePoints {
+public:
+    constexpr CodingIndependentCodePoints() = default;
+
+    constexpr CodingIndependentCodePoints(ColorPrimaries color_primaries, TransferCharacteristics transfer_characteristics, MatrixCoefficients matrix_coefficients, VideoFullRangeFlag video_full_range_flag)
+        : m_color_primaries(color_primaries)
+        , m_transfer_characteristics(transfer_characteristics)
+        , m_matrix_coefficients(matrix_coefficients)
+        , m_video_full_range_flag(video_full_range_flag)
+    {
+    }
+
+    constexpr ColorPrimaries color_primaries() const { return m_color_primaries; }
+    constexpr void set_color_primaries(ColorPrimaries value) { m_color_primaries = value; }
+    constexpr TransferCharacteristics transfer_characteristics() const { return m_transfer_characteristics; }
+    constexpr void set_transfer_characteristics(TransferCharacteristics value) { m_transfer_characteristics = value; }
+    constexpr MatrixCoefficients matrix_coefficients() const { return m_matrix_coefficients; }
+    constexpr void set_matrix_coefficients(MatrixCoefficients value) { m_matrix_coefficients = value; }
+    constexpr VideoFullRangeFlag video_full_range_flag() const { return m_video_full_range_flag; }
+    constexpr void set_video_full_range_flag(VideoFullRangeFlag value) { m_video_full_range_flag = value; }
+
+    constexpr void default_code_points_if_unspecified(CodingIndependentCodePoints cicp)
+    {
+        if (color_primaries() == ColorPrimaries::Unspecified)
+            set_color_primaries(cicp.color_primaries());
+        if (transfer_characteristics() == TransferCharacteristics::Unspecified)
+            set_transfer_characteristics(cicp.transfer_characteristics());
+        if (matrix_coefficients() == MatrixCoefficients::Unspecified)
+            set_matrix_coefficients(cicp.matrix_coefficients());
+        if (video_full_range_flag() == VideoFullRangeFlag::Unspecified)
+            set_video_full_range_flag(cicp.video_full_range_flag());
+    }
+
+    constexpr void adopt_specified_values(CodingIndependentCodePoints cicp)
+    {
+        if (cicp.color_primaries() != ColorPrimaries::Unspecified)
+            set_color_primaries(cicp.color_primaries());
+        if (cicp.transfer_characteristics() != TransferCharacteristics::Unspecified)
+            set_transfer_characteristics(cicp.transfer_characteristics());
+        if (cicp.matrix_coefficients() != MatrixCoefficients::Unspecified)
+            set_matrix_coefficients(cicp.matrix_coefficients());
+        if (cicp.video_full_range_flag() != VideoFullRangeFlag::Unspecified)
+            set_video_full_range_flag(cicp.video_full_range_flag());
+    }
+
+private:
+    ColorPrimaries m_color_primaries = ColorPrimaries::BT709;
+    TransferCharacteristics m_transfer_characteristics = TransferCharacteristics::BT709;
+    MatrixCoefficients m_matrix_coefficients = MatrixCoefficients::BT709;
+    VideoFullRangeFlag m_video_full_range_flag = VideoFullRangeFlag::Full;
+};
+
+constexpr StringView color_primaries_to_string(ColorPrimaries color_primaries)
+{
+    switch (color_primaries) {
+    case ColorPrimaries::Reserved:
+        return "Reserved"sv;
+    case ColorPrimaries::BT709:
+        return "BT.709"sv;
+    case ColorPrimaries::Unspecified:
+        return "Unspecified"sv;
+    case ColorPrimaries::BT470M:
+        return "BT.470 System M"sv;
+    case ColorPrimaries::BT470BG:
+        return "BT.470 System B, G"sv;
+    case ColorPrimaries::BT601:
+        return "BT.601"sv;
+    case ColorPrimaries::SMPTE240:
+        return "SMPTE ST 240"sv;
+    case ColorPrimaries::GenericFilm:
+        return "Generic film"sv;
+    case ColorPrimaries::BT2020:
+        return "BT.2020"sv;
+    case ColorPrimaries::XYZ:
+        return "CIE 1931 XYZ"sv;
+    case ColorPrimaries::SMPTE431:
+        return "SMPTE RP 431"sv;
+    case ColorPrimaries::SMPTE432:
+        return "SMPTE EG 432"sv;
+    case ColorPrimaries::EBU3213:
+        return "EBU Tech 3213"sv;
+    }
+    return "Reserved"sv;
+}
+
+constexpr StringView transfer_characteristics_to_string(TransferCharacteristics transfer_characteristics)
+{
+    switch (transfer_characteristics) {
+    case TransferCharacteristics::Reserved:
+        return "Reserved"sv;
+    case TransferCharacteristics::BT709:
+        return "BT.709"sv;
+    case TransferCharacteristics::Unspecified:
+        return "Unspecified"sv;
+    case TransferCharacteristics::BT470M:
+        return "BT.470 System M"sv;
+    case TransferCharacteristics::BT470BG:
+        return "BT.470 System B, G"sv;
+    case TransferCharacteristics::BT601:
+        return "BT.601"sv;
+    case TransferCharacteristics::SMPTE240:
+        return "SMPTE ST 240"sv;
+    case TransferCharacteristics::Linear:
+        return "Linear"sv;
+    case TransferCharacteristics::Log100:
+        return "Logarithmic (100:1 range)"sv;
+    case TransferCharacteristics::Log100Sqrt10:
+        return "Logarithmic (100xSqrt(10):1 range)"sv;
+    case TransferCharacteristics::IEC61966:
+        return "IEC 61966"sv;
+    case TransferCharacteristics::BT1361:
+        return "BT.1361"sv;
+    case TransferCharacteristics::SRGB:
+        return "sRGB"sv;
+    case TransferCharacteristics::BT2020BitDepth10:
+        return "BT.2020 (10-bit)"sv;
+    case TransferCharacteristics::BT2020BitDepth12:
+        return "BT.2020 (12-bit)"sv;
+    case TransferCharacteristics::SMPTE2084:
+        return "SMPTE ST 2084 (PQ)"sv;
+    case TransferCharacteristics::SMPTE428:
+        return "SMPTE ST 428"sv;
+    case TransferCharacteristics::HLG:
+        return "ARIB STD-B67 (HLG, BT.2100)"sv;
+    }
+    return "Reserved"sv;
+}
+
+constexpr StringView matrix_coefficients_to_string(MatrixCoefficients matrix_coefficients)
+{
+    switch (matrix_coefficients) {
+    case MatrixCoefficients::Identity:
+        return "Identity"sv;
+    case MatrixCoefficients::BT709:
+        return "BT.709"sv;
+    case MatrixCoefficients::Unspecified:
+        return "Unspecified"sv;
+    case MatrixCoefficients::FCC:
+        return "FCC (CFR 73.682)"sv;
+    case MatrixCoefficients::BT470BG:
+        return "BT.470 System B, G"sv;
+    case MatrixCoefficients::BT601:
+        return "BT.601"sv;
+    case MatrixCoefficients::SMPTE240:
+        return "SMPTE ST 240"sv;
+    case MatrixCoefficients::YCgCo:
+        return "YCgCo"sv;
+    case MatrixCoefficients::BT2020NonConstantLuminance:
+        return "BT.2020, non-constant luminance"sv;
+    case MatrixCoefficients::BT2020ConstantLuminance:
+        return "BT.2020, constant luminance"sv;
+    case MatrixCoefficients::SMPTE2085:
+        return "SMPTE ST 2085"sv;
+    case MatrixCoefficients::ChromaticityDerivedNonConstantLuminance:
+        return "Chromaticity-derived, non-constant luminance"sv;
+    case MatrixCoefficients::ChromaticityDerivedConstantLuminance:
+        return "Chromaticity-derived, constant luminance"sv;
+    case MatrixCoefficients::ICtCp:
+        return "BT.2100 ICtCp"sv;
+    }
+    return "Reserved"sv;
+}
+
+constexpr StringView video_full_range_flag_to_string(VideoFullRangeFlag video_full_range_flag)
+{
+    switch (video_full_range_flag) {
+    case VideoFullRangeFlag::Studio:
+        return "Studio"sv;
+    case VideoFullRangeFlag::Full:
+        return "Full"sv;
+    case VideoFullRangeFlag::Unspecified: // Not part of the spec, serenity-specific addition for convenience.
+        return "Unspecified"sv;
+    }
+    return "Unknown"sv;
+}
+
+}
+
+namespace AK {
+
+template<>
+struct Formatter<Media::ColorPrimaries> final : Formatter<StringView> {
+    ErrorOr<void> format(FormatBuilder& builder, Media::ColorPrimaries color_primaries)
+    {
+        return Formatter<StringView>::format(builder, Media::color_primaries_to_string(color_primaries));
+    }
+};
+
+template<>
+struct Formatter<Media::TransferCharacteristics> final : Formatter<StringView> {
+    ErrorOr<void> format(FormatBuilder& builder, Media::TransferCharacteristics transfer_characteristics)
+    {
+        return Formatter<StringView>::format(builder, Media::transfer_characteristics_to_string(transfer_characteristics));
+    }
+};
+
+template<>
+struct Formatter<Media::MatrixCoefficients> final : Formatter<StringView> {
+    ErrorOr<void> format(FormatBuilder& builder, Media::MatrixCoefficients matrix_coefficients)
+    {
+        return Formatter<StringView>::format(builder, Media::matrix_coefficients_to_string(matrix_coefficients));
+    }
+};
+
+template<>
+struct Formatter<Media::VideoFullRangeFlag> final : Formatter<StringView> {
+    ErrorOr<void> format(FormatBuilder& builder, Media::VideoFullRangeFlag range)
+    {
+        return Formatter<StringView>::format(builder, Media::video_full_range_flag_to_string(range));
+    }
+};
+
+template<>
+struct Formatter<Media::CodingIndependentCodePoints> final : Formatter<FormatString> {
+    ErrorOr<void> format(FormatBuilder& builder, Media::CodingIndependentCodePoints cicp)
+    {
+        return Formatter<FormatString>::format(builder, "CICP {{ CP = {}, TC = {}, MC = {}, Range = {} }}"sv, cicp.color_primaries(), cicp.transfer_characteristics(), cicp.matrix_coefficients(), cicp.video_full_range_flag());
+    }
+};
+
+}

Libraries/LibMedia/Color/ColorConverter.cpp

@@ -0,0 +1,158 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/Math.h>
+#include <AK/StdLibExtras.h>
+#include <LibGfx/Matrix4x4.h>
+#include <LibMedia/Color/ColorPrimaries.h>
+#include <LibMedia/Color/TransferCharacteristics.h>
+
+#include "ColorConverter.h"
+
+namespace Media {
+
+DecoderErrorOr<ColorConverter> ColorConverter::create(u8 bit_depth, CodingIndependentCodePoints input_cicp, CodingIndependentCodePoints output_cicp)
+{
+    // We'll need to apply tonemapping for linear HDR values.
+    bool should_tonemap = false;
+    switch (input_cicp.transfer_characteristics()) {
+    case TransferCharacteristics::SMPTE2084:
+    case TransferCharacteristics::HLG:
+        should_tonemap = true;
+        break;
+    default:
+        break;
+    }
+
+    // Conversion process:
+    // 1. Scale integer YUV values with maximum values of (1 << bit_depth) - 1 into
+    //    float 0..1 range.
+    //    This can be done with a 3x3 scaling matrix.
+    size_t maximum_value = (1u << bit_depth) - 1;
+    float scale = 1.0f / maximum_value;
+    FloatMatrix4x4 integer_scaling_matrix = {
+        scale, 0.0f, 0.0f, 0.0f, // y
+        0.0f, scale, 0.0f, 0.0f, // u
+        0.0f, 0.0f, scale, 0.0f, // v
+        0.0f, 0.0f, 0.0f, 1.0f,  // w
+    };
+
+    // 2. Scale YUV values into usable ranges.
+    //    For studio range, Y range is 16..235, and UV is 16..240.
+    //    UV values should be scaled to a range of -1..1.
+    //    This can be done in a 4x4 matrix with translation and scaling.
+    float y_min;
+    float y_max;
+    float uv_min;
+    float uv_max;
+    if (input_cicp.video_full_range_flag() == VideoFullRangeFlag::Studio) {
+        y_min = 16.0f / 255.0f;
+        y_max = 235.0f / 255.0f;
+        uv_min = y_min;
+        uv_max = 240.0f / 255.0f;
+    } else {
+        y_min = 0.0f;
+        y_max = 1.0f;
+        uv_min = 0.0f;
+        uv_max = 1.0f;
+    }
+    auto clip_y_scale = 1.0f / (y_max - y_min);
+    auto clip_uv_scale = 2.0f / (uv_max - uv_min);
+
+    FloatMatrix4x4 range_scaling_matrix = {
+        clip_y_scale, 0.0f, 0.0f, -y_min * clip_y_scale,             // y
+        0.0f, clip_uv_scale, 0.0f, -(uv_min * clip_uv_scale + 1.0f), // u
+        0.0f, 0.0f, clip_uv_scale, -(uv_min * clip_uv_scale + 1.0f), // v
+        0.0f, 0.0f, 0.0f, 1.0f,                                      // w
+    };
+
+    // 3. Convert YUV values to RGB.
+    //    This is done with coefficients that can be put into a 3x3 matrix
+    //    and combined with the above 4x4 matrix to combine steps 1 and 2.
+    FloatMatrix4x4 color_conversion_matrix;
+
+    // https://kdashg.github.io/misc/colors/from-coeffs.html
+    switch (input_cicp.matrix_coefficients()) {
+    case MatrixCoefficients::BT470BG:
+    case MatrixCoefficients::BT601:
+        color_conversion_matrix = {
+            1.0f, 0.0f, 0.70100f, 0.0f,       // y
+            1.0f, -0.17207f, -0.35707f, 0.0f, // u
+            1.0f, 0.88600f, 0.0f, 0.0f,       // v
+            0.0f, 0.0f, 0.0f, 1.0f,           // w
+        };
+        break;
+    case MatrixCoefficients::BT709:
+        color_conversion_matrix = {
+            1.0f, 0.0f, 0.78740f, 0.0f,       // y
+            1.0f, -0.09366f, -0.23406f, 0.0f, // u
+            1.0f, 0.92780f, 0.0f, 0.0f,       // v
+            0.0f, 0.0f, 0.0f, 1.0f,           // w
+        };
+        break;
+    case MatrixCoefficients::BT2020ConstantLuminance:
+    case MatrixCoefficients::BT2020NonConstantLuminance:
+        color_conversion_matrix = {
+            1.0f, 0.0f, 0.73730f, 0.0f,       // y
+            1.0f, -0.08228f, -0.28568f, 0.0f, // u
+            1.0f, 0.94070f, 0.0f, 0.0f,       // v
+            0.0f, 0.0f, 0.0f, 1.0f,           // w
+        };
+        break;
+    default:
+        return DecoderError::format(DecoderErrorCategory::Invalid, "Matrix coefficients {} not supported", matrix_coefficients_to_string(input_cicp.matrix_coefficients()));
+    }
+
+    // 4. Apply the inverse transfer function to convert RGB values to the
+    //    linear color space.
+    //    This will be turned into a lookup table and interpolated to speed
+    //    up the conversion.
+    auto to_linear_lookup_table = InterpolatedLookupTable<to_linear_size>::create(
+        [&](float value) {
+            return TransferCharacteristicsConversion::to_linear_luminance(value, input_cicp.transfer_characteristics());
+        });
+
+    // 5. Convert the RGB color to CIE XYZ coordinates using the input color
+    //    primaries and then to the output color primaries.
+    //    This is done with two 3x3 matrices that can be combined into one
+    //    matrix multiplication.
+    FloatMatrix3x3 color_primaries_matrix = TRY(get_conversion_matrix(input_cicp.color_primaries(), output_cicp.color_primaries()));
+
+    // 6. Apply the output transfer function. For HDR color spaces, this
+    //    should apply tonemapping as well.
+    //    Use a lookup table as with step 3.
+    auto to_non_linear_lookup_table = InterpolatedLookupTable<to_non_linear_size>::create(
+        [&](float value) {
+            return TransferCharacteristicsConversion::to_non_linear_luminance(value, output_cicp.transfer_characteristics());
+        });
+
+    // Expand color primaries matrix with identity elements.
+    FloatMatrix4x4 color_primaries_matrix_4x4 = {
+        color_primaries_matrix.elements()[0][0],
+        color_primaries_matrix.elements()[0][1],
+        color_primaries_matrix.elements()[0][2],
+        0.0f, // y
+        color_primaries_matrix.elements()[1][0],
+        color_primaries_matrix.elements()[1][1],
+        color_primaries_matrix.elements()[1][2],
+        0.0f, // u
+        color_primaries_matrix.elements()[2][0],
+        color_primaries_matrix.elements()[2][1],
+        color_primaries_matrix.elements()[2][2],
+        0.0f, // v
+        0.0f,
+        0.0f,
+        0.0f,
+        1.0f, // w
+    };
+
+    bool should_skip_color_remapping = output_cicp.color_primaries() == input_cicp.color_primaries() && output_cicp.transfer_characteristics() == input_cicp.transfer_characteristics();
+    FloatMatrix4x4 input_conversion_matrix = color_conversion_matrix * range_scaling_matrix * integer_scaling_matrix;
+
+    return ColorConverter(bit_depth, input_cicp, should_skip_color_remapping, should_tonemap, input_conversion_matrix, to_linear_lookup_table, color_primaries_matrix_4x4, to_non_linear_lookup_table);
+}
+
+}

Libraries/LibMedia/Color/ColorConverter.h

@@ -0,0 +1,272 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/Array.h>
+#include <AK/Function.h>
+#include <LibGfx/Color.h>
+#include <LibGfx/Matrix4x4.h>
+#include <LibMedia/Color/CodingIndependentCodePoints.h>
+#include <LibMedia/DecoderError.h>
+
+namespace Media {
+
+template<size_t N, size_t Scale = 1>
+struct InterpolatedLookupTable {
+public:
+    static InterpolatedLookupTable<N, Scale> create(Function<float(float)> transfer_function)
+    {
+        // We'll allocate one extra index to allow the values to reach 1.0.
+        InterpolatedLookupTable<N, Scale> lookup_table;
+        float index_to_value_mult = static_cast<float>(Scale) / maximum_value;
+        for (size_t i = 0; i < N; i++) {
+            float value = i * index_to_value_mult;
+            value = transfer_function(value);
+            lookup_table.m_lookup_table[i] = value;
+        }
+        return lookup_table;
+    }
+
+    ALWAYS_INLINE float do_lookup(float value) const
+    {
+        float float_index = value * (maximum_value / static_cast<float>(Scale));
+        if (float_index > maximum_value) [[unlikely]]
+            float_index = maximum_value;
+        size_t index = static_cast<size_t>(float_index);
+        float partial_index = float_index - index;
+        value = m_lookup_table[index] * (1.0f - partial_index) + m_lookup_table[index + 1] * partial_index;
+        return value;
+    }
+
+    ALWAYS_INLINE FloatVector4 do_lookup(FloatVector4 vector) const
+    {
+        return {
+            do_lookup(vector.x()),
+            do_lookup(vector.y()),
+            do_lookup(vector.z()),
+            vector.w()
+        };
+    }
+
+private:
+    static constexpr size_t maximum_value = N - 2;
+
+    Array<float, N> m_lookup_table;
+};
+
+static auto hlg_ootf_lookup_table = InterpolatedLookupTable<32, 1000>::create(
+    [](float value) {
+        return AK::pow(value, 1.2f - 1.0f);
+    });
+
+class ColorConverter final {
+
+private:
+    // Tonemapping methods are outlined here:
+    // https://64.github.io/tonemapping/
+
+    template<typename T>
+    static ALWAYS_INLINE constexpr T scalar_to_color_vector(float value)
+    {
+        if constexpr (IsSame<T, Gfx::VectorN<4, float>>) {
+            return Gfx::VectorN<4, float>(value, value, value, 1.0f);
+        } else if constexpr (IsSame<T, Gfx::VectorN<3, float>>) {
+            return Gfx::VectorN<3, float>(value, value, value);
+        } else {
+            static_assert(IsFloatingPoint<T>);
+            return static_cast<T>(value);
+        }
+    }
+
+    template<typename T>
+    static ALWAYS_INLINE constexpr T hable_tonemapping_partial(T value)
+    {
+        constexpr auto a = scalar_to_color_vector<T>(0.15f);
+        constexpr auto b = scalar_to_color_vector<T>(0.5f);
+        constexpr auto c = scalar_to_color_vector<T>(0.1f);
+        constexpr auto d = scalar_to_color_vector<T>(0.2f);
+        constexpr auto e = scalar_to_color_vector<T>(0.02f);
+        constexpr auto f = scalar_to_color_vector<T>(0.3f);
+        return ((value * (a * value + c * b) + d * e) / (value * (a * value + b) + d * f)) - e / f;
+    }
+
+    template<typename T>
+    static ALWAYS_INLINE constexpr T hable_tonemapping(T value)
+    {
+        constexpr auto exposure_bias = scalar_to_color_vector<T>(2.0f);
+        value = hable_tonemapping_partial<T>(value * exposure_bias);
+        constexpr auto scale = scalar_to_color_vector<T>(1.0f) / scalar_to_color_vector<T>(hable_tonemapping_partial(11.2f));
+        return value * scale;
+    }
+
+public:
+    static DecoderErrorOr<ColorConverter> create(u8 bit_depth, CodingIndependentCodePoints input_cicp, CodingIndependentCodePoints output_cicp);
+
+    // Referencing https://en.wikipedia.org/wiki/YCbCr
+    ALWAYS_INLINE Gfx::Color convert_yuv(u16 y, u16 u, u16 v) const
+    {
+        auto max_zero = [](FloatVector4 vector) {
+            return FloatVector4(max(0.0f, vector.x()), max(0.0f, vector.y()), max(0.0f, vector.z()), vector.w());
+        };
+
+        FloatVector4 color_vector = { static_cast<float>(y), static_cast<float>(u), static_cast<float>(v), 1.0f };
+        color_vector = m_input_conversion_matrix * color_vector;
+
+        if (m_should_skip_color_remapping) {
+            color_vector.clamp(0.0f, 1.0f);
+        } else {
+            color_vector = max_zero(color_vector);
+            color_vector = m_to_linear_lookup.do_lookup(color_vector);
+
+            if (m_cicp.transfer_characteristics() == TransferCharacteristics::HLG) {
+                // See: https://en.wikipedia.org/wiki/Hybrid_log-gamma under a bolded section "HLG reference OOTF"
+                float luminance = (0.2627f * color_vector.x() + 0.6780f * color_vector.y() + 0.0593f * color_vector.z()) * 1000.0f;
+                float coefficient = hlg_ootf_lookup_table.do_lookup(luminance);
+                color_vector = { color_vector.x() * coefficient, color_vector.y() * coefficient, color_vector.z() * coefficient, 1.0f };
+            }
+
+            // FIXME: We could implement gamut compression here:
+            //        https://github.com/jedypod/gamut-compress/blob/master/docs/gamut-compress-algorithm.md
+            //        This would allow the color values outside the output gamut to be
+            //        preserved relative to values within the gamut instead of clipping. The
+            //        downside is that this requires a pass over the image before conversion
+            //        back into gamut is done to find the maximum color values to compress.
+            //        The compression would have to be somewhat temporally consistent as well.
+            color_vector = m_color_space_conversion_matrix * color_vector;
+            color_vector = max_zero(color_vector);
+            if (m_should_tonemap)
+                color_vector = hable_tonemapping(color_vector);
+            color_vector = m_to_non_linear_lookup.do_lookup(color_vector);
+            color_vector = max_zero(color_vector);
+        }
+
+        u8 r = static_cast<u8>(color_vector.x() * 255.0f);
+        u8 g = static_cast<u8>(color_vector.y() * 255.0f);
+        u8 b = static_cast<u8>(color_vector.z() * 255.0f);
+        return Gfx::Color(r, g, b);
+    }
+
+    // Fast conversion of 8-bit YUV to full-range RGB.
+    template<MatrixCoefficients MC, VideoFullRangeFlag FR, Unsigned T>
+    static ALWAYS_INLINE Gfx::Color convert_simple_yuv_to_rgb(T y_in, T u_in, T v_in)
+    {
+        static constexpr i32 bit_depth = 8;
+        static constexpr i32 maximum_value = (1 << bit_depth) - 1;
+        static constexpr i32 one = 1 << 14;
+        static constexpr auto fraction = [](i32 numerator, i32 denominator) constexpr {
+            auto temp = static_cast<i64>(numerator) * one;
+            return static_cast<i32>(temp / denominator);
+        };
+        static constexpr auto coef = [](i32 hundred_thousandths) constexpr {
+            return fraction(hundred_thousandths, 100'000);
+        };
+        static constexpr auto multiply = [](i32 a, i32 b) constexpr {
+            return (a * b) / one;
+        };
+
+        struct RangeFactors {
+            i32 y_offset, y_scale;
+            i32 uv_offset, uv_scale;
+        };
+
+        constexpr auto range_factors = [] {
+            RangeFactors range_factors;
+
+            i32 min = 0;
+            i32 y_max = 255;
+            i32 uv_max = 255;
+
+            if constexpr (FR == VideoFullRangeFlag::Studio) {
+                min = 16;
+                y_max = 235;
+                uv_max = 240;
+            }
+
+            range_factors.y_offset = -min * maximum_value / 255;
+            range_factors.y_scale = fraction(255, y_max - min);
+            range_factors.uv_offset = -((min + uv_max) * maximum_value) / (255 * 2);
+            range_factors.uv_scale = fraction(255, uv_max - min) * 2;
+
+            range_factors.y_scale = multiply(range_factors.y_scale, fraction(255, maximum_value));
+            range_factors.uv_scale = multiply(range_factors.uv_scale, fraction(255, maximum_value));
+
+            return range_factors;
+        }();
+
+        i32 y = y_in + range_factors.y_offset;
+        i32 u = u_in + range_factors.uv_offset;
+        i32 v = v_in + range_factors.uv_offset;
+
+        i32 red;
+        i32 green;
+        i32 blue;
+
+        constexpr i32 y_scale = range_factors.y_scale;
+        constexpr i32 uv_scale = range_factors.uv_scale;
+
+        // The equations below will have the following effects:
+        //  - Scale the Y, U and V values into the range 0...maximum_value*one for these fixed-point operations.
+        //  - Scale the values by the color range defined by VideoFullRangeFlag.
+        //  - Scale the U and V values by 2 to put them in the actual YCbCr coordinate space.
+        //  - Multiply by the YCbCr coefficients to convert to RGB.
+        if constexpr (MC == MatrixCoefficients::BT709) {
+            red = y * y_scale + v * multiply(coef(78740), uv_scale);
+            green = y * y_scale + u * multiply(coef(-9366), uv_scale) + v * multiply(coef(-23406), uv_scale);
+            blue = y * y_scale + u * multiply(coef(92780), uv_scale);
+        }
+
+        if constexpr (MC == MatrixCoefficients::BT601) {
+            red = y * y_scale + v * multiply(coef(70100), uv_scale);
+            green = y * y_scale + u * multiply(coef(-17207), uv_scale) + v * multiply(coef(-35707), uv_scale);
+            blue = y * y_scale + u * multiply(coef(88600), uv_scale);
+        }
+
+        if constexpr (MC == MatrixCoefficients::BT2020ConstantLuminance) {
+            red = y * y_scale + v * multiply(coef(73730), uv_scale);
+            green = y * y_scale + u * multiply(coef(-8228), uv_scale) + v * multiply(coef(-28568), uv_scale);
+            blue = y * y_scale + u * multiply(coef(94070), uv_scale);
+        }
+
+        red = clamp(red, 0, maximum_value * one);
+        green = clamp(green, 0, maximum_value * one);
+        blue = clamp(blue, 0, maximum_value * one);
+
+        // This compiles down to a bit shift if maximum_value == 255
+        red /= fraction(maximum_value, 255);
+        green /= fraction(maximum_value, 255);
+        blue /= fraction(maximum_value, 255);
+
+        return Gfx::Color(u8(red), u8(green), u8(blue));
+    }
+
+private:
+    static constexpr size_t to_linear_size = 64;
+    static constexpr size_t to_non_linear_size = 64;
+
+    ColorConverter(u8 bit_depth, CodingIndependentCodePoints cicp, bool should_skip_color_remapping, bool should_tonemap, FloatMatrix4x4 input_conversion_matrix, InterpolatedLookupTable<to_linear_size> to_linear_lookup, FloatMatrix4x4 color_space_conversion_matrix, InterpolatedLookupTable<to_non_linear_size> to_non_linear_lookup)
+        : m_bit_depth(bit_depth)
+        , m_cicp(cicp)
+        , m_should_skip_color_remapping(should_skip_color_remapping)
+        , m_should_tonemap(should_tonemap)
+        , m_input_conversion_matrix(input_conversion_matrix)
+        , m_to_linear_lookup(move(to_linear_lookup))
+        , m_color_space_conversion_matrix(color_space_conversion_matrix)
+        , m_to_non_linear_lookup(move(to_non_linear_lookup))
+    {
+    }
+
+    u8 m_bit_depth;
+    CodingIndependentCodePoints m_cicp;
+    bool m_should_skip_color_remapping;
+    bool m_should_tonemap;
+    FloatMatrix4x4 m_input_conversion_matrix;
+    InterpolatedLookupTable<to_linear_size> m_to_linear_lookup;
+    FloatMatrix4x4 m_color_space_conversion_matrix;
+    InterpolatedLookupTable<to_non_linear_size> m_to_non_linear_lookup;
+};
+
+}

Libraries/LibMedia/Color/ColorPrimaries.cpp

@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <LibGfx/Vector2.h>
+#include <LibGfx/Vector3.h>
+
+#include "ColorPrimaries.h"
+
+namespace Media {
+
+ALWAYS_INLINE constexpr FloatVector3 primaries_to_xyz(FloatVector2 primaries)
+{
+    // https://en.wikipedia.org/wiki/CIE_1931_color_space#CIE_xy_chromaticity_diagram_and_the_CIE_xyY_color_space
+    // Luminosity is set to 1.0, so the equations are simplified.
+    auto const x = primaries.x();
+    auto const y = primaries.y();
+    return {
+        x / y,
+        1.0f,
+        (1.0f - x - y) / y
+    };
+}
+
+ALWAYS_INLINE constexpr FloatMatrix3x3 vectors_to_matrix(FloatVector3 a, FloatVector3 b, FloatVector3 c)
+{
+    return FloatMatrix3x3(
+        a.x(), a.y(), a.z(),
+        b.x(), b.y(), b.z(),
+        c.x(), c.y(), c.z());
+}
+
+ALWAYS_INLINE constexpr FloatMatrix3x3 primaries_matrix(FloatVector2 red, FloatVector2 green, FloatVector2 blue)
+{
+    return vectors_to_matrix(primaries_to_xyz(red), primaries_to_xyz(green), primaries_to_xyz(blue)).transpose();
+}
+
+ALWAYS_INLINE constexpr FloatVector3 matrix_row(FloatMatrix3x3 matrix, size_t row)
+{
+    return { matrix.elements()[row][0], matrix.elements()[row][1], matrix.elements()[row][2] };
+}
+
+ALWAYS_INLINE constexpr FloatMatrix3x3 generate_rgb_to_xyz_matrix(FloatVector2 red_xy, FloatVector2 green_xy, FloatVector2 blue_xy, FloatVector2 white_xy)
+{
+    // http://www.brucelindbloom.com/index.html?Eqn_RGB_XYZ_Matrix.html
+    FloatMatrix3x3 const matrix = primaries_matrix(red_xy, green_xy, blue_xy);
+    FloatVector3 const scale_vector = matrix.inverse() * primaries_to_xyz(white_xy);
+    return vectors_to_matrix(matrix_row(matrix, 0) * scale_vector, matrix_row(matrix, 1) * scale_vector, matrix_row(matrix, 2) * scale_vector);
+}
+
+constexpr FloatVector2 ILLUMINANT_D65 = { 0.3127f, 0.3290f };
+
+constexpr FloatVector2 BT_709_RED = { 0.64f, 0.33f };
+constexpr FloatVector2 BT_709_GREEN = { 0.30f, 0.60f };
+constexpr FloatVector2 BT_709_BLUE = { 0.15f, 0.06f };
+
+constexpr FloatVector2 BT_2020_RED = { 0.708f, 0.292f };
+constexpr FloatVector2 BT_2020_GREEN = { 0.170f, 0.797f };
+constexpr FloatVector2 BT_2020_BLUE = { 0.131f, 0.046f };
+
+constexpr FloatMatrix3x3 bt_2020_rgb_to_xyz = generate_rgb_to_xyz_matrix(BT_2020_RED, BT_2020_GREEN, BT_2020_BLUE, ILLUMINANT_D65);
+constexpr FloatMatrix3x3 bt_709_rgb_to_xyz = generate_rgb_to_xyz_matrix(BT_709_RED, BT_709_GREEN, BT_709_BLUE, ILLUMINANT_D65);
+
+DecoderErrorOr<FloatMatrix3x3> get_conversion_matrix(ColorPrimaries input_primaries, ColorPrimaries output_primaries)
+{
+    FloatMatrix3x3 input_conversion_matrix;
+    switch (input_primaries) {
+    case ColorPrimaries::BT709:
+        input_conversion_matrix = bt_709_rgb_to_xyz;
+        break;
+    case ColorPrimaries::BT2020:
+        input_conversion_matrix = bt_2020_rgb_to_xyz;
+        break;
+    default:
+        return DecoderError::format(DecoderErrorCategory::NotImplemented, "Conversion of primaries {} is not implemented", color_primaries_to_string(input_primaries));
+    }
+
+    FloatMatrix3x3 output_conversion_matrix;
+    switch (output_primaries) {
+    case ColorPrimaries::BT709:
+        output_conversion_matrix = bt_709_rgb_to_xyz.inverse();
+        break;
+    case ColorPrimaries::BT2020:
+        output_conversion_matrix = bt_2020_rgb_to_xyz.inverse();
+        break;
+    default:
+        return DecoderError::format(DecoderErrorCategory::NotImplemented, "Conversion of primaries {} is not implemented", color_primaries_to_string(output_primaries));
+    }
+
+    return output_conversion_matrix * input_conversion_matrix;
+}
+
+}

Libraries/LibMedia/Color/ColorPrimaries.h

@@ -0,0 +1,17 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <LibGfx/Matrix3x3.h>
+#include <LibMedia/Color/CodingIndependentCodePoints.h>
+#include <LibMedia/DecoderError.h>
+
+namespace Media {
+
+DecoderErrorOr<FloatMatrix3x3> get_conversion_matrix(ColorPrimaries input_primaries, ColorPrimaries output_primaries);
+
+}

Libraries/LibMedia/Color/TransferCharacteristics.cpp

@@ -0,0 +1,124 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/Format.h>
+#include <AK/Math.h>
+#include <AK/StdLibExtras.h>
+
+#include "TransferCharacteristics.h"
+
+namespace Media {
+
+// SDR maximum luminance in candelas per meter squared
+constexpr float sdr_max_luminance = 120.0f;
+
+// sRGB
+constexpr float srgb_inverse_beta = 0.0031308f;
+constexpr float srgb_inverse_linear_coef = 12.92f;
+constexpr float srgb_gamma = 2.4f;
+constexpr float srgb_alpha = 1.055f;
+
+// BT.601/BT.709/BT.2020 constants
+constexpr float bt_601_beta = 0.018053968510807f;
+constexpr float bt_601_linear_coef = 4.5f;
+constexpr float bt_601_alpha = 1.0f + 5.5f * bt_601_beta;
+constexpr float bt_601_gamma = 0.45f;
+
+// Perceptual quantizer (SMPTE ST 2084) constants
+constexpr float pq_m1 = 2610.0f / 16384.0f;
+constexpr float pq_m2 = 128.0f * 2523.0f / 4096.0f;
+constexpr float pq_c1 = 3424.0f / 4096.0f;
+constexpr float pq_c2 = 32.0f * 2413.0f / 4096.0f;
+constexpr float pq_c3 = 32.0f * 2392.0f / 4096.0f;
+constexpr float pq_max_luminance = 10000.0f;
+
+// Hybrid log-gamma constants
+constexpr float hlg_a = 0.17883277f;
+constexpr float hlg_b = 0.28466892f;
+constexpr float hlg_c = 0.55991073f;
+
+float TransferCharacteristicsConversion::to_linear_luminance(float value, TransferCharacteristics transfer_function)
+{
+    switch (transfer_function) {
+    case TransferCharacteristics::BT709:
+    case TransferCharacteristics::BT601:
+    case TransferCharacteristics::BT2020BitDepth10:
+    case TransferCharacteristics::BT2020BitDepth12:
+        // https://en.wikipedia.org/wiki/Rec._601#Transfer_characteristics
+        // https://en.wikipedia.org/wiki/Rec._709#Transfer_characteristics
+        // https://en.wikipedia.org/wiki/Rec._2020#Transfer_characteristics
+        // These three share identical OETFs.
+        if (value < bt_601_beta * bt_601_linear_coef)
+            return value / bt_601_linear_coef;
+        return AK::pow((value + (bt_601_alpha - 1.0f)) / bt_601_alpha, 1.0f / bt_601_gamma);
+    case TransferCharacteristics::SRGB:
+        // https://color.org/sRGB.pdf
+        if (value < srgb_inverse_linear_coef * srgb_inverse_beta)
+            return value / srgb_inverse_linear_coef;
+        return AK::pow((value + (srgb_alpha - 1.0f)) / srgb_alpha, srgb_gamma);
+    case TransferCharacteristics::SMPTE2084: {
+        // https://en.wikipedia.org/wiki/Perceptual_quantizer
+        auto gamma_adjusted = AK::pow(value, 1.0f / pq_m2);
+        auto numerator = max(gamma_adjusted - pq_c1, 0.0f);
+        auto denominator = pq_c2 - pq_c3 * gamma_adjusted;
+        return AK::pow(numerator / denominator, 1.0f / pq_m1) * (pq_max_luminance / sdr_max_luminance);
+    }
+    case TransferCharacteristics::HLG:
+        // https://en.wikipedia.org/wiki/Hybrid_log-gamma
+        if (value < 0.5f)
+            return (value * value) / 3.0f;
+        return (AK::exp((value - hlg_c) / hlg_a) + hlg_b) / 12.0f;
+    default:
+        dbgln("Unsupported transfer function {}", static_cast<u8>(transfer_function));
+        VERIFY_NOT_REACHED();
+    }
+}
+
+float TransferCharacteristicsConversion::to_non_linear_luminance(float value, TransferCharacteristics transfer_function)
+{
+    switch (transfer_function) {
+    case TransferCharacteristics::BT709:
+    case TransferCharacteristics::BT601:
+    case TransferCharacteristics::BT2020BitDepth10:
+    case TransferCharacteristics::BT2020BitDepth12:
+        // https://en.wikipedia.org/wiki/Rec._601#Transfer_characteristics
+        // https://en.wikipedia.org/wiki/Rec._709#Transfer_characteristics
+        // https://en.wikipedia.org/wiki/Rec._2020#Transfer_characteristics
+        // These three share identical OETFs.
+        if (value < bt_601_beta)
+            return bt_601_linear_coef * value;
+        return bt_601_alpha * AK::pow(value, bt_601_gamma) - (bt_601_alpha - 1.0f);
+    case TransferCharacteristics::SRGB:
+        // https://color.org/sRGB.pdf
+        if (value < srgb_inverse_beta)
+            return value * srgb_inverse_linear_coef;
+        return srgb_alpha * AK::pow(value, 1.0f / srgb_gamma) - (srgb_alpha - 1.0f);
+    case TransferCharacteristics::SMPTE2084: {
+        // https://en.wikipedia.org/wiki/Perceptual_quantizer
+        auto linear_value = AK::pow(value * (sdr_max_luminance / pq_max_luminance), pq_m1);
+        auto numerator = pq_c1 + pq_c2 * linear_value;
+        auto denominator = 1 + pq_c3 * linear_value;
+        return AK::pow(numerator / denominator, pq_m2);
+    }
+    case TransferCharacteristics::HLG:
+        // https://en.wikipedia.org/wiki/Hybrid_log-gamma
+        if (value < 1.0f / 12.0f)
+            return AK::sqrt(value * 3.0f);
+        return hlg_a * AK::log(12.0f * value - hlg_b) + hlg_c;
+    default:
+        dbgln("Unsupported transfer function {}", static_cast<u8>(transfer_function));
+        VERIFY_NOT_REACHED();
+    }
+}
+
+FloatVector4 TransferCharacteristicsConversion::hlg_opto_optical_transfer_function(FloatVector4 const& vector, float gamma, float gain)
+{
+    float luminance = (0.2627f * vector.x() + 0.6780f * vector.y() + 0.0593f * vector.z()) * 1000.0f;
+    float coefficient = gain * AK::pow(luminance, gamma - 1.0f);
+    return FloatVector4(vector.x() * coefficient, vector.y() * coefficient, vector.z() * coefficient, vector.w());
+}
+
+}

Libraries/LibMedia/Color/TransferCharacteristics.h

@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <LibGfx/Vector4.h>
+#include <LibMedia/Color/CodingIndependentCodePoints.h>
+
+namespace Media {
+
+class TransferCharacteristicsConversion {
+public:
+    static float to_linear_luminance(float value, TransferCharacteristics transfer_function);
+
+    static float to_non_linear_luminance(float value, TransferCharacteristics transfer_function);
+
+    // https://en.wikipedia.org/wiki/Hybrid_log-gamma
+    // See "HLG reference OOTF"
+    static FloatVector4 hlg_opto_optical_transfer_function(FloatVector4 const& vector, float gamma, float gain);
+};
+
+}

Libraries/LibMedia/Containers/Matroska/Document.h

@@ -0,0 +1,241 @@
+/*
+ * Copyright (c) 2021, Hunter Salyer <thefalsehonesty@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/ByteBuffer.h>
+#include <AK/ByteString.h>
+#include <AK/FixedArray.h>
+#include <AK/FlyString.h>
+#include <AK/HashMap.h>
+#include <AK/OwnPtr.h>
+#include <AK/Time.h>
+#include <AK/Utf8View.h>
+#include <LibMedia/Color/CodingIndependentCodePoints.h>
+
+namespace Media::Matroska {
+
+struct EBMLHeader {
+    ByteString doc_type;
+    u32 doc_type_version;
+};
+
+class SegmentInformation {
+public:
+    u64 timestamp_scale() const { return m_timestamp_scale; }
+    void set_timestamp_scale(u64 timestamp_scale) { m_timestamp_scale = timestamp_scale; }
+    Utf8View muxing_app() const { return Utf8View(m_muxing_app); }
+    void set_muxing_app(ByteString muxing_app) { m_muxing_app = move(muxing_app); }
+    Utf8View writing_app() const { return Utf8View(m_writing_app); }
+    void set_writing_app(ByteString writing_app) { m_writing_app = move(writing_app); }
+    Optional<double> duration_unscaled() const { return m_duration_unscaled; }
+    void set_duration_unscaled(double duration) { m_duration_unscaled.emplace(duration); }
+    Optional<AK::Duration> duration() const
+    {
+        if (!duration_unscaled().has_value())
+            return {};
+        return AK::Duration::from_nanoseconds(static_cast<i64>(static_cast<double>(timestamp_scale()) * duration_unscaled().value()));
+    }
+
+private:
+    u64 m_timestamp_scale { 1'000'000 };
+    ByteString m_muxing_app;
+    ByteString m_writing_app;
+    Optional<double> m_duration_unscaled;
+};
+
+class TrackEntry : public RefCounted<TrackEntry> {
+public:
+    enum TrackType : u8 {
+        Invalid = 0,
+        Video = 1,
+        Audio = 2,
+        Complex = 3,
+        Logo = 16,
+        Subtitle = 17,
+        Buttons = 18,
+        Control = 32,
+        Metadata = 33,
+    };
+
+    enum class ColorRange : u8 {
+        Unspecified = 0,
+        Broadcast = 1,
+        Full = 2,
+        UseCICP = 3, // defined by MatrixCoefficients / TransferCharacteristics
+    };
+
+    struct ColorFormat {
+        ColorPrimaries color_primaries = ColorPrimaries::Unspecified;
+        TransferCharacteristics transfer_characteristics = TransferCharacteristics::Unspecified;
+        MatrixCoefficients matrix_coefficients = MatrixCoefficients::Unspecified;
+        u64 bits_per_channel = 0;
+        ColorRange range = ColorRange::Unspecified;
+
+        CodingIndependentCodePoints to_cicp() const
+        {
+            VideoFullRangeFlag video_full_range_flag;
+            switch (range) {
+            case ColorRange::Full:
+                video_full_range_flag = VideoFullRangeFlag::Full;
+                break;
+            case ColorRange::Broadcast:
+                video_full_range_flag = VideoFullRangeFlag::Studio;
+                break;
+            case ColorRange::Unspecified:
+            case ColorRange::UseCICP:
+                // FIXME: Figure out what UseCICP should do here. Matroska specification did not
+                //        seem to explain in the 'colour' section. When this is fixed, change
+                //        replace_code_points_if_specified to match.
+                video_full_range_flag = VideoFullRangeFlag::Unspecified;
+                break;
+            }
+
+            return { color_primaries, transfer_characteristics, matrix_coefficients, video_full_range_flag };
+        }
+    };
+
+    struct VideoTrack {
+        u64 pixel_width;
+        u64 pixel_height;
+
+        ColorFormat color_format;
+    };
+
+    struct AudioTrack {
+        u64 channels;
+        u64 bit_depth;
+    };
+
+    u64 track_number() const { return m_track_number; }
+    void set_track_number(u64 track_number) { m_track_number = track_number; }
+    u64 track_uid() const { return m_track_uid; }
+    void set_track_uid(u64 track_uid) { m_track_uid = track_uid; }
+    TrackType track_type() const { return m_track_type; }
+    void set_track_type(TrackType track_type) { m_track_type = track_type; }
+    FlyString language() const { return m_language; }
+    void set_language(FlyString const& language) { m_language = language; }
+    FlyString codec_id() const { return m_codec_id; }
+    void set_codec_id(FlyString const& codec_id) { m_codec_id = codec_id; }
+    ReadonlyBytes codec_private_data() const { return m_codec_private_data.span(); }
+    ErrorOr<void> set_codec_private_data(ReadonlyBytes codec_private_data)
+    {
+        m_codec_private_data = TRY(FixedArray<u8>::create(codec_private_data));
+        return {};
+    }
+    double timestamp_scale() const { return m_timestamp_scale; }
+    void set_timestamp_scale(double timestamp_scale) { m_timestamp_scale = timestamp_scale; }
+    u64 codec_delay() const { return m_codec_delay; }
+    void set_codec_delay(u64 codec_delay) { m_codec_delay = codec_delay; }
+    u64 timestamp_offset() const { return m_timestamp_offset; }
+    void set_timestamp_offset(u64 timestamp_offset) { m_timestamp_offset = timestamp_offset; }
+    Optional<VideoTrack> video_track() const
+    {
+        if (track_type() != Video)
+            return {};
+        return m_video_track;
+    }
+    void set_video_track(VideoTrack video_track) { m_video_track = video_track; }
+    Optional<AudioTrack> audio_track() const
+    {
+        if (track_type() != Audio)
+            return {};
+        return m_audio_track;
+    }
+    void set_audio_track(AudioTrack audio_track) { m_audio_track = audio_track; }
+
+private:
+    u64 m_track_number { 0 };
+    u64 m_track_uid { 0 };
+    TrackType m_track_type { Invalid };
+    FlyString m_language = "eng"_fly_string;
+    FlyString m_codec_id;
+    FixedArray<u8> m_codec_private_data;
+    double m_timestamp_scale { 1 };
+    u64 m_codec_delay { 0 };
+    u64 m_timestamp_offset { 0 };
+
+    union {
+        VideoTrack m_video_track {};
+        AudioTrack m_audio_track;
+    };
+};
+
+class Block {
+public:
+    enum Lacing : u8 {
+        None = 0b00,
+        XIPH = 0b01,
+        FixedSize = 0b10,
+        EBML = 0b11,
+    };
+
+    u64 track_number() const { return m_track_number; }
+    void set_track_number(u64 track_number) { m_track_number = track_number; }
+    AK::Duration timestamp() const { return m_timestamp; }
+    void set_timestamp(AK::Duration timestamp) { m_timestamp = timestamp; }
+    bool only_keyframes() const { return m_only_keyframes; }
+    void set_only_keyframes(bool only_keyframes) { m_only_keyframes = only_keyframes; }
+    bool invisible() const { return m_invisible; }
+    void set_invisible(bool invisible) { m_invisible = invisible; }
+    Lacing lacing() const { return m_lacing; }
+    void set_lacing(Lacing lacing) { m_lacing = lacing; }
+    bool discardable() const { return m_discardable; }
+    void set_discardable(bool discardable) { m_discardable = discardable; }
+
+    void set_frames(Vector<ReadonlyBytes>&& frames) { m_frames = move(frames); }
+    ReadonlyBytes const& frame(size_t index) const { return frames()[index]; }
+    u64 frame_count() const { return m_frames.size(); }
+    Vector<ReadonlyBytes> const& frames() const { return m_frames; }
+
+private:
+    u64 m_track_number { 0 };
+    AK::Duration m_timestamp { AK::Duration::zero() };
+    bool m_only_keyframes { false };
+    bool m_invisible { false };
+    Lacing m_lacing { None };
+    bool m_discardable { true };
+    Vector<ReadonlyBytes> m_frames;
+};
+
+class Cluster {
+public:
+    AK::Duration timestamp() const { return m_timestamp; }
+    void set_timestamp(AK::Duration timestamp) { m_timestamp = timestamp; }
+
+private:
+    AK::Duration m_timestamp { AK::Duration::zero() };
+};
+
+class CueTrackPosition {
+public:
+    u64 track_number() const { return m_track_number; }
+    void set_track_number(u64 track_number) { m_track_number = track_number; }
+    size_t cluster_position() const { return m_cluster_position; }
+    void set_cluster_position(size_t cluster_position) { m_cluster_position = cluster_position; }
+    size_t block_offset() const { return m_block_offset; }
+    void set_block_offset(size_t block_offset) { m_block_offset = block_offset; }
+
+private:
+    u64 m_track_number { 0 };
+    size_t m_cluster_position { 0 };
+    size_t m_block_offset { 0 };
+};
+
+class CuePoint {
+public:
+    AK::Duration timestamp() const { return m_timestamp; }
+    void set_timestamp(AK::Duration timestamp) { m_timestamp = timestamp; }
+    OrderedHashMap<u64, CueTrackPosition>& track_positions() { return m_track_positions; }
+    OrderedHashMap<u64, CueTrackPosition> const& track_positions() const { return m_track_positions; }
+    Optional<CueTrackPosition const&> position_for_track(u64 track_number) const { return m_track_positions.get(track_number); }
+
+private:
+    AK::Duration m_timestamp = AK::Duration::min();
+    OrderedHashMap<u64, CueTrackPosition> m_track_positions;
+};
+
+}

Libraries/LibMedia/Containers/Matroska/MatroskaDemuxer.cpp

@@ -0,0 +1,160 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/Debug.h>
+
+#include "MatroskaDemuxer.h"
+
+namespace Media::Matroska {
+
+DecoderErrorOr<NonnullOwnPtr<MatroskaDemuxer>> MatroskaDemuxer::from_file(StringView filename)
+{
+    return make<MatroskaDemuxer>(TRY(Reader::from_file(filename)));
+}
+
+DecoderErrorOr<NonnullOwnPtr<MatroskaDemuxer>> MatroskaDemuxer::from_mapped_file(NonnullOwnPtr<Core::MappedFile> mapped_file)
+{
+    return make<MatroskaDemuxer>(TRY(Reader::from_mapped_file(move(mapped_file))));
+}
+
+DecoderErrorOr<NonnullOwnPtr<MatroskaDemuxer>> MatroskaDemuxer::from_data(ReadonlyBytes data)
+{
+    return make<MatroskaDemuxer>(TRY(Reader::from_data(data)));
+}
+
+DecoderErrorOr<Vector<Track>> MatroskaDemuxer::get_tracks_for_type(TrackType type)
+{
+    TrackEntry::TrackType matroska_track_type;
+
+    switch (type) {
+    case TrackType::Video:
+        matroska_track_type = TrackEntry::TrackType::Video;
+        break;
+    case TrackType::Audio:
+        matroska_track_type = TrackEntry::TrackType::Audio;
+        break;
+    case TrackType::Subtitles:
+        matroska_track_type = TrackEntry::TrackType::Subtitle;
+        break;
+    }
+
+    Vector<Track> tracks;
+    TRY(m_reader.for_each_track_of_type(matroska_track_type, [&](TrackEntry const& track_entry) -> DecoderErrorOr<IterationDecision> {
+        VERIFY(track_entry.track_type() == matroska_track_type);
+        Track track(type, track_entry.track_number());
+
+        switch (type) {
+        case TrackType::Video:
+            if (auto video_track = track_entry.video_track(); video_track.has_value())
+                track.set_video_data({ TRY(duration()), video_track->pixel_width, video_track->pixel_height });
+            break;
+        default:
+            break;
+        }
+
+        DECODER_TRY_ALLOC(tracks.try_append(track));
+        return IterationDecision::Continue;
+    }));
+
+    return tracks;
+}
+
+DecoderErrorOr<MatroskaDemuxer::TrackStatus*> MatroskaDemuxer::get_track_status(Track track)
+{
+    if (!m_track_statuses.contains(track)) {
+        auto iterator = TRY(m_reader.create_sample_iterator(track.identifier()));
+        DECODER_TRY_ALLOC(m_track_statuses.try_set(track, { iterator }));
+    }
+
+    return &m_track_statuses.get(track).release_value();
+}
+
+CodecID MatroskaDemuxer::get_codec_id_for_string(FlyString const& codec_id)
+{
+    dbgln_if(MATROSKA_DEBUG, "Codec ID: {}", codec_id);
+    if (codec_id == "V_VP8")
+        return CodecID::VP8;
+    if (codec_id == "V_VP9")
+        return CodecID::VP9;
+    if (codec_id == "V_MPEG1")
+        return CodecID::MPEG1;
+    if (codec_id == "V_MPEG2")
+        return CodecID::H262;
+    if (codec_id == "V_MPEG4/ISO/AVC")
+        return CodecID::H264;
+    if (codec_id == "V_MPEGH/ISO/HEVC")
+        return CodecID::H265;
+    if (codec_id == "V_AV1")
+        return CodecID::AV1;
+    if (codec_id == "V_THEORA")
+        return CodecID::Theora;
+    if (codec_id == "A_VORBIS")
+        return CodecID::Vorbis;
+    if (codec_id == "A_OPUS")
+        return CodecID::Opus;
+    return CodecID::Unknown;
+}
+
+DecoderErrorOr<CodecID> MatroskaDemuxer::get_codec_id_for_track(Track track)
+{
+    auto codec_id = TRY(m_reader.track_for_track_number(track.identifier()))->codec_id();
+    return get_codec_id_for_string(codec_id);
+}
+
+DecoderErrorOr<ReadonlyBytes> MatroskaDemuxer::get_codec_initialization_data_for_track(Track track)
+{
+    return TRY(m_reader.track_for_track_number(track.identifier()))->codec_private_data();
+}
+
+DecoderErrorOr<Optional<AK::Duration>> MatroskaDemuxer::seek_to_most_recent_keyframe(Track track, AK::Duration timestamp, Optional<AK::Duration> earliest_available_sample)
+{
+    // Removing the track status will cause us to start from the beginning.
+    if (timestamp.is_zero()) {
+        m_track_statuses.remove(track);
+        return timestamp;
+    }
+
+    auto& track_status = *TRY(get_track_status(track));
+    auto seeked_iterator = TRY(m_reader.seek_to_random_access_point(track_status.iterator, timestamp));
+    VERIFY(seeked_iterator.last_timestamp().has_value());
+
+    auto last_sample = earliest_available_sample;
+    if (!last_sample.has_value()) {
+        last_sample = track_status.iterator.last_timestamp();
+    }
+    if (last_sample.has_value()) {
+        bool skip_seek = seeked_iterator.last_timestamp().value() <= last_sample.value() && last_sample.value() <= timestamp;
+        dbgln_if(MATROSKA_DEBUG, "The last available sample at {}ms is {}closer to target timestamp {}ms than the keyframe at {}ms, {}", last_sample->to_milliseconds(), skip_seek ? ""sv : "not "sv, timestamp.to_milliseconds(), seeked_iterator.last_timestamp()->to_milliseconds(), skip_seek ? "skipping seek"sv : "seeking"sv);
+        if (skip_seek) {
+            return OptionalNone();
+        }
+    }
+
+    track_status.iterator = move(seeked_iterator);
+    return track_status.iterator.last_timestamp();
+}
+
+DecoderErrorOr<Sample> MatroskaDemuxer::get_next_sample_for_track(Track track)
+{
+    // FIXME: This makes a copy of the sample, which shouldn't be necessary.
+    //        Matroska should make a RefPtr<ByteBuffer>, probably.
+    auto& status = *TRY(get_track_status(track));
+
+    if (!status.block.has_value() || status.frame_index >= status.block->frame_count()) {
+        status.block = TRY(status.iterator.next_block());
+        status.frame_index = 0;
+    }
+    auto cicp = TRY(m_reader.track_for_track_number(track.identifier()))->video_track()->color_format.to_cicp();
+    return Sample(status.block->timestamp(), status.block->frame(status.frame_index++), VideoSampleData(cicp));
+}
+
+DecoderErrorOr<AK::Duration> MatroskaDemuxer::duration()
+{
+    auto duration = TRY(m_reader.segment_information()).duration();
+    return duration.value_or(AK::Duration::zero());
+}
+
+}

Libraries/LibMedia/Containers/Matroska/MatroskaDemuxer.h

@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/HashMap.h>
+#include <LibMedia/Demuxer.h>
+
+#include "Reader.h"
+
+namespace Media::Matroska {
+
+class MatroskaDemuxer final : public Demuxer {
+public:
+    // FIXME: We should instead accept some abstract data streaming type so that the demuxer
+    //        can work with non-contiguous data.
+    static DecoderErrorOr<NonnullOwnPtr<MatroskaDemuxer>> from_file(StringView filename);
+    static DecoderErrorOr<NonnullOwnPtr<MatroskaDemuxer>> from_mapped_file(NonnullOwnPtr<Core::MappedFile> mapped_file);
+
+    static DecoderErrorOr<NonnullOwnPtr<MatroskaDemuxer>> from_data(ReadonlyBytes data);
+
+    MatroskaDemuxer(Reader&& reader)
+        : m_reader(move(reader))
+    {
+    }
+
+    DecoderErrorOr<Vector<Track>> get_tracks_for_type(TrackType type) override;
+
+    DecoderErrorOr<Optional<AK::Duration>> seek_to_most_recent_keyframe(Track track, AK::Duration timestamp, Optional<AK::Duration> earliest_available_sample = OptionalNone()) override;
+
+    DecoderErrorOr<AK::Duration> duration() override;
+
+    DecoderErrorOr<CodecID> get_codec_id_for_track(Track track) override;
+
+    DecoderErrorOr<ReadonlyBytes> get_codec_initialization_data_for_track(Track track) override;
+
+    DecoderErrorOr<Sample> get_next_sample_for_track(Track track) override;
+
+private:
+    struct TrackStatus {
+        SampleIterator iterator;
+        Optional<Block> block {};
+        size_t frame_index { 0 };
+    };
+
+    DecoderErrorOr<TrackStatus*> get_track_status(Track track);
+    CodecID get_codec_id_for_string(FlyString const& codec_id);
+
+    Reader m_reader;
+
+    HashMap<Track, TrackStatus> m_track_statuses;
+};
+
+}

Libraries/LibMedia/Containers/Matroska/Reader.h

@@ -0,0 +1,169 @@
+/*
+ * Copyright (c) 2021, Hunter Salyer <thefalsehonesty@gmail.com>
+ * Copyright (c) 2022, Gregory Bertilson <Zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/IntegralMath.h>
+#include <AK/Optional.h>
+#include <AK/OwnPtr.h>
+#include <LibCore/MappedFile.h>
+#include <LibMedia/DecoderError.h>
+
+#include "Document.h"
+
+namespace Media::Matroska {
+
+class SampleIterator;
+class Streamer;
+
+class Reader {
+public:
+    typedef Function<DecoderErrorOr<IterationDecision>(TrackEntry const&)> TrackEntryCallback;
+
+    static DecoderErrorOr<Reader> from_file(StringView path);
+    static DecoderErrorOr<Reader> from_mapped_file(NonnullOwnPtr<Core::MappedFile> mapped_file);
+
+    static DecoderErrorOr<Reader> from_data(ReadonlyBytes data);
+
+    EBMLHeader const& header() const { return m_header.value(); }
+
+    DecoderErrorOr<SegmentInformation> segment_information();
+
+    DecoderErrorOr<void> for_each_track(TrackEntryCallback);
+    DecoderErrorOr<void> for_each_track_of_type(TrackEntry::TrackType, TrackEntryCallback);
+    DecoderErrorOr<NonnullRefPtr<TrackEntry>> track_for_track_number(u64);
+    DecoderErrorOr<size_t> track_count();
+
+    DecoderErrorOr<SampleIterator> create_sample_iterator(u64 track_number);
+    DecoderErrorOr<SampleIterator> seek_to_random_access_point(SampleIterator, AK::Duration);
+    DecoderErrorOr<Optional<Vector<CuePoint> const&>> cue_points_for_track(u64 track_number);
+    DecoderErrorOr<bool> has_cues_for_track(u64 track_number);
+
+private:
+    Reader(ReadonlyBytes data)
+        : m_data(data)
+    {
+    }
+
+    DecoderErrorOr<void> parse_initial_data();
+
+    DecoderErrorOr<Optional<size_t>> find_first_top_level_element_with_id([[maybe_unused]] StringView element_name, u32 element_id);
+
+    DecoderErrorOr<void> ensure_tracks_are_parsed();
+    DecoderErrorOr<void> parse_tracks(Streamer&);
+
+    DecoderErrorOr<void> parse_cues(Streamer&);
+    DecoderErrorOr<void> ensure_cues_are_parsed();
+    DecoderErrorOr<void> seek_to_cue_for_timestamp(SampleIterator&, AK::Duration const&);
+
+    RefPtr<Core::SharedMappedFile> m_mapped_file;
+    ReadonlyBytes m_data;
+
+    Optional<EBMLHeader> m_header;
+
+    size_t m_segment_contents_position { 0 };
+    size_t m_segment_contents_size { 0 };
+
+    HashMap<u32, size_t> m_seek_entries;
+    size_t m_last_top_level_element_position { 0 };
+
+    Optional<SegmentInformation> m_segment_information;
+
+    OrderedHashMap<u64, NonnullRefPtr<TrackEntry>> m_tracks;
+
+    // The vectors must be sorted by timestamp at all times.
+    HashMap<u64, Vector<CuePoint>> m_cues;
+    bool m_cues_have_been_parsed { false };
+};
+
+class SampleIterator {
+public:
+    DecoderErrorOr<Block> next_block();
+    Cluster const& current_cluster() const { return *m_current_cluster; }
+    Optional<AK::Duration> const& last_timestamp() const { return m_last_timestamp; }
+    TrackEntry const& track() const { return *m_track; }
+
+private:
+    friend class Reader;
+
+    SampleIterator(RefPtr<Core::SharedMappedFile> file, ReadonlyBytes data, TrackEntry& track, u64 timestamp_scale, size_t position)
+        : m_file(move(file))
+        , m_data(data)
+        , m_track(track)
+        , m_segment_timestamp_scale(timestamp_scale)
+        , m_position(position)
+    {
+    }
+
+    DecoderErrorOr<void> seek_to_cue_point(CuePoint const& cue_point);
+
+    RefPtr<Core::SharedMappedFile> m_file;
+    ReadonlyBytes m_data;
+    NonnullRefPtr<TrackEntry> m_track;
+    u64 m_segment_timestamp_scale { 0 };
+
+    // Must always point to an element ID or the end of the stream.
+    size_t m_position { 0 };
+
+    Optional<AK::Duration> m_last_timestamp;
+
+    Optional<Cluster> m_current_cluster;
+};
+
+class Streamer {
+public:
+    Streamer(ReadonlyBytes data)
+        : m_data(data)
+    {
+    }
+
+    u8 const* data() { return m_data.data() + m_position; }
+
+    char const* data_as_chars() { return reinterpret_cast<char const*>(data()); }
+
+    size_t octets_read() { return m_octets_read.last(); }
+
+    void push_octets_read() { m_octets_read.append(0); }
+
+    void pop_octets_read()
+    {
+        auto popped = m_octets_read.take_last();
+        if (!m_octets_read.is_empty())
+            m_octets_read.last() += popped;
+    }
+
+    ErrorOr<u8> read_octet();
+
+    ErrorOr<i16> read_i16();
+
+    ErrorOr<u64> read_variable_size_integer(bool mask_length = true);
+    ErrorOr<i64> read_variable_size_signed_integer();
+
+    ErrorOr<u64> read_u64();
+    ErrorOr<double> read_float();
+
+    ErrorOr<ByteString> read_string();
+
+    ErrorOr<void> read_unknown_element();
+
+    ErrorOr<ReadonlyBytes> read_raw_octets(size_t num_octets);
+
+    size_t position() const { return m_position; }
+    size_t remaining() const { return m_data.size() - position(); }
+
+    bool at_end() const { return remaining() == 0; }
+    bool has_octet() const { return remaining() >= 1; }
+
+    ErrorOr<void> seek_to_position(size_t position);
+
+private:
+    ReadonlyBytes m_data;
+    size_t m_position { 0 };
+    Vector<size_t> m_octets_read { 0 };
+};
+
+}

Libraries/LibMedia/DecoderError.h

@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/ByteString.h>
+#include <AK/Error.h>
+#include <AK/Format.h>
+#include <AK/SourceLocation.h>
+#include <LibMedia/Forward.h>
+#include <errno.h>
+
+namespace Media {
+
+template<typename T>
+using DecoderErrorOr = ErrorOr<T, DecoderError>;
+
+enum class DecoderErrorCategory : u32 {
+    Unknown,
+    IO,
+    NeedsMoreInput,
+    EndOfStream,
+    Memory,
+    // The input is corrupted.
+    Corrupted,
+    // Invalid call.
+    Invalid,
+    // The input uses features that are not yet implemented.
+    NotImplemented,
+};
+
+class DecoderError {
+public:
+    static DecoderError with_description(DecoderErrorCategory category, StringView description)
+    {
+        return DecoderError(category, description);
+    }
+
+    template<typename... Parameters>
+    static DecoderError format(DecoderErrorCategory category, CheckedFormatString<Parameters...>&& format_string, Parameters const&... parameters)
+    {
+        AK::VariadicFormatParams<AK::AllowDebugOnlyFormatters::No, Parameters...> variadic_format_params { parameters... };
+        return DecoderError::with_description(category, ByteString::vformatted(format_string.view(), variadic_format_params));
+    }
+
+    static DecoderError from_source_location(DecoderErrorCategory category, StringView description, SourceLocation location = SourceLocation::current())
+    {
+        return DecoderError::format(category, "[{} @ {}:{}]: {}", location.function_name(), location.filename(), location.line_number(), description);
+    }
+
+    static DecoderError corrupted(StringView description, SourceLocation location = SourceLocation::current())
+    {
+        return DecoderError::from_source_location(DecoderErrorCategory::Corrupted, description, location);
+    }
+
+    static DecoderError not_implemented(SourceLocation location = SourceLocation::current())
+    {
+        return DecoderError::format(DecoderErrorCategory::NotImplemented, "{} is not implemented", location.function_name());
+    }
+
+    DecoderErrorCategory category() const { return m_category; }
+    StringView description() const { return m_description; }
+    StringView string_literal() const { return m_description; }
+
+private:
+    DecoderError(DecoderErrorCategory category, ByteString description)
+        : m_category(category)
+        , m_description(move(description))
+    {
+    }
+
+    DecoderErrorCategory m_category { DecoderErrorCategory::Unknown };
+    ByteString m_description;
+};
+
+#define DECODER_TRY(category, expression)                                                  \
+    ({                                                                                     \
+        auto&& _result = ((expression));                                                   \
+        if (_result.is_error()) [[unlikely]] {                                             \
+            auto _error_string = _result.release_error().string_literal();                 \
+            return DecoderError::from_source_location(                                     \
+                ((category)), _error_string, SourceLocation::current());                   \
+        }                                                                                  \
+        static_assert(!::AK::Detail::IsLvalueReference<decltype(_result.release_value())>, \
+            "Do not return a reference from a fallible expression");                       \
+        _result.release_value();                                                           \
+    })
+
+#define DECODER_TRY_ALLOC(expression) DECODER_TRY(DecoderErrorCategory::Memory, expression)
+
+}

Libraries/LibMedia/Demuxer.h

@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/NonnullOwnPtr.h>
+#include <LibCore/EventReceiver.h>
+
+#include "CodecID.h"
+#include "DecoderError.h"
+#include "Sample.h"
+#include "Track.h"
+
+namespace Media {
+
+class Demuxer {
+public:
+    virtual ~Demuxer() = default;
+
+    virtual DecoderErrorOr<Vector<Track>> get_tracks_for_type(TrackType type) = 0;
+
+    virtual DecoderErrorOr<Sample> get_next_sample_for_track(Track track) = 0;
+
+    virtual DecoderErrorOr<CodecID> get_codec_id_for_track(Track track) = 0;
+
+    virtual DecoderErrorOr<ReadonlyBytes> get_codec_initialization_data_for_track(Track track) = 0;
+
+    // Returns the timestamp of the keyframe that was seeked to.
+    // The value is `Optional` to allow the demuxer to decide not to seek so that it can keep its position
+    // in the case that the timestamp is closer to the current time than the nearest keyframe.
+    virtual DecoderErrorOr<Optional<AK::Duration>> seek_to_most_recent_keyframe(Track track, AK::Duration timestamp, Optional<AK::Duration> earliest_available_sample = OptionalNone()) = 0;
+
+    virtual DecoderErrorOr<AK::Duration> duration() = 0;
+};
+
+}

Libraries/LibMedia/FFmpeg/FFmpegForward.h

@@ -0,0 +1,13 @@
+/*
+ * Copyright (c) 2024, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+extern "C" {
+struct AVCodecContext;
+struct AVPacket;
+struct AVFrame;
+}

Libraries/LibMedia/FFmpeg/FFmpegHelpers.h

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2024, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <LibMedia/CodecID.h>
+
+extern "C" {
+#include <libavcodec/avcodec.h>
+}
+
+namespace Media::FFmpeg {
+
+static inline AVCodecID ffmpeg_codec_id_from_serenity_codec_id(CodecID codec)
+{
+    switch (codec) {
+    case CodecID::VP8:
+        return AV_CODEC_ID_VP8;
+    case CodecID::VP9:
+        return AV_CODEC_ID_VP9;
+    case CodecID::H261:
+        return AV_CODEC_ID_H261;
+    case CodecID::MPEG1:
+    case CodecID::H262:
+        return AV_CODEC_ID_MPEG2VIDEO;
+    case CodecID::H263:
+        return AV_CODEC_ID_H263;
+    case CodecID::H264:
+        return AV_CODEC_ID_H264;
+    case CodecID::H265:
+        return AV_CODEC_ID_HEVC;
+    case CodecID::AV1:
+        return AV_CODEC_ID_AV1;
+    case CodecID::Theora:
+        return AV_CODEC_ID_THEORA;
+    case CodecID::Vorbis:
+        return AV_CODEC_ID_VORBIS;
+    case CodecID::Opus:
+        return AV_CODEC_ID_OPUS;
+    default:
+        return AV_CODEC_ID_NONE;
+    }
+}
+
+}

Libraries/LibMedia/FFmpeg/FFmpegVideoDecoder.cpp

@@ -0,0 +1,236 @@
+/*
+ * Copyright (c) 2024, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <LibCore/System.h>
+#include <LibMedia/VideoFrame.h>
+
+#include "FFmpegHelpers.h"
+#include "FFmpegVideoDecoder.h"
+
+namespace Media::FFmpeg {
+
+static AVPixelFormat negotiate_output_format(AVCodecContext*, AVPixelFormat const* formats)
+{
+    while (*formats >= 0) {
+        switch (*formats) {
+        case AV_PIX_FMT_YUV420P:
+        case AV_PIX_FMT_YUV420P10:
+        case AV_PIX_FMT_YUV420P12:
+        case AV_PIX_FMT_YUV422P:
+        case AV_PIX_FMT_YUV422P10:
+        case AV_PIX_FMT_YUV422P12:
+        case AV_PIX_FMT_YUV444P:
+        case AV_PIX_FMT_YUV444P10:
+        case AV_PIX_FMT_YUV444P12:
+            return *formats;
+        default:
+            break;
+        }
+        formats++;
+    }
+    return AV_PIX_FMT_NONE;
+}
+
+DecoderErrorOr<NonnullOwnPtr<FFmpegVideoDecoder>> FFmpegVideoDecoder::try_create(CodecID codec_id, ReadonlyBytes codec_initialization_data)
+{
+    AVCodecContext* codec_context = nullptr;
+    AVPacket* packet = nullptr;
+    AVFrame* frame = nullptr;
+    ArmedScopeGuard memory_guard {
+        [&] {
+            avcodec_free_context(&codec_context);
+            av_packet_free(&packet);
+            av_frame_free(&frame);
+        }
+    };
+
+    auto ff_codec_id = ffmpeg_codec_id_from_serenity_codec_id(codec_id);
+    auto const* codec = avcodec_find_decoder(ff_codec_id);
+    if (!codec)
+        return DecoderError::format(DecoderErrorCategory::NotImplemented, "Could not find FFmpeg decoder for codec {}", codec_id);
+
+    codec_context = avcodec_alloc_context3(codec);
+    if (!codec_context)
+        return DecoderError::format(DecoderErrorCategory::Memory, "Failed to allocate FFmpeg codec context for codec {}", codec_id);
+
+    codec_context->get_format = negotiate_output_format;
+
+    codec_context->thread_count = static_cast<int>(min(Core::System::hardware_concurrency(), 4));
+
+    if (!codec_initialization_data.is_empty()) {
+        if (codec_initialization_data.size() > NumericLimits<int>::max())
+            return DecoderError::corrupted("Codec initialization data is too large"sv);
+
+        codec_context->extradata = static_cast<u8*>(av_malloc(codec_initialization_data.size() + AV_INPUT_BUFFER_PADDING_SIZE));
+        if (!codec_context->extradata)
+            return DecoderError::with_description(DecoderErrorCategory::Memory, "Failed to allocate codec initialization data buffer for FFmpeg codec"sv);
+
+        memcpy(codec_context->extradata, codec_initialization_data.data(), codec_initialization_data.size());
+        codec_context->extradata_size = static_cast<int>(codec_initialization_data.size());
+    }
+
+    if (avcodec_open2(codec_context, codec, nullptr) < 0)
+        return DecoderError::format(DecoderErrorCategory::Unknown, "Unknown error occurred when opening FFmpeg codec {}", codec_id);
+
+    packet = av_packet_alloc();
+    if (!packet)
+        return DecoderError::with_description(DecoderErrorCategory::Memory, "Failed to allocate FFmpeg packet"sv);
+
+    frame = av_frame_alloc();
+    if (!frame)
+        return DecoderError::with_description(DecoderErrorCategory::Memory, "Failed to allocate FFmpeg frame"sv);
+
+    memory_guard.disarm();
+    return DECODER_TRY_ALLOC(try_make<FFmpegVideoDecoder>(codec_context, packet, frame));
+}
+
+FFmpegVideoDecoder::FFmpegVideoDecoder(AVCodecContext* codec_context, AVPacket* packet, AVFrame* frame)
+    : m_codec_context(codec_context)
+    , m_packet(packet)
+    , m_frame(frame)
+{
+}
+
+FFmpegVideoDecoder::~FFmpegVideoDecoder()
+{
+    av_packet_free(&m_packet);
+    av_frame_free(&m_frame);
+    avcodec_free_context(&m_codec_context);
+}
+
+DecoderErrorOr<void> FFmpegVideoDecoder::receive_sample(AK::Duration timestamp, ReadonlyBytes sample)
+{
+    VERIFY(sample.size() < NumericLimits<int>::max());
+
+    m_packet->data = const_cast<u8*>(sample.data());
+    m_packet->size = static_cast<int>(sample.size());
+    m_packet->pts = timestamp.to_microseconds();
+    m_packet->dts = m_packet->pts;
+
+    auto result = avcodec_send_packet(m_codec_context, m_packet);
+    switch (result) {
+    case 0:
+        return {};
+    case AVERROR(EAGAIN):
+        return DecoderError::with_description(DecoderErrorCategory::NeedsMoreInput, "FFmpeg decoder cannot decode any more data until frames have been retrieved"sv);
+    case AVERROR_EOF:
+        return DecoderError::with_description(DecoderErrorCategory::EndOfStream, "FFmpeg decoder has been flushed"sv);
+    case AVERROR(EINVAL):
+        return DecoderError::with_description(DecoderErrorCategory::Invalid, "FFmpeg codec has not been opened"sv);
+    case AVERROR(ENOMEM):
+        return DecoderError::with_description(DecoderErrorCategory::Memory, "FFmpeg codec ran out of internal memory"sv);
+    default:
+        return DecoderError::with_description(DecoderErrorCategory::Corrupted, "FFmpeg codec reports that the data is corrupted"sv);
+    }
+}
+
+DecoderErrorOr<NonnullOwnPtr<VideoFrame>> FFmpegVideoDecoder::get_decoded_frame()
+{
+    auto result = avcodec_receive_frame(m_codec_context, m_frame);
+
+    switch (result) {
+    case 0: {
+        auto color_primaries = static_cast<ColorPrimaries>(m_frame->color_primaries);
+        auto transfer_characteristics = static_cast<TransferCharacteristics>(m_frame->color_trc);
+        auto matrix_coefficients = static_cast<MatrixCoefficients>(m_frame->colorspace);
+        auto color_range = [&] {
+            switch (m_frame->color_range) {
+            case AVColorRange::AVCOL_RANGE_MPEG:
+                return VideoFullRangeFlag::Studio;
+            case AVColorRange::AVCOL_RANGE_JPEG:
+                return VideoFullRangeFlag::Full;
+            default:
+                return VideoFullRangeFlag::Unspecified;
+            }
+        }();
+        auto cicp = CodingIndependentCodePoints { color_primaries, transfer_characteristics, matrix_coefficients, color_range };
+
+        size_t bit_depth = [&] {
+            switch (m_frame->format) {
+            case AV_PIX_FMT_YUV420P:
+            case AV_PIX_FMT_YUV422P:
+            case AV_PIX_FMT_YUV444P:
+                return 8;
+            case AV_PIX_FMT_YUV420P10:
+            case AV_PIX_FMT_YUV422P10:
+            case AV_PIX_FMT_YUV444P10:
+                return 10;
+            case AV_PIX_FMT_YUV420P12:
+            case AV_PIX_FMT_YUV422P12:
+            case AV_PIX_FMT_YUV444P12:
+                return 12;
+            }
+            VERIFY_NOT_REACHED();
+        }();
+        size_t component_size = (bit_depth + 7) / 8;
+
+        auto subsampling = [&]() -> Subsampling {
+            switch (m_frame->format) {
+            case AV_PIX_FMT_YUV420P:
+            case AV_PIX_FMT_YUV420P10:
+            case AV_PIX_FMT_YUV420P12:
+                return { true, true };
+            case AV_PIX_FMT_YUV422P:
+            case AV_PIX_FMT_YUV422P10:
+            case AV_PIX_FMT_YUV422P12:
+                return { true, false };
+            case AV_PIX_FMT_YUV444P:
+            case AV_PIX_FMT_YUV444P10:
+            case AV_PIX_FMT_YUV444P12:
+                return { false, false };
+
+            default:
+                VERIFY_NOT_REACHED();
+            }
+        }();
+
+        auto size = Gfx::Size<u32> { m_frame->width, m_frame->height };
+
+        auto timestamp = AK::Duration::from_microseconds(m_frame->pts);
+        auto frame = DECODER_TRY_ALLOC(SubsampledYUVFrame::try_create(timestamp, size, bit_depth, cicp, subsampling));
+
+        for (u32 plane = 0; plane < 3; plane++) {
+            VERIFY(m_frame->linesize[plane] != 0);
+            if (m_frame->linesize[plane] < 0)
+                return DecoderError::with_description(DecoderErrorCategory::NotImplemented, "Reversed scanlines are not supported"sv);
+
+            bool const use_subsampling = plane > 0;
+            auto plane_size = (use_subsampling ? subsampling.subsampled_size(size) : size).to_type<size_t>();
+
+            auto output_line_size = plane_size.width() * component_size;
+            VERIFY(output_line_size <= static_cast<size_t>(m_frame->linesize[plane]));
+
+            auto const* source = m_frame->data[plane];
+            VERIFY(source != nullptr);
+            auto* destination = frame->get_raw_plane_data(plane);
+            VERIFY(destination != nullptr);
+
+            for (size_t row = 0; row < plane_size.height(); row++) {
+                memcpy(destination, source, output_line_size);
+                source += m_frame->linesize[plane];
+                destination += output_line_size;
+            }
+        }
+
+        return frame;
+    }
+    case AVERROR(EAGAIN):
+        return DecoderError::with_description(DecoderErrorCategory::NeedsMoreInput, "FFmpeg decoder has no frames available, send more input"sv);
+    case AVERROR_EOF:
+        return DecoderError::with_description(DecoderErrorCategory::EndOfStream, "FFmpeg decoder has been flushed"sv);
+    case AVERROR(EINVAL):
+        return DecoderError::with_description(DecoderErrorCategory::Invalid, "FFmpeg codec has not been opened"sv);
+    default:
+        return DecoderError::format(DecoderErrorCategory::Unknown, "FFmpeg codec encountered an unexpected error retreiving frames with code {:x}", result);
+    }
+}
+
+void FFmpegVideoDecoder::flush()
+{
+    avcodec_flush_buffers(m_codec_context);
+}
+
+}

Libraries/LibMedia/FFmpeg/FFmpegVideoDecoder.h

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2024, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <LibMedia/CodecID.h>
+#include <LibMedia/VideoDecoder.h>
+
+#include "FFmpegForward.h"
+
+namespace Media::FFmpeg {
+
+class FFmpegVideoDecoder final : public VideoDecoder {
+public:
+    static DecoderErrorOr<NonnullOwnPtr<FFmpegVideoDecoder>> try_create(CodecID, ReadonlyBytes codec_initialization_data);
+    FFmpegVideoDecoder(AVCodecContext* codec_context, AVPacket* packet, AVFrame* frame);
+    ~FFmpegVideoDecoder();
+
+    DecoderErrorOr<void> receive_sample(AK::Duration timestamp, ReadonlyBytes sample) override;
+    DecoderErrorOr<NonnullOwnPtr<VideoFrame>> get_decoded_frame() override;
+
+    void flush() override;
+
+private:
+    DecoderErrorOr<void> decode_single_sample(AK::Duration timestamp, u8* data, int size);
+
+    AVCodecContext* m_codec_context;
+    AVPacket* m_packet;
+    AVFrame* m_frame;
+};
+
+}

Libraries/LibMedia/FFmpeg/FFmpegVideoDecoderStub.cpp

@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2024, Alex Studer <alex@studer.dev>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <LibCore/System.h>
+#include <LibMedia/VideoFrame.h>
+
+#include "FFmpegForward.h"
+#include "FFmpegVideoDecoder.h"
+
+namespace Media::FFmpeg {
+
+DecoderErrorOr<NonnullOwnPtr<FFmpegVideoDecoder>> FFmpegVideoDecoder::try_create(CodecID codec_id, ReadonlyBytes codec_initialization_data)
+{
+    (void)codec_id;
+    (void)codec_initialization_data;
+    return DecoderError::format(DecoderErrorCategory::NotImplemented, "FFmpeg not available on this platform");
+}
+
+FFmpegVideoDecoder::FFmpegVideoDecoder(AVCodecContext* codec_context, AVPacket* packet, AVFrame* frame)
+    : m_codec_context(codec_context)
+    , m_packet(packet)
+    , m_frame(frame)
+{
+}
+
+FFmpegVideoDecoder::~FFmpegVideoDecoder()
+{
+}
+
+DecoderErrorOr<void> FFmpegVideoDecoder::receive_sample(AK::Duration timestamp, ReadonlyBytes sample)
+{
+    (void)timestamp;
+    (void)sample;
+    return DecoderError::format(DecoderErrorCategory::NotImplemented, "FFmpeg not available on this platform");
+}
+
+DecoderErrorOr<NonnullOwnPtr<VideoFrame>> FFmpegVideoDecoder::get_decoded_frame()
+{
+    return DecoderError::format(DecoderErrorCategory::NotImplemented, "FFmpeg not available on this platform");
+}
+
+void FFmpegVideoDecoder::flush()
+{
+}
+
+}

Libraries/LibMedia/Forward.h

@@ -0,0 +1,19 @@
+/*
+ * Copyright (c) 2023, Tim Flynn <trflynn89@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+namespace Media {
+
+class DecoderError;
+class FrameQueueItem;
+class PlaybackManager;
+class Sample;
+class Track;
+class VideoDecoder;
+class VideoFrame;
+
+}

Libraries/LibMedia/PlaybackManager.cpp

@@ -0,0 +1,736 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/Format.h>
+#include <LibCore/Timer.h>
+#include <LibMedia/Containers/Matroska/MatroskaDemuxer.h>
+#include <LibMedia/FFmpeg/FFmpegVideoDecoder.h>
+#include <LibMedia/VideoFrame.h>
+
+#include "PlaybackManager.h"
+
+namespace Media {
+
+#define TRY_OR_FATAL_ERROR(expression)                                                               \
+    ({                                                                                               \
+        auto&& _fatal_expression = (expression);                                                     \
+        if (_fatal_expression.is_error()) {                                                          \
+            dispatch_fatal_error(_fatal_expression.release_error());                                 \
+            return;                                                                                  \
+        }                                                                                            \
+        static_assert(!::AK::Detail::IsLvalueReference<decltype(_fatal_expression.release_value())>, \
+            "Do not return a reference from a fallible expression");                                 \
+        _fatal_expression.release_value();                                                           \
+    })
+
+DecoderErrorOr<NonnullOwnPtr<PlaybackManager>> PlaybackManager::from_file(StringView filename)
+{
+    auto demuxer = TRY(Matroska::MatroskaDemuxer::from_file(filename));
+    return create(move(demuxer));
+}
+
+DecoderErrorOr<NonnullOwnPtr<PlaybackManager>> PlaybackManager::from_mapped_file(NonnullOwnPtr<Core::MappedFile> mapped_file)
+{
+    auto demuxer = TRY(Matroska::MatroskaDemuxer::from_mapped_file(move(mapped_file)));
+    return create(move(demuxer));
+}
+
+DecoderErrorOr<NonnullOwnPtr<PlaybackManager>> PlaybackManager::from_data(ReadonlyBytes data)
+{
+    auto demuxer = TRY(Matroska::MatroskaDemuxer::from_data(data));
+    return create(move(demuxer));
+}
+
+PlaybackManager::PlaybackManager(NonnullOwnPtr<Demuxer>& demuxer, Track video_track, NonnullOwnPtr<VideoDecoder>&& decoder, VideoFrameQueue&& frame_queue)
+    : m_demuxer(move(demuxer))
+    , m_selected_video_track(video_track)
+    , m_frame_queue(move(frame_queue))
+    , m_decoder(move(decoder))
+    , m_decode_wait_condition(m_decode_wait_mutex)
+{
+}
+
+PlaybackManager::~PlaybackManager()
+{
+    terminate_playback();
+}
+
+void PlaybackManager::resume_playback()
+{
+    dbgln_if(PLAYBACK_MANAGER_DEBUG, "Resuming playback.");
+    TRY_OR_FATAL_ERROR(m_playback_handler->play());
+}
+
+void PlaybackManager::pause_playback()
+{
+    dbgln_if(PLAYBACK_MANAGER_DEBUG, "Pausing playback.");
+    if (!m_playback_handler->is_playing())
+        warnln("Cannot pause.");
+    TRY_OR_FATAL_ERROR(m_playback_handler->pause());
+}
+
+void PlaybackManager::terminate_playback()
+{
+    m_stop_decoding.exchange(true);
+    m_decode_wait_condition.broadcast();
+
+    if (m_decode_thread->needs_to_be_joined()) {
+        dbgln_if(PLAYBACK_MANAGER_DEBUG, "Waiting for decode thread to end...");
+        (void)m_decode_thread->join();
+        dbgln_if(PLAYBACK_MANAGER_DEBUG, "Successfully destroyed PlaybackManager.");
+    }
+}
+
+AK::Duration PlaybackManager::current_playback_time()
+{
+    return m_playback_handler->current_time();
+}
+
+AK::Duration PlaybackManager::duration()
+{
+    auto duration_result = ({
+        auto demuxer_locker = Threading::MutexLocker(m_decoder_mutex);
+        m_demuxer->duration();
+    });
+    if (duration_result.is_error()) {
+        dispatch_decoder_error(duration_result.release_error());
+        // FIXME: We should determine the last sample that the demuxer knows is available and
+        //        use that as the current duration. The duration may change if the demuxer doesn't
+        //        know there is a fixed duration.
+        return AK::Duration::zero();
+    }
+    return duration_result.release_value();
+}
+
+void PlaybackManager::dispatch_fatal_error(Error error)
+{
+    dbgln_if(PLAYBACK_MANAGER_DEBUG, "Encountered fatal error: {}", error.string_literal());
+    // FIXME: For threading, this will have to use a pre-allocated event to send to the main loop
+    //        to be able to gracefully handle OOM.
+    if (on_fatal_playback_error)
+        on_fatal_playback_error(move(error));
+}
+
+void PlaybackManager::dispatch_decoder_error(DecoderError error)
+{
+    switch (error.category()) {
+    case DecoderErrorCategory::EndOfStream:
+        dbgln_if(PLAYBACK_MANAGER_DEBUG, "{}", error.string_literal());
+        TRY_OR_FATAL_ERROR(m_playback_handler->stop());
+        break;
+    default:
+        dbgln("Playback error encountered: {}", error.string_literal());
+        TRY_OR_FATAL_ERROR(m_playback_handler->stop());
+
+        if (on_decoder_error)
+            on_decoder_error(move(error));
+
+        break;
+    }
+}
+
+void PlaybackManager::dispatch_new_frame(RefPtr<Gfx::Bitmap> frame)
+{
+    if (on_video_frame)
+        on_video_frame(move(frame));
+}
+
+bool PlaybackManager::dispatch_frame_queue_item(FrameQueueItem&& item)
+{
+    if (item.is_error()) {
+        dispatch_decoder_error(item.release_error());
+        return true;
+    }
+
+    dbgln_if(PLAYBACK_MANAGER_DEBUG, "Sent frame for presentation with timestamp {}ms, late by {}ms", item.timestamp().to_milliseconds(), (current_playback_time() - item.timestamp()).to_milliseconds());
+    dispatch_new_frame(item.bitmap());
+    return false;
+}
+
+void PlaybackManager::dispatch_state_change()
+{
+    if (on_playback_state_change)
+        on_playback_state_change();
+}
+
+void PlaybackManager::timer_callback()
+{
+    TRY_OR_FATAL_ERROR(m_playback_handler->do_timed_state_update());
+}
+
+void PlaybackManager::seek_to_timestamp(AK::Duration target_timestamp, SeekMode seek_mode)
+{
+    TRY_OR_FATAL_ERROR(m_playback_handler->seek(target_timestamp, seek_mode));
+}
+
+DecoderErrorOr<Optional<AK::Duration>> PlaybackManager::seek_demuxer_to_most_recent_keyframe(AK::Duration timestamp, Optional<AK::Duration> earliest_available_sample)
+{
+    auto seeked_timestamp = TRY(m_demuxer->seek_to_most_recent_keyframe(m_selected_video_track, timestamp, move(earliest_available_sample)));
+    if (seeked_timestamp.has_value())
+        m_decoder->flush();
+    return seeked_timestamp;
+}
+
+Optional<FrameQueueItem> PlaybackManager::dequeue_one_frame()
+{
+    auto result = m_frame_queue.dequeue();
+    m_decode_wait_condition.broadcast();
+    if (result.is_error()) {
+        if (result.error() != VideoFrameQueue::QueueStatus::Empty)
+            dispatch_fatal_error(Error::from_string_literal("Dequeue failed with an unexpected error"));
+        return {};
+    }
+    return result.release_value();
+}
+
+void PlaybackManager::set_state_update_timer(int delay_ms)
+{
+    m_state_update_timer->start(delay_ms);
+}
+
+void PlaybackManager::restart_playback()
+{
+    seek_to_timestamp(AK::Duration::zero());
+}
+
+void PlaybackManager::decode_and_queue_one_sample()
+{
+#if PLAYBACK_MANAGER_DEBUG
+    auto start_time = MonotonicTime::now();
+#endif
+
+    FrameQueueItem item_to_enqueue;
+
+    while (item_to_enqueue.is_empty()) {
+        OwnPtr<VideoFrame> decoded_frame = nullptr;
+        CodingIndependentCodePoints container_cicp;
+
+        {
+            Threading::MutexLocker decoder_locker(m_decoder_mutex);
+
+            // Get a sample to decode.
+            auto sample_result = m_demuxer->get_next_sample_for_track(m_selected_video_track);
+            if (sample_result.is_error()) {
+                item_to_enqueue = FrameQueueItem::error_marker(sample_result.release_error(), FrameQueueItem::no_timestamp);
+                break;
+            }
+            auto sample = sample_result.release_value();
+            container_cicp = sample.auxiliary_data().get<VideoSampleData>().container_cicp();
+
+            // Submit the sample to the decoder.
+            auto decode_result = m_decoder->receive_sample(sample.timestamp(), sample.data());
+            if (decode_result.is_error()) {
+                item_to_enqueue = FrameQueueItem::error_marker(decode_result.release_error(), sample.timestamp());
+                break;
+            }
+
+            // Retrieve the last available frame to present.
+            while (true) {
+                auto frame_result = m_decoder->get_decoded_frame();
+
+                if (frame_result.is_error()) {
+                    if (frame_result.error().category() == DecoderErrorCategory::NeedsMoreInput) {
+                        break;
+                    }
+
+                    item_to_enqueue = FrameQueueItem::error_marker(frame_result.release_error(), sample.timestamp());
+                    break;
+                }
+
+                decoded_frame = frame_result.release_value();
+            }
+        }
+
+        // Convert the frame for display.
+        if (decoded_frame != nullptr) {
+            auto& cicp = decoded_frame->cicp();
+            cicp.adopt_specified_values(container_cicp);
+            cicp.default_code_points_if_unspecified({ ColorPrimaries::BT709, TransferCharacteristics::BT709, MatrixCoefficients::BT709, VideoFullRangeFlag::Studio });
+
+            // BT.470 M, B/G, BT.601, BT.709 and BT.2020 have a similar transfer function to sRGB, so other applications
+            // (Chromium, VLC) forgo transfer characteristics conversion. We will emulate that behavior by
+            // handling those as sRGB instead, which causes no transfer function change in the output,
+            // unless display color management is later implemented.
+            switch (cicp.transfer_characteristics()) {
+            case TransferCharacteristics::BT470BG:
+            case TransferCharacteristics::BT470M:
+            case TransferCharacteristics::BT601:
+            case TransferCharacteristics::BT709:
+            case TransferCharacteristics::BT2020BitDepth10:
+            case TransferCharacteristics::BT2020BitDepth12:
+                cicp.set_transfer_characteristics(TransferCharacteristics::SRGB);
+                break;
+            default:
+                break;
+            }
+
+            auto bitmap_result = decoded_frame->to_bitmap();
+
+            if (bitmap_result.is_error())
+                item_to_enqueue = FrameQueueItem::error_marker(bitmap_result.release_error(), decoded_frame->timestamp());
+            else
+                item_to_enqueue = FrameQueueItem::frame(bitmap_result.release_value(), decoded_frame->timestamp());
+            break;
+        }
+    }
+
+    VERIFY(!item_to_enqueue.is_empty());
+#if PLAYBACK_MANAGER_DEBUG
+    dbgln("Media Decoder: Sample at {}ms took {}ms to decode, queue contains ~{} items", item_to_enqueue.timestamp().to_milliseconds(), (MonotonicTime::now() - start_time).to_milliseconds(), m_frame_queue.weak_used());
+#endif
+
+    auto wait = [&] {
+        auto wait_locker = Threading::MutexLocker(m_decode_wait_mutex);
+        m_decode_wait_condition.wait();
+    };
+
+    bool had_error = item_to_enqueue.is_error();
+    while (true) {
+        if (m_frame_queue.can_enqueue()) {
+            MUST(m_frame_queue.enqueue(move(item_to_enqueue)));
+            break;
+        }
+
+        if (m_stop_decoding.load()) {
+            dbgln_if(PLAYBACK_MANAGER_DEBUG, "Media Decoder: Received signal to stop, exiting decode function...");
+            return;
+        }
+
+        m_buffer_is_full.exchange(true);
+        dbgln_if(PLAYBACK_MANAGER_DEBUG, "Media Decoder: Waiting for a frame to be dequeued...");
+        wait();
+    }
+
+    if (had_error) {
+        dbgln_if(PLAYBACK_MANAGER_DEBUG, "Media Decoder: Encountered {}, waiting...", item_to_enqueue.error().category() == DecoderErrorCategory::EndOfStream ? "end of stream"sv : "error"sv);
+        m_buffer_is_full.exchange(true);
+        wait();
+    }
+
+    m_buffer_is_full.exchange(false);
+}
+
+AK::Duration PlaybackManager::PlaybackStateHandler::current_time() const
+{
+    return m_manager.m_last_present_in_media_time;
+}
+
+ErrorOr<void> PlaybackManager::PlaybackStateHandler::seek(AK::Duration target_timestamp, SeekMode seek_mode)
+{
+    return replace_handler_and_delete_this<SeekingStateHandler>(is_playing(), target_timestamp, seek_mode);
+}
+
+ErrorOr<void> PlaybackManager::PlaybackStateHandler::stop()
+{
+    return replace_handler_and_delete_this<StoppedStateHandler>();
+}
+
+template<class T, class... Args>
+ErrorOr<void> PlaybackManager::PlaybackStateHandler::replace_handler_and_delete_this(Args... args)
+{
+    OwnPtr<PlaybackStateHandler> temp_handler = TRY(try_make<T>(m_manager, args...));
+    m_manager.m_playback_handler.swap(temp_handler);
+#if PLAYBACK_MANAGER_DEBUG
+    m_has_exited = true;
+    dbgln("Changing state from {} to {}", temp_handler->name(), m_manager.m_playback_handler->name());
+#endif
+    TRY(m_manager.m_playback_handler->on_enter());
+    m_manager.dispatch_state_change();
+    return {};
+}
+
+PlaybackManager& PlaybackManager::PlaybackStateHandler::manager() const
+{
+#if PLAYBACK_MANAGER_DEBUG
+    VERIFY(!m_has_exited);
+#endif
+    return m_manager;
+}
+
+class PlaybackManager::ResumingStateHandler : public PlaybackManager::PlaybackStateHandler {
+public:
+    ResumingStateHandler(PlaybackManager& manager, bool playing)
+        : PlaybackStateHandler(manager)
+        , m_playing(playing)
+    {
+    }
+    ~ResumingStateHandler() override = default;
+
+protected:
+    ErrorOr<void> assume_next_state()
+    {
+        if (!m_playing)
+            return replace_handler_and_delete_this<PausedStateHandler>();
+        return replace_handler_and_delete_this<PlayingStateHandler>();
+    }
+
+    ErrorOr<void> play() override
+    {
+        m_playing = true;
+        manager().dispatch_state_change();
+        return {};
+    }
+    bool is_playing() const override { return m_playing; }
+    ErrorOr<void> pause() override
+    {
+        m_playing = false;
+        manager().dispatch_state_change();
+        return {};
+    }
+
+    bool m_playing { false };
+};
+
+class PlaybackManager::PlayingStateHandler : public PlaybackManager::PlaybackStateHandler {
+public:
+    PlayingStateHandler(PlaybackManager& manager)
+        : PlaybackStateHandler(manager)
+    {
+    }
+    ~PlayingStateHandler() override = default;
+
+private:
+    ErrorOr<void> on_enter() override
+    {
+        m_last_present_in_real_time = MonotonicTime::now();
+        return do_timed_state_update();
+    }
+
+    StringView name() override { return "Playing"sv; }
+
+    bool is_playing() const override { return true; }
+    PlaybackState get_state() const override { return PlaybackState::Playing; }
+    ErrorOr<void> pause() override
+    {
+        manager().m_last_present_in_media_time = current_time();
+        return replace_handler_and_delete_this<PausedStateHandler>();
+    }
+    ErrorOr<void> buffer() override
+    {
+        manager().m_last_present_in_media_time = current_time();
+        return replace_handler_and_delete_this<BufferingStateHandler>(true);
+    }
+
+    AK::Duration current_time() const override
+    {
+        return manager().m_last_present_in_media_time + (MonotonicTime::now() - m_last_present_in_real_time);
+    }
+
+    ErrorOr<void> do_timed_state_update() override
+    {
+        auto set_presentation_timer = [&]() {
+            auto frame_time_ms = (manager().m_next_frame->timestamp() - current_time()).to_milliseconds();
+            VERIFY(frame_time_ms <= NumericLimits<int>::max());
+            dbgln_if(PLAYBACK_MANAGER_DEBUG, "Time until next frame is {}ms", frame_time_ms);
+            manager().set_state_update_timer(max(static_cast<int>(frame_time_ms), 0));
+        };
+
+        if (manager().m_next_frame.has_value() && current_time() < manager().m_next_frame->timestamp()) {
+            dbgln_if(PLAYBACK_MANAGER_DEBUG, "Current time {}ms is too early to present the next frame at {}ms, delaying", current_time().to_milliseconds(), manager().m_next_frame->timestamp().to_milliseconds());
+            set_presentation_timer();
+            return {};
+        }
+
+        Optional<FrameQueueItem> future_frame_item;
+        bool should_present_frame = false;
+
+        // Skip frames until we find a frame past the current playback time, and keep the one that precedes it to display.
+        while (true) {
+            future_frame_item = manager().dequeue_one_frame();
+            if (!future_frame_item.has_value())
+                break;
+
+            if (future_frame_item->timestamp() >= current_time() || future_frame_item->timestamp() == FrameQueueItem::no_timestamp) {
+                dbgln_if(PLAYBACK_MANAGER_DEBUG, "Should present frame, future {} is error or after {}ms", future_frame_item->debug_string(), current_time().to_milliseconds());
+                should_present_frame = true;
+                break;
+            }
+
+            if (manager().m_next_frame.has_value()) {
+                dbgln_if(PLAYBACK_MANAGER_DEBUG, "At {}ms: Dropped {} in favor of {}", current_time().to_milliseconds(), manager().m_next_frame->debug_string(), future_frame_item->debug_string());
+                manager().m_skipped_frames++;
+            }
+            manager().m_next_frame.emplace(future_frame_item.release_value());
+        }
+
+        // If we don't have both of these items, we can't present, since we need to set a timer for
+        // the next frame. Check if we need to buffer based on the current state.
+        if (!manager().m_next_frame.has_value() || !future_frame_item.has_value()) {
+#if PLAYBACK_MANAGER_DEBUG
+            StringBuilder debug_string_builder;
+            debug_string_builder.append("We don't have "sv);
+            if (!manager().m_next_frame.has_value()) {
+                debug_string_builder.append("a frame to present"sv);
+                if (!future_frame_item.has_value())
+                    debug_string_builder.append(" or a future frame"sv);
+            } else {
+                debug_string_builder.append("a future frame"sv);
+            }
+            debug_string_builder.append(", checking for error and buffering"sv);
+            dbgln_if(PLAYBACK_MANAGER_DEBUG, debug_string_builder.to_byte_string());
+#endif
+            if (future_frame_item.has_value()) {
+                if (future_frame_item->is_error()) {
+                    manager().dispatch_decoder_error(future_frame_item.release_value().release_error());
+                    return {};
+                }
+                manager().m_next_frame.emplace(future_frame_item.release_value());
+            }
+            TRY(buffer());
+            return {};
+        }
+
+        // If we have a frame, send it for presentation.
+        if (should_present_frame) {
+            auto now = MonotonicTime::now();
+            manager().m_last_present_in_media_time += now - m_last_present_in_real_time;
+            m_last_present_in_real_time = now;
+
+            if (manager().dispatch_frame_queue_item(manager().m_next_frame.release_value()))
+                return {};
+        }
+
+        // Now that we've presented the current frame, we can throw whatever error is next in queue.
+        // This way, we always display a frame before the stream ends, and should also show any frames
+        // we already had when a real error occurs.
+        if (future_frame_item->is_error()) {
+            manager().dispatch_decoder_error(future_frame_item.release_value().release_error());
+            return {};
+        }
+
+        // The future frame item becomes the next one to present.
+        manager().m_next_frame.emplace(future_frame_item.release_value());
+        set_presentation_timer();
+        return {};
+    }
+
+    MonotonicTime m_last_present_in_real_time = MonotonicTime::now_coarse();
+};
+
+class PlaybackManager::PausedStateHandler : public PlaybackManager::PlaybackStateHandler {
+public:
+    PausedStateHandler(PlaybackManager& manager)
+        : PlaybackStateHandler(manager)
+    {
+    }
+    ~PausedStateHandler() override = default;
+
+private:
+    StringView name() override { return "Paused"sv; }
+
+    ErrorOr<void> play() override
+    {
+        return replace_handler_and_delete_this<PlayingStateHandler>();
+    }
+    bool is_playing() const override { return false; }
+    PlaybackState get_state() const override { return PlaybackState::Paused; }
+};
+
+// FIXME: This is a placeholder variable that could be scaled based on how long each frame decode takes to
+//        avoid triggering the timer to check the queue constantly. However, doing so may reduce the speed
+//        of seeking due to the decode thread having to wait for a signal to continue decoding.
+constexpr int buffering_or_seeking_decode_wait_time = 1;
+
+class PlaybackManager::BufferingStateHandler : public PlaybackManager::ResumingStateHandler {
+    using PlaybackManager::ResumingStateHandler::ResumingStateHandler;
+
+    ErrorOr<void> on_enter() override
+    {
+        manager().set_state_update_timer(buffering_or_seeking_decode_wait_time);
+        return {};
+    }
+
+    StringView name() override { return "Buffering"sv; }
+
+    ErrorOr<void> do_timed_state_update() override
+    {
+        auto buffer_is_full = manager().m_buffer_is_full.load();
+        dbgln_if(PLAYBACK_MANAGER_DEBUG, "Buffering timer callback has been called. Buffer is {}.", buffer_is_full ? "full, exiting"sv : "not full, waiting"sv);
+        if (buffer_is_full)
+            return assume_next_state();
+
+        manager().set_state_update_timer(buffering_or_seeking_decode_wait_time);
+        return {};
+    }
+
+    PlaybackState get_state() const override { return PlaybackState::Buffering; }
+};
+
+class PlaybackManager::SeekingStateHandler : public PlaybackManager::ResumingStateHandler {
+public:
+    SeekingStateHandler(PlaybackManager& manager, bool playing, AK::Duration target_timestamp, SeekMode seek_mode)
+        : ResumingStateHandler(manager, playing)
+        , m_target_timestamp(target_timestamp)
+        , m_seek_mode(seek_mode)
+    {
+    }
+    ~SeekingStateHandler() override = default;
+
+private:
+    ErrorOr<void> on_enter() override
+    {
+        auto earliest_available_sample = manager().m_last_present_in_media_time;
+        if (manager().m_next_frame.has_value() && manager().m_next_frame->timestamp() != FrameQueueItem::no_timestamp) {
+            earliest_available_sample = min(earliest_available_sample, manager().m_next_frame->timestamp());
+        }
+
+        {
+            Threading::MutexLocker demuxer_locker(manager().m_decoder_mutex);
+
+            auto demuxer_seek_result = manager().seek_demuxer_to_most_recent_keyframe(m_target_timestamp, earliest_available_sample);
+            if (demuxer_seek_result.is_error()) {
+                manager().dispatch_decoder_error(demuxer_seek_result.release_error());
+                return {};
+            }
+            auto keyframe_timestamp = demuxer_seek_result.release_value();
+
+#if PLAYBACK_MANAGER_DEBUG
+            auto seek_mode_name = m_seek_mode == SeekMode::Accurate ? "Accurate"sv : "Fast"sv;
+            if (keyframe_timestamp.has_value())
+                dbgln("{} seeking to timestamp target {}ms, selected keyframe at {}ms", seek_mode_name, m_target_timestamp.to_milliseconds(), keyframe_timestamp->to_milliseconds());
+            else
+                dbgln("{} seeking to timestamp target {}ms, demuxer kept its iterator position after {}ms", seek_mode_name, m_target_timestamp.to_milliseconds(), earliest_available_sample.to_milliseconds());
+#endif
+
+            if (m_seek_mode == SeekMode::Fast)
+                m_target_timestamp = keyframe_timestamp.value_or(manager().m_last_present_in_media_time);
+
+            if (keyframe_timestamp.has_value()) {
+                dbgln_if(PLAYBACK_MANAGER_DEBUG, "Keyframe is nearer to the target than the current frames, emptying queue");
+                while (manager().dequeue_one_frame().has_value()) { }
+                manager().m_next_frame.clear();
+                manager().m_last_present_in_media_time = keyframe_timestamp.value();
+            } else if (m_target_timestamp >= manager().m_last_present_in_media_time && manager().m_next_frame.has_value() && manager().m_next_frame.value().timestamp() > m_target_timestamp) {
+                dbgln_if(PLAYBACK_MANAGER_DEBUG, "Target timestamp is between the last presented frame and the next frame, exiting seek at {}ms", m_target_timestamp.to_milliseconds());
+                manager().m_last_present_in_media_time = m_target_timestamp;
+                return assume_next_state();
+            }
+        }
+
+        return skip_samples_until_timestamp();
+    }
+
+    ErrorOr<void> skip_samples_until_timestamp()
+    {
+        while (true) {
+            auto optional_item = manager().dequeue_one_frame();
+            if (!optional_item.has_value())
+                break;
+            auto item = optional_item.release_value();
+
+            dbgln_if(PLAYBACK_MANAGER_DEBUG, "Dequeuing frame at {}ms and comparing to seek target {}ms", item.timestamp().to_milliseconds(), m_target_timestamp.to_milliseconds());
+            if (manager().m_next_frame.has_value() && (item.timestamp() > m_target_timestamp || item.timestamp() == FrameQueueItem::no_timestamp)) {
+                // If the frame we're presenting is later than the target timestamp, skip the timestamp forward to it.
+                if (manager().m_next_frame->timestamp() > m_target_timestamp) {
+                    manager().m_last_present_in_media_time = manager().m_next_frame->timestamp();
+                } else {
+                    manager().m_last_present_in_media_time = m_target_timestamp;
+                }
+
+                if (manager().dispatch_frame_queue_item(manager().m_next_frame.release_value()))
+                    return {};
+
+                manager().m_next_frame.emplace(item);
+
+                dbgln_if(PLAYBACK_MANAGER_DEBUG, "Exiting seek to {} state at {}ms", m_playing ? "Playing" : "Paused", manager().m_last_present_in_media_time.to_milliseconds());
+                return assume_next_state();
+            }
+            manager().m_next_frame.emplace(item);
+        }
+
+        dbgln_if(PLAYBACK_MANAGER_DEBUG, "Frame queue is empty while seeking, waiting for buffer to fill.");
+        manager().set_state_update_timer(buffering_or_seeking_decode_wait_time);
+        return {};
+    }
+
+    StringView name() override { return "Seeking"sv; }
+
+    ErrorOr<void> seek(AK::Duration target_timestamp, SeekMode seek_mode) override
+    {
+        m_target_timestamp = target_timestamp;
+        m_seek_mode = seek_mode;
+        return on_enter();
+    }
+
+    AK::Duration current_time() const override
+    {
+        return m_target_timestamp;
+    }
+
+    // We won't need this override when threaded, the queue can pause us in on_enter().
+    ErrorOr<void> do_timed_state_update() override
+    {
+        dbgln_if(PLAYBACK_MANAGER_DEBUG, "Seeking wait finished, attempting to dequeue until timestamp.");
+        return skip_samples_until_timestamp();
+    }
+
+    PlaybackState get_state() const override { return PlaybackState::Seeking; }
+
+    AK::Duration m_target_timestamp { AK::Duration::zero() };
+    SeekMode m_seek_mode { SeekMode::Accurate };
+};
+
+class PlaybackManager::StoppedStateHandler : public PlaybackManager::PlaybackStateHandler {
+public:
+    StoppedStateHandler(PlaybackManager& manager)
+        : PlaybackStateHandler(manager)
+    {
+    }
+    ~StoppedStateHandler() override = default;
+
+private:
+    ErrorOr<void> on_enter() override
+    {
+        return {};
+    }
+
+    StringView name() override { return "Stopped"sv; }
+
+    ErrorOr<void> play() override
+    {
+        // When Stopped, the decoder thread will be waiting for a signal to start its loop going again.
+        manager().m_decode_wait_condition.broadcast();
+        return replace_handler_and_delete_this<SeekingStateHandler>(true, AK::Duration::zero(), SeekMode::Fast);
+    }
+    bool is_playing() const override { return false; }
+    PlaybackState get_state() const override { return PlaybackState::Stopped; }
+};
+
+DecoderErrorOr<NonnullOwnPtr<PlaybackManager>> PlaybackManager::create(NonnullOwnPtr<Demuxer> demuxer)
+{
+    auto video_tracks = TRY(demuxer->get_tracks_for_type(TrackType::Video));
+    if (video_tracks.is_empty())
+        return DecoderError::with_description(DecoderErrorCategory::Invalid, "No video track is present"sv);
+    auto track = video_tracks[0];
+
+    dbgln_if(PLAYBACK_MANAGER_DEBUG, "Selecting video track number {}", track.identifier());
+
+    auto codec_id = TRY(demuxer->get_codec_id_for_track(track));
+    auto codec_initialization_data = TRY(demuxer->get_codec_initialization_data_for_track(track));
+    NonnullOwnPtr<VideoDecoder> decoder = TRY(FFmpeg::FFmpegVideoDecoder::try_create(codec_id, codec_initialization_data));
+    auto frame_queue = DECODER_TRY_ALLOC(VideoFrameQueue::create());
+    auto playback_manager = DECODER_TRY_ALLOC(try_make<PlaybackManager>(demuxer, track, move(decoder), move(frame_queue)));
+
+    playback_manager->m_state_update_timer = Core::Timer::create_single_shot(0, [&self = *playback_manager] { self.timer_callback(); });
+
+    playback_manager->m_decode_thread = DECODER_TRY_ALLOC(Threading::Thread::try_create([&self = *playback_manager] {
+        while (!self.m_stop_decoding.load())
+            self.decode_and_queue_one_sample();
+
+        dbgln_if(PLAYBACK_MANAGER_DEBUG, "Media Decoder thread ended.");
+        return 0;
+    },
+        "Media Decoder"sv));
+
+    playback_manager->m_playback_handler = make<SeekingStateHandler>(*playback_manager, false, AK::Duration::zero(), SeekMode::Fast);
+    DECODER_TRY_ALLOC(playback_manager->m_playback_handler->on_enter());
+
+    playback_manager->m_decode_thread->start();
+
+    return playback_manager;
+}
+
+}

Libraries/LibMedia/PlaybackManager.h

@@ -0,0 +1,242 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/Atomic.h>
+#include <AK/Function.h>
+#include <AK/NonnullOwnPtr.h>
+#include <AK/Queue.h>
+#include <AK/Time.h>
+#include <LibCore/SharedCircularQueue.h>
+#include <LibGfx/Bitmap.h>
+#include <LibMedia/Containers/Matroska/Document.h>
+#include <LibMedia/Demuxer.h>
+#include <LibThreading/ConditionVariable.h>
+#include <LibThreading/Mutex.h>
+#include <LibThreading/Thread.h>
+
+#include "VideoDecoder.h"
+
+namespace Media {
+
+class FrameQueueItem {
+public:
+    FrameQueueItem()
+        : m_data(Empty())
+        , m_timestamp(AK::Duration::zero())
+    {
+    }
+
+    static constexpr AK::Duration no_timestamp = AK::Duration::min();
+
+    enum class Type {
+        Frame,
+        Error,
+    };
+
+    static FrameQueueItem frame(RefPtr<Gfx::Bitmap> bitmap, AK::Duration timestamp)
+    {
+        return FrameQueueItem(move(bitmap), timestamp);
+    }
+
+    static FrameQueueItem error_marker(DecoderError&& error, AK::Duration timestamp)
+    {
+        return FrameQueueItem(move(error), timestamp);
+    }
+
+    bool is_frame() const { return m_data.has<RefPtr<Gfx::Bitmap>>(); }
+    RefPtr<Gfx::Bitmap> bitmap() const { return m_data.get<RefPtr<Gfx::Bitmap>>(); }
+    AK::Duration timestamp() const { return m_timestamp; }
+
+    bool is_error() const { return m_data.has<DecoderError>(); }
+    DecoderError const& error() const { return m_data.get<DecoderError>(); }
+    DecoderError release_error()
+    {
+        auto error = move(m_data.get<DecoderError>());
+        m_data.set(Empty());
+        return error;
+    }
+
+    bool is_empty() const { return m_data.has<Empty>(); }
+
+    ByteString debug_string() const
+    {
+        if (is_error())
+            return ByteString::formatted("{} at {}ms", error().string_literal(), timestamp().to_milliseconds());
+        return ByteString::formatted("frame at {}ms", timestamp().to_milliseconds());
+    }
+
+private:
+    FrameQueueItem(RefPtr<Gfx::Bitmap> bitmap, AK::Duration timestamp)
+        : m_data(move(bitmap))
+        , m_timestamp(timestamp)
+    {
+        VERIFY(m_timestamp != no_timestamp);
+    }
+
+    FrameQueueItem(DecoderError&& error, AK::Duration timestamp)
+        : m_data(move(error))
+        , m_timestamp(timestamp)
+    {
+    }
+
+    Variant<Empty, RefPtr<Gfx::Bitmap>, DecoderError> m_data { Empty() };
+    AK::Duration m_timestamp { no_timestamp };
+};
+
+static constexpr size_t frame_buffer_count = 4;
+using VideoFrameQueue = Core::SharedSingleProducerCircularQueue<FrameQueueItem, frame_buffer_count>;
+
+enum class PlaybackState {
+    Playing,
+    Paused,
+    Buffering,
+    Seeking,
+    Stopped,
+};
+
+class PlaybackManager {
+    AK_MAKE_NONCOPYABLE(PlaybackManager);
+    AK_MAKE_NONMOVABLE(PlaybackManager);
+
+public:
+    enum class SeekMode {
+        Accurate,
+        Fast,
+    };
+
+    static constexpr SeekMode DEFAULT_SEEK_MODE = SeekMode::Accurate;
+
+    static DecoderErrorOr<NonnullOwnPtr<PlaybackManager>> from_file(StringView file);
+    static DecoderErrorOr<NonnullOwnPtr<PlaybackManager>> from_mapped_file(NonnullOwnPtr<Core::MappedFile> file);
+
+    static DecoderErrorOr<NonnullOwnPtr<PlaybackManager>> from_data(ReadonlyBytes data);
+
+    PlaybackManager(NonnullOwnPtr<Demuxer>& demuxer, Track video_track, NonnullOwnPtr<VideoDecoder>&& decoder, VideoFrameQueue&& frame_queue);
+    ~PlaybackManager();
+
+    void resume_playback();
+    void pause_playback();
+    void restart_playback();
+    void terminate_playback();
+    void seek_to_timestamp(AK::Duration, SeekMode = DEFAULT_SEEK_MODE);
+    bool is_playing() const
+    {
+        return m_playback_handler->is_playing();
+    }
+    PlaybackState get_state() const
+    {
+        return m_playback_handler->get_state();
+    }
+
+    u64 number_of_skipped_frames() const { return m_skipped_frames; }
+
+    AK::Duration current_playback_time();
+    AK::Duration duration();
+
+    Function<void(RefPtr<Gfx::Bitmap>)> on_video_frame;
+    Function<void()> on_playback_state_change;
+    Function<void(DecoderError)> on_decoder_error;
+    Function<void(Error)> on_fatal_playback_error;
+
+    Track const& selected_video_track() const { return m_selected_video_track; }
+
+private:
+    class PlaybackStateHandler;
+    // Abstract class to allow resuming play/pause after the state is completed.
+    class ResumingStateHandler;
+    class PlayingStateHandler;
+    class PausedStateHandler;
+    class BufferingStateHandler;
+    class SeekingStateHandler;
+    class StoppedStateHandler;
+
+    static DecoderErrorOr<NonnullOwnPtr<PlaybackManager>> create(NonnullOwnPtr<Demuxer> demuxer);
+
+    void timer_callback();
+    // This must be called with m_demuxer_mutex locked!
+    DecoderErrorOr<Optional<AK::Duration>> seek_demuxer_to_most_recent_keyframe(AK::Duration timestamp, Optional<AK::Duration> earliest_available_sample = OptionalNone());
+
+    Optional<FrameQueueItem> dequeue_one_frame();
+    void set_state_update_timer(int delay_ms);
+
+    void decode_and_queue_one_sample();
+
+    void dispatch_decoder_error(DecoderError error);
+    void dispatch_new_frame(RefPtr<Gfx::Bitmap> frame);
+    // Returns whether we changed playback states. If so, any PlaybackStateHandler processing must cease.
+    [[nodiscard]] bool dispatch_frame_queue_item(FrameQueueItem&&);
+    void dispatch_state_change();
+    void dispatch_fatal_error(Error);
+
+    AK::Duration m_last_present_in_media_time = AK::Duration::zero();
+
+    NonnullOwnPtr<Demuxer> m_demuxer;
+    Threading::Mutex m_decoder_mutex;
+    Track m_selected_video_track;
+
+    VideoFrameQueue m_frame_queue;
+
+    RefPtr<Core::Timer> m_state_update_timer;
+    unsigned m_decoding_buffer_time_ms = 16;
+
+    RefPtr<Threading::Thread> m_decode_thread;
+    NonnullOwnPtr<VideoDecoder> m_decoder;
+    Atomic<bool> m_stop_decoding { false };
+    Threading::Mutex m_decode_wait_mutex;
+    Threading::ConditionVariable m_decode_wait_condition;
+    Atomic<bool> m_buffer_is_full { false };
+
+    OwnPtr<PlaybackStateHandler> m_playback_handler;
+    Optional<FrameQueueItem> m_next_frame;
+
+    u64 m_skipped_frames { 0 };
+
+    // This is a nested class to allow private access.
+    class PlaybackStateHandler {
+    public:
+        PlaybackStateHandler(PlaybackManager& manager)
+            : m_manager(manager)
+        {
+        }
+        virtual ~PlaybackStateHandler() = default;
+        virtual StringView name() = 0;
+
+        virtual ErrorOr<void> on_enter() { return {}; }
+
+        virtual ErrorOr<void> play() { return {}; }
+        virtual bool is_playing() const = 0;
+        virtual PlaybackState get_state() const = 0;
+        virtual ErrorOr<void> pause() { return {}; }
+        virtual ErrorOr<void> buffer() { return {}; }
+        virtual ErrorOr<void> seek(AK::Duration target_timestamp, SeekMode);
+        virtual ErrorOr<void> stop();
+
+        virtual AK::Duration current_time() const;
+
+        virtual ErrorOr<void> do_timed_state_update() { return {}; }
+
+    protected:
+        template<class T, class... Args>
+        ErrorOr<void> replace_handler_and_delete_this(Args... args);
+
+        PlaybackManager& manager() const;
+
+        PlaybackManager& manager()
+        {
+            return const_cast<PlaybackManager&>(const_cast<PlaybackStateHandler const*>(this)->manager());
+        }
+
+    private:
+        PlaybackManager& m_manager;
+#if PLAYBACK_MANAGER_DEBUG
+        bool m_has_exited { false };
+#endif
+    };
+};
+
+}

Libraries/LibMedia/Sample.h

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/ByteBuffer.h>
+#include <AK/Time.h>
+
+#include "VideoSampleData.h"
+
+namespace Media {
+
+class Sample final {
+public:
+    using AuxiliaryData = Variant<VideoSampleData>;
+
+    Sample(AK::Duration timestamp, ReadonlyBytes data, AuxiliaryData auxiliary_data)
+        : m_timestamp(timestamp)
+        , m_data(data)
+        , m_auxiliary_data(auxiliary_data)
+    {
+    }
+
+    AK::Duration timestamp() const { return m_timestamp; }
+    ReadonlyBytes const& data() const { return m_data; }
+    AuxiliaryData const& auxiliary_data() const { return m_auxiliary_data; }
+
+private:
+    AK::Duration m_timestamp;
+    ReadonlyBytes m_data;
+    AuxiliaryData m_auxiliary_data;
+};
+
+}

Libraries/LibMedia/Subsampling.h

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2024, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <LibGfx/Size.h>
+
+namespace Media {
+
+struct Subsampling {
+public:
+    Subsampling(bool x, bool y)
+        : m_x(x)
+        , m_y(y)
+    {
+    }
+
+    Subsampling() = default;
+
+    bool x() const { return m_x; }
+    bool y() const { return m_y; }
+
+    static u32 subsampled_size(bool subsampled, u32 size)
+    {
+        u32 subsampled_as_int = static_cast<u32>(subsampled);
+        return (size + subsampled_as_int) >> subsampled_as_int;
+    }
+
+    template<Integral T>
+    Gfx::Size<T> subsampled_size(Gfx::Size<T> size) const
+    {
+        return {
+            subsampled_size(x(), size.width()),
+            subsampled_size(y(), size.height())
+        };
+    }
+
+private:
+    bool m_x = false;
+    bool m_y = false;
+};
+
+}

Libraries/LibMedia/Track.h

@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/HashFunctions.h>
+#include <AK/Time.h>
+#include <AK/Traits.h>
+#include <AK/Types.h>
+#include <AK/Variant.h>
+
+namespace Media {
+
+enum class TrackType : u32 {
+    Video,
+    Audio,
+    Subtitles,
+};
+
+class Track {
+    struct VideoData {
+        AK::Duration duration {};
+        u64 pixel_width { 0 };
+        u64 pixel_height { 0 };
+    };
+
+public:
+    Track(TrackType type, size_t identifier)
+        : m_type(type)
+        , m_identifier(identifier)
+    {
+        switch (m_type) {
+        case TrackType::Video:
+            m_track_data = VideoData {};
+            break;
+        default:
+            m_track_data = Empty {};
+            break;
+        }
+    }
+
+    TrackType type() { return m_type; }
+    size_t identifier() const { return m_identifier; }
+
+    void set_video_data(VideoData data)
+    {
+        VERIFY(m_type == TrackType::Video);
+        m_track_data = data;
+    }
+
+    VideoData const& video_data() const
+    {
+        VERIFY(m_type == TrackType::Video);
+        return m_track_data.get<VideoData>();
+    }
+
+    bool operator==(Track const& other) const
+    {
+        return m_type == other.m_type && m_identifier == other.m_identifier;
+    }
+
+    unsigned hash() const
+    {
+        return pair_int_hash(to_underlying(m_type), m_identifier);
+    }
+
+private:
+    TrackType m_type;
+    size_t m_identifier;
+
+    Variant<Empty, VideoData> m_track_data;
+};
+
+}
+
+template<>
+struct AK::Traits<Media::Track> : public DefaultTraits<Media::Track> {
+    static unsigned hash(Media::Track const& t) { return t.hash(); }
+};

Libraries/LibMedia/VideoDecoder.h

@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/ByteBuffer.h>
+#include <AK/NonnullOwnPtr.h>
+#include <AK/Time.h>
+
+#include "DecoderError.h"
+
+namespace Media {
+
+class VideoDecoder {
+public:
+    virtual ~VideoDecoder() {};
+
+    virtual DecoderErrorOr<void> receive_sample(AK::Duration timestamp, ReadonlyBytes sample) = 0;
+    DecoderErrorOr<void> receive_sample(AK::Duration timestamp, ByteBuffer const& sample) { return receive_sample(timestamp, sample.span()); }
+    virtual DecoderErrorOr<NonnullOwnPtr<VideoFrame>> get_decoded_frame() = 0;
+
+    virtual void flush() = 0;
+};
+
+}

Libraries/LibMedia/VideoFrame.cpp

@@ -0,0 +1,234 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/NonnullOwnPtr.h>
+#include <AK/OwnPtr.h>
+#include <LibMedia/Color/ColorConverter.h>
+
+#include "VideoFrame.h"
+
+namespace Media {
+
+ErrorOr<NonnullOwnPtr<SubsampledYUVFrame>> SubsampledYUVFrame::try_create(
+    AK::Duration timestamp,
+    Gfx::Size<u32> size,
+    u8 bit_depth, CodingIndependentCodePoints cicp,
+    Subsampling subsampling)
+{
+    VERIFY(bit_depth < 16);
+    size_t component_size = bit_depth > 8 ? sizeof(u16) : sizeof(u8);
+    size_t alignment_size = max(bit_depth > 8 ? sizeof(u16) : sizeof(u8), sizeof(void*));
+
+    auto alloc_buffer = [&](size_t size) -> ErrorOr<u8*> {
+        void* buffer = nullptr;
+        auto result = posix_memalign(&buffer, alignment_size, size);
+        if (result != 0)
+            return Error::from_errno(result);
+        return reinterpret_cast<u8*>(buffer);
+    };
+
+    auto y_data_size = size.to_type<size_t>().area() * component_size;
+    auto uv_data_size = subsampling.subsampled_size(size).to_type<size_t>().area() * component_size;
+    auto* y_buffer = TRY(alloc_buffer(y_data_size));
+    auto* u_buffer = TRY(alloc_buffer(uv_data_size));
+    auto* v_buffer = TRY(alloc_buffer(uv_data_size));
+
+    return adopt_nonnull_own_or_enomem(new (nothrow) SubsampledYUVFrame(timestamp, size, bit_depth, cicp, subsampling, y_buffer, u_buffer, v_buffer));
+}
+
+ErrorOr<NonnullOwnPtr<SubsampledYUVFrame>> SubsampledYUVFrame::try_create_from_data(
+    AK::Duration timestamp,
+    Gfx::Size<u32> size,
+    u8 bit_depth, CodingIndependentCodePoints cicp,
+    Subsampling subsampling,
+    ReadonlyBytes y_data, ReadonlyBytes u_data, ReadonlyBytes v_data)
+{
+    auto frame = TRY(try_create(timestamp, size, bit_depth, cicp, subsampling));
+
+    size_t component_size = bit_depth > 8 ? sizeof(u16) : sizeof(u8);
+    auto y_data_size = size.to_type<size_t>().area() * component_size;
+    auto uv_data_size = subsampling.subsampled_size(size).to_type<size_t>().area() * component_size;
+
+    VERIFY(y_data.size() >= y_data_size);
+    VERIFY(u_data.size() >= uv_data_size);
+    VERIFY(v_data.size() >= uv_data_size);
+
+    memcpy(frame->m_y_buffer, y_data.data(), y_data_size);
+    memcpy(frame->m_u_buffer, u_data.data(), uv_data_size);
+    memcpy(frame->m_v_buffer, v_data.data(), uv_data_size);
+    return frame;
+}
+
+SubsampledYUVFrame::~SubsampledYUVFrame()
+{
+    free(m_y_buffer);
+    free(m_u_buffer);
+    free(m_v_buffer);
+}
+
+template<u32 subsampling_horizontal, typename T>
+ALWAYS_INLINE void interpolate_row(u32 const row, u32 const width, T const* plane_u, T const* plane_v, T* __restrict__ u_row, T* __restrict__ v_row)
+{
+    // OPTIMIZATION: __restrict__ allows some load eliminations because the planes and the rows will not alias.
+
+    constexpr auto horizontal_step = 1u << subsampling_horizontal;
+    auto const uv_width = (width + subsampling_horizontal) >> subsampling_horizontal;
+    // Set the first column to the first chroma samples.
+    u_row[0] = plane_u[row * uv_width];
+    v_row[0] = plane_v[row * uv_width];
+
+    auto const columns_end = width - subsampling_horizontal;
+    // Interpolate the inner chroma columns.
+    for (u32 column = 1; column < columns_end; column += horizontal_step) {
+        auto uv_column = column >> subsampling_horizontal;
+        u_row[column] = plane_u[row * uv_width + uv_column];
+        v_row[column] = plane_v[row * uv_width + uv_column];
+
+        if constexpr (subsampling_horizontal != 0) {
+            u_row[column + 1] = (plane_u[row * uv_width + uv_column] + plane_u[row * uv_width + uv_column + 1]) >> 1;
+            v_row[column + 1] = (plane_v[row * uv_width + uv_column] + plane_v[row * uv_width + uv_column + 1]) >> 1;
+        }
+    }
+
+    // If there is a last chroma sample that hasn't been set above, set it now.
+    if constexpr (subsampling_horizontal != 0) {
+        if ((width & 1) == 0) {
+            u_row[width - 1] = u_row[width - 2];
+            v_row[width - 1] = v_row[width - 2];
+        }
+    }
+}
+
+template<u32 subsampling_horizontal, u32 subsampling_vertical, typename T, typename Convert>
+ALWAYS_INLINE DecoderErrorOr<void> convert_to_bitmap_subsampled(Convert convert, u32 const width, u32 const height, T const* plane_y, T const* plane_u, T const* plane_v, Gfx::Bitmap& bitmap)
+{
+    VERIFY(bitmap.width() >= 0);
+    VERIFY(bitmap.height() >= 0);
+    VERIFY(static_cast<u32>(bitmap.width()) == width);
+    VERIFY(static_cast<u32>(bitmap.height()) == height);
+
+    auto temporary_buffer = DECODER_TRY_ALLOC(FixedArray<T>::create(static_cast<size_t>(width) * 4));
+
+    // Above rows
+    auto* u_row_a = temporary_buffer.span().slice(static_cast<size_t>(width) * 0, width).data();
+    auto* v_row_a = temporary_buffer.span().slice(static_cast<size_t>(width) * 1, width).data();
+
+    // Below rows
+    auto* u_row_b = temporary_buffer.span().slice(static_cast<size_t>(width) * 2, width).data();
+    auto* v_row_b = temporary_buffer.span().slice(static_cast<size_t>(width) * 3, width).data();
+
+    u32 const vertical_step = 1 << subsampling_vertical;
+
+    interpolate_row<subsampling_horizontal>(0, width, plane_u, plane_v, u_row_a, v_row_a);
+
+    // Do interpolation for all inner rows.
+    u32 const rows_end = height - subsampling_vertical;
+    for (u32 row = 0; row < rows_end; row += vertical_step) {
+        // Horizontally scale the row if subsampled.
+        auto uv_row = row >> subsampling_vertical;
+        interpolate_row<subsampling_horizontal>(uv_row, width, plane_u, plane_v, u_row_b, v_row_b);
+
+        // If subsampled vertically, vertically interpolate the middle row between the above and below rows.
+        if constexpr (subsampling_vertical != 0) {
+            // OPTIMIZATION: Splitting these two lines into separate loops enables vectorization.
+            for (u32 column = 0; column < width; column++) {
+                u_row_a[column] = (u_row_a[column] + u_row_b[column]) >> 1;
+            }
+            for (u32 column = 0; column < width; column++) {
+                v_row_a[column] = (v_row_a[column] + v_row_b[column]) >> 1;
+            }
+        }
+
+        auto const* y_row_a = &plane_y[static_cast<size_t>(row) * width];
+        auto* scan_line_a = bitmap.scanline(static_cast<int>(row));
+
+        for (size_t column = 0; column < width; column++) {
+            scan_line_a[column] = convert(y_row_a[column], u_row_a[column], v_row_a[column]).value();
+        }
+        if constexpr (subsampling_vertical != 0) {
+            auto const* y_row_b = &plane_y[static_cast<size_t>(row + 1) * width];
+            auto* scan_line_b = bitmap.scanline(static_cast<int>(row + 1));
+            for (size_t column = 0; column < width; column++) {
+                scan_line_b[column] = convert(y_row_b[column], u_row_b[column], v_row_b[column]).value();
+            }
+        }
+
+        AK::TypedTransfer<RemoveReference<decltype(*u_row_a)>>::move(u_row_a, u_row_b, width);
+        AK::TypedTransfer<RemoveReference<decltype(*u_row_a)>>::move(v_row_a, v_row_b, width);
+    }
+
+    if constexpr (subsampling_vertical != 0) {
+        // If there is a final row that hasn't been set above, convert it now.
+        if ((height & 1) == 0) {
+            auto const* y_row = &plane_y[static_cast<size_t>(height - 1) * width];
+            auto* scan_line = bitmap.scanline(static_cast<int>(height - 1));
+            for (size_t column = 0; column < width; column++) {
+                scan_line[column] = convert(y_row[column], u_row_a[column], v_row_a[column]).value();
+            }
+        }
+    }
+
+    return {};
+}
+
+template<u32 subsampling_horizontal, u32 subsampling_vertical, typename T>
+static ALWAYS_INLINE DecoderErrorOr<void> convert_to_bitmap_selecting_converter(CodingIndependentCodePoints cicp, u8 bit_depth, u32 const width, u32 const height, void* plane_y_data, void* plane_u_data, void* plane_v_data, Gfx::Bitmap& bitmap)
+{
+    auto const* plane_y = reinterpret_cast<T const*>(plane_y_data);
+    auto const* plane_u = reinterpret_cast<T const*>(plane_u_data);
+    auto const* plane_v = reinterpret_cast<T const*>(plane_v_data);
+
+    constexpr auto output_cicp = CodingIndependentCodePoints(ColorPrimaries::BT709, TransferCharacteristics::SRGB, MatrixCoefficients::BT709, VideoFullRangeFlag::Full);
+
+    if (bit_depth == 8 && cicp.transfer_characteristics() == output_cicp.transfer_characteristics() && cicp.color_primaries() == output_cicp.color_primaries() && cicp.video_full_range_flag() == VideoFullRangeFlag::Studio) {
+        switch (cicp.matrix_coefficients()) {
+        case MatrixCoefficients::BT470BG:
+        case MatrixCoefficients::BT601:
+            return convert_to_bitmap_subsampled<subsampling_horizontal, subsampling_vertical>([](T y, T u, T v) { return ColorConverter::convert_simple_yuv_to_rgb<MatrixCoefficients::BT601, VideoFullRangeFlag::Studio>(y, u, v); }, width, height, plane_y, plane_u, plane_v, bitmap);
+        case MatrixCoefficients::BT709:
+            return convert_to_bitmap_subsampled<subsampling_horizontal, subsampling_vertical>([](T y, T u, T v) { return ColorConverter::convert_simple_yuv_to_rgb<MatrixCoefficients::BT709, VideoFullRangeFlag::Studio>(y, u, v); }, width, height, plane_y, plane_u, plane_v, bitmap);
+        default:
+            break;
+        }
+    }
+
+    auto converter = TRY(ColorConverter::create(bit_depth, cicp, output_cicp));
+    return convert_to_bitmap_subsampled<subsampling_horizontal, subsampling_vertical>([&](T y, T u, T v) { return converter.convert_yuv(y, u, v); }, width, height, plane_y, plane_u, plane_v, bitmap);
+}
+
+template<u32 subsampling_horizontal, u32 subsampling_vertical>
+static ALWAYS_INLINE DecoderErrorOr<void> convert_to_bitmap_selecting_bit_depth(CodingIndependentCodePoints cicp, u8 bit_depth, u32 const width, u32 const height, void* plane_y_data, void* plane_u_data, void* plane_v_data, Gfx::Bitmap& bitmap)
+{
+    if (bit_depth <= 8) {
+        return convert_to_bitmap_selecting_converter<subsampling_horizontal, subsampling_vertical, u8>(cicp, bit_depth, width, height, plane_y_data, plane_u_data, plane_v_data, bitmap);
+    }
+
+    return convert_to_bitmap_selecting_converter<subsampling_horizontal, subsampling_vertical, u16>(cicp, bit_depth, width, height, plane_y_data, plane_u_data, plane_v_data, bitmap);
+}
+
+static DecoderErrorOr<void> convert_to_bitmap_selecting_subsampling(Subsampling subsampling, CodingIndependentCodePoints cicp, u8 bit_depth, u32 const width, u32 const height, void* plane_y, void* plane_u, void* plane_v, Gfx::Bitmap& bitmap)
+{
+    if (subsampling.x() && subsampling.y()) {
+        return convert_to_bitmap_selecting_bit_depth<true, true>(cicp, bit_depth, width, height, plane_y, plane_u, plane_v, bitmap);
+    }
+
+    if (subsampling.x() && !subsampling.y()) {
+        return convert_to_bitmap_selecting_bit_depth<true, false>(cicp, bit_depth, width, height, plane_y, plane_u, plane_v, bitmap);
+    }
+
+    if (!subsampling.x() && subsampling.y()) {
+        return convert_to_bitmap_selecting_bit_depth<false, true>(cicp, bit_depth, width, height, plane_y, plane_u, plane_v, bitmap);
+    }
+
+    return convert_to_bitmap_selecting_bit_depth<false, false>(cicp, bit_depth, width, height, plane_y, plane_u, plane_v, bitmap);
+}
+
+DecoderErrorOr<void> SubsampledYUVFrame::output_to_bitmap(Gfx::Bitmap& bitmap)
+{
+    return convert_to_bitmap_selecting_subsampling(m_subsampling, cicp(), bit_depth(), width(), height(), m_y_buffer, m_u_buffer, m_v_buffer, bitmap);
+}
+
+}

Libraries/LibMedia/VideoFrame.h

@@ -0,0 +1,124 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/ByteBuffer.h>
+#include <AK/FixedArray.h>
+#include <AK/Time.h>
+#include <LibGfx/Bitmap.h>
+#include <LibGfx/Size.h>
+#include <LibMedia/Color/CodingIndependentCodePoints.h>
+
+#include "DecoderError.h"
+#include "Subsampling.h"
+
+namespace Media {
+
+class VideoFrame {
+
+public:
+    virtual ~VideoFrame() { }
+
+    virtual DecoderErrorOr<void> output_to_bitmap(Gfx::Bitmap& bitmap) = 0;
+    virtual DecoderErrorOr<NonnullRefPtr<Gfx::Bitmap>> to_bitmap()
+    {
+        auto bitmap = DECODER_TRY_ALLOC(Gfx::Bitmap::create(Gfx::BitmapFormat::BGRx8888, { width(), height() }));
+        TRY(output_to_bitmap(bitmap));
+        return bitmap;
+    }
+
+    inline AK::Duration timestamp() const { return m_timestamp; }
+
+    inline Gfx::Size<u32> size() const { return m_size; }
+    inline u32 width() const { return size().width(); }
+    inline u32 height() const { return size().height(); }
+
+    inline u8 bit_depth() const { return m_bit_depth; }
+    inline CodingIndependentCodePoints& cicp() { return m_cicp; }
+
+protected:
+    VideoFrame(AK::Duration timestamp,
+        Gfx::Size<u32> size,
+        u8 bit_depth, CodingIndependentCodePoints cicp)
+        : m_timestamp(timestamp)
+        , m_size(size)
+        , m_bit_depth(bit_depth)
+        , m_cicp(cicp)
+    {
+    }
+
+    AK::Duration m_timestamp;
+    Gfx::Size<u32> m_size;
+    u8 m_bit_depth;
+    CodingIndependentCodePoints m_cicp;
+};
+
+class SubsampledYUVFrame : public VideoFrame {
+
+public:
+    static ErrorOr<NonnullOwnPtr<SubsampledYUVFrame>> try_create(
+        AK::Duration timestamp,
+        Gfx::Size<u32> size,
+        u8 bit_depth, CodingIndependentCodePoints cicp,
+        Subsampling subsampling);
+
+    static ErrorOr<NonnullOwnPtr<SubsampledYUVFrame>> try_create_from_data(
+        AK::Duration timestamp,
+        Gfx::Size<u32> size,
+        u8 bit_depth, CodingIndependentCodePoints cicp,
+        Subsampling subsampling,
+        ReadonlyBytes y_data, ReadonlyBytes u_data, ReadonlyBytes v_data);
+
+    SubsampledYUVFrame(
+        AK::Duration timestamp,
+        Gfx::Size<u32> size,
+        u8 bit_depth, CodingIndependentCodePoints cicp,
+        Subsampling subsampling,
+        u8* plane_y_data, u8* plane_u_data, u8* plane_v_data)
+        : VideoFrame(timestamp, size, bit_depth, cicp)
+        , m_subsampling(subsampling)
+        , m_y_buffer(plane_y_data)
+        , m_u_buffer(plane_u_data)
+        , m_v_buffer(plane_v_data)
+    {
+        VERIFY(m_y_buffer != nullptr);
+        VERIFY(m_u_buffer != nullptr);
+        VERIFY(m_v_buffer != nullptr);
+    }
+
+    ~SubsampledYUVFrame();
+
+    DecoderErrorOr<void> output_to_bitmap(Gfx::Bitmap& bitmap) override;
+
+    u8* get_raw_plane_data(u32 plane)
+    {
+        switch (plane) {
+        case 0:
+            return m_y_buffer;
+        case 1:
+            return m_u_buffer;
+        case 2:
+            return m_v_buffer;
+        }
+        VERIFY_NOT_REACHED();
+    }
+
+    template<typename T>
+    T* get_plane_data(u32 plane)
+    {
+        VERIFY((IsSame<T, u8>) == (bit_depth() <= 8));
+        return reinterpret_cast<T*>(get_raw_plane_data(plane));
+    }
+
+protected:
+    Subsampling m_subsampling;
+    u8* m_y_buffer = nullptr;
+    u8* m_u_buffer = nullptr;
+    u8* m_v_buffer = nullptr;
+};
+
+}

Libraries/LibMedia/VideoSampleData.h

@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <LibMedia/Color/CodingIndependentCodePoints.h>
+
+namespace Media {
+
+class VideoSampleData {
+public:
+    VideoSampleData(CodingIndependentCodePoints container_cicp)
+        : m_container_cicp(container_cicp)
+    {
+    }
+
+    CodingIndependentCodePoints container_cicp() const { return m_container_cicp; }
+
+private:
+    CodingIndependentCodePoints m_container_cicp;
+};
+
+}