WIP: Make HLS work

Libraries/LibMedia/CMakeLists.txt

@@ -29,6 +29,7 @@ target_sources(LibMedia PRIVATE
         FFmpeg/FFmpegDemuxer.cpp
         FFmpeg/FFmpegIOContext.cpp
         FFmpeg/FFmpegVideoDecoder.cpp
+        FFmpeg/MSEDemuxer.cpp
 )
 
 if (NOT ANDROID)

Libraries/LibMedia/FFmpeg/MSEDemuxer.h

@@ -0,0 +1,130 @@
+/*
+ * Copyright (c) 2025, contributors
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/ByteBuffer.h>
+#include <AK/HashMap.h>
+#include <AK/Optional.h>
+#include <AK/NonnullOwnPtr.h>
+#include <AK/Vector.h>
+#include <LibMedia/Demuxer.h>
+#include <LibMedia/Export.h>
+#include <LibMedia/FFmpeg/FFmpegForward.h>
+
+extern "C" {
+#include <libavformat/avformat.h>
+}
+
+namespace Media::FFmpeg {
+
+// MSEDemuxer is a specialized demuxer for Media Source Extensions (MSE) that supports
+// progressive/streaming media playback. Unlike FFmpegDemuxer which requires complete
+// file data upfront, MSEDemuxer accepts fragmented MP4 segments incrementally.
+//
+// MSE Workflow:
+// 1. append_initialization_segment() - Parse ftyp + moov boxes to get codec info
+// 2. append_media_segment() - Repeatedly append moof + mdat boxes with actual media data
+// 3. get_next_sample_for_track() - Pull decoded packets for playback
+//
+// This demuxer uses a custom AVIOContext that reads from a growing ByteBuffer,
+// allowing FFmpeg to parse fragmented MP4 as data arrives.
+class MEDIA_API MSEDemuxer : public Demuxer {
+public:
+    static DecoderErrorOr<NonnullRefPtr<MSEDemuxer>> create();
+    virtual ~MSEDemuxer() override;
+
+    // MSE-specific methods for progressive data appending
+    DecoderErrorOr<void> append_initialization_segment(ReadonlyBytes);
+    DecoderErrorOr<void> append_media_segment(ReadonlyBytes);
+    DecoderErrorOr<void> remove(AK::Duration start, AK::Duration end);
+    void set_timestamp_offset(AK::Duration);
+    AK::Duration timestamp_offset() const { return m_timestamp_offset; }
+    AK::Duration buffered_start_time() const { return m_buffered_start; }
+    AK::Duration buffered_end_time() const { return m_buffered_end; }
+
+    // Demuxer interface implementation
+    virtual DecoderErrorOr<Vector<Track>> get_tracks_for_type(TrackType type) override;
+    virtual DecoderErrorOr<Optional<Track>> get_preferred_track_for_type(TrackType type) override;
+    virtual DecoderErrorOr<DemuxerSeekResult> seek_to_most_recent_keyframe(Track const& track, AK::Duration timestamp, DemuxerSeekOptions) override;
+    virtual DecoderErrorOr<AK::Duration> duration_of_track(Track const&) override;
+    virtual DecoderErrorOr<AK::Duration> total_duration() override;
+    virtual DecoderErrorOr<CodecID> get_codec_id_for_track(Track const& track) override;
+    virtual DecoderErrorOr<ReadonlyBytes> get_codec_initialization_data_for_track(Track const& track) override;
+    virtual DecoderErrorOr<CodedFrame> get_next_sample_for_track(Track const& track) override;
+
+private:
+    struct TrackContext {
+        TrackContext();
+        ~TrackContext();
+        TrackContext(TrackContext&&) = default;
+
+        AVFormatContext* format_context { nullptr };
+        AVPacket* packet { nullptr };
+        bool peeked_packet_already { false };
+    };
+
+    // Custom read callback for AVIOContext
+    static int avio_read_callback(void* opaque, uint8_t* buf, int buf_size);
+    static int64_t avio_seek_callback(void* opaque, int64_t offset, int whence);
+
+    MSEDemuxer();
+
+    DecoderErrorOr<void> initialize_format_context();
+    TrackContext& get_track_context(Track const&);
+    DecoderErrorOr<Track> get_track_for_stream_index(u32 stream_index);
+    AK::Duration normalize_timestamp(u32 stream_index, AK::Duration timestamp);
+    DecoderErrorOr<void> recalculate_buffered_range();
+    DecoderErrorOr<void> store_packet_for_stream(AVPacket&, u32 stream_index);
+    void ensure_stream_capacity(u32 stream_index);
+    void clear_pending_samples();
+
+    // Growing buffer that holds all appended data
+    ByteBuffer m_buffer;
+
+    // Current read position in buffer (used by avio callbacks)
+    size_t m_read_position { 0 };
+
+    // Size of the initialization segment (to skip past it when seeking to start)
+    size_t m_init_segment_size { 0 };
+
+    // FFmpeg context
+    AVFormatContext* m_format_context { nullptr };
+    AVIOContext* m_avio_context { nullptr };
+
+    // Per-track contexts for independent packet reading
+    HashMap<Track, NonnullOwnPtr<TrackContext>> m_track_contexts;
+
+    // Track whether we've processed initialization segment
+    bool m_initialized { false };
+
+    // Track whether we've returned EOF from avio_read_callback
+    // Used to detect when we need to reset FFmpeg's EOF state
+    mutable bool m_returned_eof_from_read { false };
+
+    // Estimated duration from initialization segment (may be updated as we append more data)
+    AK::Duration m_duration;
+
+    // Timestamp normalization helpers
+    Vector<Optional<AK::Duration>> m_stream_first_timestamps;
+    Vector<AK::Duration> m_stream_last_timestamps;
+    AK::Duration m_timestamp_offset {};
+
+    // Buffered range tracking
+    AK::Duration m_buffered_start {};
+    AK::Duration m_buffered_end {};
+
+    struct PendingSample {
+        ByteBuffer data;
+        i64 pts { AV_NOPTS_VALUE };
+        i64 dts { AV_NOPTS_VALUE };
+        i64 duration { 0 };
+        int flags { 0 };
+    };
+    Vector<Vector<PendingSample>> m_pending_samples;
+};
+
+}

Libraries/LibMedia/PlaybackManager.cpp

@@ -89,6 +89,69 @@ DecoderErrorOr<NonnullRefPtr<PlaybackManager>> PlaybackManager::try_create(Reado
     return playback_manager;
 }
 
+DecoderErrorOr<NonnullRefPtr<PlaybackManager>> PlaybackManager::try_create_for_mse(NonnullRefPtr<Demuxer> inner_demuxer)
+{
+    // Wrap the MSEDemuxer in a MutexedDemuxer for thread safety
+    auto demuxer = DECODER_TRY_ALLOC(try_make_ref_counted<MutexedDemuxer>(inner_demuxer));
+
+    // Create the weak wrapper
+    auto weak_playback_manager = DECODER_TRY_ALLOC(try_make_ref_counted<WeakPlaybackManager>());
+
+    // First, check for audio tracks and create audio sink if needed
+    auto all_audio_tracks = TRY(demuxer->get_tracks_for_type(TrackType::Audio));
+    auto supported_audio_tracks = AudioTracks();
+    auto supported_audio_track_datas = AudioTrackDatas();
+    supported_audio_tracks.ensure_capacity(all_audio_tracks.size());
+    supported_audio_track_datas.ensure_capacity(all_audio_tracks.size());
+    for (auto const& track : all_audio_tracks) {
+        auto audio_data_provider_result = AudioDataProvider::try_create(demuxer, track);
+        if (audio_data_provider_result.is_error())
+            continue;
+        auto audio_data_provider = audio_data_provider_result.release_value();
+        supported_audio_tracks.append(track);
+        supported_audio_track_datas.empend(AudioTrackData(track, move(audio_data_provider)));
+    }
+    supported_audio_tracks.shrink_to_fit();
+    supported_audio_track_datas.shrink_to_fit();
+
+    RefPtr<AudioMixingSink> audio_sink = nullptr;
+    if (!supported_audio_tracks.is_empty())
+        audio_sink = DECODER_TRY_ALLOC(AudioMixingSink::try_create());
+
+    // Create the time provider (BEFORE creating video providers!)
+    auto time_provider = DECODER_TRY_ALLOC([&] -> ErrorOr<NonnullRefPtr<MediaTimeProvider>> {
+        if (audio_sink)
+            return TRY(try_make_ref_counted<WrapperTimeProvider<AudioMixingSink>>(*audio_sink));
+        return TRY(try_make_ref_counted<GenericTimeProvider>());
+    }());
+
+    // Now create the video tracks and their data providers WITH the time_provider
+    auto all_video_tracks = TRY(demuxer->get_tracks_for_type(TrackType::Video));
+
+    auto supported_video_tracks = VideoTracks();
+    auto supported_video_track_datas = VideoTrackDatas();
+    supported_video_tracks.ensure_capacity(all_video_tracks.size());
+    supported_video_track_datas.ensure_capacity(all_video_tracks.size());
+    for (auto const& track : all_video_tracks) {
+        auto video_data_provider_result = VideoDataProvider::try_create(demuxer, track, time_provider);
+        if (video_data_provider_result.is_error())
+            continue;
+        supported_video_tracks.append(track);
+        supported_video_track_datas.empend(VideoTrackData(track, video_data_provider_result.release_value(), nullptr));
+    }
+    supported_video_tracks.shrink_to_fit();
+    supported_video_track_datas.shrink_to_fit();
+
+    if (supported_video_tracks.is_empty() && supported_audio_tracks.is_empty())
+        return DecoderError::with_description(DecoderErrorCategory::NotImplemented, "No supported video or audio tracks found"sv);
+
+    auto playback_manager = DECODER_TRY_ALLOC(adopt_nonnull_ref_or_enomem(new (nothrow) PlaybackManager(demuxer, weak_playback_manager, time_provider, move(supported_video_tracks), move(supported_video_track_datas), audio_sink, move(supported_audio_tracks), move(supported_audio_track_datas))));
+    weak_playback_manager->m_manager = playback_manager;
+    playback_manager->set_up_error_handlers();
+    playback_manager->m_handler = DECODER_TRY_ALLOC(try_make<PausedStateHandler>(*playback_manager));
+    return playback_manager;
+}
+
 PlaybackManager::PlaybackManager(NonnullRefPtr<MutexedDemuxer> const& demuxer, NonnullRefPtr<WeakPlaybackManager> const& weak_wrapper, NonnullRefPtr<MediaTimeProvider> const& time_provider, VideoTracks&& video_tracks, VideoTrackDatas&& video_track_datas, RefPtr<AudioMixingSink> const& audio_sink, AudioTracks&& audio_tracks, AudioTrackDatas&& audio_track_datas)
     : m_demuxer(demuxer)
     , m_weak_wrapper(weak_wrapper)
@@ -213,6 +276,74 @@ void PlaybackManager::disable_an_audio_track(Track const& track)
     m_audio_sink->set_provider(track, nullptr);
 }
 
+DecoderErrorOr<void> PlaybackManager::add_audio_track_from_demuxer(NonnullRefPtr<Demuxer> demuxer, Track const& track)
+{
+    dbgln("PlaybackManager::add_audio_track_from_demuxer() - Adding audio track from separate demuxer");
+
+    // Check if track already exists
+    for (auto const& existing_track : m_audio_tracks) {
+        if (existing_track == track) {
+            dbgln("PlaybackManager: Audio track already exists, skipping");
+            return {};
+        }
+    }
+
+    // Wrap demuxer in MutexedDemuxer if needed
+    auto mutexed_demuxer = DECODER_TRY_ALLOC(try_make_ref_counted<MutexedDemuxer>(demuxer));
+
+    // Create AudioDataProvider for this track with its demuxer
+    auto audio_data_provider = TRY(AudioDataProvider::try_create(mutexed_demuxer, track));
+
+    // Add to tracks and track datas
+    m_audio_tracks.append(track);
+    m_audio_track_datas.empend(AudioTrackData { track, move(audio_data_provider) });
+
+    // Create AudioMixingSink if we don't have one yet
+    if (!m_audio_sink) {
+        m_audio_sink = DECODER_TRY_ALLOC(AudioMixingSink::try_create());
+
+        // Update time provider to use audio sink
+        m_time_provider = DECODER_TRY_ALLOC(try_make_ref_counted<WrapperTimeProvider<AudioMixingSink>>(*m_audio_sink));
+    }
+
+    dbgln("PlaybackManager: Successfully added audio track from separate demuxer");
+    return {};
+}
+
+DecoderErrorOr<void> PlaybackManager::add_video_track_from_demuxer(NonnullRefPtr<Demuxer> demuxer, Track const& track)
+{
+    dbgln("PlaybackManager::add_video_track_from_demuxer() - Adding video track from separate demuxer");
+
+    // Check if track already exists
+    for (auto const& existing_track : m_video_tracks) {
+        if (existing_track == track) {
+            dbgln("PlaybackManager: Video track already exists, skipping");
+            return {};
+        }
+    }
+
+    // Wrap demuxer in MutexedDemuxer if needed
+    auto mutexed_demuxer = DECODER_TRY_ALLOC(try_make_ref_counted<MutexedDemuxer>(demuxer));
+
+    // Create VideoDataProvider for this track with its demuxer, using this PlaybackManager's time_provider
+    auto video_data_provider = TRY(VideoDataProvider::try_create(mutexed_demuxer, track, m_time_provider));
+
+    // Add to tracks and track datas
+    m_video_tracks.append(track);
+    m_video_track_datas.empend(VideoTrackData { track, video_data_provider, nullptr });
+
+    // Set up error handler for the new track
+    video_data_provider->set_error_handler([weak_self = m_weak_wrapper](DecoderError&& error) {
+        auto self = weak_self->take_strong();
+        if (!self)
+            return;
+        self->dispatch_error(move(error));
+    });
+
+    dbgln("PlaybackManager: Successfully added video track from separate demuxer");
+    return {};
+}
+
 void PlaybackManager::play()
 {
     m_handler->play();

Libraries/LibMedia/PlaybackManager.h

@@ -45,6 +45,7 @@ class MEDIA_API PlaybackManager final : public AtomicRefCounted<PlaybackManager>
     using AudioTracks = Vector<Track, EXPECTED_AUDIO_TRACK_COUNT>;
 
     static DecoderErrorOr<NonnullRefPtr<PlaybackManager>> try_create(ReadonlyBytes data);
+    static DecoderErrorOr<NonnullRefPtr<PlaybackManager>> try_create_for_mse(NonnullRefPtr<Demuxer> demuxer);
     ~PlaybackManager();
 
     AK::Duration duration() const;
@@ -67,6 +68,8 @@ class MEDIA_API PlaybackManager final : public AtomicRefCounted<PlaybackManager>
 
     void enable_an_audio_track(Track const& track);
     void disable_an_audio_track(Track const& track);
+    DecoderErrorOr<void> add_audio_track_from_demuxer(NonnullRefPtr<Demuxer> demuxer, Track const& track);
+    DecoderErrorOr<void> add_video_track_from_demuxer(NonnullRefPtr<Demuxer> demuxer, Track const& track);
 
     void play();
     void pause();

Libraries/LibMedia/Providers/AudioDataProvider.cpp

@@ -240,15 +240,21 @@ void AudioDataProvider::ThreadData::push_data_and_decode_a_block()
 
     auto sample_result = m_demuxer->get_next_sample_for_track(m_track);
     if (sample_result.is_error()) {
-        if (sample_result.error().category() == DecoderErrorCategory::NeedsMoreInput) {
+        auto error_category = sample_result.error().category();
+
+        // For MSE, "End of buffered data" is normal - just wait for more data to be appended
+        if (error_category == DecoderErrorCategory::NeedsMoreInput || error_category == DecoderErrorCategory::EndOfStream) {
             return;
         }
-        // FIXME: Handle the end of the stream.
+
+        // Other errors are real errors
+        dbgln("AudioDataProvider: ERROR - get_next_sample_for_track() returned: {}", sample_result.error());
         set_error_and_wait_for_seek(sample_result.release_error());
         return;
     }
 
     auto sample = sample_result.release_value();
+
     auto decode_result = m_decoder->receive_coded_data(sample.timestamp(), sample.data());
     if (decode_result.is_error()) {
         set_error_and_wait_for_seek(decode_result.release_error());