LibWeb: Implement audio media data processing through PlaybackManager

Author: https://github.com/Zaggy1024
Commit: 5456072d48
Pull-request: https://github.com/LadybirdBrowser/ladybird/pull/6410
Reviewed-by: https://github.com/R-Goc
Reviewed-by: https://github.com/gmta ✅

Libraries/LibWeb/HTML/AudioTrack.cpp

@@ -7,7 +7,6 @@
 #include <AK/IDAllocator.h>
 #include <LibJS/Runtime/Realm.h>
 #include <LibJS/Runtime/VM.h>
-#include <LibMedia/Audio/Loader.h>
 #include <LibWeb/Bindings/AudioTrackPrototype.h>
 #include <LibWeb/Bindings/Intrinsics.h>
 #include <LibWeb/DOM/Event.h>
@@ -16,8 +15,6 @@
 #include <LibWeb/HTML/EventNames.h>
 #include <LibWeb/HTML/HTMLMediaElement.h>
 #include <LibWeb/Layout/Node.h>
-#include <LibWeb/Painting/Paintable.h>
-#include <LibWeb/Platform/AudioCodecPlugin.h>
 
 namespace Web::HTML {
 
@@ -25,24 +22,11 @@ GC_DEFINE_ALLOCATOR(AudioTrack);
 
 static IDAllocator s_audio_track_id_allocator;
 
-AudioTrack::AudioTrack(JS::Realm& realm, GC::Ref<HTMLMediaElement> media_element, NonnullRefPtr<Audio::Loader> loader)
+AudioTrack::AudioTrack(JS::Realm& realm, GC::Ref<HTMLMediaElement> media_element, Media::Track const& track)
     : PlatformObject(realm)
     , m_media_element(media_element)
-    , m_audio_plugin(Platform::AudioCodecPlugin::create(move(loader)).release_value_but_fixme_should_propagate_errors())
+    , m_track_in_playback_manager(track)
 {
-    m_audio_plugin->on_playback_position_updated = [this](auto position) {
-        if (auto* paintable = m_media_element->paintable())
-            paintable->set_needs_display();
-
-        auto playback_position = static_cast<double>(position.to_milliseconds()) / 1000.0;
-        m_media_element->set_current_playback_position(playback_position);
-    };
-
-    m_audio_plugin->on_decoder_error = [this](String error_message) {
-        m_media_element->set_decoder_error(move(error_message));
-    };
-
-    update_volume();
 }
 
 AudioTrack::~AudioTrack()
@@ -62,34 +46,6 @@ void AudioTrack::initialize(JS::Realm& realm)
     m_id = String::number(id);
 }
 
-void AudioTrack::play()
-{
-    m_audio_plugin->resume_playback();
-}
-
-void AudioTrack::pause()
-{
-    m_audio_plugin->pause_playback();
-}
-
-AK::Duration AudioTrack::duration()
-{
-    return m_audio_plugin->duration();
-}
-
-void AudioTrack::seek(double position, MediaSeekMode seek_mode)
-{
-    // FIXME: Implement seeking mode.
-    (void)seek_mode;
-
-    m_audio_plugin->seek(position);
-}
-
-void AudioTrack::update_volume()
-{
-    m_audio_plugin->set_volume(m_media_element->effective_media_volume());
-}
-
 void AudioTrack::visit_edges(Cell::Visitor& visitor)
 {
     Base::visit_edges(visitor);
@@ -116,6 +72,7 @@ void AudioTrack::set_enabled(bool enabled)
     }
 
     m_enabled = enabled;
+    m_media_element->set_audio_track_enabled({}, this, enabled);
 }
 
 }

Libraries/LibWeb/HTML/AudioTrack.h

@@ -7,8 +7,8 @@
 #pragma once
 
 #include <AK/String.h>
-#include <AK/Time.h>
 #include <LibMedia/Audio/Forward.h>
+#include <LibMedia/Track.h>
 #include <LibWeb/Bindings/PlatformObject.h>
 
 namespace Web::HTML {
@@ -22,14 +22,6 @@ public:
 
     void set_audio_track_list(Badge<AudioTrackList>, GC::Ptr<AudioTrackList> audio_track_list) { m_audio_track_list = audio_track_list; }
 
-    void play();
-    void pause();
-
-    AK::Duration duration();
-    void seek(double, MediaSeekMode);
-
-    void update_volume();
-
     String const& id() const { return m_id; }
     String const& kind() const { return m_kind; }
     String const& label() const { return m_label; }
@@ -38,8 +30,10 @@ public:
     bool enabled() const { return m_enabled; }
     void set_enabled(bool enabled);
 
+    Media::Track const& track_in_playback_manager() const { return m_track_in_playback_manager; }
+
 private:
-    AudioTrack(JS::Realm&, GC::Ref<HTMLMediaElement>, NonnullRefPtr<Audio::Loader>);
+    AudioTrack(JS::Realm&, GC::Ref<HTMLMediaElement>, Media::Track const&);
 
     virtual void initialize(JS::Realm&) override;
     virtual void visit_edges(Cell::Visitor&) override;
@@ -62,7 +56,7 @@ private:
     GC::Ref<HTMLMediaElement> m_media_element;
     GC::Ptr<AudioTrackList> m_audio_track_list;
 
-    NonnullOwnPtr<Platform::AudioCodecPlugin> m_audio_plugin;
+    Media::Track m_track_in_playback_manager;
 };
 
 }

Libraries/LibWeb/HTML/AudioTrackList.h

@@ -29,11 +29,12 @@ public:
     bool has_enabled_track() const;
 
     template<typename Callback>
-    void for_each_enabled_track(Callback&& callback)
+    void for_each_track(Callback&& callback)
     {
         for (auto& audio_track : m_audio_tracks) {
-            if (audio_track->enabled())
-                callback(*audio_track);
+            auto iteration_decision = callback(*audio_track);
+            if (iteration_decision == IterationDecision::Break)
+                break;
         }
     }
 

Libraries/LibWeb/HTML/HTMLMediaElement.cpp

@@ -1126,6 +1126,14 @@ bool HTMLMediaElement::verify_response(GC::Ref<Fetch::Infrastructure::Response>
     TODO();
 }
 
+void HTMLMediaElement::set_audio_track_enabled(Badge<AudioTrack>, GC::Ptr<HTML::AudioTrack> audio_track, bool enabled)
+{
+    if (enabled)
+        m_playback_manager->enable_an_audio_track(audio_track->track_in_playback_manager());
+    else
+        m_playback_manager->disable_an_audio_track(audio_track->track_in_playback_manager());
+}
+
 void HTMLMediaElement::set_selected_video_track(Badge<VideoTrack>, GC::Ptr<HTML::VideoTrack> video_track)
 {
     set_needs_style_update(true);
@@ -1194,17 +1202,51 @@ WebIDL::ExceptionOr<void> HTMLMediaElement::process_media_data(Function<void(Str
 
     m_playback_manager = playback_manager_result.release_value();
 
-    // FIXME: -> If the media resource is found to have an audio track
-    //            1. Create an AudioTrack object to represent the audio track.
-    //            2. Update the media element's audioTracks attribute's AudioTrackList object with the new AudioTrack object.
-    //            3. Let enable be unknown.
-    //            4. If either the media resource or the URL of the current media resource indicate a particular set of audio tracks to enable, or if
-    //               the user agent has information that would facilitate the selection of specific audio tracks to improve the user's experience, then:
-    //               if this audio track is one of the ones to enable, then set enable to true, otherwise, set enable to false.
-    //            5. If enable is still unknown, then, if the media element does not yet have an enabled audio track, then set enable to true, otherwise,
-    //               set enable to false.
-    //            6. If enable is true, then enable this audio track, otherwise, do not enable this audio track.
-    //            7. Fire an event named addtrack at this AudioTrackList object, using TrackEvent, with the track attribute initialized to the new AudioTrack object.
+    // -> If the media resource is found to have an audio track
+    auto preferred_audio_track = m_playback_manager->preferred_audio_track();
+    auto has_enabled_preferred_audio_track = false;
+
+    for (auto const& track : m_playback_manager->audio_tracks()) {
+        // 1. Create an AudioTrack object to represent the audio track.
+        auto audio_track = realm.create<AudioTrack>(realm, *this, track);
+
+        // 2. Update the media element's audioTracks attribute's AudioTrackList object with the new AudioTrack object.
+        m_audio_tracks->add_track({}, audio_track);
+
+        // 3. Let enable be unknown.
+        auto enable = TriState::Unknown;
+
+        // 4. If either the media resource or the URL of the current media resource indicate a particular set of audio tracks to enable, or if
+        //    the user agent has information that would facilitate the selection of specific audio tracks to improve the user's experience, then:
+        //    if this audio track is one of the ones to enable, then set enable to true, otherwise, set enable to false.
+        if (preferred_audio_track.has_value()) {
+            if (track == preferred_audio_track && !has_enabled_preferred_audio_track) {
+                enable = TriState::True;
+                has_enabled_preferred_audio_track = true;
+            } else {
+                enable = TriState::False;
+            }
+        }
+
+        // 5. If enable is still unknown, then, if the media element does not yet have an enabled audio track, then set enable to true, otherwise,
+        //    set enable to false.
+        if (enable == TriState::Unknown)
+            enable = !m_audio_tracks->has_enabled_track() ? TriState::True : TriState::False;
+
+        // 6. If enable is true, then enable this audio track, otherwise, do not enable this audio track.
+        if (enable == TriState::True)
+            audio_track->set_enabled(true);
+
+        // 7. Fire an event named addtrack at this AudioTrackList object, using TrackEvent, with the track attribute initialized to the new AudioTrack object.
+        TrackEventInit event_init {};
+        event_init.track = GC::make_root(audio_track);
+
+        auto event = TrackEvent::create(realm, HTML::EventNames::addtrack, move(event_init));
+        m_audio_tracks->dispatch_event(event);
+    }
+
+    if (preferred_audio_track.has_value())
+        VERIFY(has_enabled_preferred_audio_track);
 
     // -> If the media resource is found to have a video track
     auto preferred_video_track = m_playback_manager->preferred_video_track();
@@ -1301,7 +1343,13 @@ WebIDL::ExceptionOr<void> HTMLMediaElement::process_media_data(Function<void(Str
     // FIXME: 11. If either the media resource or the URL of the current media resource indicate a particular start time, then set the initial playback
     //            position to that time and, if jumped is still false, seek to that time.
 
-    // FIXME: 12. If there is no enabled audio track, then enable an audio track. This will cause a change event to be fired.
+    // 12. If there is no enabled audio track, then enable an audio track. This will cause a change event to be fired.
+    if (!m_audio_tracks->has_enabled_track()) {
+        m_audio_tracks->for_each_track([](auto& track) {
+            track.set_enabled(true);
+            return IterationDecision::Break;
+        });
+    }
 
     // 13. If there is no selected video track, then select a video track. This will cause a change event to be fired.
     if (m_video_tracks->selected_index() == -1) {

Libraries/LibWeb/HTML/HTMLMediaElement.h

@@ -130,6 +130,8 @@ public:
     GC::Ref<VideoTrackList> video_tracks() const { return *m_video_tracks; }
     GC::Ref<TextTrackList> text_tracks() const { return *m_text_tracks; }
 
+    void set_audio_track_enabled(Badge<AudioTrack>, GC::Ptr<HTML::AudioTrack> audio_track, bool);
+
     void set_selected_video_track(Badge<VideoTrack>, GC::Ptr<HTML::VideoTrack> video_track);
 
     void update_video_frame_and_timeline();