Stowage/ladybird
2
0
Fork 0
mirror of https://github.com/LadybirdBrowser/ladybird.git synced 2025-12-08 06:09:58 +00:00
Code Issues Projects Releases Packages Wiki Activity

LibMedia+LibWeb: Rewrite PlaybackManager using the provider/sink model

Browse source 
With this commit, all PlaybackManager can do is autoplay a file from
start to finish, with no pausing or seeking functionality.

All audio playback functionality has been removed from HTMLMediaElement
and HTMLAudioElement in anticipation of PlaybackManager taking that
over, for both audio-only and audio/video.
This commit is contained in:
Zaggy1024 2025-10-03 00:39:39 -05:00 • committed by Jelle Raaijmakers
parent 0f9fa47352
commit 6caa2f99aa
Notes: github-actions[bot] 2025-10-28 00:35:44 +00:00
15 changed files with 371 additions and 1234 deletions
Download patch file Download diff file Expand all files Collapse all files

277
Libraries/LibWeb/HTML/HTMLMediaElement.cpp
View file

@ -1,13 +1,15 @@
/*
* Copyright (c) 2020, the SerenityOS developers.
* Copyright (c) 2023-2024, Tim Flynn <trflynn89@serenityos.org>
* Copyright (c) 2025, Gregory Bertilson <gregory@ladybird.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <LibJS/Runtime/Promise.h>
#include <LibMedia/Audio/Loader.h>
#include <LibMedia/PlaybackManager.h>
#include <LibMedia/Sinks/DisplayingVideoSink.h>
#include <LibMedia/Track.h>
#include <LibWeb/Bindings/HTMLMediaElementPrototype.h>
#include <LibWeb/Bindings/Intrinsics.h>
#include <LibWeb/DOM/Document.h>
@ -75,6 +77,13 @@ void HTMLMediaElement::initialize(JS::Realm& realm)
void HTMLMediaElement::finalize()
{
Base::finalize();
if (m_selected_video_track) {
VERIFY(m_selected_video_track_sink);
m_playback_manager->remove_the_displaying_video_sink_for_track(m_selected_video_track->track_in_playback_manager());
m_selected_video_track_sink = nullptr;
}
document().page().unregister_media_element({}, unique_id());
}
@ -97,6 +106,7 @@ void HTMLMediaElement::visit_edges(Cell::Visitor& visitor)
visitor.visit(m_source_element_selector);
visitor.visit(m_fetch_controller);
visitor.visit(m_pending_play_promises);
visitor.visit(m_selected_video_track);
}
void HTMLMediaElement::attribute_changed(FlyString const& name, Optional<String> const& old_value, Optional<String> const& value, Optional<FlyString> const& namespace_)
@ -300,9 +310,6 @@ void HTMLMediaElement::set_current_playback_position(double playback_position)
time_marches_on();
// NOTE: This notifies blocked seek_element() invocations that we have finished seeking.
m_seek_in_progress = false;
// NOTE: Invoking the following steps is not listed in the spec. Rather, the spec just describes the scenario in
// which these steps should be invoked, which is when we've reached the end of the media playback.
if (m_current_playback_position == m_duration)
@ -447,12 +454,12 @@ void HTMLMediaElement::volume_or_muted_attribute_changed()
if (auto* paintable = this->paintable())
paintable->set_needs_display();
on_volume_change();
// FIXME: Set the volume on the PlaybackManager.
}
void HTMLMediaElement::page_mute_state_changed(Badge<Page>)
{
on_volume_change();
// FIXME: Set the volume on the PlaybackManager.
}
// https://html.spec.whatwg.org/multipage/media.html#effective-media-volume
@ -1119,65 +1126,93 @@ bool HTMLMediaElement::verify_response(GC::Ref<Fetch::Infrastructure::Response>
TODO();
}
void HTMLMediaElement::set_selected_video_track(Badge<VideoTrack>, GC::Ptr<HTML::VideoTrack> video_track)
{
set_needs_style_update(true);
if (auto layout_node = this->layout_node())
layout_node->set_needs_layout_update(DOM::SetNeedsLayoutReason::HTMLVideoElementSetVideoTrack);
if (m_selected_video_track) {
VERIFY(m_selected_video_track_sink);
m_playback_manager->remove_the_displaying_video_sink_for_track(m_selected_video_track->track_in_playback_manager());
m_selected_video_track_sink = nullptr;
}
m_selected_video_track = video_track;
if (video_track)
m_selected_video_track_sink = m_playback_manager->get_or_create_the_displaying_video_sink_for_track(video_track->track_in_playback_manager());
}
void HTMLMediaElement::update_video_frame_and_timeline()
{
if (!paintable())
return;
if (!m_playback_manager)
return;
auto needs_display = false;
if (m_selected_video_track_sink) {
auto sink_update_result = m_selected_video_track_sink->update();
needs_display = sink_update_result == Media::DisplayingVideoSinkUpdateResult::NewFrameAvailable;
}
// FIXME: It might not be the best idea to send time updates at 60Hz, but this is a convenient place for this
// for now.
auto new_position = m_playback_manager->current_time().to_seconds_f64();
if (new_position != m_official_playback_position) {
set_current_playback_position(new_position);
needs_display = true;
}
if (needs_display)
paintable()->set_needs_display();
}
// https://html.spec.whatwg.org/multipage/media.html#media-data-processing-steps-list
WebIDL::ExceptionOr<void> HTMLMediaElement::process_media_data(Function<void(String)> failure_callback)
{
auto& realm = this->realm();
auto audio_loader = Audio::Loader::create(m_media_data.bytes());
auto playback_manager = Media::PlaybackManager::from_data(m_media_data);
auto playback_manager_result = Media::PlaybackManager::try_create(make<FixedMemoryStream>(m_media_data.bytes()));
// -> If the media data cannot be fetched at all, due to network errors, causing the user agent to give up trying to fetch the resource
// -> If the media data can be fetched but is found by inspection to be in an unsupported format, or can otherwise not be rendered at all
if (audio_loader.is_error() && playback_manager.is_error()) {
if (playback_manager_result.is_error()) {
// 1. The user agent should cancel the fetching process.
m_fetch_controller->stop_fetch();
// 2. Abort this subalgorithm, returning to the resource selection algorithm.
failure_callback(MUST(String::from_utf8(playback_manager.error().description())));
failure_callback(MUST(String::from_utf8(playback_manager_result.error().description())));
return {};
}
GC::Ptr<AudioTrack> audio_track;
GC::Ptr<VideoTrack> video_track;
// NOTE: The spec is unclear on whether the following media resource track conditions should trigger multiple
// times on one media resource, but it is implied to be possible by the start of the "Media elements"
// section, where it says that a "media resource can have multiple audio and video tracks."
// https://html.spec.whatwg.org/multipage/media.html#media-elements
// Therefore, we enumerate all the available tracks into our VideoTrackList and AudioTrackList.
// -> If the media resource is found to have an audio track
if (!audio_loader.is_error()) {
// 1. Create an AudioTrack object to represent the audio track.
audio_track = realm.create<AudioTrack>(realm, *this, audio_loader.release_value());
m_playback_manager = playback_manager_result.release_value();
// 2. Update the media element's audioTracks attribute's AudioTrackList object with the new AudioTrack object.
m_audio_tracks->add_track({}, *audio_track);
// 3. Let enable be unknown.
auto enable = TriState::Unknown;
// FIXME: 4. If either the media resource or the URL of the current media resource indicate a particular set of audio tracks to enable, or if
// the user agent has information that would facilitate the selection of specific audio tracks to improve the user's experience, then:
// if this audio track is one of the ones to enable, then set enable to true, otherwise, set enable to false.
// 5. If enable is still unknown, then, if the media element does not yet have an enabled audio track, then set enable to true, otherwise,
// set enable to false.
if (enable == TriState::Unknown)
enable = m_audio_tracks->has_enabled_track() ? TriState::False : TriState::True;
// 6. If enable is true, then enable this audio track, otherwise, do not enable this audio track.
if (enable == TriState::True)
audio_track->set_enabled(true);
// 7. Fire an event named addtrack at this AudioTrackList object, using TrackEvent, with the track attribute initialized to the new AudioTrack object.
TrackEventInit event_init {};
event_init.track = GC::make_root(audio_track);
auto event = TrackEvent::create(realm, HTML::EventNames::addtrack, move(event_init));
m_audio_tracks->dispatch_event(event);
}
// FIXME: -> If the media resource is found to have an audio track
// 1. Create an AudioTrack object to represent the audio track.
// 2. Update the media element's audioTracks attribute's AudioTrackList object with the new AudioTrack object.
// 3. Let enable be unknown.
// 4. If either the media resource or the URL of the current media resource indicate a particular set of audio tracks to enable, or if
// the user agent has information that would facilitate the selection of specific audio tracks to improve the user's experience, then:
// if this audio track is one of the ones to enable, then set enable to true, otherwise, set enable to false.
// 5. If enable is still unknown, then, if the media element does not yet have an enabled audio track, then set enable to true, otherwise,
// set enable to false.
// 6. If enable is true, then enable this audio track, otherwise, do not enable this audio track.
// 7. Fire an event named addtrack at this AudioTrackList object, using TrackEvent, with the track attribute initialized to the new AudioTrack object.
// -> If the media resource is found to have a video track
if (!playback_manager.is_error()) {
auto preferred_video_track = m_playback_manager->preferred_video_track();
auto has_selected_preferred_video_track = false;
for (auto const& track : m_playback_manager->video_tracks()) {
// 1. Create a VideoTrack object to represent the video track.
video_track = realm.create<VideoTrack>(realm, *this, playback_manager.release_value());
auto video_track = realm.create<VideoTrack>(realm, *this, track);
// 2. Update the media element's videoTracks attribute's VideoTrackList object with the new VideoTrack object.
m_video_tracks->add_track({}, *video_track);
@ -1185,9 +1220,17 @@ WebIDL::ExceptionOr<void> HTMLMediaElement::process_media_data(Function<void(Str
// 3. Let enable be unknown.
auto enable = TriState::Unknown;
// FIXME: 4. If either the media resource or the URL of the current media resource indicate a particular set of video tracks to enable, or if
// the user agent has information that would facilitate the selection of specific video tracks to improve the user's experience, then:
// if this video track is the first such video track, then set enable to true, otherwise, set enable to false.
// 4. If either the media resource or the URL of the current media resource indicate a particular set of video tracks to enable, or if
// the user agent has information that would facilitate the selection of specific video tracks to improve the user's experience, then:
// if this video track is the first such video track, then set enable to true, otherwise, set enable to false.
if (preferred_video_track.has_value()) {
if (track == preferred_video_track && !has_selected_preferred_video_track) {
enable = TriState::True;
has_selected_preferred_video_track = true;
} else {
enable = TriState::False;
}
}
// 5. If enable is still unknown, then, if the media element does not yet have a selected video track, then set enable to true, otherwise, set
// enable to false.
@ -1207,80 +1250,77 @@ WebIDL::ExceptionOr<void> HTMLMediaElement::process_media_data(Function<void(Str
m_video_tracks->dispatch_event(event);
}
if (preferred_video_track.has_value())
VERIFY(has_selected_preferred_video_track);
// -> Once enough of the media data has been fetched to determine the duration of the media resource, its dimensions, and other metadata
if (audio_track != nullptr || video_track != nullptr) {
// AD-HOC: After selecting a track, we do not need the source element selector anymore.
m_source_element_selector = nullptr;
// AD-HOC: After selecting a track, we do not need the source element selector anymore.
m_source_element_selector = nullptr;
// FIXME: 1. Establish the media timeline for the purposes of the current playback position and the earliest possible position, based on the media data.
// FIXME: 2. Update the timeline offset to the date and time that corresponds to the zero time in the media timeline established in the previous step,
// if any. If no explicit time and date is given by the media resource, the timeline offset must be set to Not-a-Number (NaN).
// FIXME: 1. Establish the media timeline for the purposes of the current playback position and the earliest possible position, based on the media data.
// FIXME: 2. Update the timeline offset to the date and time that corresponds to the zero time in the media timeline established in the previous step,
// if any. If no explicit time and date is given by the media resource, the timeline offset must be set to Not-a-Number (NaN).
// 3. Set the current playback position and the official playback position to the earliest possible position.
m_current_playback_position = 0;
m_official_playback_position = 0;
// 3. Set the current playback position and the official playback position to the earliest possible position.
m_current_playback_position = 0;
m_official_playback_position = 0;
// 4. Update the duration attribute with the time of the last frame of the resource, if known, on the media timeline established above. If it is
// not known (e.g. a stream that is in principle infinite), update the duration attribute to the value positive Infinity.
// FIXME: Handle unbounded media resources.
// 5. For video elements, set the videoWidth and videoHeight attributes, and queue a media element task given the media element to fire an event
// named resize at the media element.
if (video_track && is<HTMLVideoElement>(*this)) {
auto duration = video_track ? video_track->duration() : audio_track->duration();
set_duration(duration.to_seconds_f64());
// 4. Update the duration attribute with the time of the last frame of the resource, if known, on the media timeline established above. If it is
// not known (e.g. a stream that is in principle infinite), update the duration attribute to the value positive Infinity.
// FIXME: Handle unbounded media resources.
set_duration(m_playback_manager->duration().to_seconds_f64());
auto& video_element = as<HTMLVideoElement>(*this);
video_element.set_video_width(video_track->pixel_width());
video_element.set_video_height(video_track->pixel_height());
// 5. For video elements, set the videoWidth and videoHeight attributes, and queue a media element task given the media element to fire an event
// named resize at the media element.
auto* video_element = as_if<HTMLVideoElement>(*this);
if (m_selected_video_track && video_element) {
video_element->set_video_height(m_selected_video_track->track_in_playback_manager().video_data().pixel_height);
video_element->set_video_width(m_selected_video_track->track_in_playback_manager().video_data().pixel_width);
queue_a_media_element_task([this] {
dispatch_event(DOM::Event::create(this->realm(), HTML::EventNames::resize));
});
} else {
auto duration = audio_track ? audio_track->duration() : video_track->duration();
set_duration(duration.to_seconds_f64());
}
queue_a_media_element_task([this] {
dispatch_event(DOM::Event::create(this->realm(), HTML::EventNames::resize));
});
}
// 6. Set the readyState attribute to HAVE_METADATA.
set_ready_state(ReadyState::HaveMetadata);
// 6. Set the readyState attribute to HAVE_METADATA.
set_ready_state(ReadyState::HaveMetadata);
// 7. Let jumped be false.
[[maybe_unused]] auto jumped = false;
// 7. Let jumped be false.
[[maybe_unused]] auto jumped = false;
// 8. If the media element's default playback start position is greater than zero, then seek to that time, and let jumped be true.
if (m_default_playback_start_position > 0) {
seek_element(m_default_playback_start_position);
jumped = true;
}
// 8. If the media element's default playback start position is greater than zero, then seek to that time, and let jumped be true.
if (m_default_playback_start_position > 0) {
seek_element(m_default_playback_start_position);
jumped = true;
}
// 9. Set the media element's default playback start position to zero.
m_default_playback_start_position = 0;
// 9. Set the media element's default playback start position to zero.
m_default_playback_start_position = 0;
// FIXME: 10. Let the initial playback position be 0.
// FIXME: 11. If either the media resource or the URL of the current media resource indicate a particular start time, then set the initial playback
// position to that time and, if jumped is still false, seek to that time.
// FIXME: 10. Let the initial playback position be 0.
// FIXME: 11. If either the media resource or the URL of the current media resource indicate a particular start time, then set the initial playback
// position to that time and, if jumped is still false, seek to that time.
// 12. If there is no enabled audio track, then enable an audio track. This will cause a change event to be fired.
if (audio_track && !m_audio_tracks->has_enabled_track())
audio_track->set_enabled(true);
// FIXME: 12. If there is no enabled audio track, then enable an audio track. This will cause a change event to be fired.
// 13. If there is no selected video track, then select a video track. This will cause a change event to be fired.
if (video_track && m_video_tracks->selected_index() == -1)
video_track->set_selected(true);
// 13. If there is no selected video track, then select a video track. This will cause a change event to be fired.
if (m_video_tracks->selected_index() == -1) {
m_video_tracks->for_each_track([](auto& track) {
track.set_selected(true);
return IterationDecision::Break;
});
}
// -> Once the entire media resource has been fetched (but potentially before any of it has been decoded)
if (audio_track != nullptr || video_track != nullptr) {
// Fire an event named progress at the media element.
dispatch_event(DOM::Event::create(this->realm(), HTML::EventNames::progress));
// Fire an event named progress at the media element.
dispatch_event(DOM::Event::create(this->realm(), HTML::EventNames::progress));
// Set the networkState to NETWORK_IDLE and fire an event named suspend at the media element.
m_network_state = NetworkState::Idle;
dispatch_event(DOM::Event::create(this->realm(), HTML::EventNames::suspend));
// Set the networkState to NETWORK_IDLE and fire an event named suspend at the media element.
m_network_state = NetworkState::Idle;
dispatch_event(DOM::Event::create(this->realm(), HTML::EventNames::suspend));
// If the user agent ever discards any media data and then needs to resume the network activity to obtain it again, then it must queue a media
// element task given the media element to set the networkState to NETWORK_LOADING.
}
// FIXME: If the user agent ever discards any media data and then needs to resume the network activity to obtain it again, then it must queue a media
// element task given the media element to set the networkState to NETWORK_LOADING.
// FIXME: -> If the connection is interrupted after some media data has been received, causing the user agent to give up trying to fetch the resource
// FIXME: -> If the media data fetching process is aborted by the user
@ -1571,12 +1611,10 @@ void HTMLMediaElement::seek_element(double playback_position, MediaSeekMode seek
// aborted at any time by another instance of this algorithm being invoked.
// 6. If the new playback position is later than the end of the media resource, then let it be the end of the media resource instead.
if (playback_position > m_duration)
playback_position = m_duration;
playback_position = min(playback_position, m_duration);
// 7. If the new playback position is less than the earliest possible position, let it be that position instead.
if (playback_position < 0)
playback_position = 0;
playback_position = max(playback_position, 0);
// 8. If the (possibly now changed) new playback position is not in one of the ranges given in the seekable attribute,
auto time_ranges = seekable();
@ -1635,12 +1673,12 @@ void HTMLMediaElement::seek_element(double playback_position, MediaSeekMode seek
// 12. Wait until the user agent has established whether or not the media data for the new playback position is
// available, and, if it is, until it has decoded enough data to play back that position.
m_seek_in_progress = true;
on_seek(playback_position, seek_mode);
HTML::main_thread_event_loop().spin_until(GC::create_function(heap(), [&]() { return !m_seek_in_progress; }));
// FIXME: Implement seeking in PlaybackManager.
(void)seek_mode;
// FIXME: 13. Await a stable state. The synchronous section consists of all the remaining steps of this algorithm. (Steps in the
// synchronous section are marked with ⌛.)
// The following steps should be executed within the success callback of a seek promise from PlaybackManager.
// 14. ⌛ Set the seeking IDL attribute to false.
set_seeking(false);
@ -1674,7 +1712,7 @@ void HTMLMediaElement::notify_about_playing()
resolve_pending_play_promises(promises);
});
on_playing();
// FIXME: Implement resuming in PlaybackManager.
if (m_audio_tracks->has_enabled_track())
document().page().client().page_did_change_audio_play_state(AudioPlayState::Playing);
@ -1699,7 +1737,7 @@ void HTMLMediaElement::set_paused(bool paused)
m_paused = paused;
if (m_paused) {
on_paused();
// FIXME: Implement pausing in PlaybackManager.
if (m_audio_tracks->has_enabled_track())
document().page().client().page_did_change_audio_play_state(AudioPlayState::Paused);
@ -1840,14 +1878,7 @@ bool HTMLMediaElement::has_ended_playback() const
void HTMLMediaElement::reached_end_of_media_playback()
{
// 1. If the media element has a loop attribute specified, then seek to the earliest possible position of the media resource and return.
if (has_attribute(HTML::AttributeNames::loop)) {
seek_element(0);
// AD-HOC: LibVideo internally sets itself to a paused state when it reaches the end of a video. We must resume
// playing manually to actually loop. Note that we don't need to update any HTMLMediaElement state as
// it hasn't left the playing state by this point.
on_playing();
}
// FIXME: Implement this within PlaybackManager and its related classes.
// 2. As defined above, the ended IDL attribute starts returning true once the event loop returns to step 1.
@ -2068,12 +2099,12 @@ void HTMLMediaElement::set_layout_display_time(Badge<Painting::MediaPaintable>,
if (display_time.has_value() && !m_display_time.has_value()) {
if (potentially_playing()) {
m_tracking_mouse_position_while_playing = true;
on_paused();
// FIXME: Pause the PlaybackManager.
}
} else if (!display_time.has_value() && m_display_time.has_value()) {
if (m_tracking_mouse_position_while_playing) {
m_tracking_mouse_position_while_playing = false;
on_playing();
// FIXME: Resume the PlaybackManager.
}
}