Stowage/ladybird
2
0
Fork 0
mirror of https://github.com/LadybirdBrowser/ladybird.git synced 2025-12-08 06:09:58 +00:00
Code Issues Projects Releases Packages Wiki Activity
ladybird/Libraries/LibMedia/Sinks/AudioMixingSink.cpp

301 lines
11 KiB
C++
Raw Normal View History

LibMedia: Play audio through PlaybackManager using Providers/Sinks This commit implements the functionality to play back audio through PlaybackManager. To decode the audio data, AudioDataProviders are created for each track in the provided media data. These providers will fill their audio block queue, then sit idle until their corresponding tracks are enabled. In order to output the audio, one AudioMixingSink is created which manages a PlaybackStream which requests audio blocks from multiple AudioDataProviders and mixes them into one buffer with sample-perfect precision.
2025-09-24 13:40:17 -05:00
/*
* Copyright (c) 2025, Gregory Bertilson <gregory@ladybird.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <AK/Time.h>
#include <LibMedia/Audio/PlaybackStream.h>
#include <LibMedia/Providers/AudioDataProvider.h>
#include "AudioMixingSink.h"
namespace Media {
ErrorOr<NonnullRefPtr<AudioMixingSink>> AudioMixingSink::try_create()
{
auto weak_ref = TRY(try_make_ref_counted<AudioMixingSinkWeakReference>());
auto sink = TRY(try_make_ref_counted<AudioMixingSink>(weak_ref));
weak_ref->emplace(sink);
return sink;
}
AudioMixingSink::AudioMixingSink(AudioMixingSinkWeakReference& weak_ref)
: m_main_thread_event_loop(Core::EventLoop::current())
, m_weak_self(weak_ref)
{
}
AudioMixingSink::~AudioMixingSink()
{
m_weak_self->revoke();
}
void AudioMixingSink::deferred_create_playback_stream(Track const& track)
{
m_main_thread_event_loop.deferred_invoke([weak_self = m_weak_self, track = track] {
auto self = weak_self->take_strong();
if (self == nullptr)
return;
auto optional_track_mixing_data = self->m_track_mixing_datas.get(track);
if (!optional_track_mixing_data.has_value())
return;
Threading::MutexLocker locker { self->m_mutex };
auto& track_mixing_data = optional_track_mixing_data.release_value();
if (track_mixing_data.current_block.is_empty())
track_mixing_data.current_block = track_mixing_data.provider->retrieve_block();
if (!track_mixing_data.current_block.is_empty()) {
self->create_playback_stream(track_mixing_data.current_block.sample_rate(), track_mixing_data.current_block.channel_count());
return;
}
self->deferred_create_playback_stream(track);
});
}
void AudioMixingSink::set_provider(Track const& track, RefPtr<AudioDataProvider> const& provider)
{
Threading::MutexLocker locker { m_mutex };
m_track_mixing_datas.remove(track);
if (provider == nullptr)
return;
m_track_mixing_datas.set(track, TrackMixingData(*provider));
deferred_create_playback_stream(track);
}
RefPtr<AudioDataProvider> AudioMixingSink::provider(Track const& track) const
{
auto mixing_data = m_track_mixing_datas.get(track);
if (!mixing_data.has_value())
return nullptr;
return mixing_data->provider;
}
void AudioMixingSink::create_playback_stream(u32 sample_rate, u32 channel_count)
{
if (m_playback_stream_sample_rate >= sample_rate && m_playback_stream_channel_count >= channel_count) {
VERIFY(m_playback_stream);
return;
}
auto callback = [=, weak_self = m_weak_self](Bytes buffer, Audio::PcmSampleFormat format, size_t sample_count) -> ReadonlyBytes {
auto self = weak_self->take_strong();
if (!self)
return buffer.trim(0);
VERIFY(format == Audio::PcmSampleFormat::Float32);
VERIFY(!Checked<i64>::multiplication_would_overflow(sample_count, channel_count));
auto float_buffer_count = static_cast<i64>(sample_count) * channel_count;
auto float_buffer_size = float_buffer_count * sizeof(float);
VERIFY(buffer.size() >= float_buffer_size);
auto float_buffer = buffer.reinterpret<float>();
float_buffer.fill(0.0f);
Threading::MutexLocker mixing_data_locker { self->m_mutex };
if (sample_rate != self->m_playback_stream_sample_rate || channel_count != self->m_playback_stream_channel_count)
return buffer.trim(0);
LibMedia: Use AudioMixingSink as a time provider when it is available
2025-09-25 19:24:37 -05:00
auto buffer_start = self->m_next_sample_to_write.load();
LibMedia: Play audio through PlaybackManager using Providers/Sinks This commit implements the functionality to play back audio through PlaybackManager. To decode the audio data, AudioDataProviders are created for each track in the provided media data. These providers will fill their audio block queue, then sit idle until their corresponding tracks are enabled. In order to output the audio, one AudioMixingSink is created which manages a PlaybackStream which requests audio blocks from multiple AudioDataProviders and mixes them into one buffer with sample-perfect precision.
2025-09-24 13:40:17 -05:00
for (auto& [track, track_data] : self->m_track_mixing_datas) {
auto next_sample = buffer_start;
auto samples_end = next_sample + static_cast<i64>(sample_count);
auto go_to_next_block = [&] {
auto new_block = track_data.provider->retrieve_block();
if (new_block.is_empty())
return false;
track_data.current_block = move(new_block);
return true;
};
if (track_data.current_block.is_empty()) {
if (!go_to_next_block())
continue;
}
while (!track_data.current_block.is_empty()) {
auto& current_block = track_data.current_block;
auto current_block_data_count = static_cast<i64>(current_block.data_count());
auto current_block_sample_count = static_cast<i64>(current_block.sample_count());
if (current_block.sample_rate() != sample_rate || current_block.channel_count() != channel_count) {
current_block.clear();
continue;
}
LibMedia: Move overlapping audio block correction to the data provider This prevents PlaybackManager's seek while enabling an audio track from causing the AudioMixingSink to push audio blocks forward unnecessarily. Previously, the seek would cause the initial block or blocks to repeat from the perspective of AudioMixingSink, so it would think that it needs to shift the first block after the seek forward by a few samples. By moving this to the AudioDataProvider, we can clear the last sample index every time the decoder is flushed, ensuring that the block shifting always makes sense. By doing this in AudioMixingSink instead of the Decoder implementations, we avoid having to duplicate this shifting logic across multiple implementations. This also fixes an issue where multiple audio blocks occupying the same timestamp would be skipped while seeking, causing a significant break in audio.
2025-11-10 16:40:25 -06:00
auto first_sample_offset = current_block.timestamp_in_samples();
LibMedia: Play audio through PlaybackManager using Providers/Sinks This commit implements the functionality to play back audio through PlaybackManager. To decode the audio data, AudioDataProviders are created for each track in the provided media data. These providers will fill their audio block queue, then sit idle until their corresponding tracks are enabled. In order to output the audio, one AudioMixingSink is created which manages a PlaybackStream which requests audio blocks from multiple AudioDataProviders and mixes them into one buffer with sample-perfect precision.
2025-09-24 13:40:17 -05:00
if (first_sample_offset >= samples_end)
break;
auto block_end = first_sample_offset + current_block_sample_count;
if (block_end <= next_sample) {
if (!go_to_next_block())
break;
continue;
}
next_sample = max(next_sample, first_sample_offset);
auto index_in_block = (next_sample - first_sample_offset) * channel_count;
VERIFY(index_in_block < current_block_data_count);
auto index_in_buffer = (next_sample - buffer_start) * channel_count;
VERIFY(index_in_buffer < float_buffer_count);
auto write_count = current_block_data_count - index_in_block;
write_count = min(write_count, float_buffer_count - index_in_buffer);
VERIFY(write_count > 0);
VERIFY(index_in_buffer + write_count <= float_buffer_count);
VERIFY(write_count % channel_count == 0);
for (i64 i = 0; i < write_count; i++)
float_buffer[index_in_buffer + i] += current_block.data()[index_in_block + i];
auto write_end = index_in_block + write_count;
if (write_end == current_block_data_count) {
if (!go_to_next_block())
break;
continue;
}
VERIFY(write_end < current_block_data_count);
next_sample += write_count / channel_count;
if (next_sample == samples_end)
break;
VERIFY(next_sample < samples_end);
}
}
self->m_next_sample_to_write += static_cast<i64>(sample_count);
return buffer.slice(0, float_buffer_size);
};
constexpr u32 target_latency_ms = 100;
LibMedia: Use AudioMixingSink as a time provider when it is available
2025-09-25 19:24:37 -05:00
m_playback_stream = MUST(Audio::PlaybackStream::create(Audio::OutputState::Suspended, sample_rate, channel_count, target_latency_ms, move(callback)));
LibMedia: Play audio through PlaybackManager using Providers/Sinks This commit implements the functionality to play back audio through PlaybackManager. To decode the audio data, AudioDataProviders are created for each track in the provided media data. These providers will fill their audio block queue, then sit idle until their corresponding tracks are enabled. In order to output the audio, one AudioMixingSink is created which manages a PlaybackStream which requests audio blocks from multiple AudioDataProviders and mixes them into one buffer with sample-perfect precision.
2025-09-24 13:40:17 -05:00
m_playback_stream_sample_rate = sample_rate;
m_playback_stream_channel_count = channel_count;
LibMedia: Use AudioMixingSink as a time provider when it is available
2025-09-25 19:24:37 -05:00
if (m_playing)
resume();
LibMedia+LibWeb: Implement media volume/muting
2025-09-27 16:02:48 -05:00
set_volume(m_volume);
LibMedia: Use AudioMixingSink as a time provider when it is available
2025-09-25 19:24:37 -05:00
}
AK::Duration AudioMixingSink::current_time() const
{
LibMedia: Sync AudioMixingSink::set_time onto the stream thread With the previous setup setting the time directly on the main thread, the following could occur: - HTMLMediaElement temporarily pauses playback to begin a seek. - AudioMixingSink starts an audio task to drain audio and suspend. - HTMLMediaElement starts PlaybackManager seeking to a new position. - AudioDataProvider completes the seek to the new position. - The PlaybackManager tells AudioMixingSink to set the media time, which it does so synchronously on the main thread. - At this point, the time provider corresponds to the new position. - The pause completes, and a deferred invocation sets media time to its old position again. This would result in the timeline showing the wrong position after a seek on rare occasions. Instead, always queue up a drain and suspend when setting the sink's media time. This ensures that updating the time always occurs after the pause has completed. Also, since setting the time is asynchronous now, we need to store the target time until the seeking drain completes. Otherwise, we still briefly see the previous playback position after a seek.
2025-10-07 18:06:46 -05:00
if (m_temporary_time.has_value())
return m_temporary_time.value();
LibMedia: Use AudioMixingSink as a time provider when it is available
2025-09-25 19:24:37 -05:00
if (!m_playback_stream)
return m_last_media_time;
auto time = m_last_media_time + (m_playback_stream->total_time_played() - m_last_stream_time);
LibMedia: Use [from/to]_time_units in FFmpegDemuxer and AudioMixingSink
2025-11-10 16:36:45 -06:00
auto max_time = AK::Duration::from_time_units(m_next_sample_to_write.load(MemoryOrder::memory_order_acquire), 1, m_playback_stream_sample_rate);
LibMedia: Use AudioMixingSink as a time provider when it is available
2025-09-25 19:24:37 -05:00
time = min(time, max_time);
return time;
}
void AudioMixingSink::resume()
{
m_playing = true;
if (!m_playback_stream)
return;
m_playback_stream->resume()
->when_resolved([weak_self = m_weak_self, &playback_stream = *m_playback_stream](auto new_device_time) {
auto self = weak_self->take_strong();
if (!self)
return;
if (self->m_playback_stream != &playback_stream)
return;
self->m_main_thread_event_loop.deferred_invoke([self, new_device_time]() {
self->m_last_stream_time = new_device_time;
});
})
.when_rejected([](auto&& error) {
warnln("Unexpected error while resuming AudioMixingSink: {}", error.string_literal());
});
}
void AudioMixingSink::pause()
{
m_playing = false;
if (!m_playback_stream)
return;
m_playback_stream->drain_buffer_and_suspend()
->when_resolved([weak_self = m_weak_self, &playback_stream = *m_playback_stream]() {
auto self = weak_self->take_strong();
if (!self)
return;
if (self->m_playback_stream != &playback_stream)
return;
auto new_stream_time = self->m_playback_stream->total_time_played();
LibMedia: Use [from/to]_time_units in FFmpegDemuxer and AudioMixingSink
2025-11-10 16:36:45 -06:00
auto new_media_time = AK::Duration::from_time_units(self->m_next_sample_to_write, 1, self->m_playback_stream_sample_rate);
LibMedia: Use AudioMixingSink as a time provider when it is available
2025-09-25 19:24:37 -05:00
self->m_main_thread_event_loop.deferred_invoke([self, new_stream_time, new_media_time]() {
self->m_last_stream_time = new_stream_time;
self->m_last_media_time = new_media_time;
});
})
.when_rejected([](auto&& error) {
warnln("Unexpected error while pausing AudioMixingSink: {}", error.string_literal());
});
}
void AudioMixingSink::set_time(AK::Duration time)
{
LibMedia: Sync AudioMixingSink::set_time onto the stream thread With the previous setup setting the time directly on the main thread, the following could occur: - HTMLMediaElement temporarily pauses playback to begin a seek. - AudioMixingSink starts an audio task to drain audio and suspend. - HTMLMediaElement starts PlaybackManager seeking to a new position. - AudioDataProvider completes the seek to the new position. - The PlaybackManager tells AudioMixingSink to set the media time, which it does so synchronously on the main thread. - At this point, the time provider corresponds to the new position. - The pause completes, and a deferred invocation sets media time to its old position again. This would result in the timeline showing the wrong position after a seek on rare occasions. Instead, always queue up a drain and suspend when setting the sink's media time. This ensures that updating the time always occurs after the pause has completed. Also, since setting the time is asynchronous now, we need to store the target time until the seeking drain completes. Otherwise, we still briefly see the previous playback position after a seek.
2025-10-07 18:06:46 -05:00
m_temporary_time = time;
LibMedia: Use AudioMixingSink as a time provider when it is available
2025-09-25 19:24:37 -05:00
m_playback_stream->drain_buffer_and_suspend()
->when_resolved([weak_self = m_weak_self, &playback_stream = *m_playback_stream, time]() {
auto self = weak_self->take_strong();
if (!self)
return;
if (self->m_playback_stream != &playback_stream)
return;
auto new_stream_time = self->m_playback_stream->total_time_played();
self->m_main_thread_event_loop.deferred_invoke([self, new_stream_time, time]() {
{
self->m_last_stream_time = new_stream_time;
self->m_last_media_time = time;
LibMedia: Sync AudioMixingSink::set_time onto the stream thread With the previous setup setting the time directly on the main thread, the following could occur: - HTMLMediaElement temporarily pauses playback to begin a seek. - AudioMixingSink starts an audio task to drain audio and suspend. - HTMLMediaElement starts PlaybackManager seeking to a new position. - AudioDataProvider completes the seek to the new position. - The PlaybackManager tells AudioMixingSink to set the media time, which it does so synchronously on the main thread. - At this point, the time provider corresponds to the new position. - The pause completes, and a deferred invocation sets media time to its old position again. This would result in the timeline showing the wrong position after a seek on rare occasions. Instead, always queue up a drain and suspend when setting the sink's media time. This ensures that updating the time always occurs after the pause has completed. Also, since setting the time is asynchronous now, we need to store the target time until the seeking drain completes. Otherwise, we still briefly see the previous playback position after a seek.
2025-10-07 18:06:46 -05:00
self->m_temporary_time = {};
{
Threading::MutexLocker mixing_locker { self->m_mutex };
LibMedia: Use [from/to]_time_units in FFmpegDemuxer and AudioMixingSink
2025-11-10 16:36:45 -06:00
self->m_next_sample_to_write = time.to_time_units(1, self->m_playback_stream_sample_rate);
LibMedia: Sync AudioMixingSink::set_time onto the stream thread With the previous setup setting the time directly on the main thread, the following could occur: - HTMLMediaElement temporarily pauses playback to begin a seek. - AudioMixingSink starts an audio task to drain audio and suspend. - HTMLMediaElement starts PlaybackManager seeking to a new position. - AudioDataProvider completes the seek to the new position. - The PlaybackManager tells AudioMixingSink to set the media time, which it does so synchronously on the main thread. - At this point, the time provider corresponds to the new position. - The pause completes, and a deferred invocation sets media time to its old position again. This would result in the timeline showing the wrong position after a seek on rare occasions. Instead, always queue up a drain and suspend when setting the sink's media time. This ensures that updating the time always occurs after the pause has completed. Also, since setting the time is asynchronous now, we need to store the target time until the seeking drain completes. Otherwise, we still briefly see the previous playback position after a seek.
2025-10-07 18:06:46 -05:00
}
LibMedia: Use AudioMixingSink as a time provider when it is available
2025-09-25 19:24:37 -05:00
LibMedia: Move overlapping audio block correction to the data provider This prevents PlaybackManager's seek while enabling an audio track from causing the AudioMixingSink to push audio blocks forward unnecessarily. Previously, the seek would cause the initial block or blocks to repeat from the perspective of AudioMixingSink, so it would think that it needs to shift the first block after the seek forward by a few samples. By moving this to the AudioDataProvider, we can clear the last sample index every time the decoder is flushed, ensuring that the block shifting always makes sense. By doing this in AudioMixingSink instead of the Decoder implementations, we avoid having to duplicate this shifting logic across multiple implementations. This also fixes an issue where multiple audio blocks occupying the same timestamp would be skipped while seeking, causing a significant break in audio.
2025-11-10 16:40:25 -06:00
for (auto& [track, track_data] : self->m_track_mixing_datas)
LibMedia: Use AudioMixingSink as a time provider when it is available
2025-09-25 19:24:37 -05:00
track_data.current_block.clear();
}
LibMedia: Sync AudioMixingSink::set_time onto the stream thread With the previous setup setting the time directly on the main thread, the following could occur: - HTMLMediaElement temporarily pauses playback to begin a seek. - AudioMixingSink starts an audio task to drain audio and suspend. - HTMLMediaElement starts PlaybackManager seeking to a new position. - AudioDataProvider completes the seek to the new position. - The PlaybackManager tells AudioMixingSink to set the media time, which it does so synchronously on the main thread. - At this point, the time provider corresponds to the new position. - The pause completes, and a deferred invocation sets media time to its old position again. This would result in the timeline showing the wrong position after a seek on rare occasions. Instead, always queue up a drain and suspend when setting the sink's media time. This ensures that updating the time always occurs after the pause has completed. Also, since setting the time is asynchronous now, we need to store the target time until the seeking drain completes. Otherwise, we still briefly see the previous playback position after a seek.
2025-10-07 18:06:46 -05:00
if (self->m_playing)
self->resume();
LibMedia: Use AudioMixingSink as a time provider when it is available
2025-09-25 19:24:37 -05:00
});
})
.when_rejected([](auto&& error) {
warnln("Unexpected error while setting time on AudioMixingSink: {}", error.string_literal());
});
LibMedia: Play audio through PlaybackManager using Providers/Sinks This commit implements the functionality to play back audio through PlaybackManager. To decode the audio data, AudioDataProviders are created for each track in the provided media data. These providers will fill their audio block queue, then sit idle until their corresponding tracks are enabled. In order to output the audio, one AudioMixingSink is created which manages a PlaybackStream which requests audio blocks from multiple AudioDataProviders and mixes them into one buffer with sample-perfect precision.
2025-09-24 13:40:17 -05:00
}
LibMedia+LibWeb: Implement media volume/muting
2025-09-27 16:02:48 -05:00
void AudioMixingSink::set_volume(double volume)
{
m_volume = volume;
if (m_playback_stream) {
m_playback_stream->set_volume(m_volume)
->when_rejected([](Error&&) {
// FIXME: Do we even need this function to return a promise?
});
}
}
LibMedia: Play audio through PlaybackManager using Providers/Sinks This commit implements the functionality to play back audio through PlaybackManager. To decode the audio data, AudioDataProviders are created for each track in the provided media data. These providers will fill their audio block queue, then sit idle until their corresponding tracks are enabled. In order to output the audio, one AudioMixingSink is created which manages a PlaybackStream which requests audio blocks from multiple AudioDataProviders and mixes them into one buffer with sample-perfect precision.
2025-09-24 13:40:17 -05:00
}
Reference in a new issue Copy permalink