2022-10-29 18:16:14 -05:00
/*
* Copyright ( c ) 2022 , Gregory Bertilson < zaggy1024 @ gmail . com >
*
* SPDX - License - Identifier : BSD - 2 - Clause
*/
# include <AK/Format.h>
# include <LibCore/Timer.h>
2024-06-10 15:04:03 -05:00
# include <LibMedia/Containers/Matroska/MatroskaDemuxer.h>
2024-06-19 18:01:41 -05:00
# include <LibMedia/FFmpeg/FFmpegVideoDecoder.h>
# include <LibMedia/VideoFrame.h>
2022-10-29 18:16:14 -05:00
# include "PlaybackManager.h"
2024-06-10 15:04:03 -05:00
namespace Media {
2022-10-29 18:16:14 -05:00
2023-02-06 01:24:33 -06:00
# define TRY_OR_FATAL_ERROR(expression) \
( { \
2023-02-09 13:27:43 -05:00
auto & & _fatal_expression = ( expression ) ; \
2023-02-06 01:24:33 -06:00
if ( _fatal_expression . is_error ( ) ) { \
dispatch_fatal_error ( _fatal_expression . release_error ( ) ) ; \
return ; \
} \
static_assert ( ! : : AK : : Detail : : IsLvalueReference < decltype ( _fatal_expression . release_value ( ) ) > , \
" Do not return a reference from a fallible expression " ) ; \
_fatal_expression . release_value ( ) ; \
} )
2023-04-25 09:02:24 -04:00
DecoderErrorOr < NonnullOwnPtr < PlaybackManager > > PlaybackManager : : from_file ( StringView filename )
2022-10-29 18:16:14 -05:00
{
2023-04-08 21:22:15 -04:00
auto demuxer = TRY ( Matroska : : MatroskaDemuxer : : from_file ( filename ) ) ;
2023-04-14 12:30:38 -05:00
return create ( move ( demuxer ) ) ;
2023-04-08 21:22:15 -04:00
}
2023-09-26 00:54:34 +02:00
DecoderErrorOr < NonnullOwnPtr < PlaybackManager > > PlaybackManager : : from_mapped_file ( NonnullOwnPtr < Core : : MappedFile > mapped_file )
2023-05-11 12:25:52 +01:00
{
auto demuxer = TRY ( Matroska : : MatroskaDemuxer : : from_mapped_file ( move ( mapped_file ) ) ) ;
2023-04-14 12:30:38 -05:00
return create ( move ( demuxer ) ) ;
2023-05-11 12:25:52 +01:00
}
2023-04-25 09:02:24 -04:00
DecoderErrorOr < NonnullOwnPtr < PlaybackManager > > PlaybackManager : : from_data ( ReadonlyBytes data )
2023-04-08 21:22:15 -04:00
{
auto demuxer = TRY ( Matroska : : MatroskaDemuxer : : from_data ( data ) ) ;
2023-04-14 12:30:38 -05:00
return create ( move ( demuxer ) ) ;
2023-04-08 21:22:15 -04:00
}
2023-04-19 17:11:51 -05:00
PlaybackManager : : PlaybackManager ( NonnullOwnPtr < Demuxer > & demuxer , Track video_track , NonnullOwnPtr < VideoDecoder > & & decoder , VideoFrameQueue & & frame_queue )
2023-04-08 19:18:38 -04:00
: m_demuxer ( move ( demuxer ) )
2022-10-29 18:16:14 -05:00
, m_selected_video_track ( video_track )
2023-04-14 12:30:38 -05:00
, m_frame_queue ( move ( frame_queue ) )
2023-04-19 17:11:51 -05:00
, m_decoder ( move ( decoder ) )
, m_decode_wait_condition ( m_decode_wait_mutex )
2022-10-29 18:16:14 -05:00
{
}
2023-04-19 17:11:51 -05:00
PlaybackManager : : ~ PlaybackManager ( )
{
2024-04-25 08:35:56 -04:00
terminate_playback ( ) ;
2023-04-19 17:11:51 -05:00
}
2023-04-25 09:02:24 -04:00
2022-10-29 18:16:14 -05:00
void PlaybackManager : : resume_playback ( )
{
2023-02-06 01:24:33 -06:00
dbgln_if ( PLAYBACK_MANAGER_DEBUG , " Resuming playback. " ) ;
TRY_OR_FATAL_ERROR ( m_playback_handler - > play ( ) ) ;
2022-10-29 18:16:14 -05:00
}
void PlaybackManager : : pause_playback ( )
{
2023-02-06 01:24:33 -06:00
dbgln_if ( PLAYBACK_MANAGER_DEBUG , " Pausing playback. " ) ;
if ( ! m_playback_handler - > is_playing ( ) )
warnln ( " Cannot pause. " ) ;
TRY_OR_FATAL_ERROR ( m_playback_handler - > pause ( ) ) ;
2022-10-29 18:16:14 -05:00
}
2024-04-25 08:35:56 -04:00
void PlaybackManager : : terminate_playback ( )
{
m_stop_decoding . exchange ( true ) ;
m_decode_wait_condition . broadcast ( ) ;
if ( m_decode_thread - > needs_to_be_joined ( ) ) {
dbgln_if ( PLAYBACK_MANAGER_DEBUG , " Waiting for decode thread to end... " ) ;
( void ) m_decode_thread - > join ( ) ;
dbgln_if ( PLAYBACK_MANAGER_DEBUG , " Successfully destroyed PlaybackManager. " ) ;
}
}
2023-03-13 16:30:34 +01:00
Duration PlaybackManager : : current_playback_time ( )
2022-10-29 18:16:14 -05:00
{
2023-02-06 01:24:33 -06:00
return m_playback_handler - > current_time ( ) ;
2022-10-29 18:16:14 -05:00
}
2023-03-13 16:30:34 +01:00
Duration PlaybackManager : : duration ( )
2022-10-29 18:16:14 -05:00
{
2023-04-19 17:11:51 -05:00
auto duration_result = ( {
2024-06-19 22:36:27 -05:00
auto demuxer_locker = Threading : : MutexLocker ( m_decoder_mutex ) ;
2023-04-19 17:11:51 -05:00
m_demuxer - > duration ( ) ;
} ) ;
2023-05-30 14:49:17 -05:00
if ( duration_result . is_error ( ) ) {
2023-02-12 00:44:16 -06:00
dispatch_decoder_error ( duration_result . release_error ( ) ) ;
2023-05-30 14:49:17 -05:00
// FIXME: We should determine the last sample that the demuxer knows is available and
// use that as the current duration. The duration may change if the demuxer doesn't
// know there is a fixed duration.
return Duration : : zero ( ) ;
}
2022-11-11 17:14:27 -06:00
return duration_result . release_value ( ) ;
2022-10-29 18:16:14 -05:00
}
2023-02-06 01:24:33 -06:00
void PlaybackManager : : dispatch_fatal_error ( Error error )
2022-11-09 06:52:37 -06:00
{
2023-02-06 01:24:33 -06:00
dbgln_if ( PLAYBACK_MANAGER_DEBUG , " Encountered fatal error: {} " , error . string_literal ( ) ) ;
// FIXME: For threading, this will have to use a pre-allocated event to send to the main loop
// to be able to gracefully handle OOM.
2023-04-08 19:18:38 -04:00
if ( on_fatal_playback_error )
on_fatal_playback_error ( move ( error ) ) ;
2023-02-06 01:24:33 -06:00
}
2022-11-27 00:45:14 -06:00
2023-02-12 00:44:16 -06:00
void PlaybackManager : : dispatch_decoder_error ( DecoderError error )
2023-02-06 01:24:33 -06:00
{
2022-11-09 06:52:37 -06:00
switch ( error . category ( ) ) {
case DecoderErrorCategory : : EndOfStream :
2022-11-11 22:04:38 -06:00
dbgln_if ( PLAYBACK_MANAGER_DEBUG , " {} " , error . string_literal ( ) ) ;
2023-02-06 01:24:33 -06:00
TRY_OR_FATAL_ERROR ( m_playback_handler - > stop ( ) ) ;
2022-11-09 06:52:37 -06:00
break ;
default :
2022-11-11 22:04:38 -06:00
dbgln ( " Playback error encountered: {} " , error . string_literal ( ) ) ;
2023-02-06 01:24:33 -06:00
TRY_OR_FATAL_ERROR ( m_playback_handler - > stop ( ) ) ;
2023-04-08 19:18:38 -04:00
if ( on_decoder_error )
on_decoder_error ( move ( error ) ) ;
2022-11-09 06:52:37 -06:00
break ;
}
}
2023-02-12 00:44:16 -06:00
void PlaybackManager : : dispatch_new_frame ( RefPtr < Gfx : : Bitmap > frame )
{
2023-04-08 19:18:38 -04:00
if ( on_video_frame )
on_video_frame ( move ( frame ) ) ;
2023-02-12 00:44:16 -06:00
}
bool PlaybackManager : : dispatch_frame_queue_item ( FrameQueueItem & & item )
{
if ( item . is_error ( ) ) {
dispatch_decoder_error ( item . release_error ( ) ) ;
return true ;
}
2023-04-19 17:11:51 -05:00
dbgln_if ( PLAYBACK_MANAGER_DEBUG , " Sent frame for presentation with timestamp {}ms, late by {}ms " , item . timestamp ( ) . to_milliseconds ( ) , ( current_playback_time ( ) - item . timestamp ( ) ) . to_milliseconds ( ) ) ;
2023-02-12 00:44:16 -06:00
dispatch_new_frame ( item . bitmap ( ) ) ;
return false ;
}
2023-02-12 02:25:25 -06:00
void PlaybackManager : : dispatch_state_change ( )
{
2023-04-08 19:18:38 -04:00
if ( on_playback_state_change )
on_playback_state_change ( ) ;
2023-02-12 02:25:25 -06:00
}
2023-02-06 01:24:33 -06:00
void PlaybackManager : : timer_callback ( )
2022-11-14 00:54:21 -06:00
{
2023-05-18 23:30:49 -05:00
TRY_OR_FATAL_ERROR ( m_playback_handler - > do_timed_state_update ( ) ) ;
2022-11-14 00:54:21 -06:00
}
2023-03-13 16:30:34 +01:00
void PlaybackManager : : seek_to_timestamp ( Duration target_timestamp , SeekMode seek_mode )
2022-10-29 18:16:14 -05:00
{
2023-02-06 04:11:32 -06:00
TRY_OR_FATAL_ERROR ( m_playback_handler - > seek ( target_timestamp , seek_mode ) ) ;
2022-10-29 18:16:14 -05:00
}
2023-06-24 04:57:27 -05:00
DecoderErrorOr < Optional < Duration > > PlaybackManager : : seek_demuxer_to_most_recent_keyframe ( Duration timestamp , Optional < Duration > earliest_available_sample )
2022-10-29 18:16:14 -05:00
{
2024-06-19 22:36:27 -05:00
auto seeked_timestamp = TRY ( m_demuxer - > seek_to_most_recent_keyframe ( m_selected_video_track , timestamp , move ( earliest_available_sample ) ) ) ;
if ( seeked_timestamp . has_value ( ) )
m_decoder - > flush ( ) ;
return seeked_timestamp ;
2022-10-29 18:16:14 -05:00
}
2023-04-19 17:11:51 -05:00
Optional < FrameQueueItem > PlaybackManager : : dequeue_one_frame ( )
2022-11-11 22:26:19 -06:00
{
2023-04-19 17:11:51 -05:00
auto result = m_frame_queue . dequeue ( ) ;
m_decode_wait_condition . broadcast ( ) ;
if ( result . is_error ( ) ) {
if ( result . error ( ) ! = VideoFrameQueue : : QueueStatus : : Empty )
dispatch_fatal_error ( Error : : from_string_literal ( " Dequeue failed with an unexpected error " ) ) ;
return { } ;
}
return result . release_value ( ) ;
2022-11-11 22:26:19 -06:00
}
2023-05-18 23:30:49 -05:00
void PlaybackManager : : set_state_update_timer ( int delay_ms )
{
m_state_update_timer - > start ( delay_ms ) ;
}
2023-04-19 17:11:51 -05:00
void PlaybackManager : : restart_playback ( )
2023-02-06 01:24:33 -06:00
{
2023-03-13 16:30:34 +01:00
seek_to_timestamp ( Duration : : zero ( ) ) ;
2023-02-06 01:24:33 -06:00
}
2023-04-19 17:11:51 -05:00
void PlaybackManager : : decode_and_queue_one_sample ( )
2022-10-29 18:16:14 -05:00
{
# if PLAYBACK_MANAGER_DEBUG
2023-03-17 19:50:39 +01:00
auto start_time = MonotonicTime : : now ( ) ;
2022-10-29 18:16:14 -05:00
# endif
2023-04-19 17:11:51 -05:00
FrameQueueItem item_to_enqueue ;
while ( item_to_enqueue . is_empty ( ) ) {
2024-06-19 22:36:27 -05:00
OwnPtr < VideoFrame > decoded_frame = nullptr ;
CodingIndependentCodePoints container_cicp ;
2022-11-09 06:52:37 -06:00
2024-06-19 22:36:27 -05:00
{
Threading : : MutexLocker decoder_locker ( m_decoder_mutex ) ;
2022-11-03 19:18:38 -05:00
2024-06-19 22:36:27 -05:00
// Get a sample to decode.
auto sample_result = m_demuxer - > get_next_sample_for_track ( m_selected_video_track ) ;
if ( sample_result . is_error ( ) ) {
item_to_enqueue = FrameQueueItem : : error_marker ( sample_result . release_error ( ) , FrameQueueItem : : no_timestamp ) ;
break ;
}
auto sample = sample_result . release_value ( ) ;
container_cicp = sample . auxiliary_data ( ) . get < Video : : VideoSampleData > ( ) . container_cicp ( ) ;
// Submit the sample to the decoder.
auto decode_result = m_decoder - > receive_sample ( sample . timestamp ( ) , sample . data ( ) ) ;
if ( decode_result . is_error ( ) ) {
item_to_enqueue = FrameQueueItem : : error_marker ( decode_result . release_error ( ) , sample . timestamp ( ) ) ;
break ;
}
2022-11-03 19:18:38 -05:00
2024-06-19 22:36:27 -05:00
// Retrieve the last available frame to present.
while ( true ) {
auto frame_result = m_decoder - > get_decoded_frame ( ) ;
if ( frame_result . is_error ( ) ) {
if ( frame_result . error ( ) . category ( ) = = DecoderErrorCategory : : NeedsMoreInput ) {
break ;
}
item_to_enqueue = FrameQueueItem : : error_marker ( frame_result . release_error ( ) , sample . timestamp ( ) ) ;
2022-11-03 19:18:38 -05:00
break ;
2023-04-19 17:11:51 -05:00
}
2022-10-29 18:16:14 -05:00
2024-06-19 22:36:27 -05:00
decoded_frame = frame_result . release_value ( ) ;
2022-11-03 19:18:38 -05:00
}
}
2022-10-29 18:16:14 -05:00
2023-04-19 17:11:51 -05:00
// Convert the frame for display.
if ( decoded_frame ! = nullptr ) {
auto & cicp = decoded_frame - > cicp ( ) ;
2024-06-19 22:36:27 -05:00
cicp . adopt_specified_values ( container_cicp ) ;
2023-04-19 17:11:51 -05:00
cicp . default_code_points_if_unspecified ( { ColorPrimaries : : BT709 , TransferCharacteristics : : BT709 , MatrixCoefficients : : BT709 , VideoFullRangeFlag : : Studio } ) ;
// BT.601, BT.709 and BT.2020 have a similar transfer function to sRGB, so other applications
// (Chromium, VLC) forgo transfer characteristics conversion. We will emulate that behavior by
// handling those as sRGB instead, which causes no transfer function change in the output,
// unless display color management is later implemented.
switch ( cicp . transfer_characteristics ( ) ) {
case TransferCharacteristics : : BT601 :
case TransferCharacteristics : : BT709 :
case TransferCharacteristics : : BT2020BitDepth10 :
case TransferCharacteristics : : BT2020BitDepth12 :
cicp . set_transfer_characteristics ( TransferCharacteristics : : SRGB ) ;
break ;
default :
break ;
}
2022-10-29 18:16:14 -05:00
2023-04-19 17:11:51 -05:00
auto bitmap_result = decoded_frame - > to_bitmap ( ) ;
if ( bitmap_result . is_error ( ) )
2024-06-19 18:29:12 -05:00
item_to_enqueue = FrameQueueItem : : error_marker ( bitmap_result . release_error ( ) , decoded_frame - > timestamp ( ) ) ;
2023-04-19 17:11:51 -05:00
else
2024-06-19 18:29:12 -05:00
item_to_enqueue = FrameQueueItem : : frame ( bitmap_result . release_value ( ) , decoded_frame - > timestamp ( ) ) ;
2023-04-19 17:11:51 -05:00
break ;
}
}
2022-10-29 18:16:14 -05:00
2023-04-19 17:11:51 -05:00
VERIFY ( ! item_to_enqueue . is_empty ( ) ) ;
2022-10-29 18:16:14 -05:00
# if PLAYBACK_MANAGER_DEBUG
2023-03-17 19:50:39 +01:00
dbgln ( " Media Decoder: Sample at {}ms took {}ms to decode, queue contains ~{} items " , item_to_enqueue . timestamp ( ) . to_milliseconds ( ) , ( MonotonicTime : : now ( ) - start_time ) . to_milliseconds ( ) , m_frame_queue . weak_used ( ) ) ;
2022-10-29 18:16:14 -05:00
# endif
2023-04-19 17:11:51 -05:00
auto wait = [ & ] {
auto wait_locker = Threading : : MutexLocker ( m_decode_wait_mutex ) ;
m_decode_wait_condition . wait ( ) ;
} ;
2022-10-29 18:16:14 -05:00
2023-04-19 17:11:51 -05:00
bool had_error = item_to_enqueue . is_error ( ) ;
while ( true ) {
if ( m_frame_queue . can_enqueue ( ) ) {
MUST ( m_frame_queue . enqueue ( move ( item_to_enqueue ) ) ) ;
break ;
}
if ( m_stop_decoding . load ( ) ) {
dbgln_if ( PLAYBACK_MANAGER_DEBUG , " Media Decoder: Received signal to stop, exiting decode function... " ) ;
return ;
}
m_buffer_is_full . exchange ( true ) ;
dbgln_if ( PLAYBACK_MANAGER_DEBUG , " Media Decoder: Waiting for a frame to be dequeued... " ) ;
wait ( ) ;
2022-10-29 18:16:14 -05:00
}
2023-04-19 17:11:51 -05:00
if ( had_error ) {
dbgln_if ( PLAYBACK_MANAGER_DEBUG , " Media Decoder: Encountered {}, waiting... " , item_to_enqueue . error ( ) . category ( ) = = DecoderErrorCategory : : EndOfStream ? " end of stream " sv : " error " sv ) ;
m_buffer_is_full . exchange ( true ) ;
wait ( ) ;
}
m_buffer_is_full . exchange ( false ) ;
2023-02-06 01:24:33 -06:00
}
2023-03-13 16:30:34 +01:00
Duration PlaybackManager : : PlaybackStateHandler : : current_time ( ) const
2023-02-06 01:24:33 -06:00
{
return m_manager . m_last_present_in_media_time ;
}
2023-03-13 16:30:34 +01:00
ErrorOr < void > PlaybackManager : : PlaybackStateHandler : : seek ( Duration target_timestamp , SeekMode seek_mode )
2023-02-06 01:24:33 -06:00
{
return replace_handler_and_delete_this < SeekingStateHandler > ( is_playing ( ) , target_timestamp , seek_mode ) ;
2022-10-29 18:16:14 -05:00
}
2023-02-06 01:24:33 -06:00
ErrorOr < void > PlaybackManager : : PlaybackStateHandler : : stop ( )
{
return replace_handler_and_delete_this < StoppedStateHandler > ( ) ;
}
template < class T , class . . . Args >
ErrorOr < void > PlaybackManager : : PlaybackStateHandler : : replace_handler_and_delete_this ( Args . . . args )
{
2023-04-14 12:30:38 -05:00
OwnPtr < PlaybackStateHandler > temp_handler = TRY ( try_make < T > ( m_manager , args . . . ) ) ;
2023-02-06 01:24:33 -06:00
m_manager . m_playback_handler . swap ( temp_handler ) ;
# if PLAYBACK_MANAGER_DEBUG
m_has_exited = true ;
dbgln ( " Changing state from {} to {} " , temp_handler - > name ( ) , m_manager . m_playback_handler - > name ( ) ) ;
# endif
TRY ( m_manager . m_playback_handler - > on_enter ( ) ) ;
2023-04-12 00:21:43 -05:00
m_manager . dispatch_state_change ( ) ;
2023-02-06 01:24:33 -06:00
return { } ;
}
PlaybackManager & PlaybackManager : : PlaybackStateHandler : : manager ( ) const
{
# if PLAYBACK_MANAGER_DEBUG
VERIFY ( ! m_has_exited ) ;
# endif
return m_manager ;
}
2023-02-12 02:36:06 -06:00
class PlaybackManager : : ResumingStateHandler : public PlaybackManager : : PlaybackStateHandler {
2023-02-12 00:48:27 -06:00
public :
2023-02-12 02:36:06 -06:00
ResumingStateHandler ( PlaybackManager & manager , bool playing )
2023-02-12 00:48:27 -06:00
: PlaybackStateHandler ( manager )
, m_playing ( playing )
{
}
2023-02-12 02:36:06 -06:00
~ ResumingStateHandler ( ) override = default ;
protected :
ErrorOr < void > assume_next_state ( )
{
if ( ! m_playing )
return replace_handler_and_delete_this < PausedStateHandler > ( ) ;
return replace_handler_and_delete_this < PlayingStateHandler > ( ) ;
}
ErrorOr < void > play ( ) override
{
m_playing = true ;
manager ( ) . dispatch_state_change ( ) ;
return { } ;
}
2023-04-12 01:01:29 -05:00
bool is_playing ( ) const override { return m_playing ; }
2023-02-12 02:36:06 -06:00
ErrorOr < void > pause ( ) override
{
m_playing = false ;
manager ( ) . dispatch_state_change ( ) ;
return { } ;
}
2023-04-09 09:39:48 -04:00
bool m_playing { false } ;
2023-02-12 02:36:06 -06:00
} ;
2023-02-06 01:24:33 -06:00
class PlaybackManager : : PlayingStateHandler : public PlaybackManager : : PlaybackStateHandler {
public :
PlayingStateHandler ( PlaybackManager & manager )
: PlaybackStateHandler ( manager )
{
}
~ PlayingStateHandler ( ) override = default ;
private :
ErrorOr < void > on_enter ( ) override
{
2023-03-17 19:50:39 +01:00
m_last_present_in_real_time = MonotonicTime : : now ( ) ;
2023-05-18 23:30:49 -05:00
return do_timed_state_update ( ) ;
2023-02-06 01:24:33 -06:00
}
StringView name ( ) override { return " Playing " sv ; }
2023-07-07 22:48:11 -04:00
bool is_playing ( ) const override { return true ; }
2023-04-12 01:01:29 -05:00
PlaybackState get_state ( ) const override { return PlaybackState : : Playing ; }
2023-02-06 01:24:33 -06:00
ErrorOr < void > pause ( ) override
{
manager ( ) . m_last_present_in_media_time = current_time ( ) ;
return replace_handler_and_delete_this < PausedStateHandler > ( ) ;
}
ErrorOr < void > buffer ( ) override
{
manager ( ) . m_last_present_in_media_time = current_time ( ) ;
return replace_handler_and_delete_this < BufferingStateHandler > ( true ) ;
}
2023-03-13 16:30:34 +01:00
Duration current_time ( ) const override
2023-02-06 01:24:33 -06:00
{
2023-03-17 19:50:39 +01:00
return manager ( ) . m_last_present_in_media_time + ( MonotonicTime : : now ( ) - m_last_present_in_real_time ) ;
2023-02-06 01:24:33 -06:00
}
2023-05-18 23:30:49 -05:00
ErrorOr < void > do_timed_state_update ( ) override
2023-02-06 01:24:33 -06:00
{
2023-02-12 02:00:59 -06:00
auto set_presentation_timer = [ & ] ( ) {
auto frame_time_ms = ( manager ( ) . m_next_frame - > timestamp ( ) - current_time ( ) ) . to_milliseconds ( ) ;
VERIFY ( frame_time_ms < = NumericLimits < int > : : max ( ) ) ;
dbgln_if ( PLAYBACK_MANAGER_DEBUG , " Time until next frame is {}ms " , frame_time_ms ) ;
2023-05-18 23:30:49 -05:00
manager ( ) . set_state_update_timer ( max ( static_cast < int > ( frame_time_ms ) , 0 ) ) ;
2023-02-12 02:00:59 -06:00
} ;
2023-02-12 00:48:27 -06:00
if ( manager ( ) . m_next_frame . has_value ( ) & & current_time ( ) < manager ( ) . m_next_frame - > timestamp ( ) ) {
2023-02-12 02:00:59 -06:00
dbgln_if ( PLAYBACK_MANAGER_DEBUG , " Current time {}ms is too early to present the next frame at {}ms, delaying " , current_time ( ) . to_milliseconds ( ) , manager ( ) . m_next_frame - > timestamp ( ) . to_milliseconds ( ) ) ;
set_presentation_timer ( ) ;
return { } ;
}
2023-02-06 01:24:33 -06:00
Optional < FrameQueueItem > future_frame_item ;
bool should_present_frame = false ;
// Skip frames until we find a frame past the current playback time, and keep the one that precedes it to display.
2023-04-19 17:11:51 -05:00
while ( true ) {
future_frame_item = manager ( ) . dequeue_one_frame ( ) ;
if ( ! future_frame_item . has_value ( ) )
break ;
2023-02-06 01:24:33 -06:00
2023-02-12 00:50:30 -06:00
if ( future_frame_item - > timestamp ( ) > = current_time ( ) | | future_frame_item - > timestamp ( ) = = FrameQueueItem : : no_timestamp ) {
2023-02-06 01:24:33 -06:00
dbgln_if ( PLAYBACK_MANAGER_DEBUG , " Should present frame, future {} is error or after {}ms " , future_frame_item - > debug_string ( ) , current_time ( ) . to_milliseconds ( ) ) ;
should_present_frame = true ;
break ;
}
if ( manager ( ) . m_next_frame . has_value ( ) ) {
dbgln_if ( PLAYBACK_MANAGER_DEBUG , " At {}ms: Dropped {} in favor of {} " , current_time ( ) . to_milliseconds ( ) , manager ( ) . m_next_frame - > debug_string ( ) , future_frame_item - > debug_string ( ) ) ;
manager ( ) . m_skipped_frames + + ;
}
manager ( ) . m_next_frame . emplace ( future_frame_item . release_value ( ) ) ;
}
// If we don't have both of these items, we can't present, since we need to set a timer for
// the next frame. Check if we need to buffer based on the current state.
if ( ! manager ( ) . m_next_frame . has_value ( ) | | ! future_frame_item . has_value ( ) ) {
# if PLAYBACK_MANAGER_DEBUG
StringBuilder debug_string_builder ;
debug_string_builder . append ( " We don't have " sv ) ;
if ( ! manager ( ) . m_next_frame . has_value ( ) ) {
debug_string_builder . append ( " a frame to present " sv ) ;
if ( ! future_frame_item . has_value ( ) )
debug_string_builder . append ( " or a future frame " sv ) ;
} else {
debug_string_builder . append ( " a future frame " sv ) ;
}
debug_string_builder . append ( " , checking for error and buffering " sv ) ;
2023-12-16 17:49:34 +03:30
dbgln_if ( PLAYBACK_MANAGER_DEBUG , debug_string_builder . to_byte_string ( ) ) ;
2023-02-06 01:24:33 -06:00
# endif
if ( future_frame_item . has_value ( ) ) {
if ( future_frame_item - > is_error ( ) ) {
2023-02-12 00:44:16 -06:00
manager ( ) . dispatch_decoder_error ( future_frame_item . release_value ( ) . release_error ( ) ) ;
2023-02-06 01:24:33 -06:00
return { } ;
}
manager ( ) . m_next_frame . emplace ( future_frame_item . release_value ( ) ) ;
}
TRY ( buffer ( ) ) ;
return { } ;
}
// If we have a frame, send it for presentation.
if ( should_present_frame ) {
2023-03-17 19:50:39 +01:00
auto now = MonotonicTime : : now ( ) ;
2023-02-06 01:24:33 -06:00
manager ( ) . m_last_present_in_media_time + = now - m_last_present_in_real_time ;
m_last_present_in_real_time = now ;
2023-02-12 00:44:16 -06:00
if ( manager ( ) . dispatch_frame_queue_item ( manager ( ) . m_next_frame . release_value ( ) ) )
return { } ;
2023-02-06 01:24:33 -06:00
}
// Now that we've presented the current frame, we can throw whatever error is next in queue.
// This way, we always display a frame before the stream ends, and should also show any frames
// we already had when a real error occurs.
if ( future_frame_item - > is_error ( ) ) {
2023-02-12 00:44:16 -06:00
manager ( ) . dispatch_decoder_error ( future_frame_item . release_value ( ) . release_error ( ) ) ;
2023-02-06 01:24:33 -06:00
return { } ;
}
// The future frame item becomes the next one to present.
manager ( ) . m_next_frame . emplace ( future_frame_item . release_value ( ) ) ;
2023-02-12 02:00:59 -06:00
set_presentation_timer ( ) ;
2023-02-06 01:24:33 -06:00
return { } ;
}
2023-03-17 19:50:39 +01:00
MonotonicTime m_last_present_in_real_time = MonotonicTime : : now_coarse ( ) ;
2023-02-06 01:24:33 -06:00
} ;
class PlaybackManager : : PausedStateHandler : public PlaybackManager : : PlaybackStateHandler {
public :
PausedStateHandler ( PlaybackManager & manager )
: PlaybackStateHandler ( manager )
{
}
~ PausedStateHandler ( ) override = default ;
private :
StringView name ( ) override { return " Paused " sv ; }
ErrorOr < void > play ( ) override
{
return replace_handler_and_delete_this < PlayingStateHandler > ( ) ;
}
2023-07-07 22:48:11 -04:00
bool is_playing ( ) const override { return false ; }
2023-04-12 01:01:29 -05:00
PlaybackState get_state ( ) const override { return PlaybackState : : Paused ; }
2023-02-06 01:24:33 -06:00
} ;
2023-04-19 17:11:51 -05:00
// FIXME: This is a placeholder variable that could be scaled based on how long each frame decode takes to
// avoid triggering the timer to check the queue constantly. However, doing so may reduce the speed
// of seeking due to the decode thread having to wait for a signal to continue decoding.
constexpr int buffering_or_seeking_decode_wait_time = 1 ;
2023-02-12 02:36:06 -06:00
class PlaybackManager : : BufferingStateHandler : public PlaybackManager : : ResumingStateHandler {
using PlaybackManager : : ResumingStateHandler : : ResumingStateHandler ;
2023-02-06 01:24:33 -06:00
ErrorOr < void > on_enter ( ) override
{
2023-05-18 23:30:49 -05:00
manager ( ) . set_state_update_timer ( buffering_or_seeking_decode_wait_time ) ;
2023-02-06 01:24:33 -06:00
return { } ;
}
StringView name ( ) override { return " Buffering " sv ; }
2023-05-18 23:30:49 -05:00
ErrorOr < void > do_timed_state_update ( ) override
2023-02-06 01:24:33 -06:00
{
2023-04-19 17:11:51 -05:00
auto buffer_is_full = manager ( ) . m_buffer_is_full . load ( ) ;
dbgln_if ( PLAYBACK_MANAGER_DEBUG , " Buffering timer callback has been called. Buffer is {}. " , buffer_is_full ? " full, exiting " sv : " not full, waiting " sv ) ;
if ( buffer_is_full )
return assume_next_state ( ) ;
2023-05-18 23:30:49 -05:00
manager ( ) . set_state_update_timer ( buffering_or_seeking_decode_wait_time ) ;
2023-04-19 17:11:51 -05:00
return { } ;
2023-02-06 01:24:33 -06:00
}
2023-04-12 01:01:29 -05:00
PlaybackState get_state ( ) const override { return PlaybackState : : Buffering ; }
2023-02-06 01:24:33 -06:00
} ;
2023-02-12 02:36:06 -06:00
class PlaybackManager : : SeekingStateHandler : public PlaybackManager : : ResumingStateHandler {
2023-02-06 01:24:33 -06:00
public :
2023-03-13 16:30:34 +01:00
SeekingStateHandler ( PlaybackManager & manager , bool playing , Duration target_timestamp , SeekMode seek_mode )
2023-02-12 02:36:06 -06:00
: ResumingStateHandler ( manager , playing )
2023-02-06 01:24:33 -06:00
, m_target_timestamp ( target_timestamp )
, m_seek_mode ( seek_mode )
{
}
~ SeekingStateHandler ( ) override = default ;
private :
ErrorOr < void > on_enter ( ) override
{
2023-02-06 01:25:02 -06:00
auto earliest_available_sample = manager ( ) . m_last_present_in_media_time ;
2023-02-12 00:50:30 -06:00
if ( manager ( ) . m_next_frame . has_value ( ) & & manager ( ) . m_next_frame - > timestamp ( ) ! = FrameQueueItem : : no_timestamp ) {
2023-02-06 01:25:02 -06:00
earliest_available_sample = min ( earliest_available_sample , manager ( ) . m_next_frame - > timestamp ( ) ) ;
}
2023-04-19 17:11:51 -05:00
{
2024-06-19 22:36:27 -05:00
Threading : : MutexLocker demuxer_locker ( manager ( ) . m_decoder_mutex ) ;
2023-06-24 04:57:27 -05:00
auto demuxer_seek_result = manager ( ) . seek_demuxer_to_most_recent_keyframe ( m_target_timestamp , earliest_available_sample ) ;
if ( demuxer_seek_result . is_error ( ) ) {
manager ( ) . dispatch_decoder_error ( demuxer_seek_result . release_error ( ) ) ;
return { } ;
}
auto keyframe_timestamp = demuxer_seek_result . release_value ( ) ;
2023-02-06 01:25:02 -06:00
# if PLAYBACK_MANAGER_DEBUG
2023-04-19 17:11:51 -05:00
auto seek_mode_name = m_seek_mode = = SeekMode : : Accurate ? " Accurate " sv : " Fast " sv ;
if ( keyframe_timestamp . has_value ( ) )
dbgln ( " {} seeking to timestamp target {}ms, selected keyframe at {}ms " , seek_mode_name , m_target_timestamp . to_milliseconds ( ) , keyframe_timestamp - > to_milliseconds ( ) ) ;
else
dbgln ( " {} seeking to timestamp target {}ms, demuxer kept its iterator position after {}ms " , seek_mode_name , m_target_timestamp . to_milliseconds ( ) , earliest_available_sample . to_milliseconds ( ) ) ;
2023-02-06 01:25:02 -06:00
# endif
2023-02-06 01:24:33 -06:00
2023-04-19 17:11:51 -05:00
if ( m_seek_mode = = SeekMode : : Fast )
m_target_timestamp = keyframe_timestamp . value_or ( manager ( ) . m_last_present_in_media_time ) ;
if ( keyframe_timestamp . has_value ( ) ) {
dbgln_if ( PLAYBACK_MANAGER_DEBUG , " Keyframe is nearer to the target than the current frames, emptying queue " ) ;
while ( manager ( ) . dequeue_one_frame ( ) . has_value ( ) ) { }
manager ( ) . m_next_frame . clear ( ) ;
manager ( ) . m_last_present_in_media_time = keyframe_timestamp . value ( ) ;
} else if ( m_target_timestamp > = manager ( ) . m_last_present_in_media_time & & manager ( ) . m_next_frame . has_value ( ) & & manager ( ) . m_next_frame . value ( ) . timestamp ( ) > m_target_timestamp ) {
dbgln_if ( PLAYBACK_MANAGER_DEBUG , " Target timestamp is between the last presented frame and the next frame, exiting seek at {}ms " , m_target_timestamp . to_milliseconds ( ) ) ;
manager ( ) . m_last_present_in_media_time = m_target_timestamp ;
return assume_next_state ( ) ;
}
2023-02-06 01:24:33 -06:00
}
return skip_samples_until_timestamp ( ) ;
}
ErrorOr < void > skip_samples_until_timestamp ( )
{
2023-04-19 17:11:51 -05:00
while ( true ) {
auto optional_item = manager ( ) . dequeue_one_frame ( ) ;
if ( ! optional_item . has_value ( ) )
break ;
auto item = optional_item . release_value ( ) ;
2023-02-06 01:24:33 -06:00
2023-02-12 00:50:30 -06:00
dbgln_if ( PLAYBACK_MANAGER_DEBUG , " Dequeuing frame at {}ms and comparing to seek target {}ms " , item . timestamp ( ) . to_milliseconds ( ) , m_target_timestamp . to_milliseconds ( ) ) ;
2023-04-12 00:24:05 -05:00
if ( manager ( ) . m_next_frame . has_value ( ) & & ( item . timestamp ( ) > m_target_timestamp | | item . timestamp ( ) = = FrameQueueItem : : no_timestamp ) ) {
// If the frame we're presenting is later than the target timestamp, skip the timestamp forward to it.
if ( manager ( ) . m_next_frame - > timestamp ( ) > m_target_timestamp ) {
manager ( ) . m_last_present_in_media_time = manager ( ) . m_next_frame - > timestamp ( ) ;
} else {
2023-02-12 00:50:30 -06:00
manager ( ) . m_last_present_in_media_time = m_target_timestamp ;
2023-02-06 01:24:33 -06:00
}
2023-02-12 00:50:30 -06:00
2023-04-12 00:24:05 -05:00
if ( manager ( ) . dispatch_frame_queue_item ( manager ( ) . m_next_frame . release_value ( ) ) )
return { } ;
2023-02-06 01:24:33 -06:00
manager ( ) . m_next_frame . emplace ( item ) ;
2023-02-12 00:50:30 -06:00
dbgln_if ( PLAYBACK_MANAGER_DEBUG , " Exiting seek to {} state at {}ms " , m_playing ? " Playing " : " Paused " , manager ( ) . m_last_present_in_media_time . to_milliseconds ( ) ) ;
2023-02-12 02:36:06 -06:00
return assume_next_state ( ) ;
2023-02-12 00:50:30 -06:00
}
manager ( ) . m_next_frame . emplace ( item ) ;
2023-02-06 01:24:33 -06:00
}
2023-04-19 17:11:51 -05:00
dbgln_if ( PLAYBACK_MANAGER_DEBUG , " Frame queue is empty while seeking, waiting for buffer to fill. " ) ;
2023-05-18 23:30:49 -05:00
manager ( ) . set_state_update_timer ( buffering_or_seeking_decode_wait_time ) ;
2023-02-06 01:24:33 -06:00
return { } ;
}
StringView name ( ) override { return " Seeking " sv ; }
2023-03-13 16:30:34 +01:00
ErrorOr < void > seek ( Duration target_timestamp , SeekMode seek_mode ) override
2023-02-06 01:24:33 -06:00
{
m_target_timestamp = target_timestamp ;
m_seek_mode = seek_mode ;
return on_enter ( ) ;
}
2023-03-13 16:30:34 +01:00
Duration current_time ( ) const override
2023-02-06 01:24:33 -06:00
{
return m_target_timestamp ;
}
// We won't need this override when threaded, the queue can pause us in on_enter().
2023-05-18 23:30:49 -05:00
ErrorOr < void > do_timed_state_update ( ) override
2023-02-06 01:24:33 -06:00
{
2023-04-19 17:11:51 -05:00
dbgln_if ( PLAYBACK_MANAGER_DEBUG , " Seeking wait finished, attempting to dequeue until timestamp. " ) ;
2023-02-06 01:24:33 -06:00
return skip_samples_until_timestamp ( ) ;
}
2023-04-12 01:01:29 -05:00
PlaybackState get_state ( ) const override { return PlaybackState : : Seeking ; }
2023-03-13 16:30:34 +01:00
Duration m_target_timestamp { Duration : : zero ( ) } ;
2023-02-06 01:24:33 -06:00
SeekMode m_seek_mode { SeekMode : : Accurate } ;
} ;
class PlaybackManager : : StoppedStateHandler : public PlaybackManager : : PlaybackStateHandler {
public :
StoppedStateHandler ( PlaybackManager & manager )
: PlaybackStateHandler ( manager )
{
}
~ StoppedStateHandler ( ) override = default ;
private :
ErrorOr < void > on_enter ( ) override
{
return { } ;
}
StringView name ( ) override { return " Stopped " sv ; }
ErrorOr < void > play ( ) override
{
2023-04-19 17:11:51 -05:00
// When Stopped, the decoder thread will be waiting for a signal to start its loop going again.
manager ( ) . m_decode_wait_condition . broadcast ( ) ;
2023-03-13 16:30:34 +01:00
return replace_handler_and_delete_this < SeekingStateHandler > ( true , Duration : : zero ( ) , SeekMode : : Fast ) ;
2023-02-06 01:24:33 -06:00
}
2023-07-07 22:48:11 -04:00
bool is_playing ( ) const override { return false ; }
2023-04-12 01:01:29 -05:00
PlaybackState get_state ( ) const override { return PlaybackState : : Stopped ; }
2023-02-06 01:24:33 -06:00
} ;
2024-05-17 12:39:29 -04:00
DecoderErrorOr < NonnullOwnPtr < PlaybackManager > > PlaybackManager : : create ( NonnullOwnPtr < Demuxer > demuxer )
{
auto video_tracks = TRY ( demuxer - > get_tracks_for_type ( TrackType : : Video ) ) ;
if ( video_tracks . is_empty ( ) )
return DecoderError : : with_description ( DecoderErrorCategory : : Invalid , " No video track is present " sv ) ;
auto track = video_tracks [ 0 ] ;
dbgln_if ( PLAYBACK_MANAGER_DEBUG , " Selecting video track number {} " , track . identifier ( ) ) ;
auto codec_id = TRY ( demuxer - > get_codec_id_for_track ( track ) ) ;
2024-06-19 18:22:21 -05:00
auto codec_initialization_data = TRY ( demuxer - > get_codec_initialization_data_for_track ( track ) ) ;
NonnullOwnPtr < VideoDecoder > decoder = TRY ( FFmpeg : : FFmpegVideoDecoder : : try_create ( codec_id , codec_initialization_data ) ) ;
2024-05-17 12:39:29 -04:00
auto frame_queue = DECODER_TRY_ALLOC ( VideoFrameQueue : : create ( ) ) ;
2024-06-19 18:01:41 -05:00
auto playback_manager = DECODER_TRY_ALLOC ( try_make < PlaybackManager > ( demuxer , track , move ( decoder ) , move ( frame_queue ) ) ) ;
2024-05-17 12:39:29 -04:00
playback_manager - > m_state_update_timer = Core : : Timer : : create_single_shot ( 0 , [ & self = * playback_manager ] { self . timer_callback ( ) ; } ) ;
playback_manager - > m_decode_thread = DECODER_TRY_ALLOC ( Threading : : Thread : : try_create ( [ & self = * playback_manager ] {
while ( ! self . m_stop_decoding . load ( ) )
self . decode_and_queue_one_sample ( ) ;
dbgln_if ( PLAYBACK_MANAGER_DEBUG , " Media Decoder thread ended. " ) ;
return 0 ;
} ,
" Media Decoder " sv ) ) ;
playback_manager - > m_playback_handler = make < SeekingStateHandler > ( * playback_manager , false , Duration : : zero ( ) , SeekMode : : Fast ) ;
DECODER_TRY_ALLOC ( playback_manager - > m_playback_handler - > on_enter ( ) ) ;
playback_manager - > m_decode_thread - > start ( ) ;
return playback_manager ;
}
2022-10-29 18:16:14 -05:00
}