From 457a69786b788d3158813156d11e70980f50f7b7 Mon Sep 17 00:00:00 2001 From: Zaggy1024 Date: Wed, 19 Jun 2024 18:22:21 -0500 Subject: [PATCH] LibMedia: Pass Matroska codec private data to the FFmpeg decoder This is necessary to give the H.264 decoder the data it needs to initialize, including frame size and profile. --- .../LibMedia/Containers/Matroska/Document.h | 8 ++++++++ .../Containers/Matroska/MatroskaDemuxer.cpp | 5 +++++ .../LibMedia/Containers/Matroska/MatroskaDemuxer.h | 2 ++ .../LibMedia/Containers/Matroska/Reader.cpp | 7 +++++++ Userland/Libraries/LibMedia/Demuxer.h | 2 ++ .../LibMedia/FFmpeg/FFmpegVideoDecoder.cpp | 14 +++++++++++++- .../Libraries/LibMedia/FFmpeg/FFmpegVideoDecoder.h | 2 +- Userland/Libraries/LibMedia/PlaybackManager.cpp | 3 ++- 8 files changed, 40 insertions(+), 3 deletions(-) diff --git a/Userland/Libraries/LibMedia/Containers/Matroska/Document.h b/Userland/Libraries/LibMedia/Containers/Matroska/Document.h index 97e5d57df58..4acb45bce24 100644 --- a/Userland/Libraries/LibMedia/Containers/Matroska/Document.h +++ b/Userland/Libraries/LibMedia/Containers/Matroska/Document.h @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -119,6 +120,12 @@ public: void set_language(FlyString const& language) { m_language = language; } FlyString codec_id() const { return m_codec_id; } void set_codec_id(FlyString const& codec_id) { m_codec_id = codec_id; } + ReadonlyBytes codec_private_data() const { return m_codec_private_data.span(); } + ErrorOr set_codec_private_data(ReadonlyBytes codec_private_data) + { + m_codec_private_data = TRY(FixedArray::create(codec_private_data)); + return {}; + } double timestamp_scale() const { return m_timestamp_scale; } void set_timestamp_scale(double timestamp_scale) { m_timestamp_scale = timestamp_scale; } u64 codec_delay() const { return m_codec_delay; } @@ -146,6 +153,7 @@ private: TrackType m_track_type { Invalid }; FlyString m_language = "eng"_fly_string; FlyString m_codec_id; + FixedArray m_codec_private_data; double m_timestamp_scale { 1 }; u64 m_codec_delay { 0 }; u64 m_timestamp_offset { 0 }; diff --git a/Userland/Libraries/LibMedia/Containers/Matroska/MatroskaDemuxer.cpp b/Userland/Libraries/LibMedia/Containers/Matroska/MatroskaDemuxer.cpp index e829ea39a59..9539c263dc5 100644 --- a/Userland/Libraries/LibMedia/Containers/Matroska/MatroskaDemuxer.cpp +++ b/Userland/Libraries/LibMedia/Containers/Matroska/MatroskaDemuxer.cpp @@ -104,6 +104,11 @@ DecoderErrorOr MatroskaDemuxer::get_codec_id_for_track(Track track) return get_codec_id_for_string(codec_id); } +DecoderErrorOr MatroskaDemuxer::get_codec_initialization_data_for_track(Track track) +{ + return TRY(m_reader.track_for_track_number(track.identifier()))->codec_private_data(); +} + DecoderErrorOr> MatroskaDemuxer::seek_to_most_recent_keyframe(Track track, Duration timestamp, Optional earliest_available_sample) { // Removing the track status will cause us to start from the beginning. diff --git a/Userland/Libraries/LibMedia/Containers/Matroska/MatroskaDemuxer.h b/Userland/Libraries/LibMedia/Containers/Matroska/MatroskaDemuxer.h index d01ddd09ba2..8ac3d973ba8 100644 --- a/Userland/Libraries/LibMedia/Containers/Matroska/MatroskaDemuxer.h +++ b/Userland/Libraries/LibMedia/Containers/Matroska/MatroskaDemuxer.h @@ -35,6 +35,8 @@ public: DecoderErrorOr get_codec_id_for_track(Track track) override; + DecoderErrorOr get_codec_initialization_data_for_track(Track track) override; + DecoderErrorOr get_next_sample_for_track(Track track) override; private: diff --git a/Userland/Libraries/LibMedia/Containers/Matroska/Reader.cpp b/Userland/Libraries/LibMedia/Containers/Matroska/Reader.cpp index 4d65803761f..ee36c75544d 100644 --- a/Userland/Libraries/LibMedia/Containers/Matroska/Reader.cpp +++ b/Userland/Libraries/LibMedia/Containers/Matroska/Reader.cpp @@ -50,6 +50,7 @@ constexpr u32 TRACK_UID_ID = 0x73C5; constexpr u32 TRACK_TYPE_ID = 0x83; constexpr u32 TRACK_LANGUAGE_ID = 0x22B59C; constexpr u32 TRACK_CODEC_ID = 0x86; +constexpr u32 TRACK_CODEC_PRIVATE = 0x63A2; constexpr u32 TRACK_TIMESTAMP_SCALE_ID = 0x23314F; constexpr u32 TRACK_OFFSET_ID = 0x537F; constexpr u32 TRACK_VIDEO_ID = 0xE0; @@ -480,6 +481,12 @@ static DecoderErrorOr> parse_track_entry(Streamer& str track_entry->set_codec_id(DECODER_TRY_ALLOC(String::from_byte_string(TRY_READ(streamer.read_string())))); dbgln_if(MATROSKA_TRACE_DEBUG, "Read Track's CodecID attribute: {}", track_entry->codec_id()); break; + case TRACK_CODEC_PRIVATE: { + auto codec_private_data = TRY_READ(streamer.read_raw_octets(TRY_READ(streamer.read_variable_size_integer()))); + DECODER_TRY_ALLOC(track_entry->set_codec_private_data(codec_private_data)); + dbgln_if(MATROSKA_TRACE_DEBUG, "Read Track's CodecID attribute: {}", track_entry->codec_id()); + break; + } case TRACK_TIMESTAMP_SCALE_ID: track_entry->set_timestamp_scale(TRY_READ(streamer.read_float())); dbgln_if(MATROSKA_TRACE_DEBUG, "Read Track's TrackTimestampScale attribute: {}", track_entry->timestamp_scale()); diff --git a/Userland/Libraries/LibMedia/Demuxer.h b/Userland/Libraries/LibMedia/Demuxer.h index b62b43e9c7b..163242db8b8 100644 --- a/Userland/Libraries/LibMedia/Demuxer.h +++ b/Userland/Libraries/LibMedia/Demuxer.h @@ -26,6 +26,8 @@ public: virtual DecoderErrorOr get_codec_id_for_track(Track track) = 0; + virtual DecoderErrorOr get_codec_initialization_data_for_track(Track track) = 0; + // Returns the timestamp of the keyframe that was seeked to. // The value is `Optional` to allow the demuxer to decide not to seek so that it can keep its position // in the case that the timestamp is closer to the current time than the nearest keyframe. diff --git a/Userland/Libraries/LibMedia/FFmpeg/FFmpegVideoDecoder.cpp b/Userland/Libraries/LibMedia/FFmpeg/FFmpegVideoDecoder.cpp index 601135e5791..f3f5af64c5a 100644 --- a/Userland/Libraries/LibMedia/FFmpeg/FFmpegVideoDecoder.cpp +++ b/Userland/Libraries/LibMedia/FFmpeg/FFmpegVideoDecoder.cpp @@ -34,7 +34,7 @@ static AVPixelFormat negotiate_output_format(AVCodecContext*, AVPixelFormat cons return AV_PIX_FMT_NONE; } -DecoderErrorOr> FFmpegVideoDecoder::try_create(CodecID codec_id) +DecoderErrorOr> FFmpegVideoDecoder::try_create(CodecID codec_id, ReadonlyBytes codec_initialization_data) { AVCodecContext* codec_context = nullptr; AVPacket* packet = nullptr; @@ -60,6 +60,18 @@ DecoderErrorOr> FFmpegVideoDecoder::try_create codec_context->thread_count = static_cast(min(Core::System::hardware_concurrency(), 4)); + if (!codec_initialization_data.is_empty()) { + if (codec_initialization_data.size() > NumericLimits::max()) + return DecoderError::corrupted("Codec initialization data is too large"sv); + + codec_context->extradata = static_cast(av_malloc(codec_initialization_data.size() + AV_INPUT_BUFFER_PADDING_SIZE)); + if (!codec_context->extradata) + return DecoderError::with_description(DecoderErrorCategory::Memory, "Failed to allocate codec initialization data buffer for FFmpeg codec"sv); + + memcpy(codec_context->extradata, codec_initialization_data.data(), codec_initialization_data.size()); + codec_context->extradata_size = static_cast(codec_initialization_data.size()); + } + if (avcodec_open2(codec_context, codec, nullptr) < 0) return DecoderError::format(DecoderErrorCategory::Unknown, "Unknown error occurred when opening FFmpeg codec {}", codec_id); diff --git a/Userland/Libraries/LibMedia/FFmpeg/FFmpegVideoDecoder.h b/Userland/Libraries/LibMedia/FFmpeg/FFmpegVideoDecoder.h index be1baa283e1..3ec165ba36a 100644 --- a/Userland/Libraries/LibMedia/FFmpeg/FFmpegVideoDecoder.h +++ b/Userland/Libraries/LibMedia/FFmpeg/FFmpegVideoDecoder.h @@ -15,7 +15,7 @@ namespace Media::FFmpeg { class FFmpegVideoDecoder final : public VideoDecoder { public: - static DecoderErrorOr> try_create(CodecID); + static DecoderErrorOr> try_create(CodecID, ReadonlyBytes codec_initialization_data); FFmpegVideoDecoder(AVCodecContext* codec_context, AVPacket* packet, AVFrame* frame); ~FFmpegVideoDecoder(); diff --git a/Userland/Libraries/LibMedia/PlaybackManager.cpp b/Userland/Libraries/LibMedia/PlaybackManager.cpp index 93ee932788e..579ddb416e8 100644 --- a/Userland/Libraries/LibMedia/PlaybackManager.cpp +++ b/Userland/Libraries/LibMedia/PlaybackManager.cpp @@ -701,7 +701,8 @@ DecoderErrorOr> PlaybackManager::create(NonnullOw dbgln_if(PLAYBACK_MANAGER_DEBUG, "Selecting video track number {}", track.identifier()); auto codec_id = TRY(demuxer->get_codec_id_for_track(track)); - NonnullOwnPtr decoder = TRY(FFmpeg::FFmpegVideoDecoder::try_create(codec_id)); + auto codec_initialization_data = TRY(demuxer->get_codec_initialization_data_for_track(track)); + NonnullOwnPtr decoder = TRY(FFmpeg::FFmpegVideoDecoder::try_create(codec_id, codec_initialization_data)); auto frame_queue = DECODER_TRY_ALLOC(VideoFrameQueue::create()); auto playback_manager = DECODER_TRY_ALLOC(try_make(demuxer, track, move(decoder), move(frame_queue)));