LibMedia: Use FFmpeg to decode more video formats

VP9 continues to function, but this also allows AV1 to be decoded. With
this commit, H.264 is still non-functional, as the decoder requires
some extra initial data from the track definition in the Matroska file.

.github/actions/setup/action.yml

@@ -25,7 +25,7 @@ runs:
         sudo add-apt-repository ppa:ubuntu-toolchain-r/test
 
         sudo apt-get update
-        sudo apt-get install autoconf autoconf-archive automake build-essential cmake fonts-liberation2 zip curl tar ccache clang-18 clang++-18 lld-18 gcc-13 g++-13 libstdc++-13-dev ninja-build unzip qt6-base-dev qt6-tools-dev-tools libqt6svg6-dev qt6-multimedia-dev libgl1-mesa-dev libpulse-dev libssl-dev libegl1-mesa-dev
+        sudo apt-get install autoconf autoconf-archive automake build-essential cmake libavcodec-dev fonts-liberation2 zip curl tar ccache clang-18 clang++-18 lld-18 gcc-13 g++-13 libstdc++-13-dev ninja-build unzip qt6-base-dev qt6-tools-dev-tools libqt6svg6-dev qt6-multimedia-dev libgl1-mesa-dev libpulse-dev libssl-dev libegl1-mesa-dev
 
         sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 100
         sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-18 100
@@ -51,7 +51,7 @@ runs:
         set -e
         sudo xcode-select --switch /Applications/Xcode_15.4.app
         brew update
-        brew install autoconf autoconf-archive automake coreutils bash ninja wabt ccache unzip qt llvm@18
+        brew install autoconf autoconf-archive automake coreutils bash ffmpeg ninja wabt ccache unzip qt llvm@18
 
     - name: 'Install vcpkg'
       shell: bash

Documentation/BuildInstructionsLadybird.md

@@ -9,7 +9,7 @@ NOTE: In all of the below lists of packages, the Qt6 multimedia package is not n
 On Debian/Ubuntu required packages include, but are not limited to:
 
 ```
-sudo apt install autoconf autoconf-archive automake build-essential cmake libgl1-mesa-dev ninja-build qt6-base-dev qt6-tools-dev-tools qt6-multimedia-dev ccache fonts-liberation2 zip unzip curl tar
+sudo apt install autoconf autoconf-archive automake build-essential cmake libavcodec-dev libgl1-mesa-dev ninja-build qt6-base-dev qt6-tools-dev-tools qt6-multimedia-dev ccache fonts-liberation2 zip unzip curl tar
 ```
 
 For Ubuntu 20.04 and above, ensure that the Qt6 Wayland packages are available:
@@ -21,7 +21,7 @@ sudo apt install qt6-wayland
 On Arch Linux/Manjaro:
 
 ```
-sudo pacman -S --needed base-devel cmake libgl ninja qt6-base qt6-tools qt6-wayland qt6-multimedia ccache ttf-liberation curl unzip zip tar autoconf-archive
+sudo pacman -S --needed base-devel cmake ffmpeg libgl ninja qt6-base qt6-tools qt6-wayland qt6-multimedia ccache ttf-liberation curl unzip zip tar autoconf-archive
 ```
 
 On Fedora or derivatives:
@@ -56,7 +56,7 @@ Xcode 14 versions before 14.3 might crash while building ladybird. Xcode 14.3 or
 
 ```
 xcode-select --install
-brew install autoconf autoconf-archive automake cmake ninja ccache pkg-config
+brew install autoconf autoconf-archive automake cmake ffmpeg ninja ccache pkg-config
 ```
 
 If you also plan to use the Qt chrome on macOS:

Meta/CMake/lagom_compile_options.cmake

@@ -21,6 +21,10 @@ if (LINUX)
     add_compile_options(-D_FILE_OFFSET_BITS=64)
 endif()
 
+if (APPLE)
+    list(APPEND CMAKE_PREFIX_PATH /opt/homebrew)
+endif()
+
 if (CMAKE_BUILD_TYPE STREQUAL "Debug")
     if (NOT MSVC)
         add_compile_options(-ggdb3)

Userland/Libraries/LibMedia/CMakeLists.txt

@@ -4,6 +4,7 @@ set(SOURCES
     Color/TransferCharacteristics.cpp
     Containers/Matroska/MatroskaDemuxer.cpp
     Containers/Matroska/Reader.cpp
+    FFmpeg/FFmpegVideoDecoder.cpp
     PlaybackManager.cpp
     VideoFrame.cpp
     Video/VP9/Decoder.cpp
@@ -15,3 +16,9 @@ set(SOURCES
 
 serenity_lib(LibMedia media)
 target_link_libraries(LibMedia PRIVATE LibCore LibIPC LibGfx LibThreading)
+
+find_package(PkgConfig REQUIRED)
+pkg_check_modules(AVCODEC REQUIRED IMPORTED_TARGET libavcodec)
+target_include_directories(LibMedia PRIVATE ${AVCODEC_INCLUDE_DIRS})
+target_link_directories(LibMedia PRIVATE ${AVCODEC_LIBRARY_DIRS})
+target_link_libraries(LibMedia PRIVATE ${AVCODEC_LIBRARIES})

Userland/Libraries/LibMedia/FFmpeg/FFmpegForward.h

@@ -0,0 +1,13 @@
+/*
+ * Copyright (c) 2024, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+extern "C" {
+struct AVCodecContext;
+struct AVPacket;
+struct AVFrame;
+}

Userland/Libraries/LibMedia/FFmpeg/FFmpegHelpers.h

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2024, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <LibMedia/CodecID.h>
+
+extern "C" {
+#include <libavcodec/avcodec.h>
+}
+
+namespace Media::FFmpeg {
+
+static inline AVCodecID ffmpeg_codec_id_from_serenity_codec_id(CodecID codec)
+{
+    switch (codec) {
+    case CodecID::VP8:
+        return AV_CODEC_ID_VP8;
+    case CodecID::VP9:
+        return AV_CODEC_ID_VP9;
+    case CodecID::H261:
+        return AV_CODEC_ID_H261;
+    case CodecID::MPEG1:
+    case CodecID::H262:
+        return AV_CODEC_ID_MPEG2VIDEO;
+    case CodecID::H263:
+        return AV_CODEC_ID_H263;
+    case CodecID::H264:
+        return AV_CODEC_ID_H264;
+    case CodecID::H265:
+        return AV_CODEC_ID_HEVC;
+    case CodecID::AV1:
+        return AV_CODEC_ID_AV1;
+    case CodecID::Theora:
+        return AV_CODEC_ID_THEORA;
+    case CodecID::Vorbis:
+        return AV_CODEC_ID_VORBIS;
+    case CodecID::Opus:
+        return AV_CODEC_ID_OPUS;
+    default:
+        return AV_CODEC_ID_NONE;
+    }
+}
+
+}

Userland/Libraries/LibMedia/FFmpeg/FFmpegVideoDecoder.cpp

@@ -0,0 +1,216 @@
+/*
+ * Copyright (c) 2024, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <LibCore/System.h>
+#include <LibMedia/VideoFrame.h>
+
+#include "FFmpegHelpers.h"
+#include "FFmpegVideoDecoder.h"
+
+namespace Media::FFmpeg {
+
+static AVPixelFormat negotiate_output_format(AVCodecContext*, AVPixelFormat const* formats)
+{
+    while (*formats >= 0) {
+        switch (*formats) {
+        case AV_PIX_FMT_YUV420P:
+        case AV_PIX_FMT_YUV420P10:
+        case AV_PIX_FMT_YUV420P12:
+        case AV_PIX_FMT_YUV422P:
+        case AV_PIX_FMT_YUV422P10:
+        case AV_PIX_FMT_YUV422P12:
+        case AV_PIX_FMT_YUV444P:
+        case AV_PIX_FMT_YUV444P10:
+        case AV_PIX_FMT_YUV444P12:
+            return *formats;
+        default:
+            break;
+        }
+        formats++;
+    }
+    return AV_PIX_FMT_NONE;
+}
+
+DecoderErrorOr<NonnullOwnPtr<FFmpegVideoDecoder>> FFmpegVideoDecoder::try_create(CodecID codec_id)
+{
+    AVCodecContext* codec_context = nullptr;
+    AVPacket* packet = nullptr;
+    AVFrame* frame = nullptr;
+    ArmedScopeGuard memory_guard {
+        [&] {
+            avcodec_free_context(&codec_context);
+            av_packet_free(&packet);
+            av_frame_free(&frame);
+        }
+    };
+
+    auto ff_codec_id = ffmpeg_codec_id_from_serenity_codec_id(codec_id);
+    auto const* codec = avcodec_find_decoder(ff_codec_id);
+    if (!codec)
+        return DecoderError::format(DecoderErrorCategory::NotImplemented, "Could not find FFmpeg decoder for codec {}", codec_id);
+
+    codec_context = avcodec_alloc_context3(codec);
+    if (!codec_context)
+        return DecoderError::format(DecoderErrorCategory::Memory, "Failed to allocate FFmpeg codec context for codec {}", codec_id);
+
+    codec_context->get_format = negotiate_output_format;
+
+    codec_context->thread_count = static_cast<int>(min(Core::System::hardware_concurrency(), 4));
+
+    if (avcodec_open2(codec_context, codec, nullptr) < 0)
+        return DecoderError::format(DecoderErrorCategory::Unknown, "Unknown error occurred when opening FFmpeg codec {}", codec_id);
+
+    packet = av_packet_alloc();
+    if (!packet)
+        return DecoderError::with_description(DecoderErrorCategory::Memory, "Failed to allocate FFmpeg packet"sv);
+
+    frame = av_frame_alloc();
+    if (!frame)
+        return DecoderError::with_description(DecoderErrorCategory::Memory, "Failed to allocate FFmpeg frame"sv);
+
+    memory_guard.disarm();
+    return DECODER_TRY_ALLOC(try_make<FFmpegVideoDecoder>(codec_context, packet, frame));
+}
+
+FFmpegVideoDecoder::FFmpegVideoDecoder(AVCodecContext* codec_context, AVPacket* packet, AVFrame* frame)
+    : m_codec_context(codec_context)
+    , m_packet(packet)
+    , m_frame(frame)
+{
+}
+
+FFmpegVideoDecoder::~FFmpegVideoDecoder()
+{
+    av_packet_free(&m_packet);
+    av_frame_free(&m_frame);
+    avcodec_free_context(&m_codec_context);
+}
+
+DecoderErrorOr<void> FFmpegVideoDecoder::receive_sample(ReadonlyBytes sample)
+{
+    VERIFY(sample.size() < NumericLimits<int>::max());
+
+    m_packet->data = const_cast<u8*>(sample.data());
+    m_packet->size = static_cast<int>(sample.size());
+
+    auto result = avcodec_send_packet(m_codec_context, m_packet);
+    switch (result) {
+    case 0:
+        return {};
+    case AVERROR(EAGAIN):
+        return DecoderError::with_description(DecoderErrorCategory::NeedsMoreInput, "FFmpeg decoder cannot decode any more data until frames have been retrieved"sv);
+    case AVERROR_EOF:
+        return DecoderError::with_description(DecoderErrorCategory::EndOfStream, "FFmpeg decoder has been flushed"sv);
+    case AVERROR(EINVAL):
+        return DecoderError::with_description(DecoderErrorCategory::Invalid, "FFmpeg codec has not been opened"sv);
+    case AVERROR(ENOMEM):
+        return DecoderError::with_description(DecoderErrorCategory::Memory, "FFmpeg codec ran out of internal memory"sv);
+    default:
+        return DecoderError::with_description(DecoderErrorCategory::Corrupted, "FFmpeg codec reports that the data is corrupted"sv);
+    }
+}
+
+DecoderErrorOr<NonnullOwnPtr<VideoFrame>> FFmpegVideoDecoder::get_decoded_frame()
+{
+    auto result = avcodec_receive_frame(m_codec_context, m_frame);
+
+    switch (result) {
+    case 0: {
+        auto color_primaries = static_cast<ColorPrimaries>(m_frame->color_primaries);
+        auto transfer_characteristics = static_cast<TransferCharacteristics>(m_frame->color_trc);
+        auto matrix_coefficients = static_cast<MatrixCoefficients>(m_frame->colorspace);
+        auto color_range = [&] {
+            switch (m_frame->color_range) {
+            case AVColorRange::AVCOL_RANGE_MPEG:
+                return VideoFullRangeFlag::Studio;
+            case AVColorRange::AVCOL_RANGE_JPEG:
+                return VideoFullRangeFlag::Full;
+            default:
+                return VideoFullRangeFlag::Unspecified;
+            }
+        }();
+        auto cicp = CodingIndependentCodePoints { color_primaries, transfer_characteristics, matrix_coefficients, color_range };
+
+        size_t bit_depth = [&] {
+            switch (m_frame->format) {
+            case AV_PIX_FMT_YUV420P:
+            case AV_PIX_FMT_YUV422P:
+            case AV_PIX_FMT_YUV444P:
+                return 8;
+            case AV_PIX_FMT_YUV420P10:
+            case AV_PIX_FMT_YUV422P10:
+            case AV_PIX_FMT_YUV444P10:
+                return 10;
+            case AV_PIX_FMT_YUV420P12:
+            case AV_PIX_FMT_YUV422P12:
+            case AV_PIX_FMT_YUV444P12:
+                return 12;
+            }
+            VERIFY_NOT_REACHED();
+        }();
+        size_t component_size = (bit_depth + 7) / 8;
+
+        auto subsampling = [&]() -> Subsampling {
+            switch (m_frame->format) {
+            case AV_PIX_FMT_YUV420P:
+            case AV_PIX_FMT_YUV420P10:
+            case AV_PIX_FMT_YUV420P12:
+                return { true, true };
+            case AV_PIX_FMT_YUV422P:
+            case AV_PIX_FMT_YUV422P10:
+            case AV_PIX_FMT_YUV422P12:
+                return { true, false };
+            case AV_PIX_FMT_YUV444P:
+            case AV_PIX_FMT_YUV444P10:
+            case AV_PIX_FMT_YUV444P12:
+                return { false, false };
+
+            default:
+                VERIFY_NOT_REACHED();
+            }
+        }();
+
+        auto size = Gfx::Size<u32> { m_frame->width, m_frame->height };
+
+        auto frame = DECODER_TRY_ALLOC(SubsampledYUVFrame::try_create(size, bit_depth, cicp, subsampling));
+
+        for (u32 plane = 0; plane < 3; plane++) {
+            VERIFY(m_frame->linesize[plane] != 0);
+            if (m_frame->linesize[plane] < 0)
+                return DecoderError::with_description(DecoderErrorCategory::NotImplemented, "Reversed scanlines are not supported"sv);
+
+            bool const use_subsampling = plane > 0;
+            auto plane_size = (use_subsampling ? subsampling.subsampled_size(size) : size).to_type<size_t>();
+
+            auto output_line_size = plane_size.width() * component_size;
+            VERIFY(output_line_size <= static_cast<size_t>(m_frame->linesize[plane]));
+
+            auto const* source = m_frame->data[plane];
+            VERIFY(source != nullptr);
+            auto* destination = frame->get_raw_plane_data(plane);
+            VERIFY(destination != nullptr);
+
+            for (size_t row = 0; row < plane_size.height(); row++) {
+                memcpy(destination, source, output_line_size);
+                source += m_frame->linesize[plane];
+                destination += output_line_size;
+            }
+        }
+
+        return frame;
+    }
+    case AVERROR(EAGAIN):
+        return DecoderError::with_description(DecoderErrorCategory::NeedsMoreInput, "FFmpeg decoder has no frames available, send more input"sv);
+    case AVERROR_EOF:
+        return DecoderError::with_description(DecoderErrorCategory::EndOfStream, "FFmpeg decoder has been flushed"sv);
+    case AVERROR(EINVAL):
+        return DecoderError::with_description(DecoderErrorCategory::Invalid, "FFmpeg codec has not been opened"sv);
+    default:
+        return DecoderError::format(DecoderErrorCategory::Unknown, "FFmpeg codec encountered an unexpected error retreiving frames with code {:x}", result);
+    }
+}
+
+}

Userland/Libraries/LibMedia/FFmpeg/FFmpegVideoDecoder.h

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2024, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <LibMedia/CodecID.h>
+#include <LibMedia/VideoDecoder.h>
+
+#include "FFmpegForward.h"
+
+namespace Media::FFmpeg {
+
+class FFmpegVideoDecoder final : public VideoDecoder {
+public:
+    static DecoderErrorOr<NonnullOwnPtr<FFmpegVideoDecoder>> try_create(CodecID);
+    FFmpegVideoDecoder(AVCodecContext* codec_context, AVPacket* packet, AVFrame* frame);
+    ~FFmpegVideoDecoder();
+
+    DecoderErrorOr<void> receive_sample(ReadonlyBytes sample) override;
+    DecoderErrorOr<NonnullOwnPtr<VideoFrame>> get_decoded_frame() override;
+
+private:
+    DecoderErrorOr<void> decode_single_sample(Duration timestamp, u8* data, int size);
+
+    AVCodecContext* m_codec_context;
+    AVPacket* m_packet;
+    AVFrame* m_frame;
+};
+
+}

Userland/Libraries/LibMedia/PlaybackManager.cpp

@@ -7,7 +7,8 @@
 #include <AK/Format.h>
 #include <LibCore/Timer.h>
 #include <LibMedia/Containers/Matroska/MatroskaDemuxer.h>
-#include <LibMedia/Video/VP9/Decoder.h>
+#include <LibMedia/FFmpeg/FFmpegVideoDecoder.h>
+#include <LibMedia/VideoFrame.h>
 
 #include "PlaybackManager.h"
 
@@ -700,18 +701,9 @@ DecoderErrorOr<NonnullOwnPtr<PlaybackManager>> PlaybackManager::create(NonnullOw
     dbgln_if(PLAYBACK_MANAGER_DEBUG, "Selecting video track number {}", track.identifier());
 
     auto codec_id = TRY(demuxer->get_codec_id_for_track(track));
-    OwnPtr<VideoDecoder> decoder;
+    NonnullOwnPtr<VideoDecoder> decoder = TRY(FFmpeg::FFmpegVideoDecoder::try_create(codec_id));
-    switch (codec_id) {
-    case CodecID::VP9:
-        decoder = DECODER_TRY_ALLOC(try_make<Video::VP9::Decoder>());
-        break;
-
-    default:
-        return DecoderError::format(DecoderErrorCategory::Invalid, "Unsupported codec: {}", codec_id);
-    }
-    auto decoder_non_null = decoder.release_nonnull();
     auto frame_queue = DECODER_TRY_ALLOC(VideoFrameQueue::create());
-    auto playback_manager = DECODER_TRY_ALLOC(try_make<PlaybackManager>(demuxer, track, move(decoder_non_null), move(frame_queue)));
+    auto playback_manager = DECODER_TRY_ALLOC(try_make<PlaybackManager>(demuxer, track, move(decoder), move(frame_queue)));
 
     playback_manager->m_state_update_timer = Core::Timer::create_single_shot(0, [&self = *playback_manager] { self.timer_callback(); });