LibMedia: Use FFmpeg to decode more video formats

VP9 continues to function, but this also allows AV1 to be decoded. With
this commit, H.264 is still non-functional, as the decoder requires
some extra initial data from the track definition in the Matroska file.
This commit is contained in:
Zaggy1024 2024-06-19 18:01:41 -05:00 committed by Andrew Kaster
parent bf1e0fac94
commit 81001b37ce
Notes: sideshowbarker 2024-07-17 22:01:16 +09:00
9 changed files with 330 additions and 17 deletions

View file

@ -25,7 +25,7 @@ runs:
sudo add-apt-repository ppa:ubuntu-toolchain-r/test
sudo apt-get update
sudo apt-get install autoconf autoconf-archive automake build-essential cmake fonts-liberation2 zip curl tar ccache clang-18 clang++-18 lld-18 gcc-13 g++-13 libstdc++-13-dev ninja-build unzip qt6-base-dev qt6-tools-dev-tools libqt6svg6-dev qt6-multimedia-dev libgl1-mesa-dev libpulse-dev libssl-dev libegl1-mesa-dev
sudo apt-get install autoconf autoconf-archive automake build-essential cmake libavcodec-dev fonts-liberation2 zip curl tar ccache clang-18 clang++-18 lld-18 gcc-13 g++-13 libstdc++-13-dev ninja-build unzip qt6-base-dev qt6-tools-dev-tools libqt6svg6-dev qt6-multimedia-dev libgl1-mesa-dev libpulse-dev libssl-dev libegl1-mesa-dev
sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 100
sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-18 100
@ -51,7 +51,7 @@ runs:
set -e
sudo xcode-select --switch /Applications/Xcode_15.4.app
brew update
brew install autoconf autoconf-archive automake coreutils bash ninja wabt ccache unzip qt llvm@18
brew install autoconf autoconf-archive automake coreutils bash ffmpeg ninja wabt ccache unzip qt llvm@18
- name: 'Install vcpkg'
shell: bash

View file

@ -9,7 +9,7 @@ NOTE: In all of the below lists of packages, the Qt6 multimedia package is not n
On Debian/Ubuntu required packages include, but are not limited to:
```
sudo apt install autoconf autoconf-archive automake build-essential cmake libgl1-mesa-dev ninja-build qt6-base-dev qt6-tools-dev-tools qt6-multimedia-dev ccache fonts-liberation2 zip unzip curl tar
sudo apt install autoconf autoconf-archive automake build-essential cmake libavcodec-dev libgl1-mesa-dev ninja-build qt6-base-dev qt6-tools-dev-tools qt6-multimedia-dev ccache fonts-liberation2 zip unzip curl tar
```
For Ubuntu 20.04 and above, ensure that the Qt6 Wayland packages are available:
@ -21,7 +21,7 @@ sudo apt install qt6-wayland
On Arch Linux/Manjaro:
```
sudo pacman -S --needed base-devel cmake libgl ninja qt6-base qt6-tools qt6-wayland qt6-multimedia ccache ttf-liberation curl unzip zip tar autoconf-archive
sudo pacman -S --needed base-devel cmake ffmpeg libgl ninja qt6-base qt6-tools qt6-wayland qt6-multimedia ccache ttf-liberation curl unzip zip tar autoconf-archive
```
On Fedora or derivatives:
@ -56,7 +56,7 @@ Xcode 14 versions before 14.3 might crash while building ladybird. Xcode 14.3 or
```
xcode-select --install
brew install autoconf autoconf-archive automake cmake ninja ccache pkg-config
brew install autoconf autoconf-archive automake cmake ffmpeg ninja ccache pkg-config
```
If you also plan to use the Qt chrome on macOS:

View file

@ -21,6 +21,10 @@ if (LINUX)
add_compile_options(-D_FILE_OFFSET_BITS=64)
endif()
if (APPLE)
list(APPEND CMAKE_PREFIX_PATH /opt/homebrew)
endif()
if (CMAKE_BUILD_TYPE STREQUAL "Debug")
if (NOT MSVC)
add_compile_options(-ggdb3)

View file

@ -4,6 +4,7 @@ set(SOURCES
Color/TransferCharacteristics.cpp
Containers/Matroska/MatroskaDemuxer.cpp
Containers/Matroska/Reader.cpp
FFmpeg/FFmpegVideoDecoder.cpp
PlaybackManager.cpp
VideoFrame.cpp
Video/VP9/Decoder.cpp
@ -15,3 +16,9 @@ set(SOURCES
serenity_lib(LibMedia media)
target_link_libraries(LibMedia PRIVATE LibCore LibIPC LibGfx LibThreading)
find_package(PkgConfig REQUIRED)
pkg_check_modules(AVCODEC REQUIRED IMPORTED_TARGET libavcodec)
target_include_directories(LibMedia PRIVATE ${AVCODEC_INCLUDE_DIRS})
target_link_directories(LibMedia PRIVATE ${AVCODEC_LIBRARY_DIRS})
target_link_libraries(LibMedia PRIVATE ${AVCODEC_LIBRARIES})

View file

@ -0,0 +1,13 @@
/*
* Copyright (c) 2024, Gregory Bertilson <zaggy1024@gmail.com>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
extern "C" {
struct AVCodecContext;
struct AVPacket;
struct AVFrame;
}

View file

@ -0,0 +1,48 @@
/*
* Copyright (c) 2024, Gregory Bertilson <zaggy1024@gmail.com>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <LibMedia/CodecID.h>
extern "C" {
#include <libavcodec/avcodec.h>
}
namespace Media::FFmpeg {
static inline AVCodecID ffmpeg_codec_id_from_serenity_codec_id(CodecID codec)
{
switch (codec) {
case CodecID::VP8:
return AV_CODEC_ID_VP8;
case CodecID::VP9:
return AV_CODEC_ID_VP9;
case CodecID::H261:
return AV_CODEC_ID_H261;
case CodecID::MPEG1:
case CodecID::H262:
return AV_CODEC_ID_MPEG2VIDEO;
case CodecID::H263:
return AV_CODEC_ID_H263;
case CodecID::H264:
return AV_CODEC_ID_H264;
case CodecID::H265:
return AV_CODEC_ID_HEVC;
case CodecID::AV1:
return AV_CODEC_ID_AV1;
case CodecID::Theora:
return AV_CODEC_ID_THEORA;
case CodecID::Vorbis:
return AV_CODEC_ID_VORBIS;
case CodecID::Opus:
return AV_CODEC_ID_OPUS;
default:
return AV_CODEC_ID_NONE;
}
}
}

View file

@ -0,0 +1,216 @@
/*
* Copyright (c) 2024, Gregory Bertilson <zaggy1024@gmail.com>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <LibCore/System.h>
#include <LibMedia/VideoFrame.h>
#include "FFmpegHelpers.h"
#include "FFmpegVideoDecoder.h"
namespace Media::FFmpeg {
static AVPixelFormat negotiate_output_format(AVCodecContext*, AVPixelFormat const* formats)
{
while (*formats >= 0) {
switch (*formats) {
case AV_PIX_FMT_YUV420P:
case AV_PIX_FMT_YUV420P10:
case AV_PIX_FMT_YUV420P12:
case AV_PIX_FMT_YUV422P:
case AV_PIX_FMT_YUV422P10:
case AV_PIX_FMT_YUV422P12:
case AV_PIX_FMT_YUV444P:
case AV_PIX_FMT_YUV444P10:
case AV_PIX_FMT_YUV444P12:
return *formats;
default:
break;
}
formats++;
}
return AV_PIX_FMT_NONE;
}
DecoderErrorOr<NonnullOwnPtr<FFmpegVideoDecoder>> FFmpegVideoDecoder::try_create(CodecID codec_id)
{
AVCodecContext* codec_context = nullptr;
AVPacket* packet = nullptr;
AVFrame* frame = nullptr;
ArmedScopeGuard memory_guard {
[&] {
avcodec_free_context(&codec_context);
av_packet_free(&packet);
av_frame_free(&frame);
}
};
auto ff_codec_id = ffmpeg_codec_id_from_serenity_codec_id(codec_id);
auto const* codec = avcodec_find_decoder(ff_codec_id);
if (!codec)
return DecoderError::format(DecoderErrorCategory::NotImplemented, "Could not find FFmpeg decoder for codec {}", codec_id);
codec_context = avcodec_alloc_context3(codec);
if (!codec_context)
return DecoderError::format(DecoderErrorCategory::Memory, "Failed to allocate FFmpeg codec context for codec {}", codec_id);
codec_context->get_format = negotiate_output_format;
codec_context->thread_count = static_cast<int>(min(Core::System::hardware_concurrency(), 4));
if (avcodec_open2(codec_context, codec, nullptr) < 0)
return DecoderError::format(DecoderErrorCategory::Unknown, "Unknown error occurred when opening FFmpeg codec {}", codec_id);
packet = av_packet_alloc();
if (!packet)
return DecoderError::with_description(DecoderErrorCategory::Memory, "Failed to allocate FFmpeg packet"sv);
frame = av_frame_alloc();
if (!frame)
return DecoderError::with_description(DecoderErrorCategory::Memory, "Failed to allocate FFmpeg frame"sv);
memory_guard.disarm();
return DECODER_TRY_ALLOC(try_make<FFmpegVideoDecoder>(codec_context, packet, frame));
}
FFmpegVideoDecoder::FFmpegVideoDecoder(AVCodecContext* codec_context, AVPacket* packet, AVFrame* frame)
: m_codec_context(codec_context)
, m_packet(packet)
, m_frame(frame)
{
}
FFmpegVideoDecoder::~FFmpegVideoDecoder()
{
av_packet_free(&m_packet);
av_frame_free(&m_frame);
avcodec_free_context(&m_codec_context);
}
DecoderErrorOr<void> FFmpegVideoDecoder::receive_sample(ReadonlyBytes sample)
{
VERIFY(sample.size() < NumericLimits<int>::max());
m_packet->data = const_cast<u8*>(sample.data());
m_packet->size = static_cast<int>(sample.size());
auto result = avcodec_send_packet(m_codec_context, m_packet);
switch (result) {
case 0:
return {};
case AVERROR(EAGAIN):
return DecoderError::with_description(DecoderErrorCategory::NeedsMoreInput, "FFmpeg decoder cannot decode any more data until frames have been retrieved"sv);
case AVERROR_EOF:
return DecoderError::with_description(DecoderErrorCategory::EndOfStream, "FFmpeg decoder has been flushed"sv);
case AVERROR(EINVAL):
return DecoderError::with_description(DecoderErrorCategory::Invalid, "FFmpeg codec has not been opened"sv);
case AVERROR(ENOMEM):
return DecoderError::with_description(DecoderErrorCategory::Memory, "FFmpeg codec ran out of internal memory"sv);
default:
return DecoderError::with_description(DecoderErrorCategory::Corrupted, "FFmpeg codec reports that the data is corrupted"sv);
}
}
DecoderErrorOr<NonnullOwnPtr<VideoFrame>> FFmpegVideoDecoder::get_decoded_frame()
{
auto result = avcodec_receive_frame(m_codec_context, m_frame);
switch (result) {
case 0: {
auto color_primaries = static_cast<ColorPrimaries>(m_frame->color_primaries);
auto transfer_characteristics = static_cast<TransferCharacteristics>(m_frame->color_trc);
auto matrix_coefficients = static_cast<MatrixCoefficients>(m_frame->colorspace);
auto color_range = [&] {
switch (m_frame->color_range) {
case AVColorRange::AVCOL_RANGE_MPEG:
return VideoFullRangeFlag::Studio;
case AVColorRange::AVCOL_RANGE_JPEG:
return VideoFullRangeFlag::Full;
default:
return VideoFullRangeFlag::Unspecified;
}
}();
auto cicp = CodingIndependentCodePoints { color_primaries, transfer_characteristics, matrix_coefficients, color_range };
size_t bit_depth = [&] {
switch (m_frame->format) {
case AV_PIX_FMT_YUV420P:
case AV_PIX_FMT_YUV422P:
case AV_PIX_FMT_YUV444P:
return 8;
case AV_PIX_FMT_YUV420P10:
case AV_PIX_FMT_YUV422P10:
case AV_PIX_FMT_YUV444P10:
return 10;
case AV_PIX_FMT_YUV420P12:
case AV_PIX_FMT_YUV422P12:
case AV_PIX_FMT_YUV444P12:
return 12;
}
VERIFY_NOT_REACHED();
}();
size_t component_size = (bit_depth + 7) / 8;
auto subsampling = [&]() -> Subsampling {
switch (m_frame->format) {
case AV_PIX_FMT_YUV420P:
case AV_PIX_FMT_YUV420P10:
case AV_PIX_FMT_YUV420P12:
return { true, true };
case AV_PIX_FMT_YUV422P:
case AV_PIX_FMT_YUV422P10:
case AV_PIX_FMT_YUV422P12:
return { true, false };
case AV_PIX_FMT_YUV444P:
case AV_PIX_FMT_YUV444P10:
case AV_PIX_FMT_YUV444P12:
return { false, false };
default:
VERIFY_NOT_REACHED();
}
}();
auto size = Gfx::Size<u32> { m_frame->width, m_frame->height };
auto frame = DECODER_TRY_ALLOC(SubsampledYUVFrame::try_create(size, bit_depth, cicp, subsampling));
for (u32 plane = 0; plane < 3; plane++) {
VERIFY(m_frame->linesize[plane] != 0);
if (m_frame->linesize[plane] < 0)
return DecoderError::with_description(DecoderErrorCategory::NotImplemented, "Reversed scanlines are not supported"sv);
bool const use_subsampling = plane > 0;
auto plane_size = (use_subsampling ? subsampling.subsampled_size(size) : size).to_type<size_t>();
auto output_line_size = plane_size.width() * component_size;
VERIFY(output_line_size <= static_cast<size_t>(m_frame->linesize[plane]));
auto const* source = m_frame->data[plane];
VERIFY(source != nullptr);
auto* destination = frame->get_raw_plane_data(plane);
VERIFY(destination != nullptr);
for (size_t row = 0; row < plane_size.height(); row++) {
memcpy(destination, source, output_line_size);
source += m_frame->linesize[plane];
destination += output_line_size;
}
}
return frame;
}
case AVERROR(EAGAIN):
return DecoderError::with_description(DecoderErrorCategory::NeedsMoreInput, "FFmpeg decoder has no frames available, send more input"sv);
case AVERROR_EOF:
return DecoderError::with_description(DecoderErrorCategory::EndOfStream, "FFmpeg decoder has been flushed"sv);
case AVERROR(EINVAL):
return DecoderError::with_description(DecoderErrorCategory::Invalid, "FFmpeg codec has not been opened"sv);
default:
return DecoderError::format(DecoderErrorCategory::Unknown, "FFmpeg codec encountered an unexpected error retreiving frames with code {:x}", result);
}
}
}

View file

@ -0,0 +1,33 @@
/*
* Copyright (c) 2024, Gregory Bertilson <zaggy1024@gmail.com>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <LibMedia/CodecID.h>
#include <LibMedia/VideoDecoder.h>
#include "FFmpegForward.h"
namespace Media::FFmpeg {
class FFmpegVideoDecoder final : public VideoDecoder {
public:
static DecoderErrorOr<NonnullOwnPtr<FFmpegVideoDecoder>> try_create(CodecID);
FFmpegVideoDecoder(AVCodecContext* codec_context, AVPacket* packet, AVFrame* frame);
~FFmpegVideoDecoder();
DecoderErrorOr<void> receive_sample(ReadonlyBytes sample) override;
DecoderErrorOr<NonnullOwnPtr<VideoFrame>> get_decoded_frame() override;
private:
DecoderErrorOr<void> decode_single_sample(Duration timestamp, u8* data, int size);
AVCodecContext* m_codec_context;
AVPacket* m_packet;
AVFrame* m_frame;
};
}

View file

@ -7,7 +7,8 @@
#include <AK/Format.h>
#include <LibCore/Timer.h>
#include <LibMedia/Containers/Matroska/MatroskaDemuxer.h>
#include <LibMedia/Video/VP9/Decoder.h>
#include <LibMedia/FFmpeg/FFmpegVideoDecoder.h>
#include <LibMedia/VideoFrame.h>
#include "PlaybackManager.h"
@ -700,18 +701,9 @@ DecoderErrorOr<NonnullOwnPtr<PlaybackManager>> PlaybackManager::create(NonnullOw
dbgln_if(PLAYBACK_MANAGER_DEBUG, "Selecting video track number {}", track.identifier());
auto codec_id = TRY(demuxer->get_codec_id_for_track(track));
OwnPtr<VideoDecoder> decoder;
switch (codec_id) {
case CodecID::VP9:
decoder = DECODER_TRY_ALLOC(try_make<Video::VP9::Decoder>());
break;
default:
return DecoderError::format(DecoderErrorCategory::Invalid, "Unsupported codec: {}", codec_id);
}
auto decoder_non_null = decoder.release_nonnull();
NonnullOwnPtr<VideoDecoder> decoder = TRY(FFmpeg::FFmpegVideoDecoder::try_create(codec_id));
auto frame_queue = DECODER_TRY_ALLOC(VideoFrameQueue::create());
auto playback_manager = DECODER_TRY_ALLOC(try_make<PlaybackManager>(demuxer, track, move(decoder_non_null), move(frame_queue)));
auto playback_manager = DECODER_TRY_ALLOC(try_make<PlaybackManager>(demuxer, track, move(decoder), move(frame_queue)));
playback_manager->m_state_update_timer = Core::Timer::create_single_shot(0, [&self = *playback_manager] { self.timer_callback(); });