Ladybird+LibWeb+WebContent: Create a platform plugin for playing audio

This creates (and installs upon WebContent startup) a platform plugin to
play audio data.

On Serenity, we use AudioServer to play audio over IPC. Unfortunately,
AudioServer is currently coupled with Serenity's audio devices, and thus
cannot be used in Ladybird on Lagom. Instead, we use a Qt audio device
to play the audio, which requires the Qt multimedia package.

While we use Qt to play the audio, note that we can still use LibAudio
to decode the audio data and retrieve samples - we simply send Qt the
raw PCM signals.
This commit is contained in:
Timothy Flynn 2023-06-12 13:44:10 -04:00 committed by Andreas Kling
parent ee48d7514f
commit a34e369252
Notes: sideshowbarker 2024-07-17 03:05:16 +09:00
15 changed files with 317 additions and 9 deletions

View file

@ -7,7 +7,7 @@ Qt6 development packages and a C++20 capable compiler are required. gcc-12 or cl
On Debian/Ubuntu required packages include, but are not limited to: On Debian/Ubuntu required packages include, but are not limited to:
``` ```
sudo apt install build-essential cmake libgl1-mesa-dev ninja-build qt6-base-dev libqt6svg6-dev qt6-tools-dev-tools sudo apt install build-essential cmake libgl1-mesa-dev ninja-build qt6-base-dev libqt6svg6-dev qt6-tools-dev-tools qt6-multimedia-dev
``` ```
For Ubuntu 20.04 and above, ensure that the Qt6 Wayland packages are available: For Ubuntu 20.04 and above, ensure that the Qt6 Wayland packages are available:
@ -19,12 +19,12 @@ sudo apt install qt6-wayland
On Arch Linux/Manjaro: On Arch Linux/Manjaro:
``` ```
sudo pacman -S --needed base-devel cmake libgl ninja qt6-base qt6-svg qt6-tools qt6-wayland sudo pacman -S --needed base-devel cmake libgl ninja qt6-base qt6-svg qt6-tools qt6-wayland qt6-multimedia
``` ```
On Fedora or derivatives: On Fedora or derivatives:
``` ```
sudo dnf install cmake libglvnd-devel ninja-build qt6-qtbase-devel qt6-qtsvg-devel qt6-qttools-devel qt6-qtwayland-devel sudo dnf install cmake libglvnd-devel ninja-build qt6-qtbase-devel qt6-qtsvg-devel qt6-qttools-devel qt6-qtwayland-devel qt6-qtmultimedia-devel
``` ```
On openSUSE: On openSUSE:

View file

@ -0,0 +1,89 @@
/*
* Copyright (c) 2023, Tim Flynn <trflynn89@serenityos.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include "AudioCodecPluginLadybird.h"
#include <AK/Endian.h>
#include <AK/MemoryStream.h>
#include <LibAudio/Sample.h>
#include <QAudioFormat>
#include <QAudioSink>
#include <QBuffer>
#include <QMediaDevices>
namespace Ladybird {
ErrorOr<NonnullOwnPtr<AudioCodecPluginLadybird>> AudioCodecPluginLadybird::create()
{
auto devices = TRY(adopt_nonnull_own_or_enomem(new (nothrow) QMediaDevices()));
auto const& device_info = devices->defaultAudioOutput();
auto format = device_info.preferredFormat();
format.setSampleFormat(QAudioFormat::Int16);
format.setChannelCount(2);
if (!device_info.isFormatSupported(format))
return Error::from_string_literal("Audio device format not supported");
auto audio_output = TRY(adopt_nonnull_own_or_enomem(new (nothrow) QAudioSink(device_info, format)));
return adopt_nonnull_own_or_enomem(new (nothrow) AudioCodecPluginLadybird(move(devices), move(audio_output)));
}
AudioCodecPluginLadybird::AudioCodecPluginLadybird(NonnullOwnPtr<QMediaDevices> devices, NonnullOwnPtr<QAudioSink> audio_output)
: m_devices(move(devices))
, m_audio_output(move(audio_output))
, m_io_device(m_audio_output->start())
{
}
AudioCodecPluginLadybird::~AudioCodecPluginLadybird() = default;
size_t AudioCodecPluginLadybird::device_sample_rate()
{
return m_audio_output->format().sampleRate();
}
void AudioCodecPluginLadybird::enqueue_samples(FixedArray<Audio::Sample> samples)
{
QByteArray buffer;
buffer.resize(samples.size() * 2 * sizeof(u16));
FixedMemoryStream stream { Bytes { buffer.data(), static_cast<size_t>(buffer.size()) } };
for (auto& sample : samples) {
LittleEndian<i16> pcm;
pcm = static_cast<i16>(sample.left * NumericLimits<i16>::max());
MUST(stream.write_value(pcm));
pcm = static_cast<i16>(sample.right * NumericLimits<i16>::max());
MUST(stream.write_value(pcm));
}
m_io_device->write(buffer.data(), buffer.size());
}
size_t AudioCodecPluginLadybird::remaining_samples() const
{
return 0;
}
void AudioCodecPluginLadybird::resume_playback()
{
m_audio_output->resume();
}
void AudioCodecPluginLadybird::pause_playback()
{
m_audio_output->suspend();
}
void AudioCodecPluginLadybird::playback_ended()
{
m_audio_output->suspend();
}
}

View file

@ -0,0 +1,42 @@
/*
* Copyright (c) 2023, Tim Flynn <trflynn89@serenityos.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <AK/Error.h>
#include <AK/NonnullOwnPtr.h>
#include <LibAudio/Forward.h>
#include <LibWeb/Platform/AudioCodecPlugin.h>
class QAudioSink;
class QIODevice;
class QMediaDevices;
namespace Ladybird {
class AudioCodecPluginLadybird final : public Web::Platform::AudioCodecPlugin {
public:
static ErrorOr<NonnullOwnPtr<AudioCodecPluginLadybird>> create();
virtual ~AudioCodecPluginLadybird() override;
virtual size_t device_sample_rate() override;
virtual void enqueue_samples(FixedArray<Audio::Sample>) override;
virtual size_t remaining_samples() const override;
virtual void resume_playback() override;
virtual void pause_playback() override;
virtual void playback_ended() override;
private:
AudioCodecPluginLadybird(NonnullOwnPtr<QMediaDevices>, NonnullOwnPtr<QAudioSink>);
NonnullOwnPtr<QMediaDevices> m_devices;
NonnullOwnPtr<QAudioSink> m_audio_output;
QIODevice* m_io_device { nullptr };
};
}

View file

@ -73,7 +73,7 @@ add_compile_options(-Wno-user-defined-literals)
set(CMAKE_AUTOMOC ON) set(CMAKE_AUTOMOC ON)
set(CMAKE_AUTORCC ON) set(CMAKE_AUTORCC ON)
set(CMAKE_AUTOUIC ON) set(CMAKE_AUTOUIC ON)
find_package(Qt6 REQUIRED COMPONENTS Core Widgets Network Svg) find_package(Qt6 REQUIRED COMPONENTS Core Widgets Network Svg Multimedia)
set(BROWSER_SOURCE_DIR ${SERENITY_SOURCE_DIR}/Userland/Applications/Browser/) set(BROWSER_SOURCE_DIR ${SERENITY_SOURCE_DIR}/Userland/Applications/Browser/)

View file

@ -6,6 +6,7 @@ set(WEBCONTENT_SOURCES
${WEBCONTENT_SOURCE_DIR}/PageHost.cpp ${WEBCONTENT_SOURCE_DIR}/PageHost.cpp
${WEBCONTENT_SOURCE_DIR}/WebContentConsoleClient.cpp ${WEBCONTENT_SOURCE_DIR}/WebContentConsoleClient.cpp
${WEBCONTENT_SOURCE_DIR}/WebDriverConnection.cpp ${WEBCONTENT_SOURCE_DIR}/WebDriverConnection.cpp
../AudioCodecPluginLadybird.cpp
../EventLoopImplementationQt.cpp ../EventLoopImplementationQt.cpp
../FontPluginQt.cpp ../FontPluginQt.cpp
../ImageCodecPluginLadybird.cpp ../ImageCodecPluginLadybird.cpp
@ -25,4 +26,4 @@ qt_add_executable(WebContent ${WEBCONTENT_SOURCES})
target_include_directories(WebContent PRIVATE ${SERENITY_SOURCE_DIR}/Userland/Services/) target_include_directories(WebContent PRIVATE ${SERENITY_SOURCE_DIR}/Userland/Services/)
target_include_directories(WebContent PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/..) target_include_directories(WebContent PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/..)
target_link_libraries(WebContent PRIVATE Qt::Core Qt::Gui Qt::Network LibCore LibFileSystem LibGfx LibIPC LibJS LibMain LibWeb LibWebSocket) target_link_libraries(WebContent PRIVATE Qt::Core Qt::Gui Qt::Network Qt::Multimedia LibAudio LibCore LibFileSystem LibGfx LibIPC LibJS LibMain LibWeb LibWebSocket)

View file

@ -4,6 +4,7 @@
* SPDX-License-Identifier: BSD-2-Clause * SPDX-License-Identifier: BSD-2-Clause
*/ */
#include "../AudioCodecPluginLadybird.h"
#include "../EventLoopImplementationQt.h" #include "../EventLoopImplementationQt.h"
#include "../FontPluginQt.h" #include "../FontPluginQt.h"
#include "../ImageCodecPluginLadybird.h" #include "../ImageCodecPluginLadybird.h"
@ -57,6 +58,10 @@ ErrorOr<int> serenity_main(Main::Arguments arguments)
Web::Platform::EventLoopPlugin::install(*new Web::Platform::EventLoopPluginSerenity); Web::Platform::EventLoopPlugin::install(*new Web::Platform::EventLoopPluginSerenity);
Web::Platform::ImageCodecPlugin::install(*new Ladybird::ImageCodecPluginLadybird); Web::Platform::ImageCodecPlugin::install(*new Ladybird::ImageCodecPluginLadybird);
Web::Platform::AudioCodecPlugin::install_creation_hook([] {
return Ladybird::AudioCodecPluginLadybird::create();
});
Web::ResourceLoader::initialize(RequestManagerQt::create()); Web::ResourceLoader::initialize(RequestManagerQt::create());
Web::WebSockets::WebSocketClientManager::initialize(Ladybird::WebSocketClientManagerLadybird::create()); Web::WebSockets::WebSocketClientManager::initialize(Ladybird::WebSocketClientManagerLadybird::create());

View file

@ -21,7 +21,7 @@ steps:
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
sudo add-apt-repository 'deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-15 main' sudo add-apt-repository 'deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-15 main'
sudo apt-get update sudo apt-get update
sudo apt-get install ccache gcc-12 g++-12 clang-15 libstdc++-12-dev ninja-build unzip qt6-base-dev qt6-tools-dev-tools libqt6svg6-dev libgl1-mesa-dev sudo apt-get install ccache gcc-12 g++-12 clang-15 libstdc++-12-dev ninja-build unzip qt6-base-dev qt6-tools-dev-tools libqt6svg6-dev qt6-multimedia-dev libgl1-mesa-dev
sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-15 100 sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-15 100
sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-15 100 sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-15 100

View file

@ -473,6 +473,7 @@ set(SOURCES
PerformanceTimeline/EntryTypes.cpp PerformanceTimeline/EntryTypes.cpp
PerformanceTimeline/PerformanceEntry.cpp PerformanceTimeline/PerformanceEntry.cpp
PermissionsPolicy/AutoplayAllowlist.cpp PermissionsPolicy/AutoplayAllowlist.cpp
Platform/AudioCodecPlugin.cpp
Platform/EventLoopPlugin.cpp Platform/EventLoopPlugin.cpp
Platform/EventLoopPluginSerenity.cpp Platform/EventLoopPluginSerenity.cpp
Platform/FontPlugin.cpp Platform/FontPlugin.cpp
@ -595,7 +596,7 @@ set(GENERATED_SOURCES
serenity_lib(LibWeb web) serenity_lib(LibWeb web)
# NOTE: We link with LibSoftGPU here instead of lazy loading it via dlopen() so that we do not have to unveil the library and pledge prot_exec. # NOTE: We link with LibSoftGPU here instead of lazy loading it via dlopen() so that we do not have to unveil the library and pledge prot_exec.
target_link_libraries(LibWeb PRIVATE LibCore LibCrypto LibJS LibMarkdown LibHTTP LibGemini LibGL LibGUI LibGfx LibIPC LibLocale LibRegex LibSoftGPU LibSyntax LibTextCodec LibUnicode LibVideo LibWasm LibXML LibIDL) target_link_libraries(LibWeb PRIVATE LibCore LibCrypto LibJS LibMarkdown LibHTTP LibGemini LibGL LibGUI LibGfx LibIPC LibLocale LibRegex LibSoftGPU LibSyntax LibTextCodec LibUnicode LibAudio LibVideo LibWasm LibXML LibIDL)
link_with_locale_data(LibWeb) link_with_locale_data(LibWeb)
generate_js_bindings(LibWeb) generate_js_bindings(LibWeb)

View file

@ -509,6 +509,7 @@ class AutoplayAllowlist;
} }
namespace Web::Platform { namespace Web::Platform {
class AudioCodecPlugin;
class Timer; class Timer;
} }

View file

@ -0,0 +1,28 @@
/*
* Copyright (c) 2023, Tim Flynn <trflynn89@serenityos.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <LibWeb/Platform/AudioCodecPlugin.h>
namespace Web::Platform {
static Function<ErrorOr<NonnullOwnPtr<AudioCodecPlugin>>()> s_creation_hook;
AudioCodecPlugin::AudioCodecPlugin() = default;
AudioCodecPlugin::~AudioCodecPlugin() = default;
void AudioCodecPlugin::install_creation_hook(Function<ErrorOr<NonnullOwnPtr<AudioCodecPlugin>>()> creation_hook)
{
VERIFY(!s_creation_hook);
s_creation_hook = move(creation_hook);
}
ErrorOr<NonnullOwnPtr<AudioCodecPlugin>> AudioCodecPlugin::create()
{
VERIFY(s_creation_hook);
return s_creation_hook();
}
}

View file

@ -0,0 +1,37 @@
/*
* Copyright (c) 2023, Tim Flynn <trflynn89@serenityos.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <AK/FixedArray.h>
#include <AK/Function.h>
#include <AK/NonnullOwnPtr.h>
#include <AK/Optional.h>
#include <LibAudio/Forward.h>
namespace Web::Platform {
class AudioCodecPlugin {
public:
static void install_creation_hook(Function<ErrorOr<NonnullOwnPtr<AudioCodecPlugin>>()>);
static ErrorOr<NonnullOwnPtr<AudioCodecPlugin>> create();
virtual ~AudioCodecPlugin();
virtual size_t device_sample_rate() = 0;
virtual void enqueue_samples(FixedArray<Audio::Sample>) = 0;
virtual size_t remaining_samples() const = 0;
virtual void resume_playback() = 0;
virtual void pause_playback() = 0;
virtual void playback_ended() = 0;
protected:
AudioCodecPlugin();
};
}

View file

@ -0,0 +1,59 @@
/*
* Copyright (c) 2023, Tim Flynn <trflynn89@serenityos.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <LibAudio/ConnectionToServer.h>
#include <WebContent/AudioCodecPluginSerenity.h>
namespace WebContent {
ErrorOr<NonnullOwnPtr<AudioCodecPluginSerenity>> AudioCodecPluginSerenity::create()
{
auto connection = TRY(Audio::ConnectionToServer::try_create());
return adopt_nonnull_own_or_enomem(new (nothrow) AudioCodecPluginSerenity(move(connection)));
}
AudioCodecPluginSerenity::AudioCodecPluginSerenity(NonnullRefPtr<Audio::ConnectionToServer> connection)
: m_connection(move(connection))
{
}
AudioCodecPluginSerenity::~AudioCodecPluginSerenity() = default;
size_t AudioCodecPluginSerenity::device_sample_rate()
{
if (!m_device_sample_rate.has_value())
m_device_sample_rate = m_connection->get_sample_rate();
return *m_device_sample_rate;
}
void AudioCodecPluginSerenity::enqueue_samples(FixedArray<Audio::Sample> samples)
{
m_connection->async_enqueue(move(samples)).release_value_but_fixme_should_propagate_errors();
}
size_t AudioCodecPluginSerenity::remaining_samples() const
{
return m_connection->remaining_samples();
}
void AudioCodecPluginSerenity::resume_playback()
{
m_connection->async_start_playback();
}
void AudioCodecPluginSerenity::pause_playback()
{
m_connection->async_start_playback();
}
void AudioCodecPluginSerenity::playback_ended()
{
m_connection->async_pause_playback();
m_connection->clear_client_buffer();
m_connection->async_clear_buffer();
}
}

View file

@ -0,0 +1,38 @@
/*
* Copyright (c) 2023, Tim Flynn <trflynn89@serenityos.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <AK/Error.h>
#include <AK/NonnullRefPtr.h>
#include <AK/Optional.h>
#include <LibAudio/Forward.h>
#include <LibWeb/Platform/AudioCodecPlugin.h>
namespace WebContent {
class AudioCodecPluginSerenity final : public Web::Platform::AudioCodecPlugin {
public:
static ErrorOr<NonnullOwnPtr<AudioCodecPluginSerenity>> create();
virtual ~AudioCodecPluginSerenity() override;
virtual size_t device_sample_rate() override;
virtual void enqueue_samples(FixedArray<Audio::Sample>) override;
virtual size_t remaining_samples() const override;
virtual void resume_playback() override;
virtual void pause_playback() override;
virtual void playback_ended() override;
private:
explicit AudioCodecPluginSerenity(NonnullRefPtr<Audio::ConnectionToServer>);
NonnullRefPtr<Audio::ConnectionToServer> m_connection;
Optional<size_t> m_device_sample_rate;
};
}

View file

@ -11,6 +11,7 @@ compile_ipc(WebDriverClient.ipc WebDriverClientEndpoint.h)
compile_ipc(WebDriverServer.ipc WebDriverServerEndpoint.h) compile_ipc(WebDriverServer.ipc WebDriverServerEndpoint.h)
set(SOURCES set(SOURCES
AudioCodecPluginSerenity.cpp
ConnectionFromClient.cpp ConnectionFromClient.cpp
ConsoleGlobalEnvironmentExtensions.cpp ConsoleGlobalEnvironmentExtensions.cpp
ImageCodecPluginSerenity.cpp ImageCodecPluginSerenity.cpp
@ -28,5 +29,5 @@ set(GENERATED_SOURCES
) )
serenity_bin(WebContent) serenity_bin(WebContent)
target_link_libraries(WebContent PRIVATE LibCore LibFileSystem LibIPC LibGfx LibImageDecoderClient LibJS LibWebView LibWeb LibLocale LibMain) target_link_libraries(WebContent PRIVATE LibCore LibFileSystem LibIPC LibGfx LibAudio LibImageDecoderClient LibJS LibWebView LibWeb LibLocale LibMain)
link_with_locale_data(WebContent) link_with_locale_data(WebContent)

View file

@ -4,6 +4,7 @@
* SPDX-License-Identifier: BSD-2-Clause * SPDX-License-Identifier: BSD-2-Clause
*/ */
#include "AudioCodecPluginSerenity.h"
#include "ImageCodecPluginSerenity.h" #include "ImageCodecPluginSerenity.h"
#include <LibCore/EventLoop.h> #include <LibCore/EventLoop.h>
#include <LibCore/LocalServer.h> #include <LibCore/LocalServer.h>
@ -25,7 +26,7 @@
ErrorOr<int> serenity_main(Main::Arguments) ErrorOr<int> serenity_main(Main::Arguments)
{ {
Core::EventLoop event_loop; Core::EventLoop event_loop;
TRY(Core::System::pledge("stdio recvfd sendfd accept unix rpath thread")); TRY(Core::System::pledge("stdio recvfd sendfd accept unix rpath thread proc"));
// This must be first; we can't check if /tmp/webdriver exists once we've unveiled other paths. // This must be first; we can't check if /tmp/webdriver exists once we've unveiled other paths.
auto webdriver_socket_path = DeprecatedString::formatted("{}/webdriver", TRY(Core::StandardPaths::runtime_directory())); auto webdriver_socket_path = DeprecatedString::formatted("{}/webdriver", TRY(Core::StandardPaths::runtime_directory()));
@ -35,6 +36,7 @@ ErrorOr<int> serenity_main(Main::Arguments)
TRY(Core::System::unveil("/res", "r")); TRY(Core::System::unveil("/res", "r"));
TRY(Core::System::unveil("/etc/timezone", "r")); TRY(Core::System::unveil("/etc/timezone", "r"));
TRY(Core::System::unveil("/usr/lib", "r")); TRY(Core::System::unveil("/usr/lib", "r"));
TRY(Core::System::unveil("/tmp/session/%sid/portal/audio", "rw"));
TRY(Core::System::unveil("/tmp/session/%sid/portal/request", "rw")); TRY(Core::System::unveil("/tmp/session/%sid/portal/request", "rw"));
TRY(Core::System::unveil("/tmp/session/%sid/portal/image", "rw")); TRY(Core::System::unveil("/tmp/session/%sid/portal/image", "rw"));
TRY(Core::System::unveil("/tmp/session/%sid/portal/websocket", "rw")); TRY(Core::System::unveil("/tmp/session/%sid/portal/websocket", "rw"));
@ -44,6 +46,10 @@ ErrorOr<int> serenity_main(Main::Arguments)
Web::Platform::ImageCodecPlugin::install(*new WebContent::ImageCodecPluginSerenity); Web::Platform::ImageCodecPlugin::install(*new WebContent::ImageCodecPluginSerenity);
Web::Platform::FontPlugin::install(*new Web::Platform::FontPluginSerenity); Web::Platform::FontPlugin::install(*new Web::Platform::FontPluginSerenity);
Web::Platform::AudioCodecPlugin::install_creation_hook([] {
return WebContent::AudioCodecPluginSerenity::create();
});
Web::WebSockets::WebSocketClientManager::initialize(TRY(WebView::WebSocketClientManagerAdapter::try_create())); Web::WebSockets::WebSocketClientManager::initialize(TRY(WebView::WebSocketClientManagerAdapter::try_create()));
Web::ResourceLoader::initialize(TRY(WebView::RequestServerAdapter::try_create())); Web::ResourceLoader::initialize(TRY(WebView::RequestServerAdapter::try_create()));
TRY(Web::Bindings::initialize_main_thread_vm()); TRY(Web::Bindings::initialize_main_thread_vm());