ladybird/Tests/LibWeb/test-web.cpp
Ali Mohammad Pur a91a49337c LibCore+Everywhere: Move OpenMode out of IODevice
...and make it an enum class so people don't omit "OpenMode".
2021-05-12 11:00:45 +01:00

668 lines
23 KiB
C++
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/*
* Copyright (c) 2020-2021, the SerenityOS developers.
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <AK/Function.h>
#include <AK/JsonObject.h>
#include <AK/JsonValue.h>
#include <AK/QuickSort.h>
#include <AK/URL.h>
#include <LibCore/ArgsParser.h>
#include <LibCore/DirIterator.h>
#include <LibCore/File.h>
#include <LibGUI/Application.h>
#include <LibGUI/BoxLayout.h>
#include <LibGUI/Widget.h>
#include <LibGUI/Window.h>
#include <LibJS/Interpreter.h>
#include <LibJS/Lexer.h>
#include <LibJS/Parser.h>
#include <LibJS/Runtime/Array.h>
#include <LibJS/Runtime/JSONObject.h>
#include <LibTest/Results.h>
#include <LibWeb/HTML/Parser/HTMLDocumentParser.h>
#include <LibWeb/InProcessWebView.h>
#include <LibWeb/Loader/ResourceLoader.h>
#include <signal.h>
#include <sys/time.h>
#define TOP_LEVEL_TEST_NAME "__$$TOP_LEVEL$$__"
struct ParserError {
JS::Parser::Error error;
String hint;
};
struct JSFileResult {
String name;
Optional<ParserError> error {};
double time_taken { 0 };
// A failed test takes precedence over a skipped test, which both have
// precedence over a passed test
Test::Result most_severe_test_result { Test::Result::Pass };
Vector<Test::Suite> suites {};
Vector<String> logged_messages {};
};
Function<void(const URL&)> g_on_page_change;
class TestRunnerObject final : public JS::Object {
JS_OBJECT(TestRunnerObject, JS::Object);
public:
explicit TestRunnerObject(JS::GlobalObject&);
virtual void initialize(JS::GlobalObject&) override;
virtual ~TestRunnerObject() override;
private:
JS_DECLARE_NATIVE_FUNCTION(change_page);
};
TestRunnerObject::TestRunnerObject(JS::GlobalObject& global_object)
: Object(*global_object.object_prototype())
{
}
void TestRunnerObject::initialize(JS::GlobalObject& global_object)
{
Object::initialize(global_object);
define_native_function("changePage", change_page, 1);
}
TestRunnerObject::~TestRunnerObject()
{
}
JS_DEFINE_NATIVE_FUNCTION(TestRunnerObject::change_page)
{
auto url = vm.argument(0).to_string(global_object);
if (vm.exception())
return {};
if (g_on_page_change)
g_on_page_change(url);
return JS::js_undefined();
}
class TestRunner {
public:
TestRunner(String web_test_root, String js_test_root, Web::InProcessWebView& page_view, bool print_times)
: m_web_test_root(move(web_test_root))
, m_js_test_root(move(js_test_root))
, m_print_times(print_times)
, m_page_view(page_view)
{
}
void run();
private:
JSFileResult run_file_test(const String& test_path);
void print_file_result(const JSFileResult& file_result) const;
void print_test_results() const;
String m_web_test_root;
String m_js_test_root;
bool m_print_times;
double m_total_elapsed_time_in_ms { 0 };
Test::Counts m_counts;
RefPtr<Web::InProcessWebView> m_page_view;
RefPtr<JS::Program> m_js_test_common;
RefPtr<JS::Program> m_web_test_common;
};
static void cleanup_and_exit()
{
// Clear the taskbar progress.
#ifdef __serenity__
fprintf(stderr, "\033]9;-1;\033\\");
#endif
exit(1);
}
#if 0
static void handle_sigabrt(int)
{
dbgln("test-web: SIGABRT received, cleaning up.");
cleanup_and_exit();
}
#endif
static double get_time_in_ms()
{
struct timeval tv1;
auto return_code = gettimeofday(&tv1, nullptr);
VERIFY(return_code >= 0);
return static_cast<double>(tv1.tv_sec) * 1000.0 + static_cast<double>(tv1.tv_usec) / 1000.0;
}
template<typename Callback>
void iterate_directory_recursively(const String& directory_path, Callback callback)
{
Core::DirIterator directory_iterator(directory_path, Core::DirIterator::Flags::SkipDots);
while (directory_iterator.has_next()) {
auto file_path = directory_iterator.next_full_path();
if (Core::File::is_directory(file_path)) {
iterate_directory_recursively(file_path, callback);
} else {
callback(move(file_path));
}
}
}
static Vector<String> get_test_paths(const String& test_root)
{
Vector<String> paths;
iterate_directory_recursively(test_root, [&](const String& file_path) {
if (!file_path.ends_with("test-common.js") && !file_path.ends_with(".html") && !file_path.ends_with(".ts"))
paths.append(file_path);
});
quick_sort(paths);
return paths;
}
void TestRunner::run()
{
size_t progress_counter = 0;
auto test_paths = get_test_paths(m_web_test_root);
g_on_page_change = [this](auto& page_to_load) {
if (!page_to_load.is_valid()) {
printf("Invalid page URL (%s) on page change", page_to_load.to_string().characters());
cleanup_and_exit();
}
VERIFY(m_page_view->document());
// We want to keep the same document since the interpreter is tied to the document,
// and we don't want to lose the test state. So, we just clear the document and
// give a new parser the existing document to work on.
m_page_view->document()->remove_all_children();
Web::LoadRequest request;
request.set_url(page_to_load);
Web::ResourceLoader::the().load_sync(
request,
[&](auto data, auto&, auto) {
Web::HTML::HTMLDocumentParser parser(*m_page_view->document(), data, "utf-8");
parser.run(page_to_load);
},
[page_to_load](auto& error, auto) {
printf("Failed to load test page: %s (%s)", page_to_load.to_string().characters(), error.characters());
cleanup_and_exit();
});
};
for (auto& path : test_paths) {
++progress_counter;
print_file_result(run_file_test(path));
#ifdef __serenity__
fprintf(stderr, "\033]9;%zu;%zu;\033\\", progress_counter, test_paths.size());
#endif
}
#ifdef __serenity__
fprintf(stderr, "\033]9;-1;\033\\");
#endif
print_test_results();
}
static Result<NonnullRefPtr<JS::Program>, ParserError> parse_file(const String& file_path)
{
auto file = Core::File::construct(file_path);
auto result = file->open(Core::OpenMode::ReadOnly);
if (!result) {
printf("Failed to open the following file: \"%s\"\n", file_path.characters());
cleanup_and_exit();
}
auto contents = file->read_all();
String test_file_string(reinterpret_cast<const char*>(contents.data()), contents.size());
file->close();
auto parser = JS::Parser(JS::Lexer(test_file_string));
auto program = parser.parse_program();
if (parser.has_errors()) {
auto error = parser.errors()[0];
return Result<NonnullRefPtr<JS::Program>, ParserError>(ParserError { error, error.source_location_hint(test_file_string) });
}
return Result<NonnullRefPtr<JS::Program>, ParserError>(program);
}
static Optional<JsonValue> get_test_results(JS::Interpreter& interpreter)
{
auto result = interpreter.vm().get_variable("__TestResults__", interpreter.global_object());
auto json_string = JS::JSONObject::stringify_impl(interpreter.global_object(), result, JS::js_undefined(), JS::js_undefined());
auto json = JsonValue::from_string(json_string);
if (!json.has_value())
return {};
return json.value();
}
JSFileResult TestRunner::run_file_test(const String& test_path)
{
double start_time = get_time_in_ms();
VERIFY(m_page_view->document());
auto& old_interpreter = m_page_view->document()->interpreter();
if (!m_js_test_common) {
auto result = parse_file(String::formatted("{}/test-common.js", m_js_test_root));
if (result.is_error()) {
printf("Unable to parse %s/test-common.js\n", m_js_test_root.characters());
printf("%s\n", result.error().error.to_string().characters());
printf("%s\n", result.error().hint.characters());
cleanup_and_exit();
}
m_js_test_common = result.value();
}
if (!m_web_test_common) {
auto result = parse_file(String::formatted("{}/test-common.js", m_web_test_root));
if (result.is_error()) {
printf("Unable to parse %s/test-common.js\n", m_web_test_root.characters());
printf("%s\n", result.error().error.to_string().characters());
printf("%s\n", result.error().hint.characters());
cleanup_and_exit();
}
m_web_test_common = result.value();
}
auto file_program = parse_file(test_path);
if (file_program.is_error())
return { test_path, file_program.error() };
// Setup the test on the current page to get "__PageToLoad__".
old_interpreter.run(old_interpreter.global_object(), *m_web_test_common);
old_interpreter.run(old_interpreter.global_object(), *file_program.value());
auto page_to_load = URL(old_interpreter.vm().get_variable("__PageToLoad__", old_interpreter.global_object()).as_string().string());
if (!page_to_load.is_valid()) {
printf("Invalid page URL for %s", test_path.characters());
cleanup_and_exit();
}
JSFileResult file_result;
Web::LoadRequest request;
request.set_url(page_to_load);
Web::ResourceLoader::the().load_sync(
request,
[&](auto data, auto&, auto) {
// Create a new parser and immediately get its document to replace the old interpreter.
auto document = Web::DOM::Document::create();
Web::HTML::HTMLDocumentParser parser(document, data, "utf-8");
auto& new_interpreter = parser.document().interpreter();
// FIXME: This is a hack while we're refactoring Interpreter/VM stuff.
JS::VM::InterpreterExecutionScope scope(new_interpreter);
// Setup the test environment and call "__BeforeInitialPageLoad__"
new_interpreter.global_object().define_property(
"libweb_tester",
new_interpreter.heap().allocate<TestRunnerObject>(new_interpreter.global_object(), new_interpreter.global_object()),
JS::Attribute::Enumerable | JS::Attribute::Configurable);
new_interpreter.run(new_interpreter.global_object(), *m_js_test_common);
new_interpreter.run(new_interpreter.global_object(), *m_web_test_common);
new_interpreter.run(new_interpreter.global_object(), *file_program.value());
auto& before_initial_page_load = new_interpreter.vm().get_variable("__BeforeInitialPageLoad__", new_interpreter.global_object()).as_function();
[[maybe_unused]] auto rc_before = new_interpreter.vm().call(before_initial_page_load, JS::js_undefined());
if (new_interpreter.exception())
new_interpreter.vm().clear_exception();
// Now parse the HTML page.
parser.run(page_to_load);
m_page_view->set_document(&parser.document());
// Finally run the test by calling "__AfterInitialPageLoad__"
auto& after_initial_page_load = new_interpreter.vm().get_variable("__AfterInitialPageLoad__", new_interpreter.global_object()).as_function();
[[maybe_unused]] auto rc_after = new_interpreter.vm().call(after_initial_page_load, JS::js_undefined());
if (new_interpreter.exception())
new_interpreter.vm().clear_exception();
auto test_json = get_test_results(new_interpreter);
if (!test_json.has_value()) {
printf("Received malformed JSON from test \"%s\"\n", test_path.characters());
cleanup_and_exit();
}
file_result = { test_path.substring(m_web_test_root.length() + 1, test_path.length() - m_web_test_root.length() - 1) };
// Collect logged messages
auto& arr = new_interpreter.vm().get_variable("__UserOutput__", new_interpreter.global_object()).as_array();
for (auto& entry : arr.indexed_properties()) {
auto message = entry.value_and_attributes(&new_interpreter.global_object()).value;
file_result.logged_messages.append(message.to_string_without_side_effects());
}
test_json.value().as_object().for_each_member([&](const String& suite_name, const JsonValue& suite_value) {
Test::Suite suite { suite_name };
VERIFY(suite_value.is_object());
suite_value.as_object().for_each_member([&](const String& test_name, const JsonValue& test_value) {
Test::Case test { test_name, Test::Result::Fail, "" };
VERIFY(test_value.is_object());
VERIFY(test_value.as_object().has("result"));
auto result = test_value.as_object().get("result");
VERIFY(result.is_string());
auto result_string = result.as_string();
if (result_string == "pass") {
test.result = Test::Result::Pass;
m_counts.tests_passed++;
} else if (result_string == "fail") {
test.result = Test::Result::Fail;
m_counts.tests_failed++;
suite.most_severe_test_result = Test::Result::Fail;
VERIFY(test_value.as_object().has("details"));
auto details = test_value.as_object().get("details");
VERIFY(result.is_string());
test.details = details.as_string();
} else {
test.result = Test::Result::Skip;
if (suite.most_severe_test_result == Test::Result::Pass)
suite.most_severe_test_result = Test::Result::Skip;
m_counts.tests_skipped++;
}
suite.tests.append(test);
});
if (suite.most_severe_test_result == Test::Result::Fail) {
m_counts.suites_failed++;
file_result.most_severe_test_result = Test::Result::Fail;
} else {
if (suite.most_severe_test_result == Test::Result::Skip && file_result.most_severe_test_result == Test::Result::Pass)
file_result.most_severe_test_result = Test::Result::Skip;
m_counts.suites_passed++;
}
file_result.suites.append(suite);
});
m_counts.files_total++;
file_result.time_taken = get_time_in_ms() - start_time;
m_total_elapsed_time_in_ms += file_result.time_taken;
},
[page_to_load](auto& error, auto) {
printf("Failed to load test page: %s (%s)", page_to_load.to_string().characters(), error.characters());
cleanup_and_exit();
});
return file_result;
}
enum Modifier {
BG_RED,
BG_GREEN,
FG_RED,
FG_GREEN,
FG_ORANGE,
FG_GRAY,
FG_BLACK,
FG_BOLD,
ITALIC,
CLEAR,
};
static void print_modifiers(Vector<Modifier> modifiers)
{
for (auto& modifier : modifiers) {
auto code = [&]() -> String {
switch (modifier) {
case BG_RED:
return "\033[48;2;255;0;102m";
case BG_GREEN:
return "\033[48;2;102;255;0m";
case FG_RED:
return "\033[38;2;255;0;102m";
case FG_GREEN:
return "\033[38;2;102;255;0m";
case FG_ORANGE:
return "\033[38;2;255;102;0m";
case FG_GRAY:
return "\033[38;2;135;139;148m";
case FG_BLACK:
return "\033[30m";
case FG_BOLD:
return "\033[1m";
case ITALIC:
return "\033[3m";
case CLEAR:
return "\033[0m";
}
VERIFY_NOT_REACHED();
};
printf("%s", code().characters());
}
}
void TestRunner::print_file_result(const JSFileResult& file_result) const
{
if (file_result.most_severe_test_result == Test::Result::Fail || file_result.error.has_value()) {
print_modifiers({ BG_RED, FG_BLACK, FG_BOLD });
printf(" FAIL ");
print_modifiers({ CLEAR });
} else {
if (m_print_times || file_result.most_severe_test_result != Test::Result::Pass) {
print_modifiers({ BG_GREEN, FG_BLACK, FG_BOLD });
printf(" PASS ");
print_modifiers({ CLEAR });
} else {
return;
}
}
printf(" %s", file_result.name.characters());
if (m_print_times) {
print_modifiers({ CLEAR, ITALIC, FG_GRAY });
if (file_result.time_taken < 1000) {
printf(" (%dms)\n", static_cast<int>(file_result.time_taken));
} else {
printf(" (%.3fs)\n", file_result.time_taken / 1000.0);
}
print_modifiers({ CLEAR });
} else {
printf("\n");
}
if (!file_result.logged_messages.is_empty()) {
print_modifiers({ FG_GRAY, FG_BOLD });
#ifdef __serenity__
printf(" Console output:\n");
#else
// This emoji has a second invisible byte after it. The one above does not
printf(" Console output:\n");
#endif
print_modifiers({ CLEAR, FG_GRAY });
for (auto& message : file_result.logged_messages)
printf(" %s\n", message.characters());
}
if (file_result.error.has_value()) {
auto test_error = file_result.error.value();
print_modifiers({ FG_RED });
#ifdef __serenity__
printf(" ❌ The file failed to parse\n\n");
#else
// No invisible byte here, but the spacing still needs to be altered on the host
printf(" ❌ The file failed to parse\n\n");
#endif
print_modifiers({ FG_GRAY });
for (auto& message : test_error.hint.split('\n', true)) {
printf(" %s\n", message.characters());
}
print_modifiers({ FG_RED });
printf(" %s\n\n", test_error.error.to_string().characters());
return;
}
if (file_result.most_severe_test_result != Test::Result::Pass) {
for (auto& suite : file_result.suites) {
if (suite.most_severe_test_result == Test::Result::Pass)
continue;
bool failed = suite.most_severe_test_result == Test::Result::Fail;
print_modifiers({ FG_GRAY, FG_BOLD });
if (failed) {
#ifdef __serenity__
printf(" ❌ Suite: ");
#else
// No invisible byte here, but the spacing still needs to be altered on the host
printf(" ❌ Suite: ");
#endif
} else {
#ifdef __serenity__
printf(" ⚠ Suite: ");
#else
// This emoji has a second invisible byte after it. The one above does not
printf(" ⚠️ Suite: ");
#endif
}
print_modifiers({ CLEAR, FG_GRAY });
if (suite.name == TOP_LEVEL_TEST_NAME) {
printf("<top-level>\n");
} else {
printf("%s\n", suite.name.characters());
}
print_modifiers({ CLEAR });
for (auto& test : suite.tests) {
if (test.result == Test::Result::Pass)
continue;
print_modifiers({ FG_GRAY, FG_BOLD });
printf(" Test: ");
if (test.result == Test::Result::Fail) {
print_modifiers({ CLEAR, FG_RED });
printf("%s (failed):\n", test.name.characters());
printf(" %s\n", test.details.characters());
} else {
print_modifiers({ CLEAR, FG_ORANGE });
printf("%s (skipped)\n", test.name.characters());
}
print_modifiers({ CLEAR });
}
}
}
}
void TestRunner::print_test_results() const
{
printf("\nTest Suites: ");
if (m_counts.suites_failed) {
print_modifiers({ FG_RED });
printf("%d failed, ", m_counts.suites_failed);
print_modifiers({ CLEAR });
}
if (m_counts.suites_passed) {
print_modifiers({ FG_GREEN });
printf("%d passed, ", m_counts.suites_passed);
print_modifiers({ CLEAR });
}
printf("%d total\n", m_counts.suites_failed + m_counts.suites_passed);
printf("Tests: ");
if (m_counts.tests_failed) {
print_modifiers({ FG_RED });
printf("%d failed, ", m_counts.tests_failed);
print_modifiers({ CLEAR });
}
if (m_counts.tests_skipped) {
print_modifiers({ FG_ORANGE });
printf("%d skipped, ", m_counts.tests_skipped);
print_modifiers({ CLEAR });
}
if (m_counts.tests_passed) {
print_modifiers({ FG_GREEN });
printf("%d passed, ", m_counts.tests_passed);
print_modifiers({ CLEAR });
}
printf("%d total\n", m_counts.tests_failed + m_counts.tests_passed);
printf("Files: %d total\n", m_counts.files_total);
printf("Time: ");
if (m_total_elapsed_time_in_ms < 1000.0) {
printf("%dms\n\n", static_cast<int>(m_total_elapsed_time_in_ms));
} else {
printf("%-.3fs\n\n", m_total_elapsed_time_in_ms / 1000.0);
}
}
int main(int argc, char** argv)
{
bool print_times = false;
bool show_window = false;
#if 0
struct sigaction act;
memset(&act, 0, sizeof(act));
act.sa_flags = SA_NOCLDWAIT;
act.sa_handler = handle_sigabrt;
int rc = sigaction(SIGABRT, &act, nullptr);
if (rc < 0) {
perror("sigaction");
return 1;
}
#endif
Core::ArgsParser args_parser;
args_parser.add_option(print_times, "Show duration of each test", "show-time", 't');
args_parser.add_option(show_window, "Show window while running tests", "window", 'w');
args_parser.parse(argc, argv);
auto app = GUI::Application::construct(argc, argv);
auto window = GUI::Window::construct();
auto& main_widget = window->set_main_widget<GUI::Widget>();
main_widget.set_fill_with_background_color(true);
main_widget.set_layout<GUI::VerticalBoxLayout>();
auto& view = main_widget.add<Web::InProcessWebView>();
view.set_document(Web::DOM::Document::create());
if (show_window) {
window->set_title("LibWeb Test Window");
window->resize(640, 480);
window->show();
}
#ifdef __serenity__
TestRunner("/home/anon/web-tests", "/home/anon/js-tests", view, print_times).run();
#else
char* serenity_source_dir = getenv("SERENITY_SOURCE_DIR");
if (!serenity_source_dir) {
printf("test-web requires the SERENITY_SOURCE_DIR environment variable to be set");
return 1;
}
TestRunner(String::formatted("{}/Userland/Libraries/LibWeb/Tests", serenity_source_dir), String::formatted("{}/Userland/Libraries/LibJS/Tests", serenity_source_dir), view, print_times).run();
#endif
return 0;
}