beenull/ladybird
1
0
Fork 0
mirror of https://github.com/LadybirdBrowser/ladybird.git synced 2024-09-30 00:31:14 +00:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

LibCpp: Do lexing in the Preprocessor

Browse source 
We now call Preprocessor::process_and_lex() and pass the result to the
parser.

Doing the lexing in the preprocessor will allow us to maintain the
original position information of tokens after substituting definitions.
This commit is contained in:
Itamar 2021-08-06 10:29:57 +03:00 committed by Andreas Kling
parent bf7262681e
commit 4673a517f6
Notes: sideshowbarker 2024-07-18 07:18:16 +09:00 
Author: https://github.com/itamar8910
Commit: https://github.com/SerenityOS/serenity/commit/4673a517f62
Pull-request: https://github.com/SerenityOS/serenity/pull/9230
Reviewed-by: https://github.com/alimpfard
9 changed files with 42 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
Tests/LibCpp/test-cpp-parser.cpp
View file

@ -43,7 +43,8 @@ TEST_CASE(test_regression)
auto target_ast = read_all(ast_file_path);
StringView source_view(source);
::Cpp::Parser parser(source_view, file_path);
Cpp::Preprocessor preprocessor(file_path, source_view);
Cpp::Parser parser(preprocessor.process_and_lex(), file_path);
auto root = parser.parse();
EXPECT(parser.errors().is_empty());

4
Userland/DevTools/HackStudio/LanguageServers/Cpp/CppComprehensionEngine.cpp
View file

@ -569,7 +569,7 @@ OwnPtr<CppComprehensionEngine::DocumentData> CppComprehensionEngine::create_docu
document_data->m_preprocessor = make<Preprocessor>(document_data->m_filename, document_data->text());
document_data->preprocessor().set_ignore_unsupported_keywords(true);
document_data->preprocessor().set_keep_include_statements(true);
document_data->preprocessor().process();
auto tokens = document_data->preprocessor().process_and_lex();
Preprocessor::Definitions preprocessor_definitions;
for (auto item : document_data->preprocessor().definitions())
@ -590,7 +590,7 @@ OwnPtr<CppComprehensionEngine::DocumentData> CppComprehensionEngine::create_docu
preprocessor_definitions.set(move(item.key), move(item.value));
}
document_data->m_parser = make<Parser>(document_data->preprocessor().processed_text(), filename, move(preprocessor_definitions));
document_data->m_parser = make<Parser>(move(tokens), filename, move(preprocessor_definitions));
auto root = document_data->parser().parse();

4
Userland/DevTools/HackStudio/LanguageServers/Cpp/Tests.cpp
View file

@ -134,14 +134,14 @@ void test_complete_includes()
add_file(filedb, "sample_header.h");
CppComprehensionEngine autocomplete(filedb);
auto suggestions = autocomplete.get_suggestions("complete_includes.cpp", { 0, 23 });
auto suggestions = autocomplete.get_suggestions("complete_includes.cpp", { 0, 22 });
if (suggestions.size() != 1)
FAIL(project include - bad size);
if (suggestions[0].completion != "sample_header.h")
FAIL("project include - wrong results");
suggestions = autocomplete.get_suggestions("complete_includes.cpp", { 1, 19 });
suggestions = autocomplete.get_suggestions("complete_includes.cpp", { 1, 18 });
if (suggestions.size() != 1)
FAIL(global include - bad size);

11
Userland/Libraries/LibCpp/Parser.cpp
View file

@ -15,11 +15,11 @@
namespace Cpp {
Parser::Parser(const StringView& program, const String& filename, Preprocessor::Definitions&& definitions)
Parser::Parser(Vector<Token> const& tokens, const String& filename, Preprocessor::Definitions&& definitions)
: m_preprocessor_definitions(move(definitions))
, m_filename(filename)
{
initialize_program_tokens(program);
initialize_program_tokens(tokens);
if constexpr (CPP_DEBUG) {
dbgln("Tokens:");
for (size_t i = 0; i < m_tokens.size(); ++i) {
@ -28,10 +28,9 @@ Parser::Parser(const StringView& program, const String& filename, Preprocessor::
}
}
void Parser::initialize_program_tokens(const StringView& program)
void Parser::initialize_program_tokens(Vector<Token> const& tokens)
{
Lexer lexer(program);
for (auto& token : lexer.lex()) {
for (auto& token : tokens) {
if (token.type() == Token::Type::Whitespace)
continue;
if (token.type() == Token::Type::Identifier) {
@ -1397,7 +1396,7 @@ bool Parser::match_ellipsis()
return false;
return peek().type() == Token::Type::Dot && peek(1).type() == Token::Type::Dot && peek(2).type() == Token::Type::Dot;
}
void Parser::add_tokens_for_preprocessor(Token& replaced_token, Preprocessor::DefinedValue& definition)
void Parser::add_tokens_for_preprocessor(Token const& replaced_token, Preprocessor::DefinedValue& definition)
{
if (!definition.value.has_value())
return;

6
Userland/Libraries/LibCpp/Parser.h
View file

@ -18,7 +18,7 @@ class Parser final {
AK_MAKE_NONCOPYABLE(Parser);
public:
explicit Parser(const StringView& program, const String& filename, Preprocessor::Definitions&& = {});
explicit Parser(Vector<Token> const& tokens, const String& filename, Preprocessor::Definitions&& = {});
~Parser() = default;
NonnullRefPtr<TranslationUnit> parse();
@ -185,8 +185,8 @@ private:
void consume_attribute_specification();
void consume_access_specifier();
bool match_ellipsis();
void initialize_program_tokens(const StringView& program);
void add_tokens_for_preprocessor(Token& replaced_token, Preprocessor::DefinedValue&);
void initialize_program_tokens(Vector<Token> const& tokens);
void add_tokens_for_preprocessor(Token const& replaced_token, Preprocessor::DefinedValue&);
Vector<StringView> parse_type_qualifiers();
Vector<StringView> parse_function_qualifiers();

24
Userland/Libraries/LibCpp/Preprocessor.cpp
View file

@ -8,6 +8,7 @@
#include <AK/Assertions.h>
#include <AK/GenericLexer.h>
#include <AK/StringBuilder.h>
#include <LibCpp/Lexer.h>
#include <ctype.h>
namespace Cpp {
@ -36,8 +37,9 @@ Preprocessor::Preprocessor(const String& filename, const StringView& program)
}
}
const String& Preprocessor::process()
Vector<Token> Preprocessor::process_and_lex()
{
Vector<Token> all_tokens;
for (; m_line_index < m_lines.size(); ++m_line_index) {
auto& line = m_lines[m_line_index];
@ -51,14 +53,14 @@ const String& Preprocessor::process()
}
if (include_in_processed_text) {
m_builder.append(line);
for (auto& token : process_line(line)) {
if (token.type() != Token::Type::Whitespace)
all_tokens.append(token);
}
}
m_builder.append("\n");
}
m_processed_text = m_builder.to_string();
return m_processed_text;
return all_tokens;
}
static void consume_whitespace(GenericLexer& lexer)
@ -224,10 +226,14 @@ void Preprocessor::handle_preprocessor_keyword(const StringView& keyword, Generi
}
}
const String& Preprocessor::processed_text()
Vector<Token> Preprocessor::process_line(const StringView& line)
{
VERIFY(!m_processed_text.is_null());
return m_processed_text;
Lexer line_lexer { line, m_line_index };
auto tokens = line_lexer.lex();
// TODO: Go over tokens of line, do token substitution
return tokens;
}
};

14
Userland/Libraries/LibCpp/Preprocessor.h
View file

@ -12,6 +12,7 @@
#include <AK/String.h>
#include <AK/StringView.h>
#include <AK/Vector.h>
#include <LibCpp/Token.h>
namespace Cpp {
@ -19,8 +20,7 @@ class Preprocessor {
public:
explicit Preprocessor(const String& filename, const StringView& program);
const String& process();
const String& processed_text();
Vector<Token> process_and_lex();
Vector<StringView> included_paths() const { return m_included_paths; }
struct DefinedValue {
@ -38,13 +38,13 @@ public:
private:
using PreprocessorKeyword = StringView;
PreprocessorKeyword handle_preprocessor_line(const StringView&);
void handle_preprocessor_keyword(const StringView& keyword, GenericLexer& line_lexer);
PreprocessorKeyword handle_preprocessor_line(StringView const&);
void handle_preprocessor_keyword(StringView const& keyword, GenericLexer& line_lexer);
Vector<Token> process_line(StringView const& line);
String m_filename;
String m_program;
Definitions m_definitions;
const String m_filename;
const StringView m_program;
StringBuilder m_builder;
Vector<StringView> m_lines;
size_t m_line_index { 0 };
size_t m_current_depth { 0 };

5
Userland/Utilities/PreprocessorTest.cpp
View file

@ -16,5 +16,8 @@ int main(int, char**)
}
auto content = file->read_all();
Cpp::Preprocessor cpp("other.h", StringView { content });
dbgln("{}", cpp.process());
auto tokens = cpp.process_and_lex();
for (auto& token : tokens) {
dbgln("{}", token.to_string());
}
}

10
Userland/Utilities/cpp-parser.cpp
View file

@ -13,9 +13,7 @@ int main(int argc, char** argv)
Core::ArgsParser args_parser;
const char* path = nullptr;
bool tokens_mode = false;
bool preprocess_mode = false;
args_parser.add_option(tokens_mode, "Print Tokens", "tokens", 'T');
args_parser.add_option(preprocess_mode, "Print Preprocessed source", "preprocessed", 'P');
args_parser.add_positional_argument(path, "Cpp File", "cpp-file", Core::ArgsParser::Required::No);
args_parser.parse(argc, argv);
@ -30,13 +28,9 @@ int main(int argc, char** argv)
StringView content_view(content);
::Cpp::Preprocessor processor(path, content_view);
auto preprocessed_content = processor.process();
if (preprocess_mode) {
outln(preprocessed_content);
return 0;
}
auto tokens = processor.process_and_lex();
::Cpp::Parser parser(preprocessed_content, path);
::Cpp::Parser parser(tokens, path);
if (tokens_mode) {
parser.print_tokens();
return 0;