beenull/ladybird
1
0
Fork 0
mirror of https://github.com/LadybirdBrowser/ladybird.git synced 2024-09-30 00:31:14 +00:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

LibCpp: Do macro substitution in the preprocessor instead of the parser

Browse source 
After this change, the parser is completely separated from preprocessor
concepts.
This commit is contained in:
Itamar 2021-08-06 12:23:20 +03:00 committed by Andreas Kling
parent 0c4dc00f01
commit 9da9398bf0
Notes: sideshowbarker 2024-07-18 07:18:08 +09:00 
Author: https://github.com/itamar8910
Commit: https://github.com/SerenityOS/serenity/commit/9da9398bf0c
Pull-request: https://github.com/SerenityOS/serenity/pull/9230
Reviewed-by: https://github.com/alimpfard
5 changed files with 55 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
Userland/DevTools/HackStudio/LanguageServers/Cpp/CppComprehensionEngine.cpp
View file

@ -163,7 +163,7 @@ Vector<GUI::AutocompleteProvider::Entry> CppComprehensionEngine::autocomplete_na
}
if (reference_scope.is_empty()) {
for (auto& preprocessor_name : document.parser().preprocessor_definitions().keys()) {
for (auto& preprocessor_name : document.preprocessor().definitions().keys()) {
if (preprocessor_name.starts_with(partial_text)) {
suggestions.append({ preprocessor_name.to_string(), partial_text.length(), GUI::AutocompleteProvider::CompletionKind::PreprocessorDefinition });
}
@ -412,13 +412,13 @@ Optional<GUI::AutocompleteProvider::ProjectLocation> CppComprehensionEngine::fin
Position cpp_position { text_position.line(), text_position.column() };
// Search for a replaced preprocessor token that intersects with text_position
for (auto& replaced_token : document.parser().replaced_preprocessor_tokens()) {
if (replaced_token.token.start() > cpp_position)
for (auto& substitution : document.preprocessor().substitutions()) {
if (substitution.original_token.start() > cpp_position)
continue;
if (replaced_token.token.end() < cpp_position)
if (substitution.original_token.end() < cpp_position)
continue;
return GUI::AutocompleteProvider::ProjectLocation { replaced_token.preprocessor_value.filename, replaced_token.preprocessor_value.line, replaced_token.preprocessor_value.column };
return GUI::AutocompleteProvider::ProjectLocation { substitution.defined_value.filename, substitution.defined_value.line, substitution.defined_value.column };
}
return {};
}
@ -592,7 +592,7 @@ OwnPtr<CppComprehensionEngine::DocumentData> CppComprehensionEngine::create_docu
document_data->m_available_headers.set(header);
}
document_data->m_parser = make<Parser>(move(tokens), filename, document_data->preprocessor().definitions());
document_data->m_parser = make<Parser>(move(tokens), filename);
auto root = document_data->parser().parse();

36
Userland/Libraries/LibCpp/Parser.cpp
View file

@ -15,11 +15,10 @@
namespace Cpp {
Parser::Parser(Vector<Token> const& tokens, const String& filename, Preprocessor::Definitions const& definitions)
: m_preprocessor_definitions(move(definitions))
, m_filename(filename)
Parser::Parser(Vector<Token> tokens, String const& filename)
: m_filename(filename)
, m_tokens(move(tokens))
{
initialize_program_tokens(tokens);
if constexpr (CPP_DEBUG) {
dbgln("Tokens:");
for (size_t i = 0; i < m_tokens.size(); ++i) {
@ -28,22 +27,6 @@ Parser::Parser(Vector<Token> const& tokens, const String& filename, Preprocessor
}
}
void Parser::initialize_program_tokens(Vector<Token> const& tokens)
{
for (auto& token : tokens) {
if (token.type() == Token::Type::Whitespace)
continue;
if (token.type() == Token::Type::Identifier) {
if (auto defined_value = m_preprocessor_definitions.find(text_of_token(token)); defined_value != m_preprocessor_definitions.end()) {
add_tokens_for_preprocessor(token, defined_value->value);
m_replaced_preprocessor_tokens.append({ token, defined_value->value });
continue;
}
}
m_tokens.append(move(token));
}
}
NonnullRefPtr<TranslationUnit> Parser::parse()
{
LOG_SCOPE();
@ -1396,19 +1379,6 @@ bool Parser::match_ellipsis()
return false;
return peek().type() == Token::Type::Dot && peek(1).type() == Token::Type::Dot && peek(2).type() == Token::Type::Dot;
}
void Parser::add_tokens_for_preprocessor(Token const& replaced_token, Preprocessor::DefinedValue& definition)
{
if (!definition.value.has_value())
return;
Lexer lexer(definition.value.value());
for (auto token : lexer.lex()) {
if (token.type() == Token::Type::Whitespace)
continue;
token.set_start(replaced_token.start());
token.set_end(replaced_token.end());
m_tokens.append(move(token));
}
}
NonnullRefPtr<NamespaceDeclaration> Parser::parse_namespace_declaration(ASTNode& parent, bool is_nested_namespace)
{

13
Userland/Libraries/LibCpp/Parser.h
View file

@ -18,7 +18,7 @@ class Parser final {
AK_MAKE_NONCOPYABLE(Parser);
public:
explicit Parser(Vector<Token> const& tokens, const String& filename, Preprocessor::Definitions const& = {});
explicit Parser(Vector<Token> tokens, String const& filename);
~Parser() = default;
NonnullRefPtr<TranslationUnit> parse();
@ -33,7 +33,6 @@ public:
StringView text_of_token(const Cpp::Token& token) const;
void print_tokens() const;
const Vector<String>& errors() const { return m_errors; }
const Preprocessor::Definitions& preprocessor_definitions() const { return m_preprocessor_definitions; }
struct TodoEntry {
String content;
@ -43,11 +42,6 @@ public:
};
Vector<TodoEntry> get_todo_entries() const;
struct TokenAndPreprocessorDefinition {
Token token;
Preprocessor::DefinedValue preprocessor_value;
};
const Vector<TokenAndPreprocessorDefinition>& replaced_preprocessor_tokens() const { return m_replaced_preprocessor_tokens; }
Vector<Token> tokens_in_range(Position start, Position end) const;
private:
@ -185,8 +179,6 @@ private:
void consume_attribute_specification();
void consume_access_specifier();
bool match_ellipsis();
void initialize_program_tokens(Vector<Token> const& tokens);
void add_tokens_for_preprocessor(Token const& replaced_token, Preprocessor::DefinedValue&);
Vector<StringView> parse_type_qualifiers();
Vector<StringView> parse_function_qualifiers();
@ -196,7 +188,6 @@ private:
};
void parse_constructor_or_destructor_impl(FunctionDeclaration&, CtorOrDtor);
Preprocessor::Definitions m_preprocessor_definitions;
String m_filename;
Vector<Token> m_tokens;
State m_state;
@ -204,8 +195,6 @@ private:
RefPtr<TranslationUnit> m_root_node;
Vector<String> m_errors;
NonnullRefPtrVector<ASTNode> m_nodes;
Vector<TokenAndPreprocessorDefinition> m_replaced_preprocessor_tokens;
};
}

39
Userland/Libraries/LibCpp/Preprocessor.cpp
View file

@ -39,7 +39,6 @@ Preprocessor::Preprocessor(const String& filename, const StringView& program)
Vector<Token> Preprocessor::process_and_lex()
{
Vector<Token> all_tokens;
for (; m_line_index < m_lines.size(); ++m_line_index) {
auto& line = m_lines[m_line_index];
@ -53,14 +52,11 @@ Vector<Token> Preprocessor::process_and_lex()
}
if (include_in_processed_text) {
for (auto& token : process_line(line)) {
if (token.type() != Token::Type::Whitespace)
all_tokens.append(token);
}
process_line(line);
}
}
return all_tokens;
return m_tokens;
}
static void consume_whitespace(GenericLexer& lexer)
@ -231,14 +227,39 @@ void Preprocessor::handle_preprocessor_keyword(const StringView& keyword, Generi
}
}
Vector<Token> Preprocessor::process_line(const StringView& line)
void Preprocessor::process_line(StringView const& line)
{
Lexer line_lexer { line, m_line_index };
auto tokens = line_lexer.lex();
// TODO: Go over tokens of line, do token substitution
for (auto& token : tokens) {
if (token.type() == Token::Type::Whitespace)
continue;
if (token.type() == Token::Type::Identifier) {
if (auto defined_value = m_definitions.find(token.text()); defined_value != m_definitions.end()) {
do_substitution(token, defined_value->value);
continue;
}
}
m_tokens.append(token);
}
}
return tokens;
void Preprocessor::do_substitution(Token const& replaced_token, DefinedValue const& defined_value)
{
m_substitutions.append({ replaced_token, defined_value });
if (defined_value.value.is_null())
return;
Lexer lexer(m_substitutions.last().defined_value.value);
for (auto& token : lexer.lex()) {
if (token.type() == Token::Type::Whitespace)
continue;
token.set_start(replaced_token.start());
token.set_end(replaced_token.end());
m_tokens.append(token);
}
}
};

20
Userland/Libraries/LibCpp/Preprocessor.h
View file

@ -25,14 +25,20 @@ public:
Vector<StringView> included_paths() const { return m_included_paths; }
struct DefinedValue {
Optional<StringView> value;
String value;
FlyString filename;
size_t line { 0 };
size_t column { 0 };
};
using Definitions = HashMap<StringView, DefinedValue>;
const Definitions& definitions() const { return m_definitions; }
struct Substitution {
Token original_token;
DefinedValue defined_value;
};
Definitions const& definitions() const { return m_definitions; }
Vector<Substitution> const& substitutions() const { return m_substitutions; }
void set_ignore_unsupported_keywords(bool ignore) { m_options.ignore_unsupported_keywords = ignore; }
void set_keep_include_statements(bool keep) { m_options.keep_include_statements = keep; }
@ -43,12 +49,17 @@ private:
using PreprocessorKeyword = StringView;
PreprocessorKeyword handle_preprocessor_line(StringView const&);
void handle_preprocessor_keyword(StringView const& keyword, GenericLexer& line_lexer);
Vector<Token> process_line(StringView const& line);
void process_line(StringView const& line);
void do_substitution(Token const& replaced_token, DefinedValue const&);
String m_filename;
String m_program;
Definitions m_definitions;
Vector<StringView> m_lines;
Vector<Token> m_tokens;
Definitions m_definitions;
Vector<Substitution> m_substitutions;
size_t m_line_index { 0 };
size_t m_current_depth { 0 };
Vector<size_t> m_depths_of_taken_branches;
@ -62,7 +73,6 @@ private:
State m_state { State::Normal };
Vector<StringView> m_included_paths;
String m_processed_text;
struct Options {
bool ignore_unsupported_keywords { false };