Ladybird/ladybird
1
0
Fork 0
mirror of https://github.com/LadybirdBrowser/ladybird.git synced 2025-06-10 18:10:56 +09:00
Code Activity

LibWeb: Move Token and Tokenizer into Parser namespace

Browse source
This commit is contained in:
Sam Atkins 2022-04-12 15:10:08 +01:00 • committed by Andreas Kling
parent 7272997b1b
commit bf786d66b1
Notes: sideshowbarker 2024-07-17 11:52:11 +09:00
8 changed files with 52 additions and 51 deletions
Download patch file Download diff file Expand all files Collapse all files

5
Userland/Libraries/LibWeb/CSS/Number.h
View file

@ -7,14 +7,13 @@
#pragma once
#include <AK/Types.h>
#include <LibWeb/Forward.h>
#include <math.h>
namespace Web::CSS {
class Tokenizer;
class Number {
friend class Tokenizer;
friend class Parser::Tokenizer;
public:
enum class Type {

2
Userland/Libraries/LibWeb/CSS/Parser/Token.cpp
View file

@ -9,7 +9,7 @@
#include <LibWeb/CSS/Parser/Token.h>
#include <LibWeb/CSS/Serialize.h>
namespace Web::CSS {
namespace Web::CSS::Parser {
String Token::to_string() const
{

2
Userland/Libraries/LibWeb/CSS/Parser/Token.h
View file

@ -12,7 +12,7 @@
#include <AK/Utf8View.h>
#include <LibWeb/CSS/Number.h>
namespace Web::CSS {
namespace Web::CSS::Parser {
class Token {
friend class Tokenizer;

4
Userland/Libraries/LibWeb/CSS/Parser/Tokenizer.cpp
View file

@ -13,6 +13,8 @@
#include <LibWeb/CSS/Parser/Tokenizer.h>
#include <math.h>
namespace Web::CSS::Parser {
// U+FFFD REPLACEMENT CHARACTER (<28>)
#define REPLACEMENT_CHARACTER 0xFFFD
static constexpr u32 TOKENIZER_EOF = 0xFFFFFFFF;
@ -190,8 +192,6 @@ static inline bool is_E(u32 code_point)
return code_point == 0x45;
}
namespace Web::CSS {
Tokenizer::Tokenizer(StringView input, String const& encoding)
{
auto* decoder = TextCodec::decoder_for(encoding);

2
Userland/Libraries/LibWeb/CSS/Parser/Tokenizer.h
View file

@ -14,7 +14,7 @@
#include <LibWeb/CSS/Parser/Token.h>
#include <LibWeb/Forward.h>
namespace Web::CSS {
namespace Web::CSS::Parser {
class U32Twin {
public:

10
Userland/Libraries/LibWeb/CSS/StyleComputer.cpp
View file

@ -557,7 +557,7 @@ bool StyleComputer::expand_unresolved_values(DOM::Element& element, StringView p
return false;
auto const& custom_property_name_token = var_contents.first();
if (!custom_property_name_token.is(Token::Type::Ident))
if (!custom_property_name_token.is(Parser::Token::Type::Ident))
return false;
auto custom_property_name = custom_property_name_token.token().ident();
if (!custom_property_name.starts_with("--"))
@ -582,7 +582,7 @@ bool StyleComputer::expand_unresolved_values(DOM::Element& element, StringView p
}
// Use the provided fallback value, if any.
if (var_contents.size() > 2 && var_contents[1].is(Token::Type::Comma)) {
if (var_contents.size() > 2 && var_contents[1].is(Parser::Token::Type::Comma)) {
if (!expand_unresolved_values(element, property_name, dependencies, var_contents, dest, 2))
return false;
continue;
@ -594,7 +594,7 @@ bool StyleComputer::expand_unresolved_values(DOM::Element& element, StringView p
return false;
auto const& attr_name_token = attr_contents.first();
if (!attr_name_token.is(Token::Type::Ident))
if (!attr_name_token.is(Parser::Token::Type::Ident))
return false;
auto attr_name = attr_name_token.token().ident();
@ -602,13 +602,13 @@ bool StyleComputer::expand_unresolved_values(DOM::Element& element, StringView p
// 1. If the attr() function has a substitution value, replace the attr() function by the substitution value.
if (!attr_value.is_null()) {
// FIXME: attr() should also accept an optional type argument, not just strings.
dest.empend(Token::of_string(attr_value));
dest.empend(Parser::Token::of_string(attr_value));
continue;
}
// 2. Otherwise, if the attr() function has a fallback value as its last argument, replace the attr() function by the fallback value.
// If there are any var() or attr() references in the fallback, substitute them as well.
if (attr_contents.size() > 2 && attr_contents[1].is(Token::Type::Comma)) {
if (attr_contents.size() > 2 && attr_contents[1].is(Parser::Token::Type::Comma)) {
if (!expand_unresolved_values(element, property_name, dependencies, attr_contents, dest, 2))
return false;
continue;

76
Userland/Libraries/LibWeb/CSS/SyntaxHighlighter/SyntaxHighlighter.cpp
View file

@ -12,7 +12,7 @@ namespace Web::CSS {
bool SyntaxHighlighter::is_identifier(u64 token) const
{
return static_cast<CSS::Token::Type>(token) == CSS::Token::Type::Ident;
return static_cast<CSS::Parser::Token::Type>(token) == CSS::Parser::Token::Type::Ident;
}
bool SyntaxHighlighter::is_navigatable(u64) const
@ -27,7 +27,7 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
Vector<GUI::TextDocumentSpan> spans;
auto highlight = [&](auto start_line, auto start_column, auto end_line, auto end_column, Gfx::TextAttributes attributes, CSS::Token::Type type) {
auto highlight = [&](auto start_line, auto start_column, auto end_line, auto end_column, Gfx::TextAttributes attributes, CSS::Parser::Token::Type type) {
if (start_line > end_line || (start_line == end_line && start_column >= end_column)) {
dbgln_if(SYNTAX_HIGHLIGHTING_DEBUG, "(CSS::SyntaxHighlighter) discarding ({}-{}) to ({}-{}) because it has zero or negative length", start_line, start_column, end_line, end_column);
return;
@ -43,85 +43,85 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
false);
};
CSS::Tokenizer tokenizer { text, "utf-8" };
CSS::Parser::Tokenizer tokenizer { text, "utf-8" };
auto tokens = tokenizer.parse();
for (auto const& token : tokens) {
if (token.is(Token::Type::EndOfFile))
if (token.is(Parser::Token::Type::EndOfFile))
break;
switch (token.type()) {
case Token::Type::Ident:
case Parser::Token::Type::Ident:
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_identifier(), {} }, token.type());
break;
case Token::Type::String:
case Parser::Token::Type::String:
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_string(), {} }, token.type());
break;
case Token::Type::Whitespace:
case Parser::Token::Type::Whitespace:
// CSS doesn't produce comment tokens, they're just included as part of whitespace.
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_comment(), {} }, token.type());
break;
case Token::Type::AtKeyword:
case Parser::Token::Type::AtKeyword:
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_keyword(), {} }, token.type());
break;
case Token::Type::Function:
case Parser::Token::Type::Function:
// Function tokens include the opening '(', so we split that into two tokens for highlighting purposes.
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column - 1, { palette.syntax_keyword(), {} }, token.type());
highlight(token.end_position().line, token.end_position().column - 1, token.end_position().line, token.end_position().column, { palette.syntax_punctuation(), {} }, Token::Type::OpenParen);
highlight(token.end_position().line, token.end_position().column - 1, token.end_position().line, token.end_position().column, { palette.syntax_punctuation(), {} }, Parser::Token::Type::OpenParen);
break;
case Token::Type::Url:
case Parser::Token::Type::Url:
// An Url token is a `url()` function with its parameter string unquoted.
// url
highlight(token.start_position().line, token.start_position().column, token.start_position().line, token.start_position().column + 3, { palette.syntax_keyword(), {} }, token.type());
// (
highlight(token.start_position().line, token.start_position().column + 3, token.start_position().line, token.start_position().column + 4, { palette.syntax_punctuation(), {} }, Token::Type::OpenParen);
highlight(token.start_position().line, token.start_position().column + 3, token.start_position().line, token.start_position().column + 4, { palette.syntax_punctuation(), {} }, Parser::Token::Type::OpenParen);
// <string>
highlight(token.start_position().line, token.start_position().column + 4, token.end_position().line, token.end_position().column - 1, { palette.syntax_string(), {} }, Token::Type::String);
highlight(token.start_position().line, token.start_position().column + 4, token.end_position().line, token.end_position().column - 1, { palette.syntax_string(), {} }, Parser::Token::Type::String);
// )
highlight(token.end_position().line, token.end_position().column - 1, token.end_position().line, token.end_position().column, { palette.syntax_punctuation(), {} }, Token::Type::CloseParen);
highlight(token.end_position().line, token.end_position().column - 1, token.end_position().line, token.end_position().column, { palette.syntax_punctuation(), {} }, Parser::Token::Type::CloseParen);
break;
case Token::Type::Number:
case Token::Type::Dimension:
case Token::Type::Percentage:
case Parser::Token::Type::Number:
case Parser::Token::Type::Dimension:
case Parser::Token::Type::Percentage:
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_number(), {} }, token.type());
break;
case Token::Type::Delim:
case Token::Type::Colon:
case Token::Type::Comma:
case Token::Type::Semicolon:
case Token::Type::OpenCurly:
case Token::Type::OpenParen:
case Token::Type::OpenSquare:
case Token::Type::CloseCurly:
case Token::Type::CloseParen:
case Token::Type::CloseSquare:
case Parser::Token::Type::Delim:
case Parser::Token::Type::Colon:
case Parser::Token::Type::Comma:
case Parser::Token::Type::Semicolon:
case Parser::Token::Type::OpenCurly:
case Parser::Token::Type::OpenParen:
case Parser::Token::Type::OpenSquare:
case Parser::Token::Type::CloseCurly:
case Parser::Token::Type::CloseParen:
case Parser::Token::Type::CloseSquare:
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_punctuation(), {} }, token.type());
break;
case Token::Type::CDO:
case Token::Type::CDC:
case Parser::Token::Type::CDO:
case Parser::Token::Type::CDC:
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_comment(), {} }, token.type());
break;
case Token::Type::Hash:
case Parser::Token::Type::Hash:
// FIXME: Hash tokens can be ID selectors or colors, we don't know which without parsing properly.
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_number(), {} }, token.type());
break;
case Token::Type::Invalid:
case Token::Type::BadUrl:
case Token::Type::BadString:
case Parser::Token::Type::Invalid:
case Parser::Token::Type::BadUrl:
case Parser::Token::Type::BadString:
// FIXME: Error highlighting color in palette?
highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { Color(Color::NamedColor::Red), {}, false, true }, token.type());
break;
case Token::Type::EndOfFile:
case Parser::Token::Type::EndOfFile:
default:
break;
}
@ -144,10 +144,10 @@ Vector<Syntax::Highlighter::MatchingTokenPair> SyntaxHighlighter::matching_token
{
static Vector<Syntax::Highlighter::MatchingTokenPair> pairs;
if (pairs.is_empty()) {
pairs.append({ static_cast<u64>(CSS::Token::Type::OpenCurly), static_cast<u64>(CSS::Token::Type::CloseCurly) });
pairs.append({ static_cast<u64>(CSS::Token::Type::OpenParen), static_cast<u64>(CSS::Token::Type::CloseParen) });
pairs.append({ static_cast<u64>(CSS::Token::Type::OpenSquare), static_cast<u64>(CSS::Token::Type::CloseSquare) });
pairs.append({ static_cast<u64>(CSS::Token::Type::CDO), static_cast<u64>(CSS::Token::Type::CDC) });
pairs.append({ static_cast<u64>(CSS::Parser::Token::Type::OpenCurly), static_cast<u64>(CSS::Parser::Token::Type::CloseCurly) });
pairs.append({ static_cast<u64>(CSS::Parser::Token::Type::OpenParen), static_cast<u64>(CSS::Parser::Token::Type::CloseParen) });
pairs.append({ static_cast<u64>(CSS::Parser::Token::Type::OpenSquare), static_cast<u64>(CSS::Parser::Token::Type::CloseSquare) });
pairs.append({ static_cast<u64>(CSS::Parser::Token::Type::CDO), static_cast<u64>(CSS::Parser::Token::Type::CDC) });
}
return pairs;
}

2
Userland/Libraries/LibWeb/Forward.h
View file

@ -106,6 +106,8 @@ class DeclarationOrAtRule;
class Function;
class Parser;
class StyleRule;
class Token;
class Tokenizer;
}
namespace Web::DOM {