Ladybird/ladybird
1
0
Fork 0
mirror of https://github.com/LadybirdBrowser/ladybird.git synced 2025-06-10 18:10:56 +09:00
Code Activity

LibJS: Unify syntax highlighting

Browse source 
So far we have three different syntax highlighters for LibJS:

- js's Line::Editor stylization
- JS::MarkupGenerator
- GUI::JSSyntaxHighlighter

This not only caused repetition of most token types in each highlighter
but also a lot of inconsistency regarding the styling of certain tokens:

- JSSyntaxHighlighter was considering TokenType::Period to be an
  operator whereas MarkupGenerator categorized it as punctuation.
- MarkupGenerator was considering TokenType::{Break,Case,Continue,
  Default,Switch,With} control keywords whereas JSSyntaxHighlighter just
  disregarded them
- MarkupGenerator considered some future reserved keywords invalid and
  others not. JSSyntaxHighlighter and js disregarded most

Adding a new token type meant adding it to ENUMERATE_JS_TOKENS as well
as each individual highlighter's switch/case construct.

I added a TokenCategory enum, and each TokenType is now associated to a
certain category, which the syntax highlighters then can use for styling
rather than operating on the token type directly. This also makes
changing a token's category everywhere easier, should we need to do that
(e.g. I decided to make TokenType::{Period,QuestionMarkPeriod}
TokenCategory::Operator for now, but we might want to change them to
Punctuation.
This commit is contained in:
Linus Groh 2020-10-04 22:28:59 +01:00 • committed by Andreas Kling
parent 6b55b007dd
commit e80217a746
Notes: sideshowbarker 2024-07-19 02:02:51 +09:00
5 changed files with 195 additions and 437 deletions
Download patch file Download diff file Expand all files Collapse all files

105
Libraries/LibGUI/JSSyntaxHighlighter.cpp
View file

@ -35,110 +35,23 @@ namespace GUI {
static TextStyle style_for_token_type(Gfx::Palette palette, JS::TokenType type) static TextStyle style_for_token_type(Gfx::Palette palette, JS::TokenType type)
{ {
switch (type) { switch (JS::Token::category(type)) {
case JS::TokenType::Invalid: case JS::TokenCategory::Invalid:
case JS::TokenType::Eof:
return { palette.syntax_comment() }; return { palette.syntax_comment() };
case JS::TokenType::NumericLiteral: case JS::TokenCategory::Number:
case JS::TokenType::BigIntLiteral:
return { palette.syntax_number() }; return { palette.syntax_number() };
case JS::TokenType::StringLiteral: case JS::TokenCategory::String:
case JS::TokenType::TemplateLiteralStart:
case JS::TokenType::TemplateLiteralEnd:
case JS::TokenType::TemplateLiteralString:
case JS::TokenType::RegexLiteral:
case JS::TokenType::RegexFlags:
case JS::TokenType::UnterminatedStringLiteral:
case JS::TokenType::UnterminatedRegexLiteral:
return { palette.syntax_string() }; return { palette.syntax_string() };
case JS::TokenType::BracketClose: case JS::TokenCategory::Punctuation:
case JS::TokenType::BracketOpen:
case JS::TokenType::Comma:
case JS::TokenType::CurlyClose:
case JS::TokenType::CurlyOpen:
case JS::TokenType::ParenClose:
case JS::TokenType::ParenOpen:
case JS::TokenType::Semicolon:
case JS::TokenType::TemplateLiteralExprStart:
case JS::TokenType::TemplateLiteralExprEnd:
return { palette.syntax_punctuation() }; return { palette.syntax_punctuation() };
case JS::TokenType::Ampersand: case JS::TokenCategory::Operator:
case JS::TokenType::AmpersandEquals:
case JS::TokenType::Asterisk:
case JS::TokenType::DoubleAsteriskEquals:
case JS::TokenType::AsteriskEquals:
case JS::TokenType::Caret:
case JS::TokenType::CaretEquals:
case JS::TokenType::DoubleAmpersand:
case JS::TokenType::DoubleAsterisk:
case JS::TokenType::DoublePipe:
case JS::TokenType::DoubleQuestionMark:
case JS::TokenType::Equals:
case JS::TokenType::EqualsEquals:
case JS::TokenType::EqualsEqualsEquals:
case JS::TokenType::ExclamationMark:
case JS::TokenType::ExclamationMarkEquals:
case JS::TokenType::ExclamationMarkEqualsEquals:
case JS::TokenType::GreaterThan:
case JS::TokenType::GreaterThanEquals:
case JS::TokenType::LessThan:
case JS::TokenType::LessThanEquals:
case JS::TokenType::Minus:
case JS::TokenType::MinusEquals:
case JS::TokenType::MinusMinus:
case JS::TokenType::Percent:
case JS::TokenType::PercentEquals:
case JS::TokenType::Period:
case JS::TokenType::Pipe:
case JS::TokenType::PipeEquals:
case JS::TokenType::Plus:
case JS::TokenType::PlusEquals:
case JS::TokenType::PlusPlus:
case JS::TokenType::QuestionMark:
case JS::TokenType::QuestionMarkPeriod:
case JS::TokenType::ShiftLeft:
case JS::TokenType::ShiftLeftEquals:
case JS::TokenType::ShiftRight:
case JS::TokenType::ShiftRightEquals:
case JS::TokenType::Slash:
case JS::TokenType::SlashEquals:
case JS::TokenType::Tilde:
case JS::TokenType::UnsignedShiftRight:
case JS::TokenType::UnsignedShiftRightEquals:
return { palette.syntax_operator() }; return { palette.syntax_operator() };
case JS::TokenType::BoolLiteral: case JS::TokenCategory::Keyword:
case JS::TokenType::Class:
case JS::TokenType::Const:
case JS::TokenType::Delete:
case JS::TokenType::Debugger:
case JS::TokenType::Extends:
case JS::TokenType::Function:
case JS::TokenType::In:
case JS::TokenType::Instanceof:
case JS::TokenType::Interface:
case JS::TokenType::Let:
case JS::TokenType::New:
case JS::TokenType::NullLiteral:
case JS::TokenType::Super:
case JS::TokenType::Typeof:
case JS::TokenType::Var:
case JS::TokenType::Void:
return { palette.syntax_keyword(), &Gfx::Font::default_bold_fixed_width_font() }; return { palette.syntax_keyword(), &Gfx::Font::default_bold_fixed_width_font() };
case JS::TokenType::Await: case JS::TokenCategory::ControlKeyword:
case JS::TokenType::Catch:
case JS::TokenType::Do:
case JS::TokenType::Else:
case JS::TokenType::Finally:
case JS::TokenType::For:
case JS::TokenType::If:
case JS::TokenType::Return:
case JS::TokenType::Try:
case JS::TokenType::While:
case JS::TokenType::Yield:
return { palette.syntax_control_keyword(), &Gfx::Font::default_bold_fixed_width_font() }; return { palette.syntax_control_keyword(), &Gfx::Font::default_bold_fixed_width_font() };
case JS::TokenType::Identifier: case JS::TokenCategory::Identifier:
return { palette.syntax_identifier() }; return { palette.syntax_identifier() };
default: default:
return { palette.base_text() }; return { palette.base_text() };
} }

135
Libraries/LibJS/MarkupGenerator.cpp
View file

@ -204,131 +204,28 @@ String MarkupGenerator::style_from_style_type(StyleType type)
MarkupGenerator::StyleType MarkupGenerator::style_type_for_token(Token token) MarkupGenerator::StyleType MarkupGenerator::style_type_for_token(Token token)
{ {
switch (token.type()) { switch (token.category()) {
case TokenType::Enum: case TokenCategory::Invalid:
case TokenType::Eof:
case TokenType::Implements:
case TokenType::Invalid:
case TokenType::Package:
case TokenType::Private:
case TokenType::Protected:
case TokenType::Public:
case TokenType::Static:
case TokenType::UnterminatedTemplateLiteral:
return StyleType::Invalid; return StyleType::Invalid;
case TokenType::NumericLiteral: case TokenCategory::Number:
case TokenType::BigIntLiteral:
return StyleType::Number; return StyleType::Number;
case TokenType::StringLiteral: case TokenCategory::String:
case TokenType::TemplateLiteralStart:
case TokenType::TemplateLiteralEnd:
case TokenType::TemplateLiteralString:
case TokenType::RegexLiteral:
case TokenType::RegexFlags:
case TokenType::UnterminatedStringLiteral:
case TokenType::UnterminatedRegexLiteral:
return StyleType::String; return StyleType::String;
case TokenType::BracketClose: case TokenCategory::Punctuation:
case TokenType::BracketOpen:
case TokenType::Comma:
case TokenType::CurlyClose:
case TokenType::CurlyOpen:
case TokenType::ParenClose:
case TokenType::ParenOpen:
case TokenType::Semicolon:
case TokenType::Colon:
case TokenType::Period:
return StyleType::Punctuation; return StyleType::Punctuation;
case TokenType::Ampersand: case TokenCategory::Operator:
case TokenType::AmpersandEquals:
case TokenType::Arrow:
case TokenType::Asterisk:
case TokenType::AsteriskEquals:
case TokenType::Caret:
case TokenType::CaretEquals:
case TokenType::DoubleAmpersand:
case TokenType::DoubleAsterisk:
case TokenType::DoubleAsteriskEquals:
case TokenType::DoublePipe:
case TokenType::DoubleQuestionMark:
case TokenType::Equals:
case TokenType::EqualsEquals:
case TokenType::EqualsEqualsEquals:
case TokenType::ExclamationMark:
case TokenType::ExclamationMarkEquals:
case TokenType::ExclamationMarkEqualsEquals:
case TokenType::GreaterThan:
case TokenType::GreaterThanEquals:
case TokenType::LessThan:
case TokenType::LessThanEquals:
case TokenType::Minus:
case TokenType::MinusEquals:
case TokenType::MinusMinus:
case TokenType::Percent:
case TokenType::PercentEquals:
case TokenType::Pipe:
case TokenType::PipeEquals:
case TokenType::Plus:
case TokenType::PlusEquals:
case TokenType::PlusPlus:
case TokenType::QuestionMark:
case TokenType::QuestionMarkPeriod:
case TokenType::ShiftLeft:
case TokenType::ShiftLeftEquals:
case TokenType::ShiftRight:
case TokenType::ShiftRightEquals:
case TokenType::Slash:
case TokenType::SlashEquals:
case TokenType::Tilde:
case TokenType::TripleDot:
case TokenType::UnsignedShiftRight:
case TokenType::UnsignedShiftRightEquals:
return StyleType::Operator; return StyleType::Operator;
case TokenType::BoolLiteral: case TokenCategory::Keyword:
case TokenType::NullLiteral: switch (token.type()) {
return StyleType::KeywordBold; case TokenType::BoolLiteral:
case TokenType::Async: case TokenType::NullLiteral:
case TokenType::Class: return StyleType::KeywordBold;
case TokenType::Const: default:
case TokenType::Debugger: return StyleType::Keyword;
case TokenType::Delete: }
case TokenType::Export: case TokenCategory::ControlKeyword:
case TokenType::Extends:
case TokenType::Function:
case TokenType::Import:
case TokenType::In:
case TokenType::Instanceof:
case TokenType::Interface:
case TokenType::Let:
case TokenType::New:
case TokenType::Super:
case TokenType::TemplateLiteralExprStart:
case TokenType::TemplateLiteralExprEnd:
case TokenType::This:
case TokenType::Throw:
case TokenType::Typeof:
case TokenType::Var:
case TokenType::Void:
return StyleType::Keyword;
case TokenType::Await:
case TokenType::Break:
case TokenType::Case:
case TokenType::Catch:
case TokenType::Continue:
case TokenType::Default:
case TokenType::Do:
case TokenType::Else:
case TokenType::Finally:
case TokenType::For:
case TokenType::If:
case TokenType::Return:
case TokenType::Switch:
case TokenType::Try:
case TokenType::While:
case TokenType::With:
case TokenType::Yield:
return StyleType::ControlKeyword; return StyleType::ControlKeyword;
case TokenType::Identifier: case TokenCategory::Identifier:
return StyleType::Identifier; return StyleType::Identifier;
default: default:
dbgln("Unknown style type for token {}", token.name()); dbgln("Unknown style type for token {}", token.name());

24
Libraries/LibJS/Token.cpp
View file

@ -35,9 +35,9 @@ namespace JS {
const char* Token::name(TokenType type) const char* Token::name(TokenType type)
{ {
switch (type) { switch (type) {
#define __ENUMERATE_JS_TOKEN(x) \ #define __ENUMERATE_JS_TOKEN(type, category) \
case TokenType::x: \ case TokenType::type: \
return #x; return #type;
ENUMERATE_JS_TOKENS ENUMERATE_JS_TOKENS
#undef __ENUMERATE_JS_TOKEN #undef __ENUMERATE_JS_TOKEN
default: default:
@ -51,6 +51,24 @@ const char* Token::name() const
return name(m_type); return name(m_type);
} }
TokenCategory Token::category(TokenType type)
{
switch (type) {
#define __ENUMERATE_JS_TOKEN(type, category) \
case TokenType::type: \
return TokenCategory::category;
ENUMERATE_JS_TOKENS
#undef __ENUMERATE_JS_TOKEN
default:
ASSERT_NOT_REACHED();
}
}
TokenCategory Token::category() const
{
return category(m_type);
}
double Token::double_value() const double Token::double_value() const
{ {
ASSERT(type() == TokenType::NumericLiteral); ASSERT(type() == TokenType::NumericLiteral);

249
Libraries/LibJS/Token.h
View file

@ -31,132 +31,143 @@
namespace JS { namespace JS {
#define ENUMERATE_JS_TOKENS \ #define ENUMERATE_JS_TOKENS \
__ENUMERATE_JS_TOKEN(Ampersand) \ __ENUMERATE_JS_TOKEN(Ampersand, Operator) \
__ENUMERATE_JS_TOKEN(AmpersandEquals) \ __ENUMERATE_JS_TOKEN(AmpersandEquals, Operator) \
__ENUMERATE_JS_TOKEN(Arrow) \ __ENUMERATE_JS_TOKEN(Arrow, Operator) \
__ENUMERATE_JS_TOKEN(Asterisk) \ __ENUMERATE_JS_TOKEN(Asterisk, Operator) \
__ENUMERATE_JS_TOKEN(DoubleAsteriskEquals) \ __ENUMERATE_JS_TOKEN(DoubleAsteriskEquals, Operator) \
__ENUMERATE_JS_TOKEN(AsteriskEquals) \ __ENUMERATE_JS_TOKEN(AsteriskEquals, Operator) \
__ENUMERATE_JS_TOKEN(Async) \ __ENUMERATE_JS_TOKEN(Async, Keyword) \
__ENUMERATE_JS_TOKEN(Await) \ __ENUMERATE_JS_TOKEN(Await, Keyword) \
__ENUMERATE_JS_TOKEN(BigIntLiteral) \ __ENUMERATE_JS_TOKEN(BigIntLiteral, Number) \
__ENUMERATE_JS_TOKEN(BoolLiteral) \ __ENUMERATE_JS_TOKEN(BoolLiteral, Keyword) \
__ENUMERATE_JS_TOKEN(BracketClose) \ __ENUMERATE_JS_TOKEN(BracketClose, Punctuation) \
__ENUMERATE_JS_TOKEN(BracketOpen) \ __ENUMERATE_JS_TOKEN(BracketOpen, Punctuation) \
__ENUMERATE_JS_TOKEN(Break) \ __ENUMERATE_JS_TOKEN(Break, Keyword) \
__ENUMERATE_JS_TOKEN(Caret) \ __ENUMERATE_JS_TOKEN(Caret, Operator) \
__ENUMERATE_JS_TOKEN(CaretEquals) \ __ENUMERATE_JS_TOKEN(CaretEquals, Operator) \
__ENUMERATE_JS_TOKEN(Case) \ __ENUMERATE_JS_TOKEN(Case, ControlKeyword) \
__ENUMERATE_JS_TOKEN(Catch) \ __ENUMERATE_JS_TOKEN(Catch, ControlKeyword) \
__ENUMERATE_JS_TOKEN(Class) \ __ENUMERATE_JS_TOKEN(Class, Keyword) \
__ENUMERATE_JS_TOKEN(Colon) \ __ENUMERATE_JS_TOKEN(Colon, Punctuation) \
__ENUMERATE_JS_TOKEN(Comma) \ __ENUMERATE_JS_TOKEN(Comma, Punctuation) \
__ENUMERATE_JS_TOKEN(Const) \ __ENUMERATE_JS_TOKEN(Const, Keyword) \
__ENUMERATE_JS_TOKEN(Continue) \ __ENUMERATE_JS_TOKEN(Continue, ControlKeyword) \
__ENUMERATE_JS_TOKEN(CurlyClose) \ __ENUMERATE_JS_TOKEN(CurlyClose, Punctuation) \
__ENUMERATE_JS_TOKEN(CurlyOpen) \ __ENUMERATE_JS_TOKEN(CurlyOpen, Punctuation) \
__ENUMERATE_JS_TOKEN(Debugger) \ __ENUMERATE_JS_TOKEN(Debugger, Keyword) \
__ENUMERATE_JS_TOKEN(Default) \ __ENUMERATE_JS_TOKEN(Default, ControlKeyword) \
__ENUMERATE_JS_TOKEN(Delete) \ __ENUMERATE_JS_TOKEN(Delete, Keyword) \
__ENUMERATE_JS_TOKEN(Do) \ __ENUMERATE_JS_TOKEN(Do, ControlKeyword) \
__ENUMERATE_JS_TOKEN(DoubleAmpersand) \ __ENUMERATE_JS_TOKEN(DoubleAmpersand, Operator) \
__ENUMERATE_JS_TOKEN(DoubleAsterisk) \ __ENUMERATE_JS_TOKEN(DoubleAsterisk, Operator) \
__ENUMERATE_JS_TOKEN(DoublePipe) \ __ENUMERATE_JS_TOKEN(DoublePipe, Operator) \
__ENUMERATE_JS_TOKEN(DoubleQuestionMark) \ __ENUMERATE_JS_TOKEN(DoubleQuestionMark, Operator) \
__ENUMERATE_JS_TOKEN(Else) \ __ENUMERATE_JS_TOKEN(Else, ControlKeyword) \
__ENUMERATE_JS_TOKEN(Enum) \ __ENUMERATE_JS_TOKEN(Enum, Keyword) \
__ENUMERATE_JS_TOKEN(Eof) \ __ENUMERATE_JS_TOKEN(Eof, Invalid) \
__ENUMERATE_JS_TOKEN(Equals) \ __ENUMERATE_JS_TOKEN(Equals, Operator) \
__ENUMERATE_JS_TOKEN(EqualsEquals) \ __ENUMERATE_JS_TOKEN(EqualsEquals, Operator) \
__ENUMERATE_JS_TOKEN(EqualsEqualsEquals) \ __ENUMERATE_JS_TOKEN(EqualsEqualsEquals, Operator) \
__ENUMERATE_JS_TOKEN(ExclamationMark) \ __ENUMERATE_JS_TOKEN(ExclamationMark, Operator) \
__ENUMERATE_JS_TOKEN(ExclamationMarkEquals) \ __ENUMERATE_JS_TOKEN(ExclamationMarkEquals, Operator) \
__ENUMERATE_JS_TOKEN(ExclamationMarkEqualsEquals) \ __ENUMERATE_JS_TOKEN(ExclamationMarkEqualsEquals, Operator) \
__ENUMERATE_JS_TOKEN(Export) \ __ENUMERATE_JS_TOKEN(Export, Keyword) \
__ENUMERATE_JS_TOKEN(Extends) \ __ENUMERATE_JS_TOKEN(Extends, Keyword) \
__ENUMERATE_JS_TOKEN(Finally) \ __ENUMERATE_JS_TOKEN(Finally, ControlKeyword) \
__ENUMERATE_JS_TOKEN(For) \ __ENUMERATE_JS_TOKEN(For, ControlKeyword) \
__ENUMERATE_JS_TOKEN(Function) \ __ENUMERATE_JS_TOKEN(Function, Keyword) \
__ENUMERATE_JS_TOKEN(GreaterThan) \ __ENUMERATE_JS_TOKEN(GreaterThan, Operator) \
__ENUMERATE_JS_TOKEN(GreaterThanEquals) \ __ENUMERATE_JS_TOKEN(GreaterThanEquals, Operator) \
__ENUMERATE_JS_TOKEN(Identifier) \ __ENUMERATE_JS_TOKEN(Identifier, Identifier) \
__ENUMERATE_JS_TOKEN(If) \ __ENUMERATE_JS_TOKEN(If, ControlKeyword) \
__ENUMERATE_JS_TOKEN(Implements) \ __ENUMERATE_JS_TOKEN(Implements, Keyword) \
__ENUMERATE_JS_TOKEN(Import) \ __ENUMERATE_JS_TOKEN(Import, Keyword) \
__ENUMERATE_JS_TOKEN(In) \ __ENUMERATE_JS_TOKEN(In, Keyword) \
__ENUMERATE_JS_TOKEN(Instanceof) \ __ENUMERATE_JS_TOKEN(Instanceof, Keyword) \
__ENUMERATE_JS_TOKEN(Interface) \ __ENUMERATE_JS_TOKEN(Interface, Keyword) \
__ENUMERATE_JS_TOKEN(Invalid) \ __ENUMERATE_JS_TOKEN(Invalid, Invalid) \
__ENUMERATE_JS_TOKEN(LessThan) \ __ENUMERATE_JS_TOKEN(LessThan, Operator) \
__ENUMERATE_JS_TOKEN(LessThanEquals) \ __ENUMERATE_JS_TOKEN(LessThanEquals, Operator) \
__ENUMERATE_JS_TOKEN(Let) \ __ENUMERATE_JS_TOKEN(Let, Keyword) \
__ENUMERATE_JS_TOKEN(Minus) \ __ENUMERATE_JS_TOKEN(Minus, Operator) \
__ENUMERATE_JS_TOKEN(MinusEquals) \ __ENUMERATE_JS_TOKEN(MinusEquals, Operator) \
__ENUMERATE_JS_TOKEN(MinusMinus) \ __ENUMERATE_JS_TOKEN(MinusMinus, Operator) \
__ENUMERATE_JS_TOKEN(New) \ __ENUMERATE_JS_TOKEN(New, Keyword) \
__ENUMERATE_JS_TOKEN(NullLiteral) \ __ENUMERATE_JS_TOKEN(NullLiteral, Keyword) \
__ENUMERATE_JS_TOKEN(NumericLiteral) \ __ENUMERATE_JS_TOKEN(NumericLiteral, Number) \
__ENUMERATE_JS_TOKEN(Package) \ __ENUMERATE_JS_TOKEN(Package, Keyword) \
__ENUMERATE_JS_TOKEN(ParenClose) \ __ENUMERATE_JS_TOKEN(ParenClose, Punctuation) \
__ENUMERATE_JS_TOKEN(ParenOpen) \ __ENUMERATE_JS_TOKEN(ParenOpen, Punctuation) \
__ENUMERATE_JS_TOKEN(Percent) \ __ENUMERATE_JS_TOKEN(Percent, Operator) \
__ENUMERATE_JS_TOKEN(PercentEquals) \ __ENUMERATE_JS_TOKEN(PercentEquals, Operator) \
__ENUMERATE_JS_TOKEN(Period) \ __ENUMERATE_JS_TOKEN(Period, Operator) \
__ENUMERATE_JS_TOKEN(Pipe) \ __ENUMERATE_JS_TOKEN(Pipe, Operator) \
__ENUMERATE_JS_TOKEN(PipeEquals) \ __ENUMERATE_JS_TOKEN(PipeEquals, Operator) \
__ENUMERATE_JS_TOKEN(Plus) \ __ENUMERATE_JS_TOKEN(Plus, Operator) \
__ENUMERATE_JS_TOKEN(PlusEquals) \ __ENUMERATE_JS_TOKEN(PlusEquals, Operator) \
__ENUMERATE_JS_TOKEN(PlusPlus) \ __ENUMERATE_JS_TOKEN(PlusPlus, Operator) \
__ENUMERATE_JS_TOKEN(Private) \ __ENUMERATE_JS_TOKEN(Private, Keyword) \
__ENUMERATE_JS_TOKEN(Protected) \ __ENUMERATE_JS_TOKEN(Protected, Keyword) \
__ENUMERATE_JS_TOKEN(Public) \ __ENUMERATE_JS_TOKEN(Public, Keyword) \
__ENUMERATE_JS_TOKEN(QuestionMark) \ __ENUMERATE_JS_TOKEN(QuestionMark, Operator) \
__ENUMERATE_JS_TOKEN(QuestionMarkPeriod) \ __ENUMERATE_JS_TOKEN(QuestionMarkPeriod, Operator) \
__ENUMERATE_JS_TOKEN(RegexLiteral) \ __ENUMERATE_JS_TOKEN(RegexLiteral, String) \
__ENUMERATE_JS_TOKEN(RegexFlags) \ __ENUMERATE_JS_TOKEN(RegexFlags, String) \
__ENUMERATE_JS_TOKEN(Return) \ __ENUMERATE_JS_TOKEN(Return, ControlKeyword) \
__ENUMERATE_JS_TOKEN(Semicolon) \ __ENUMERATE_JS_TOKEN(Semicolon, Punctuation) \
__ENUMERATE_JS_TOKEN(ShiftLeft) \ __ENUMERATE_JS_TOKEN(ShiftLeft, Operator) \
__ENUMERATE_JS_TOKEN(ShiftLeftEquals) \ __ENUMERATE_JS_TOKEN(ShiftLeftEquals, Operator) \
__ENUMERATE_JS_TOKEN(ShiftRight) \ __ENUMERATE_JS_TOKEN(ShiftRight, Operator) \
__ENUMERATE_JS_TOKEN(ShiftRightEquals) \ __ENUMERATE_JS_TOKEN(ShiftRightEquals, Operator) \
__ENUMERATE_JS_TOKEN(Slash) \ __ENUMERATE_JS_TOKEN(Slash, Operator) \
__ENUMERATE_JS_TOKEN(SlashEquals) \ __ENUMERATE_JS_TOKEN(SlashEquals, Operator) \
__ENUMERATE_JS_TOKEN(Static) \ __ENUMERATE_JS_TOKEN(Static, Keyword) \
__ENUMERATE_JS_TOKEN(StringLiteral) \ __ENUMERATE_JS_TOKEN(StringLiteral, String) \
__ENUMERATE_JS_TOKEN(Super) \ __ENUMERATE_JS_TOKEN(Super, Keyword) \
__ENUMERATE_JS_TOKEN(Switch) \ __ENUMERATE_JS_TOKEN(Switch, ControlKeyword) \
__ENUMERATE_JS_TOKEN(TemplateLiteralEnd) \ __ENUMERATE_JS_TOKEN(TemplateLiteralEnd, String) \
__ENUMERATE_JS_TOKEN(TemplateLiteralExprEnd) \ __ENUMERATE_JS_TOKEN(TemplateLiteralExprEnd, Punctuation) \
__ENUMERATE_JS_TOKEN(TemplateLiteralExprStart) \ __ENUMERATE_JS_TOKEN(TemplateLiteralExprStart, Punctuation) \
__ENUMERATE_JS_TOKEN(TemplateLiteralStart) \ __ENUMERATE_JS_TOKEN(TemplateLiteralStart, String) \
__ENUMERATE_JS_TOKEN(TemplateLiteralString) \ __ENUMERATE_JS_TOKEN(TemplateLiteralString, String) \
__ENUMERATE_JS_TOKEN(This) \ __ENUMERATE_JS_TOKEN(This, Keyword) \
__ENUMERATE_JS_TOKEN(Throw) \ __ENUMERATE_JS_TOKEN(Throw, ControlKeyword) \
__ENUMERATE_JS_TOKEN(Tilde) \ __ENUMERATE_JS_TOKEN(Tilde, Operator) \
__ENUMERATE_JS_TOKEN(TripleDot) \ __ENUMERATE_JS_TOKEN(TripleDot, Operator) \
__ENUMERATE_JS_TOKEN(Try) \ __ENUMERATE_JS_TOKEN(Try, ControlKeyword) \
__ENUMERATE_JS_TOKEN(Typeof) \ __ENUMERATE_JS_TOKEN(Typeof, Keyword) \
__ENUMERATE_JS_TOKEN(UnsignedShiftRight) \ __ENUMERATE_JS_TOKEN(UnsignedShiftRight, Operator) \
__ENUMERATE_JS_TOKEN(UnsignedShiftRightEquals) \ __ENUMERATE_JS_TOKEN(UnsignedShiftRightEquals, Operator) \
__ENUMERATE_JS_TOKEN(UnterminatedRegexLiteral) \ __ENUMERATE_JS_TOKEN(UnterminatedRegexLiteral, String) \
__ENUMERATE_JS_TOKEN(UnterminatedStringLiteral) \ __ENUMERATE_JS_TOKEN(UnterminatedStringLiteral, String) \
__ENUMERATE_JS_TOKEN(UnterminatedTemplateLiteral) \ __ENUMERATE_JS_TOKEN(UnterminatedTemplateLiteral, String) \
__ENUMERATE_JS_TOKEN(Var) \ __ENUMERATE_JS_TOKEN(Var, Keyword) \
__ENUMERATE_JS_TOKEN(Void) \ __ENUMERATE_JS_TOKEN(Void, Keyword) \
__ENUMERATE_JS_TOKEN(While) \ __ENUMERATE_JS_TOKEN(While, ControlKeyword) \
__ENUMERATE_JS_TOKEN(With) \ __ENUMERATE_JS_TOKEN(With, ControlKeyword) \
__ENUMERATE_JS_TOKEN(Yield) __ENUMERATE_JS_TOKEN(Yield, ControlKeyword)
enum class TokenType { enum class TokenType {
#define __ENUMERATE_JS_TOKEN(x) x, #define __ENUMERATE_JS_TOKEN(type, category) type,
ENUMERATE_JS_TOKENS ENUMERATE_JS_TOKENS
#undef __ENUMERATE_JS_TOKEN #undef __ENUMERATE_JS_TOKEN
_COUNT_OF_TOKENS _COUNT_OF_TOKENS
}; };
constexpr size_t cs_num_of_js_tokens = static_cast<size_t>(TokenType::_COUNT_OF_TOKENS); constexpr size_t cs_num_of_js_tokens = static_cast<size_t>(TokenType::_COUNT_OF_TOKENS);
enum class TokenCategory {
Invalid,
Number,
String,
Punctuation,
Operator,
Keyword,
ControlKeyword,
Identifier
};
class Token { class Token {
public: public:
Token(TokenType type, StringView trivia, StringView value, size_t line_number, size_t line_column) Token(TokenType type, StringView trivia, StringView value, size_t line_number, size_t line_column)
@ -169,6 +180,8 @@ public:
} }
TokenType type() const { return m_type; } TokenType type() const { return m_type; }
TokenCategory category() const;
static TokenCategory category(TokenType);
const char* name() const; const char* name() const;
static const char* name(TokenType); static const char* name(TokenType);

119
Userland/js.cpp
View file

@ -600,118 +600,35 @@ int main(int argc, char** argv)
} }
} }
switch (token.type()) { switch (token.category()) {
case JS::TokenType::Invalid: case JS::TokenCategory::Invalid:
case JS::TokenType::Eof:
stylize({ start, end }, { Line::Style::Foreground(Line::Style::XtermColor::Red), Line::Style::Underline }); stylize({ start, end }, { Line::Style::Foreground(Line::Style::XtermColor::Red), Line::Style::Underline });
break; break;
case JS::TokenType::NumericLiteral: case JS::TokenCategory::Number:
case JS::TokenType::BigIntLiteral:
stylize({ start, end }, { Line::Style::Foreground(Line::Style::XtermColor::Magenta) }); stylize({ start, end }, { Line::Style::Foreground(Line::Style::XtermColor::Magenta) });
break; break;
case JS::TokenType::StringLiteral: case JS::TokenCategory::String:
case JS::TokenType::TemplateLiteralStart:
case JS::TokenType::TemplateLiteralEnd:
case JS::TokenType::TemplateLiteralString:
case JS::TokenType::RegexLiteral:
case JS::TokenType::RegexFlags:
case JS::TokenType::UnterminatedStringLiteral:
case JS::TokenType::UnterminatedRegexLiteral:
stylize({ start, end }, { Line::Style::Foreground(Line::Style::XtermColor::Green), Line::Style::Bold }); stylize({ start, end }, { Line::Style::Foreground(Line::Style::XtermColor::Green), Line::Style::Bold });
break; break;
case JS::TokenType::BracketClose: case JS::TokenCategory::Punctuation:
case JS::TokenType::BracketOpen:
case JS::TokenType::Comma:
case JS::TokenType::CurlyClose:
case JS::TokenType::CurlyOpen:
case JS::TokenType::ParenClose:
case JS::TokenType::ParenOpen:
case JS::TokenType::Semicolon:
case JS::TokenType::Period:
break; break;
case JS::TokenType::Ampersand: case JS::TokenCategory::Operator:
case JS::TokenType::AmpersandEquals:
case JS::TokenType::Asterisk:
case JS::TokenType::DoubleAsteriskEquals:
case JS::TokenType::AsteriskEquals:
case JS::TokenType::Caret:
case JS::TokenType::CaretEquals:
case JS::TokenType::DoubleAmpersand:
case JS::TokenType::DoubleAsterisk:
case JS::TokenType::DoublePipe:
case JS::TokenType::DoubleQuestionMark:
case JS::TokenType::Equals:
case JS::TokenType::EqualsEquals:
case JS::TokenType::EqualsEqualsEquals:
case JS::TokenType::ExclamationMark:
case JS::TokenType::ExclamationMarkEquals:
case JS::TokenType::ExclamationMarkEqualsEquals:
case JS::TokenType::GreaterThan:
case JS::TokenType::GreaterThanEquals:
case JS::TokenType::LessThan:
case JS::TokenType::LessThanEquals:
case JS::TokenType::Minus:
case JS::TokenType::MinusEquals:
case JS::TokenType::MinusMinus:
case JS::TokenType::Percent:
case JS::TokenType::PercentEquals:
case JS::TokenType::Pipe:
case JS::TokenType::PipeEquals:
case JS::TokenType::Plus:
case JS::TokenType::PlusEquals:
case JS::TokenType::PlusPlus:
case JS::TokenType::QuestionMark:
case JS::TokenType::QuestionMarkPeriod:
case JS::TokenType::ShiftLeft:
case JS::TokenType::ShiftLeftEquals:
case JS::TokenType::ShiftRight:
case JS::TokenType::ShiftRightEquals:
case JS::TokenType::Slash:
case JS::TokenType::SlashEquals:
case JS::TokenType::Tilde:
case JS::TokenType::UnsignedShiftRight:
case JS::TokenType::UnsignedShiftRightEquals:
break; break;
case JS::TokenType::BoolLiteral: case JS::TokenCategory::Keyword:
case JS::TokenType::NullLiteral: switch (token.type()) {
stylize({ start, end }, { Line::Style::Foreground(Line::Style::XtermColor::Yellow), Line::Style::Bold }); case JS::TokenType::BoolLiteral:
case JS::TokenType::NullLiteral:
stylize({ start, end }, { Line::Style::Foreground(Line::Style::XtermColor::Yellow), Line::Style::Bold });
break;
default:
stylize({ start, end }, { Line::Style::Foreground(Line::Style::XtermColor::Blue), Line::Style::Bold });
break;
}
break; break;
case JS::TokenType::Class: case JS::TokenCategory::ControlKeyword:
case JS::TokenType::Const:
case JS::TokenType::Debugger:
case JS::TokenType::Delete:
case JS::TokenType::Extends:
case JS::TokenType::Function:
case JS::TokenType::In:
case JS::TokenType::Instanceof:
case JS::TokenType::Interface:
case JS::TokenType::Let:
case JS::TokenType::New:
case JS::TokenType::Super:
case JS::TokenType::TemplateLiteralExprStart:
case JS::TokenType::TemplateLiteralExprEnd:
case JS::TokenType::Throw:
case JS::TokenType::Typeof:
case JS::TokenType::Var:
case JS::TokenType::Void:
stylize({ start, end }, { Line::Style::Foreground(Line::Style::XtermColor::Blue), Line::Style::Bold });
break;
case JS::TokenType::Await:
case JS::TokenType::Case:
case JS::TokenType::Catch:
case JS::TokenType::Do:
case JS::TokenType::Else:
case JS::TokenType::Finally:
case JS::TokenType::For:
case JS::TokenType::If:
case JS::TokenType::Return:
case JS::TokenType::Switch:
case JS::TokenType::Try:
case JS::TokenType::While:
case JS::TokenType::Yield:
stylize({ start, end }, { Line::Style::Foreground(Line::Style::XtermColor::Cyan), Line::Style::Italic }); stylize({ start, end }, { Line::Style::Foreground(Line::Style::XtermColor::Cyan), Line::Style::Italic });
break; break;
case JS::TokenType::Identifier: case JS::TokenCategory::Identifier:
stylize({ start, end }, { Line::Style::Foreground(Line::Style::XtermColor::White), Line::Style::Bold }); stylize({ start, end }, { Line::Style::Foreground(Line::Style::XtermColor::White), Line::Style::Bold });
default: default:
break; break;