157 lines
3.9 KiB
C++
157 lines
3.9 KiB
C++
enum class TokenType
|
|
{
|
|
Unrecognized=0,
|
|
|
|
End,
|
|
|
|
WhiteSpace, // ' ','\t','\r','\n'
|
|
OnelineComment, // //
|
|
MultilineComment, // /* */
|
|
|
|
// Atoms
|
|
Identifier, // abc123
|
|
Integer, // 1234
|
|
Float, // 12.34
|
|
String, // "123"
|
|
RawString, // """text"""
|
|
|
|
// Math operators
|
|
Plus, // +
|
|
Minus, // -
|
|
Star, // *
|
|
Slash, // /
|
|
Percent, // %
|
|
|
|
Handle, // #
|
|
At, // @
|
|
|
|
AddAssign, // +=
|
|
SubAssign, // -=
|
|
MulAssign, // *=
|
|
DivAssign, // /=
|
|
ModAssign, // %=
|
|
|
|
OrAssign, // |=
|
|
AndAssign, // &=
|
|
XorAssign, // ^=
|
|
ShiftLeftAssign, // <<=
|
|
ShiftRightAssign, // >>=
|
|
|
|
Inc, // ++
|
|
Dec, // --
|
|
|
|
Dot, // .
|
|
|
|
// Statement tokens
|
|
Assignment, // =
|
|
EndStatement, // ;
|
|
ListSeparator, // ,
|
|
StartStatementBlock,// {
|
|
EndStatementBlock, // }
|
|
OpenParanthesis, // (
|
|
CloseParanthesis, // )
|
|
OpenBracket, // [
|
|
CloseBracket, // ]
|
|
Amp, // &
|
|
|
|
Not, // !
|
|
Or, // ||
|
|
And, // &&
|
|
Xor, // ^^
|
|
|
|
// Bitwise operators
|
|
BitOr, // |
|
|
BitNot, // ~
|
|
BitXor, // ^
|
|
BitShiftLeft, // <<
|
|
BitShiftRight, // >>
|
|
|
|
// Compare operators
|
|
Equal, // ==
|
|
NotEqual, // !=
|
|
LessThan, // <
|
|
GreaterThan, // >
|
|
LessThanOrEqual, // <=
|
|
GreaterThanOrEqual, // >=
|
|
|
|
Question, // ?
|
|
Colon, // :
|
|
};//
|
|
|
|
struct TokenWord
|
|
{
|
|
const char *word;
|
|
uint work_size;
|
|
uint token_type;
|
|
};
|
|
|
|
constexpr TokenWord token_words[]=
|
|
{
|
|
#define DEFINE_TOKEN(str,type) {str,sizeof(str)-1,uint(TokenType::type)}
|
|
|
|
DEFINE_TOKEN("+", Plus),
|
|
DEFINE_TOKEN("-", Minus),
|
|
DEFINE_TOKEN("*", Star),
|
|
DEFINE_TOKEN("/", Slash),
|
|
DEFINE_TOKEN("%", Percent),
|
|
DEFINE_TOKEN("=", Assignment),
|
|
DEFINE_TOKEN(".", Dot),
|
|
DEFINE_TOKEN("+=", AddAssign),
|
|
DEFINE_TOKEN("-=", SubAssign),
|
|
DEFINE_TOKEN("*=", MulAssign),
|
|
DEFINE_TOKEN("/=", DivAssign),
|
|
DEFINE_TOKEN("%=", ModAssign),
|
|
DEFINE_TOKEN("|=", OrAssign),
|
|
DEFINE_TOKEN("&=", AndAssign),
|
|
DEFINE_TOKEN("^=", XorAssign),
|
|
DEFINE_TOKEN("<<=", ShiftLeftAssign),
|
|
DEFINE_TOKEN(">>=", ShiftRightAssign),
|
|
DEFINE_TOKEN("|", BitOr),
|
|
DEFINE_TOKEN("~", BitNot),
|
|
DEFINE_TOKEN("^", BitXor),
|
|
DEFINE_TOKEN("<<", BitShiftLeft),
|
|
DEFINE_TOKEN(">>", BitShiftRight),
|
|
DEFINE_TOKEN(";", EndStatement),
|
|
DEFINE_TOKEN(",", ListSeparator),
|
|
DEFINE_TOKEN("{", StartStatementBlock),
|
|
DEFINE_TOKEN(")", EndStatementBlock),
|
|
DEFINE_TOKEN("(", OpenParanthesis),
|
|
DEFINE_TOKEN(")", CloseParanthesis),
|
|
DEFINE_TOKEN("[", OpenBracket),
|
|
DEFINE_TOKEN("]", CloseBracket),
|
|
DEFINE_TOKEN("?", Question),
|
|
DEFINE_TOKEN(":", Colon),
|
|
DEFINE_TOKEN("==", Equal),
|
|
DEFINE_TOKEN("!=", NotEqual),
|
|
DEFINE_TOKEN("<", LessThan),
|
|
DEFINE_TOKEN(">", GreaterThan),
|
|
DEFINE_TOKEN("<=", LessThanOrEqual),
|
|
DEFINE_TOKEN(">=", GreaterThanOrEqual),
|
|
DEFINE_TOKEN("++", Inc),
|
|
DEFINE_TOKEN("--", Dec),
|
|
DEFINE_TOKEN("&", Amp),
|
|
DEFINE_TOKEN("!", Not),
|
|
DEFINE_TOKEN("||", Or),
|
|
DEFINE_TOKEN("&&", And),
|
|
DEFINE_TOKEN("^^", Xor),
|
|
DEFINE_TOKEN("#", Handle),
|
|
DEFINE_TOKEN("@", At)
|
|
|
|
#undef DEFINE_TOKEN
|
|
};
|
|
|
|
constexpr uint TokenWordsNumber =sizeof(token_words)/sizeof(TokenWord);
|
|
constexpr char WhiteSpaces[] =" \t\r\n";
|
|
constexpr uint WhiteSpacesNumber =sizeof(WhiteSpaces)-1;
|
|
|
|
/**
|
|
* 通用词法解析器
|
|
*/
|
|
class Tokenizer
|
|
{
|
|
|
|
public:
|
|
|
|
|
|
};//class Tokenizer
|