From f93c19afdab6590ee57496607b170f951d97ab61 Mon Sep 17 00:00:00 2001 From: hyzboy Date: Fri, 17 Jan 2020 14:09:56 +0800 Subject: [PATCH 1/7] add Tokenizer.h --- inc/hgl/util/Tokenizer/Tokenizer.h | 156 +++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 inc/hgl/util/Tokenizer/Tokenizer.h diff --git a/inc/hgl/util/Tokenizer/Tokenizer.h b/inc/hgl/util/Tokenizer/Tokenizer.h new file mode 100644 index 0000000..1af5672 --- /dev/null +++ b/inc/hgl/util/Tokenizer/Tokenizer.h @@ -0,0 +1,156 @@ +enum class TokenType +{ + Unrecognized=0, + + End, + + WhiteSpace, // ' ','\t','\r','\n' + OnelineComment, // // + MultilineComment, // /* */ + + // Atoms + Identifier, // abc123 + Integer, // 1234 + Float, // 12.34 + String, // "123" + RawString, // """text""" + + // Math operators + Plus, // + + Minus, // - + Star, // * + Slash, // / + Percent, // % + + Handle, // # + At, // @ + + AddAssign, // += + SubAssign, // -= + MulAssign, // *= + DivAssign, // /= + ModAssign, // %= + + OrAssign, // |= + AndAssign, // &= + XorAssign, // ^= + ShiftLeftAssign, // <<= + ShiftRightAssign, // >>= + + Inc, // ++ + Dec, // -- + + Dot, // . + + // Statement tokens + Assignment, // = + EndStatement, // ; + ListSeparator, // , + StartStatementBlock,// { + EndStatementBlock, // } + OpenParanthesis, // ( + CloseParanthesis, // ) + OpenBracket, // [ + CloseBracket, // ] + Amp, // & + + Not, // ! + Or, // || + And, // && + Xor, // ^^ + + // Bitwise operators + BitOr, // | + BitNot, // ~ + BitXor, // ^ + BitShiftLeft, // << + BitShiftRight, // >> + + // Compare operators + Equal, // == + NotEqual, // != + LessThan, // < + GreaterThan, // > + LessThanOrEqual, // <= + GreaterThanOrEqual, // >= + + Question, // ? + Colon, // : +};// + +struct TokenWord +{ + const char *word; + uint work_size; + uint token_type; +}; + +constexpr TokenWord token_words[]= +{ +#define DEFINE_TOKEN(str,type) {str,sizeof(str)-1,uint(TokenType::type)} + + DEFINE_TOKEN("+", Plus), + DEFINE_TOKEN("-", Minus), + DEFINE_TOKEN("*", Star), + DEFINE_TOKEN("/", Slash), + DEFINE_TOKEN("%", Percent), + DEFINE_TOKEN("=", Assignment), + DEFINE_TOKEN(".", Dot), + DEFINE_TOKEN("+=", AddAssign), + DEFINE_TOKEN("-=", SubAssign), + DEFINE_TOKEN("*=", MulAssign), + DEFINE_TOKEN("/=", DivAssign), + DEFINE_TOKEN("%=", ModAssign), + DEFINE_TOKEN("|=", OrAssign), + DEFINE_TOKEN("&=", AndAssign), + DEFINE_TOKEN("^=", XorAssign), + DEFINE_TOKEN("<<=", ShiftLeftAssign), + DEFINE_TOKEN(">>=", ShiftRightAssign), + DEFINE_TOKEN("|", BitOr), + DEFINE_TOKEN("~", BitNot), + DEFINE_TOKEN("^", BitXor), + DEFINE_TOKEN("<<", BitShiftLeft), + DEFINE_TOKEN(">>", BitShiftRight), + DEFINE_TOKEN(";", EndStatement), + DEFINE_TOKEN(",", ListSeparator), + DEFINE_TOKEN("{", StartStatementBlock), + DEFINE_TOKEN(")", EndStatementBlock), + DEFINE_TOKEN("(", OpenParanthesis), + DEFINE_TOKEN(")", CloseParanthesis), + DEFINE_TOKEN("[", OpenBracket), + DEFINE_TOKEN("]", CloseBracket), + DEFINE_TOKEN("?", Question), + DEFINE_TOKEN(":", Colon), + DEFINE_TOKEN("==", Equal), + DEFINE_TOKEN("!=", NotEqual), + DEFINE_TOKEN("<", LessThan), + DEFINE_TOKEN(">", GreaterThan), + DEFINE_TOKEN("<=", LessThanOrEqual), + DEFINE_TOKEN(">=", GreaterThanOrEqual), + DEFINE_TOKEN("++", Inc), + DEFINE_TOKEN("--", Dec), + DEFINE_TOKEN("&", Amp), + DEFINE_TOKEN("!", Not), + DEFINE_TOKEN("||", Or), + DEFINE_TOKEN("&&", And), + DEFINE_TOKEN("^^", Xor), + DEFINE_TOKEN("#", Handle), + DEFINE_TOKEN("@", At) + +#undef DEFINE_TOKEN +}; + +constexpr uint TokenWordsNumber =sizeof(token_words)/sizeof(TokenWord); +constexpr char WhiteSpaces[] =" \t\r\n"; +constexpr uint WhiteSpacesNumber =sizeof(WhiteSpaces)-1; + +/** + * 通用词法解析器 + */ +class Tokenizer +{ + +public: + + +};//class Tokenizer From db0b17029caea927d9bd175ede8d23d908b0438f Mon Sep 17 00:00:00 2001 From: hyzboy Date: Fri, 24 Jan 2020 00:17:26 +0800 Subject: [PATCH 2/7] use "JSONCPP_STRING" instead "Json::String" --- src/json/JsonTool.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/json/JsonTool.cpp b/src/json/JsonTool.cpp index 615e6a0..46840c0 100644 --- a/src/json/JsonTool.cpp +++ b/src/json/JsonTool.cpp @@ -40,7 +40,7 @@ namespace hgl Json::CharReaderBuilder builder; Json::CharReader *reader=builder.newCharReader(); - Json::String errs; + JSONCPP_STRING errs; const bool result=reader->parse(txt,txt+size,&root,&errs); @@ -75,7 +75,7 @@ namespace hgl int SaveJson(Json::Value &root,const OSString &filename,OSString &error_info) { UTF8String txt; - + if(!JsonToString(root,txt,error_info)) return(false); From 66212fca8db68a36a0aa1d35a41972acc369d09b Mon Sep 17 00:00:00 2001 From: hyzboy Date: Fri, 24 Jan 2020 00:17:50 +0800 Subject: [PATCH 3/7] update submodules expat and jsoncpp. --- 3rdpty/expat | 2 +- 3rdpty/jsoncpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/3rdpty/expat b/3rdpty/expat index 83e1688..20f3da2 160000 --- a/3rdpty/expat +++ b/3rdpty/expat @@ -1 +1 @@ -Subproject commit 83e1688c74ef4d9034e6b47a6595dee480be6147 +Subproject commit 20f3da24e5b5dd49ec0785cae68dcc4b149331c0 diff --git a/3rdpty/jsoncpp b/3rdpty/jsoncpp index 9e0d70a..6317f9a 160000 --- a/3rdpty/jsoncpp +++ b/3rdpty/jsoncpp @@ -1 +1 @@ -Subproject commit 9e0d70aa66e6ba993dd05723ca64c26ab00f3572 +Subproject commit 6317f9a406c43e264e0ae0164fa701362ef04c9a From a1a6ad0f25880905ab60e7ae3a5bab623200e323 Mon Sep 17 00:00:00 2001 From: hyzboy Date: Wed, 11 Mar 2020 16:27:40 +0800 Subject: [PATCH 4/7] add BOM Fileheader --- src/xml/XMLParseClass.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/xml/XMLParseClass.cpp b/src/xml/XMLParseClass.cpp index 00e6afd..f1e58a7 100644 --- a/src/xml/XMLParseClass.cpp +++ b/src/xml/XMLParseClass.cpp @@ -1,4 +1,4 @@ -//取名为XMLParseClass是为了避免与expat的xmlparse.c编译造成obj冲突 +//取名为XMLParseClass是为了避免与expat的xmlparse.c编译造成obj冲突 #include #include From ef72a3ff8d1e5194643b892be08806fd3d7cfcfc Mon Sep 17 00:00:00 2001 From: hyzboy Date: Tue, 7 Apr 2020 14:39:24 +0800 Subject: [PATCH 5/7] update submodules --- 3rdpty/expat | 2 +- 3rdpty/jsoncpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/3rdpty/expat b/3rdpty/expat index 20f3da2..c0bfb97 160000 --- a/3rdpty/expat +++ b/3rdpty/expat @@ -1 +1 @@ -Subproject commit 20f3da24e5b5dd49ec0785cae68dcc4b149331c0 +Subproject commit c0bfb97ddd8d3beb7912b877b6a4278148b2a0a4 diff --git a/3rdpty/jsoncpp b/3rdpty/jsoncpp index 6317f9a..3beb37e 160000 --- a/3rdpty/jsoncpp +++ b/3rdpty/jsoncpp @@ -1 +1 @@ -Subproject commit 6317f9a406c43e264e0ae0164fa701362ef04c9a +Subproject commit 3beb37ea14aec1bdce1a6d542dc464d00f4a6cec From bf9866041fb7944ed52f4d09936c85901de6e21f Mon Sep 17 00:00:00 2001 From: hyzboy Date: Mon, 13 Apr 2020 21:17:39 +0800 Subject: [PATCH 6/7] =?UTF-8?q?=E5=A2=9E=E5=8A=A0jsoncpp=20include=20path?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- path_config.cmake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/path_config.cmake b/path_config.cmake index d1e7d81..38a9493 100644 --- a/path_config.cmake +++ b/path_config.cmake @@ -8,4 +8,6 @@ macro(CMUtilSetup source_path) set(CMUTIL_ROOT_3RDPTY_PATH ${source_path}/3rdpty) include_directories(${CMUTIL_ROOT_INCLUDE_PATH}) + include_directories(${CMUTIL_ROOT_3RDPTY_PATH}/jsoncpp/include) endmacro() + From dbdb48efdcc10173b9edf01916c79a38661f80a1 Mon Sep 17 00:00:00 2001 From: hyzboy Date: Fri, 24 Apr 2020 21:12:17 +0800 Subject: [PATCH 7/7] support new BaseString --- src/json/JsonTool.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/json/JsonTool.cpp b/src/json/JsonTool.cpp index 46840c0..ae49e63 100644 --- a/src/json/JsonTool.cpp +++ b/src/json/JsonTool.cpp @@ -83,7 +83,7 @@ namespace hgl if(result!=txt.Length()) { - error_info=OS_TEXT("[ERROR][SaveJson] Save file failed, only write ")+OSString(result)+OS_TEXT("/")+OSString(txt.Length()); + error_info=OS_TEXT("[ERROR][SaveJson] Save file failed, only write ")+OSString::valueOf(result)+OS_TEXT("/")+OSString::valueOf(txt.Length()); return(false); }