vaclavt/mlisp
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

usql updates, lexer changes

Browse Source
This commit is contained in:
VaclavT
2021-09-15 21:22:24 +02:00
parent 7e4a4b5583
commit 1e0506e6cd
21 changed files with 986 additions and 777 deletions
Download Patch File Download Diff File Expand all files Collapse all files

388
usql/lexer.cpp
View File

@@ -13,8 +13,8 @@ namespace usql {
Lexer::Lexer() {
k_words_regex =
"[-+]?[0-9]+\\.[0-9]+|[-+]?[0-9][0-9_]+[0-9]|[0-9]+|[A-Za-z]+[A-Za-z0-9_#]*|[\\(\\)\\[\\]\\{\\}]|[-\\+\\*/"
",;:\?]|!=|<>|==|>=|<=|~=|>|<|=|;|~|\\||or|and|\n|\r|\r\n|'([^']|'')*'|\".*?\"|%.*?\n";
"[-+]?[0-9]+\\.[0-9]+|[-+]?[0-9]+|[A-Za-z]+[A-Za-z0-9_#]*|[\\(\\)\\[\\]\\{\\}]|[-\\+\\*/"
",;:\?]|!=|<>|==|>=|<=|~=|>|<|=|;|~|\\||\n|\r|\r\n|'([^']|'')*'|\".*?\"|%.*?\n";
k_int_regex = "[-+]?[0-9]+";
k_int_underscored_regex = "[-+]?[0-9][0-9_]+[0-9]";
k_double_regex = "[-+]?[0-9]+\\.[0-9]+";
@@ -104,7 +104,8 @@ namespace usql {
bool Lexer::isRelationalOperator(TokenType token_type) {
return (token_type == TokenType::equal || token_type == TokenType::not_equal ||
token_type == TokenType::greater || token_type == TokenType::greater_equal ||
token_type == TokenType::lesser || token_type == TokenType::lesser_equal);
token_type == TokenType::lesser || token_type == TokenType::lesser_equal ||
token_type == TokenType::is);
}
bool Lexer::isLogicalOperator(TokenType token_type) {
@@ -119,105 +120,59 @@ namespace usql {
TokenType Lexer::type(const std::string &token) {
// FIXME 'one is evaluated as identifier
if (token == ";")
return TokenType::semicolon;
if (token == "+")
return TokenType::plus;
if (token == "-")
return TokenType::minus;
if (token == "*")
return TokenType::multiply;
if (token == "/")
return TokenType::divide;
if (token == "(")
return TokenType::open_paren;
if (token == ")")
return TokenType::close_paren;
if (token == "=")
return TokenType::equal;
if (token == "!=" || token == "<>")
return TokenType::not_equal;
if (token == ">")
return TokenType::greater;
if (token == ">=")
return TokenType::greater_equal;
if (token == "<")
return TokenType::lesser;
if (token == "<=")
return TokenType::lesser_equal;
if (token == "as")
return TokenType::keyword_as;
if (token == "create")
return TokenType::keyword_create;
if (token == "drop")
return TokenType::keyword_drop;
if (token == "where")
return TokenType::keyword_where;
if (token == "order")
return TokenType::keyword_order;
if (token == "by")
return TokenType::keyword_by;
if (token == "offset")
return TokenType::keyword_offset;
if (token == "limit")
return TokenType::keyword_limit;
if (token == "asc")
return TokenType::keyword_asc;
if (token == "desc")
return TokenType::keyword_desc;
if (token == "from")
return TokenType::keyword_from;
if (token == "delete")
return TokenType::keyword_delete;
if (token == "table")
return TokenType::keyword_table;
if (token == "insert")
return TokenType::keyword_insert;
if (token == "into")
return TokenType::keyword_into;
if (token == "values")
return TokenType::keyword_values;
if (token == "select")
return TokenType::keyword_select;
if (token == "set")
return TokenType::keyword_set;
if (token == "copy")
return TokenType::keyword_copy;
if (token == "update")
return TokenType::keyword_update;
if (token == "load")
return TokenType::keyword_load;
if (token == "save")
return TokenType::keyword_save;
if (token == "not")
return TokenType::keyword_not;
if (token == "null")
return TokenType::keyword_null;
if (token == "integer")
return TokenType::keyword_integer;
if (token == "float")
return TokenType::keyword_float;
if (token == "varchar")
return TokenType::keyword_varchar;
if (token == "date")
return TokenType::keyword_date;
if (token == "boolean")
return TokenType::keyword_bool;
if (token == "distinct")
return TokenType::keyword_distinct;
if (token == "show")
return TokenType::keyword_show;
if (token == "or")
return TokenType::logical_or;
if (token == "and")
return TokenType::logical_and;
if (token == ",")
return TokenType::comma;
if (token == "\n" || token == "\r\n" || token == "\r")
return TokenType::newline;
if (token == ";") return TokenType::semicolon;
if (token == "+") return TokenType::plus;
if (token == "-") return TokenType::minus;
if (token == "*") return TokenType::multiply;
if (token == "/") return TokenType::divide;
if (token == "(") return TokenType::open_paren;
if (token == ")") return TokenType::close_paren;
if (token == "=") return TokenType::equal;
if (token == "!=" || token == "<>") return TokenType::not_equal;
if (token == ">") return TokenType::greater;
if (token == ">=") return TokenType::greater_equal;
if (token == "<") return TokenType::lesser;
if (token == "<=") return TokenType::lesser_equal;
if (token == "is") return TokenType::is;
if (token == "as") return TokenType::keyword_as;
if (token == "create") return TokenType::keyword_create;
if (token == "drop") return TokenType::keyword_drop;
if (token == "where") return TokenType::keyword_where;
if (token == "order") return TokenType::keyword_order;
if (token == "by") return TokenType::keyword_by;
if (token == "offset") return TokenType::keyword_offset;
if (token == "limit") return TokenType::keyword_limit;
if (token == "asc") return TokenType::keyword_asc;
if (token == "desc") return TokenType::keyword_desc;
if (token == "from") return TokenType::keyword_from;
if (token == "delete") return TokenType::keyword_delete;
if (token == "table") return TokenType::keyword_table;
if (token == "insert") return TokenType::keyword_insert;
if (token == "into") return TokenType::keyword_into;
if (token == "values") return TokenType::keyword_values;
if (token == "select") return TokenType::keyword_select;
if (token == "set") return TokenType::keyword_set;
if (token == "copy") return TokenType::keyword_copy;
if (token == "update") return TokenType::keyword_update;
if (token == "load") return TokenType::keyword_load;
if (token == "save") return TokenType::keyword_save;
if (token == "not") return TokenType::keyword_not;
if (token == "null") return TokenType::keyword_null;
if (token == "integer") return TokenType::keyword_integer;
if (token == "float") return TokenType::keyword_float;
if (token == "varchar") return TokenType::keyword_varchar;
if (token == "date") return TokenType::keyword_date;
if (token == "boolean") return TokenType::keyword_bool;
if (token == "true") return TokenType::keyword_true;
if (token == "false") return TokenType::keyword_false;
if (token == "distinct") return TokenType::keyword_distinct;
if (token == "show") return TokenType::keyword_show;
if (token == "or") return TokenType::logical_or;
if (token == "and") return TokenType::logical_and;
if (token == ",") return TokenType::comma;
if (token == "\n" || token == "\r\n" || token == "\r") return TokenType::newline;
if (token.length() > 1 && token.at(0) == '%' &&
(token.at(token.length() - 1) == '\n' || token.at(token.length() - 1) == '\r'))
if (token.length() > 1 && token.at(0) == '%' && (token.at(token.length() - 1) == '\n' || token.at(token.length() - 1) == '\r'))
return TokenType::comment;
if (token.length() >= 2 && token.at(0) == '"' && token.at(token.length() - 1) == '"')
@@ -226,17 +181,10 @@ namespace usql {
if (token.length() >= 2 && token.at(0) == '\'' && token.at(token.length() - 1) == '\'')
return TokenType::string_literal;
if (std::regex_match(token, k_int_regex))
return TokenType::int_number;
if (std::regex_match(token, k_int_underscored_regex))
return TokenType::int_number;
if (std::regex_match(token, k_double_regex))
return TokenType::double_number;
if (std::regex_match(token, k_identifier_regex))
return TokenType::identifier;
if (std::regex_match(token, k_int_regex)) return TokenType::int_number;
if (std::regex_match(token, k_int_underscored_regex)) return TokenType::int_number;
if (std::regex_match(token, k_double_regex)) return TokenType::double_number;
if (std::regex_match(token, k_identifier_regex)) return TokenType::identifier;
return TokenType::undef;
}
@@ -279,169 +227,65 @@ namespace usql {
}
std::string Lexer::typeToString(TokenType token_type) {
std::string txt;
switch (token_type) {
case TokenType::undef:
txt = "undef";
break;
case TokenType::identifier:
txt = "identifier";
break;
case TokenType::plus:
txt = "+";
break;
case TokenType::minus:
txt = "-";
break;
case TokenType::multiply:
txt = "*";
break;
case TokenType::divide:
txt = "/";
break;
case TokenType::equal:
txt = "==";
break;
case TokenType::not_equal:
txt = "!=";
break;
case TokenType::greater:
txt = ">";
break;
case TokenType::greater_equal:
txt = ">=";
break;
case TokenType::lesser:
txt = "<";
break;
case TokenType::lesser_equal:
txt = "<=";
break;
case TokenType::keyword_as:
txt = "as";
break;
case TokenType::keyword_create:
txt = "create";
break;
case TokenType::keyword_drop:
txt = "drop";
break;
case TokenType::keyword_where:
txt = "where";
break;
case TokenType::keyword_order:
txt = "order";
break;
case TokenType::keyword_by:
txt = "by";
break;
case TokenType::keyword_offset:
txt = "offset";
break;
case TokenType::keyword_limit:
txt = "limit";
break;
case TokenType::keyword_asc:
txt = "asc";
break;
case TokenType::keyword_desc:
txt = "desc";
break;
case TokenType::keyword_table:
txt = "table";
break;
case TokenType::keyword_into:
txt = "into";
break;
case TokenType::keyword_values:
txt = "values";
break;
case TokenType::keyword_select:
txt = "select";
break;
case TokenType::keyword_set:
txt = "set";
break;
case TokenType::keyword_copy:
txt = "copy";
break;
case TokenType::keyword_update:
txt = "update";
break;
case TokenType::keyword_load:
txt = "load";
break;
case TokenType::keyword_save:
txt = "save";
break;
case TokenType::keyword_not:
txt = "not";
break;
case TokenType::keyword_null:
txt = "null";
break;
case TokenType::keyword_integer:
txt = "integer";
break;
case TokenType::keyword_float:
txt = "float";
break;
case TokenType::keyword_varchar:
txt = "varchar";
break;
case TokenType::keyword_date:
txt = "date";
break;
case TokenType::keyword_bool:
txt = "boolean";
break;
case TokenType::keyword_distinct:
txt = "distinct";
break;
case TokenType::keyword_show:
txt = "show";
break;
case TokenType::int_number:
txt = "int number";
break;
case TokenType::double_number:
txt = "double number";
break;
case TokenType::string_literal:
txt = "string literal";
break;
case TokenType::open_paren:
txt = "(";
break;
case TokenType::close_paren:
txt = ")";
break;
case TokenType::logical_and:
txt = "and";
break;
case TokenType::logical_or:
txt = "or";
break;
case TokenType::semicolon:
txt = ";";
break;
case TokenType::comma:
txt = ",";
break;
case TokenType::newline:
txt = "newline";
break;
case TokenType::comment:
txt = "comment";
break;
case TokenType::eof:
txt = "eof";
break;
case TokenType::undef: return "undef";
case TokenType::identifier: return "identifier";
case TokenType::plus: return "+";
case TokenType::minus: return "-";
case TokenType::multiply: return "*";
case TokenType::divide: return "/";
case TokenType::equal: return "==";
case TokenType::not_equal: return "!=";
case TokenType::greater: return ">";
case TokenType::greater_equal: return ">=";
case TokenType::lesser: return "<";
case TokenType::lesser_equal: return "<=";
case TokenType::is: return "is";
case TokenType::keyword_as: return "as";
case TokenType::keyword_create: return "create";
case TokenType::keyword_drop: return "drop";
case TokenType::keyword_where: return "where";
case TokenType::keyword_order: return "order";
case TokenType::keyword_by: return "by";
case TokenType::keyword_offset: return "offset";
case TokenType::keyword_limit: return "limit";
case TokenType::keyword_asc: return "asc";
case TokenType::keyword_desc: return "desc";
case TokenType::keyword_table: return "table";
case TokenType::keyword_into: return "into";
case TokenType::keyword_values: return "values";
case TokenType::keyword_select: return "select";
case TokenType::keyword_set: return "set";
case TokenType::keyword_copy: return "copy";
case TokenType::keyword_update: return "update";
case TokenType::keyword_load: return "load";
case TokenType::keyword_save: return "save";
case TokenType::keyword_not: return "not";
case TokenType::keyword_null: return "null";
case TokenType::keyword_integer: return "integer";
case TokenType::keyword_float: return "float";
case TokenType::keyword_varchar: return "varchar";
case TokenType::keyword_date: return "date";
case TokenType::keyword_bool: return "boolean";
case TokenType::keyword_true: return "true";
case TokenType::keyword_false: return "false";
case TokenType::keyword_distinct: return "distinct";
case TokenType::keyword_show: return "show";
case TokenType::int_number: return "int number";
case TokenType::double_number: return "double number";
case TokenType::string_literal: return "string literal";
case TokenType::open_paren: return "(";
case TokenType::close_paren: return ")";
case TokenType::logical_and: return "and";
case TokenType::logical_or: return "or";
case TokenType::semicolon: return ";";
case TokenType::comma: return ",";
case TokenType::newline: return "newline";
case TokenType::comment: return "comment";
case TokenType::eof: return "eof";
default:
txt = "FIXME, unknown token type";
break;
return "FIXME, unknown token type";
}
return txt;
}
}