vaclavt/mlisp
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

usql update

Browse Source
This commit is contained in:
VaclavT
2021-08-23 18:14:05 +02:00
parent 8d220356f2
commit 0e90d6047c
28 changed files with 1623 additions and 3774 deletions
Download Patch File Download Diff File Expand all files Collapse all files

116
usql/lexer.cpp
View File

@@ -14,7 +14,7 @@ namespace usql {
Lexer::Lexer() {
k_words_regex =
"[-+]?[0-9]+\\.[0-9]+|[-+]?[0-9][0-9_]+[0-9]|[0-9]+|[A-Za-z]+[A-Za-z0-9_#]*|[\\(\\)\\[\\]\\{\\}]|[-\\+\\*/"
",;:\?]|==|>=|<=|~=|>|<|=|;|~|\\||or|and|\n|\r|\r\n|'([^']|'')*'|\".*?\"|%.*?\n";
",;:\?]|!=|<>|==|>=|<=|~=|>|<|=|;|~|\\||or|and|\n|\r|\r\n|'([^']|'')*'|\".*?\"|%.*?\n";
k_int_regex = "[-+]?[0-9]+";
k_int_underscored_regex = "[-+]?[0-9][0-9_]+[0-9]";
k_double_regex = "[-+]?[0-9]+\\.[0-9]+";
@@ -22,16 +22,15 @@ namespace usql {
}
void Lexer::parse(const std::string &code) {
// TODO handle empty code
m_tokens.clear();
if (code.empty())
throw Exception("empty code");
m_tokens.clear();
m_tokens.reserve(64);
// PERF something like this to preallocate ??
if (code.size() > 100) {
m_tokens.reserve(code.size() / 10);
}
m_code_str = code;
if (!m_code_str.empty() && m_code_str.back() != '\n') {
m_code_str.append("\n"); // TODO temp solution to prevent possible situation when last line is a comment
m_code_str.append("\n"); // temp solution to prevent possible situation when last line is a comment
}
auto words_begin = std::sregex_iterator(m_code_str.begin(), m_code_str.end(), k_words_regex);
@@ -45,7 +44,7 @@ namespace usql {
match_str = stringLiteral(match_str);
if (token_type != TokenType::newline)
m_tokens.push_back(Token{match_str, token_type});
m_tokens.emplace_back(match_str, token_type);
}
// DEBUG IT
@@ -56,8 +55,8 @@ namespace usql {
void Lexer::debugTokens() {
int i = 0;
for (std::vector<Token>::iterator it = m_tokens.begin(); it != m_tokens.end(); ++it) {
std::cerr << i << "\t" << it->token_string << std::endl;
for (auto & m_token : m_tokens) {
std::cerr << i << "\t" << m_token.token_string << std::endl;
i++;
}
}
@@ -97,8 +96,6 @@ namespace usql {
return m_index < m_tokens.size() - 1 ? m_tokens[m_index + 1].type : TokenType::eof;
}
TokenType Lexer::prevTokenType() { return m_index > 0 ? m_tokens[m_index - 1].type : TokenType::undef; }
bool Lexer::isRelationalOperator(TokenType token_type) {
return (token_type == TokenType::equal || token_type == TokenType::not_equal ||
token_type == TokenType::greater || token_type == TokenType::greater_equal ||
@@ -116,118 +113,101 @@ namespace usql {
}
TokenType Lexer::type(const std::string &token) {
// TODO, FIXME 'one is evaluated as identifier
// FIXME 'one is evaluated as identifier
if (token == ";")
return TokenType::semicolon;
if (token == "+")
return TokenType::plus;
if (token == "-")
return TokenType::minus;
if (token == "*")
return TokenType::multiply;
if (token == "/")
return TokenType::divide;
if (token == "(")
return TokenType::open_paren;
if (token == ")")
return TokenType::close_paren;
if (token == "=")
return TokenType::equal;
if (token == "!=")
if (token == "!=" || token == "<>")
return TokenType::not_equal;
if (token == ">")
return TokenType::greater;
if (token == ">=")
return TokenType::greater_equal;
if (token == "<")
return TokenType::lesser;
if (token == "<=")
return TokenType::lesser_equal;
if (token == "as")
return TokenType::keyword_as;
if (token == "create")
return TokenType::keyword_create;
if (token == "drop")
return TokenType::keyword_drop;
if (token == "where")
return TokenType::keyword_where;
if (token == "order")
return TokenType::keyword_order;
if (token == "by")
return TokenType::keyword_by;
if (token == "offset")
return TokenType::keyword_offset;
if (token == "limit")
return TokenType::keyword_limit;
if (token == "asc")
return TokenType::keyword_asc;
if (token == "desc")
return TokenType::keyword_desc;
if (token == "from")
return TokenType::keyword_from;
if (token == "delete")
return TokenType::keyword_delete;
if (token == "table")
return TokenType::keyword_table;
if (token == "insert")
return TokenType::keyword_insert;
if (token == "into")
return TokenType::keyword_into;
if (token == "values")
return TokenType::keyword_values;
if (token == "select")
return TokenType::keyword_select;
if (token == "set")
return TokenType::keyword_set;
if (token == "copy")
return TokenType::keyword_copy;
if (token == "update")
return TokenType::keyword_update;
if (token == "load")
return TokenType::keyword_load;
if (token == "save")
return TokenType::keyword_save;
if (token == "not")
return TokenType::keyword_not;
if (token == "null")
return TokenType::keyword_null;
if (token == "integer")
return TokenType::keyword_integer;
if (token == "float")
return TokenType::keyword_float;
if (token == "varchar")
return TokenType::keyword_varchar;
if (token == "date")
return TokenType::keyword_date;
if (token == "boolean")
return TokenType::keyword_bool;
if (token == "distinct")
return TokenType::keyword_distinct;
if (token == "show")
return TokenType::keyword_show;
if (token == "or")
return TokenType::logical_or;
if (token == "and")
return TokenType::logical_and;
if (token == ",")
return TokenType::comma;
if (token == "\n" || token == "\r\n" || token == "\r")
return TokenType::newline;
@@ -267,7 +247,7 @@ namespace usql {
if (!replace) {
return str;
}
std::string out = "";
std::string out;
out.reserve(str.size());
@@ -347,6 +327,24 @@ namespace usql {
case TokenType::keyword_where:
txt = "where";
break;
case TokenType::keyword_order:
txt = "order";
break;
case TokenType::keyword_by:
txt = "by";
break;
case TokenType::keyword_offset:
txt = "offset";
break;
case TokenType::keyword_limit:
txt = "limit";
break;
case TokenType::keyword_asc:
txt = "asc";
break;
case TokenType::keyword_desc:
txt = "desc";
break;
case TokenType::keyword_table:
txt = "table";
break;
@@ -389,6 +387,18 @@ namespace usql {
case TokenType::keyword_varchar:
txt = "varchar";
break;
case TokenType::keyword_date:
txt = "date";
break;
case TokenType::keyword_bool:
txt = "boolean";
break;
case TokenType::keyword_distinct:
txt = "distinct";
break;
case TokenType::keyword_show:
txt = "show";
break;
case TokenType::int_number:
txt = "int number";
break;