vaclavt/usql
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

small refactoring

Browse Source
This commit is contained in:
VaclavT
2021-12-19 12:58:17 +01:00
parent 04c0ed3f03
commit 35cba3b0c4
8 changed files with 753 additions and 746 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
csvreader.cpp
View File

@@ -1,4 +1,4 @@
#include <errno.h>
#include <cerrno>
#include "exception.h"
#include "csvreader.h"
@@ -7,7 +7,7 @@
namespace usql {
CsvReader::CsvReader(bool skip_hdr, char field_sep, char quote_ch, char line_sep, char line_sep2) {
CsvReader::CsvReader(bool skip_hdr, char field_sep, char quote_ch, char line_sep, char line_sep2) {
skip_header = skip_hdr;
field_separator = field_sep;
quote_character = quote_ch;
@@ -15,7 +15,7 @@ namespace usql {
line_separator2 = line_sep2;
header_skiped = !skip_hdr;
}
}
int CsvReader::parseCSV(const std::string &filename, std::vector<ColDefNode> &cols_def, Table &table) {
@@ -41,7 +41,7 @@ int CsvReader::parseCSV(const std::string &filename, std::vector<ColDefNode> &co
size_t len = 0;
int read_chars;
long read_chars;
while ((read_chars = getline(&line_str, &len, fp)) != -1) {
if (skip_header && !header_skiped) {
header_skiped = true;
@@ -59,7 +59,7 @@ int CsvReader::parseCSV(const std::string &filename, std::vector<ColDefNode> &co
if (*aChar == quote_character) {
inQuote = !inQuote;
} else if (*aChar == field_separator) {
if (inQuote == true) {
if (inQuote) {
field += *aChar;
} else {
line.push_back(field);
@@ -80,9 +80,6 @@ int CsvReader::parseCSV(const std::string &filename, std::vector<ColDefNode> &co
field.clear();
line.clear();
// DEBUG
// if (row_cnt > 50000) break;
//
}
fclose(fp);
@@ -93,7 +90,7 @@ int CsvReader::parseCSV(const std::string &filename, std::vector<ColDefNode> &co
return row_cnt;
}
int CsvReader::parseCSV2(const std::string &csvSource, std::vector<ColDefNode> &cols_def, Table& table) {
int CsvReader::parseCSV2(const std::string &csvSource, std::vector<ColDefNode> &cols_def, Table& table) {
int row_cnt = 0;
bool inQuote(false);
bool newLine(false);
@@ -109,17 +106,17 @@ int CsvReader::parseCSV(const std::string &filename, std::vector<ColDefNode> &co
inQuote = !inQuote;
} else if (*aChar == field_separator) {
newLine = false;
if (inQuote == true) {
if (inQuote) {
field += *aChar;
} else {
line.push_back(field);
field.clear();
}
} else if (*aChar == line_separator || *aChar == line_separator2) {
if (inQuote == true) {
if (inQuote) {
field += *aChar;
} else {
if (newLine == false) {
if (!newLine) {
line.push_back(field);
if (header_skiped) {
table.create_row_from_vector(cols_def, line);

2
csvreader.h
View File

@@ -24,7 +24,7 @@ namespace usql {
bool header_skiped;
public:
CsvReader(bool skip_hdr = true, char field_sep = ',', char quote_ch = '"', char line_sep = '\r', char line_sep2 = '\n');
explicit CsvReader(bool skip_hdr = true, char field_sep = ',', char quote_ch = '"', char line_sep = '\r', char line_sep2 = '\n');
int parseCSV2(const std::string &csvSource, std::vector<ColDefNode> &cols_def, Table& table);

2
debug.h
View File

@@ -25,6 +25,8 @@ std::vector<std::string> k_debug_sql_commands {
"insert into a (i, s) values(2, 'two')",
"insert into a (i, s) values(2, 'second two')",
"insert into a (i, s) values(3, 'three')",
"insert into a (i, s) values(4, 'four')",
"save a into '/tmp/a.csv'",
"set 'USE_INDEXSCAN' = 'true'",
// "select * from a where 1 = i",
// "delete from a where i = 2 and s ='two'",

70
lexer.cpp
View File

@@ -5,13 +5,13 @@
namespace usql {
Token::Token(const std::string &token_str, TokenType typ) {
Token::Token(const std::string &token_str, TokenType typ) {
token_string = token_str;
type = typ;
}
}
Lexer::Lexer() {
Lexer::Lexer() {
k_words_regex =
"[-+]?[0-9]+\\.[0-9]+|[-+]?[0-9]+|[A-Za-z]+[A-Za-z0-9_#]*|[\\(\\)\\[\\]\\{\\}]|[-\\+\\*/"
",;:\?]|!=|<>|==|>=|<=|~=|>|<|=|;|~|\\||\n|\r|\r\n|'([^']|'')*'|\".*?\"|%.*?\n";
@@ -19,9 +19,9 @@ namespace usql {
k_int_underscored_regex = "[-+]?[0-9][0-9_]+[0-9]";
k_double_regex = "[-+]?[0-9]+\\.[0-9]+";
k_identifier_regex = "[A-Za-z]+[A-Za-z0-9_#]*";
}
}
void Lexer::parse(const std::string &code) {
void Lexer::parse(const std::string &code) {
if (code.empty())
throw Exception("Lexer.parse empty code");
@@ -53,74 +53,74 @@ namespace usql {
// debugTokens();
m_index = 0;
}
}
void Lexer::debugTokens() {
void Lexer::debugTokens() {
int i = 0;
for (auto & m_token : m_tokens) {
std::cerr << i << "\t" << m_token.token_string << std::endl;
i++;
}
}
}
Token Lexer::currentToken() { return m_tokens[m_index]; }
Token Lexer::currentToken() { return m_tokens[m_index]; }
Token Lexer::consumeToken() {
Token Lexer::consumeToken() {
int i = m_index;
nextToken();
return m_tokens[i];
}
}
Token Lexer::consumeToken(TokenType type) {
Token Lexer::consumeToken(TokenType type) {
int i = m_index;
skipToken(type);
return m_tokens[i];
}
}
void Lexer::nextToken() {
void Lexer::nextToken() {
if (m_index < m_tokens.size()) {
m_index++;
}
}
}
void Lexer::skipToken(TokenType type) {
void Lexer::skipToken(TokenType type) {
if (tokenType() == type) {
nextToken();
} else {
throw Exception("ERROR unexpected token " + consumeToken().token_string + ", instead of " + typeToString(type));
}
}
}
void Lexer::skipTokenOptional(TokenType type) {
void Lexer::skipTokenOptional(TokenType type) {
if (tokenType() == type) {
nextToken();
}
}
}
TokenType Lexer::tokenType() { return m_index < m_tokens.size() ? currentToken().type : TokenType::eof; }
TokenType Lexer::tokenType() { return m_index < m_tokens.size() ? currentToken().type : TokenType::eof; }
TokenType Lexer::nextTokenType() {
TokenType Lexer::nextTokenType() {
return m_index < m_tokens.size() - 1 ? m_tokens[m_index + 1].type : TokenType::eof;
}
}
bool Lexer::isRelationalOperator(TokenType token_type) {
bool Lexer::isRelationalOperator(TokenType token_type) {
return (token_type == TokenType::equal || token_type == TokenType::not_equal ||
token_type == TokenType::greater || token_type == TokenType::greater_equal ||
token_type == TokenType::lesser || token_type == TokenType::lesser_equal ||
token_type == TokenType::is);
}
}
bool Lexer::isLogicalOperator(TokenType token_type) {
bool Lexer::isLogicalOperator(TokenType token_type) {
return (token_type == TokenType::logical_and || token_type == TokenType::logical_or);
}
}
bool Lexer::isArithmeticalOperator(TokenType token_type) {
bool Lexer::isArithmeticalOperator(TokenType token_type) {
return (token_type == TokenType::plus || token_type == TokenType::minus ||
token_type == TokenType::multiply ||
token_type == TokenType::divide);
}
}
TokenType Lexer::type(const std::string &token) {
TokenType Lexer::type(const std::string &token) {
if (token == ";") return TokenType::semicolon;
if (token == "+") return TokenType::plus;
if (token == "-") return TokenType::minus;
@@ -190,9 +190,9 @@ namespace usql {
if (std::regex_match(token, k_identifier_regex)) return TokenType::identifier;
return TokenType::undef;
}
}
std::string Lexer::stringLiteral(std::string token) {
std::string Lexer::stringLiteral(std::string token) {
// remove ' or " from the literal ends
bool replace = token[0] == '\'' && token[token.size() - 1] == '\'';
@@ -227,9 +227,9 @@ namespace usql {
}
}
return out;
}
}
std::string Lexer::typeToString(TokenType token_type) {
std::string Lexer::typeToString(TokenType token_type) {
switch (token_type) {
case TokenType::undef: return "undef";
case TokenType::identifier: return "identifier";
@@ -291,6 +291,6 @@ namespace usql {
default:
return "FIXME, unknown token type";
}
}
}
} // namespace usql

148
parser.h
View File

@@ -13,15 +13,15 @@ static const int FUNCTION_CALL = -1;
namespace usql {
enum class ColumnType {
enum class ColumnType {
integer_type,
float_type,
varchar_type,
date_type,
bool_type
};
};
enum class NodeType {
enum class NodeType {
true_node,
null_value,
int_value,
@@ -49,9 +49,9 @@ namespace usql {
function,
column_def,
error
};
};
struct Node {
struct Node {
NodeType node_type;
explicit Node(const NodeType type) : node_type(type) {}
@@ -60,10 +60,10 @@ namespace usql {
virtual void dump() const {
std::cout << "type: Node" << std::endl;
}
};
};
struct ColOrderNode : Node {
struct ColOrderNode : Node {
std::string col_name;
int col_index;
bool ascending;
@@ -74,11 +74,11 @@ namespace usql {
void dump() const override {
std::cout << "type: ColOrderNode, col_name: " << col_name << ", col_index: " << col_index << ", asc: " << ascending << std::endl;
}
};
};
struct OffsetLimitNode : Node {
struct OffsetLimitNode : Node {
int offset;
int limit;
@@ -87,10 +87,10 @@ namespace usql {
void dump() const override {
std::cout << "type: OffsetLimitNode, offset: " << offset << ", limit: " << limit << std::endl;
}
};
};
struct SelectColNode : Node {
struct SelectColNode : Node {
std::unique_ptr<Node> value;
std::string name;
@@ -101,9 +101,9 @@ namespace usql {
std::cout << "type: SelectColNode, name:" << name << "value:" << std::endl;
value->dump();
}
};
};
struct ColDefNode : Node {
struct ColDefNode : Node {
std::string name;
ColumnType type;
int order;
@@ -117,9 +117,9 @@ namespace usql {
void dump() const override {
std::cout << "type: ColDefNode, name: " << name << ", type: " << (int)type << " TODO add more" << std::endl;
}
};
};
struct FunctionNode : Node {
struct FunctionNode : Node {
std::string function; // TODO use enum
std::vector<std::unique_ptr<Node>> params;
@@ -129,17 +129,17 @@ namespace usql {
void dump() const override {
std::cout << "type: FunctionNode, function: " << function << " TODO add more" << std::endl;
}
};
};
struct TrueNode : Node {
struct TrueNode : Node {
TrueNode() : Node(NodeType::true_node) {}
void dump() const override {
std::cout << "type: TrueNode," << std::endl;
}
};
};
struct ValueNode : Node {
struct ValueNode : Node {
explicit ValueNode(NodeType type) : Node(type) {}
virtual bool isNull() const { return false; }
@@ -150,9 +150,9 @@ namespace usql {
virtual bool getBooleanValue() const = 0;
~ValueNode() override = default;
};
};
struct NullValueNode : ValueNode {
struct NullValueNode : ValueNode {
NullValueNode() : ValueNode(NodeType::null_value) {}
@@ -167,9 +167,9 @@ namespace usql {
void dump() const override {
std::cout << "type: NullValueNode," << std::endl;
}
};
};
struct IntValueNode : ValueNode {
struct IntValueNode : ValueNode {
long value;
explicit IntValueNode(long value) : ValueNode(NodeType::int_value), value(value) {}
@@ -183,9 +183,9 @@ namespace usql {
void dump() const override {
std::cout << "type: IntValueNode, value: " << value << std::endl;
}
};
};
struct DoubleValueNode : ValueNode {
struct DoubleValueNode : ValueNode {
double value;
explicit DoubleValueNode(double value) : ValueNode(NodeType::float_value), value(value) {}
@@ -199,9 +199,9 @@ namespace usql {
void dump() const override {
std::cout << "type: DoubleValueNode, value: " << value << std::endl;
}
};
};
struct StringValueNode : ValueNode {
struct StringValueNode : ValueNode {
std::string value;
explicit StringValueNode(std::string value) : ValueNode(NodeType::string_value), value(std::move(value)) {}
@@ -215,9 +215,9 @@ namespace usql {
void dump() const override {
std::cout << "type: StringValueNode, value: " << value << std::endl;
}
};
};
struct BooleanValueNode : ValueNode {
struct BooleanValueNode : ValueNode {
bool value;
explicit BooleanValueNode(bool value) : ValueNode(NodeType::bool_value), value(value) {}
@@ -231,10 +231,10 @@ namespace usql {
void dump() const override {
std::cout << "type: BooleanValueNode, value: " << value << std::endl;
}
};
};
struct DatabaseValueNode : Node {
struct DatabaseValueNode : Node {
std::string col_name;
explicit DatabaseValueNode(std::string name) : Node(NodeType::database_value), col_name(std::move(name)) {}
@@ -242,15 +242,15 @@ namespace usql {
void dump() const override {
std::cout << "type: DatabaseValueNode, col_name: " << col_name << std::endl;
}
};
};
enum class LogicalOperatorType {
enum class LogicalOperatorType {
and_operator,
or_operator
// not_operator
};
};
struct LogicalOperatorNode : Node {
struct LogicalOperatorNode : Node {
LogicalOperatorType op;
std::unique_ptr<Node> left;
std::unique_ptr<Node> right;
@@ -263,9 +263,9 @@ namespace usql {
left->dump();
right->dump();
}
};
};
enum class RelationalOperatorType {
enum class RelationalOperatorType {
equal,
greater,
greater_equal,
@@ -275,9 +275,9 @@ namespace usql {
is,
is_not
// like
};
};
struct RelationalOperatorNode : Node {
struct RelationalOperatorNode : Node {
RelationalOperatorType op;
std::unique_ptr<Node> left;
@@ -291,17 +291,17 @@ namespace usql {
left->dump();
right->dump();
}
};
};
enum class ArithmeticalOperatorType {
enum class ArithmeticalOperatorType {
copy_value, // just copy lef value and do nothing with it
plus_operator,
minus_operator,
multiply_operator,
divide_operator
};
};
struct ArithmeticalOperatorNode : Node {
struct ArithmeticalOperatorNode : Node {
ArithmeticalOperatorType op;
std::unique_ptr<Node> left;
@@ -315,9 +315,9 @@ namespace usql {
left->dump();
right->dump();
}
};
};
struct CreateTableNode : Node {
struct CreateTableNode : Node {
std::string table_name;
std::vector<ColDefNode> cols_defs;
@@ -327,9 +327,9 @@ namespace usql {
void dump() const override {
std::cout << "type: CreateTableNode, table_name: " << table_name << "TODO complete me" << std::endl;
}
};
};
struct InsertIntoTableNode : Node {
struct InsertIntoTableNode : Node {
std::string table_name;
std::vector<DatabaseValueNode> cols_names;
std::vector<std::unique_ptr<Node>> cols_values;
@@ -340,9 +340,9 @@ namespace usql {
void dump() const override {
std::cout << "type: InsertIntoTableNode, table_name: " << table_name << "TODO complete me" << std::endl;
}
};
};
struct SelectFromTableNode : Node {
struct SelectFromTableNode : Node {
std::string table_name;
std::unique_ptr<std::vector<SelectColNode>> cols_names;
std::unique_ptr<Node> where;
@@ -357,9 +357,9 @@ namespace usql {
std::cout << "type: SelectFromTableNode, table_name: " << table_name << "TODO complete me" << std::endl;
where->dump();
}
};
};
struct CreateTableAsSelectNode : Node {
struct CreateTableAsSelectNode : Node {
std::string table_name;
std::unique_ptr<Node> select_table;
@@ -370,9 +370,9 @@ namespace usql {
std::cout << "type: CreateTableAsSelectNode, table_name: " << table_name << std::endl;
select_table->dump();
}
};
};
struct UpdateTableNode : Node {
struct UpdateTableNode : Node {
std::string table_name;
std::vector<DatabaseValueNode> cols_names;
std::vector<std::unique_ptr<Node>> values;
@@ -387,9 +387,9 @@ namespace usql {
std::cout << "type: UpdateTableNode, table_name: " << table_name << "TODO complete me" << std::endl;
where->dump();
}
};
};
struct LoadIntoTableNode : Node {
struct LoadIntoTableNode : Node {
std::string table_name;
std::string filename;
@@ -399,9 +399,9 @@ namespace usql {
void dump() const override {
std::cout << "type: LoadIntoTableNode, table_name: " << table_name << ", filename" << filename << std::endl;
}
};
};
struct SaveTableNode : Node {
struct SaveTableNode : Node {
std::string table_name;
std::string filename;
@@ -411,9 +411,9 @@ namespace usql {
void dump() const override {
std::cout << "type: SaveTableNode, table_name: " << table_name << ", filename" << filename << std::endl;
}
};
};
struct DropTableNode : Node {
struct DropTableNode : Node {
std::string table_name;
explicit DropTableNode(std::string name) : Node(NodeType::drop_table), table_name(std::move(name)) {}
@@ -421,9 +421,9 @@ namespace usql {
void dump() const override {
std::cout << "type: SelectFromTableNode, table_name: " << table_name << std::endl;
}
};
};
struct DeleteFromTableNode : Node {
struct DeleteFromTableNode : Node {
std::string table_name;
std::unique_ptr<Node> where;
@@ -434,9 +434,9 @@ namespace usql {
std::cout << "type: DeleteFromTableNode, table_name: " << table_name << std::endl;
where->dump();
}
};
};
struct SetNode : Node {
struct SetNode : Node {
std::string name;
std::string value;
@@ -446,9 +446,9 @@ namespace usql {
void dump() const override {
std::cout << "type: SetNode, name: " << name << ", value: " << value << std::endl;
}
};
};
struct ShowNode : Node {
struct ShowNode : Node {
std::string name;
explicit ShowNode(std::string node_name) : Node(NodeType::show), name(std::move(node_name)) {}
@@ -456,9 +456,9 @@ namespace usql {
void dump() const override {
std::cout << "type: ShowNode, name: " << name << std::endl;
}
};
};
struct CreateIndexNode : Node {
struct CreateIndexNode : Node {
std::string index_name;
std::string table_name;
std::string column_name;
@@ -469,16 +469,16 @@ namespace usql {
void dump() const override {
std::cout << "type: CreateIndexNode, table_name: " << table_name << ", index_name: " << index_name << ", column_name: " << column_name << std::endl;
}
};
};
class Parser {
private:
public:
class Parser {
private:
public:
Parser();
std::unique_ptr<Node> parse(const std::string &code);
private:
private:
std::unique_ptr<Node> parse_create_table();
std::unique_ptr<Node> parse_drop_table();
std::unique_ptr<Node> parse_load_table();
@@ -504,8 +504,8 @@ namespace usql {
LogicalOperatorType parse_logical_operator();
ArithmeticalOperatorType parse_arithmetical_operator();
private:
private:
Lexer m_lexer;
};
};
} // namespace

12
row.cpp
View File

@@ -29,6 +29,18 @@ int ColStringValue::compare(ColValue &other) const {
return other.isNull() ? 1 : m_string->compare(other.getStringValue()); // null goes to end
}
std::string ColStringValue::getCsvStringValue() const {
auto src_str = getStringValue();
std::string toSearch{"\""}, replaceStr{"\\\""};
size_t pos = src_str.find(toSearch);
while(pos != std::string::npos) {
src_str.replace(pos, toSearch.size(), replaceStr);
pos =src_str.find(toSearch, pos + replaceStr.size());
}
return src_str;
}
int ColDateValue::compare(ColValue &other) const {
long r = m_date - other.getIntegerValue();
return other.isNull() ? 1 : r > 0 ? 1 : r == 0 ? 0 : -1;

6
row.h
View File

@@ -88,11 +88,7 @@ struct ColStringValue : ColValue {
long getIntegerValue() const override { return std::stoi(*m_string); };
double getDoubleValue() const override { return std::stod(*m_string); };
std::string getStringValue() const override { return *m_string; };
std::string getCsvStringValue() const override {
// TODO handle correctly CSV string
// ?? return std::regex_replace(getStringValue(), std::regex( "\"" ), "\\\"" );
return getStringValue();
};
std::string getCsvStringValue() const override;;
long getDateValue() const override { return std::stoi(*m_string); };
bool getBoolValue() const override { throw Exception("Not supported on ColStringValue"); };

2
utils/cp_to_mlisp.sh Normal file → Executable file
View File

@@ -1,4 +1,4 @@
#!/usr/bin/env bash
cp \
csvreader.h \