better table print, order by/offset/limit improvements

.vscode/settings.json

@@ -70,7 +70,9 @@
 		"mutex": "cpp",
 		"ostream": "cpp",
 		"streambuf": "cpp",
-		"thread": "cpp"
+		"thread": "cpp",
+		"compare": "cpp",
+		"complex": "cpp"
 	},
 	"C_Cpp.intelliSenseEngineFallback": "Disabled",
 	"cmake.configureOnOpen": true,

CMakeLists.txt

@@ -11,8 +11,10 @@ project(usql)
 
 set(PROJECT_NAME usql)
 
+include_directories(${CMAKE_SOURCE_DIR}/clib ${CMAKE_SOURCE_DIR})
+
 set(SOURCE
-	exception.cpp lexer.cpp parser.cpp usql.cpp main.cpp table.cpp table.h row.cpp row.h csvreader.cpp csvreader.h ml_date.cpp)
+	exception.cpp lexer.cpp parser.cpp usql.cpp main.cpp table.cpp table.h row.cpp row.h csvreader.cpp csvreader.h ml_date.cpp clib/ml_string.cpp)
 
 add_executable(${PROJECT_NAME} ${SOURCE})
 

Readme.md

@@ -1,11 +1,16 @@
 
 ### TODO
-- support for order by, offset, limit
+- support for distinct
+- command line interface
+- better table print
+- support for order by, offset, limit (allow column name in order by, validate)
+- add count min and max functions, eg aggregate functions
+- support for uniqueue indexes
 - support for btree indexes
 - support for parenthesis
+- functions rtrim, ltrim, rpad, lpad
 - support for *
 - add pipe | token
-- add count min and max functions, eg aggregate functions
 - maybe to create iterator on table
 - add exceptions and rename it to UsqlException
 - class members should have prefix m_

clib/ml_string.cpp

@@ -0,0 +1,104 @@
+
+#include "ml_string.h"
+
+
+// Replace a substring with a replacement string in a source string
+void replace_substring(std::string &src, const std::string &substr, const std::string &replacement) {
+	size_t i = 0;
+	for (i = src.find(substr, i); i != std::string::npos; i = src.find(substr, i)) {
+		src.replace(i, substr.size(), replacement);
+		i += replacement.size();
+	}
+}
+
+
+// Returns true if where contains regex
+bool regexp_search(const std::string &where, const std::string &regex_str) {
+	// online tester https://www.regextester.com/97722
+	std::regex regex(regex_str);
+	std::smatch match;
+
+	if (std::regex_search(where, match, regex)) {
+		// std::cout << "matches for '" << where << "'\n";
+		// std::cout << "Prefix: '" << match.prefix() << "'\n";
+		// for (size_t i = 0; i < match.size(); ++i)
+		// 	std::cout << i << ": " << match[i] << '\n';
+		// std::cout << "Suffix: '" << match.suffix() << "\'\n\n";
+
+		return true;
+	}
+	return false;
+}
+
+std::vector<std::string> regexp_strsplit(const std::string &string_to_split, const std::string &rgx_str) {
+	std::vector<std::string> elems;
+
+	std::regex rgx(rgx_str);
+	std::sregex_token_iterator iter(string_to_split.begin(), string_to_split.end(), rgx, -1);
+	std::sregex_token_iterator end;
+
+	for (; iter != end; ++iter)
+		elems.push_back(*iter);
+
+	return elems;
+}
+
+std::string string_lucase(std::string s, const std::string &strcase) {
+	if (strcase == "upper")
+		std::transform(s.begin(), s.end(),s.begin(), ::toupper);
+	
+	if (strcase == "lower")
+		std::transform(s.begin(), s.end(),s.begin(), ::tolower);
+	
+	return s;
+}
+
+std::string string_trim(std::string s, const std::string &chars_to_trim, const std::string &rltrim) {
+	if (rltrim == "ltrim" || rltrim == "trim")
+		s.erase(0, s.find_first_not_of(chars_to_trim));
+	
+	if (rltrim == "rtrim" || rltrim == "trim")
+		s.erase(s.find_last_not_of(chars_to_trim)+1);
+
+	return s;
+}
+
+std::string string_padd(const std::string &str, int pad_len, char fill_char, bool from_right) {
+	int str_len = str.length();
+
+	if (str_len == pad_len)
+		return str;
+	else if (str_len > pad_len) {
+		return (from_right ? str.substr(0, pad_len) : str.substr(str_len - pad_len, std::string::npos));
+	}
+
+	if (from_right)
+		return str + std::string(pad_len - str.size(), fill_char);
+	else
+		return std::string(pad_len - str.size(), fill_char) + str;
+}
+
+
+std::string string_substring(const std::string & str, long pos, long count) {
+	size_t	start_pos = pos;
+
+	if (pos < 0) {
+		start_pos = str.size() - abs(pos);
+	}
+	
+	if ( (start_pos >= str.size()) || (count < 1) || (start_pos < 0) ) {
+		throw std::invalid_argument("Invalid parameter(s) for string-substr.");
+	}
+
+	return str.substr(start_pos, count);
+}
+
+size_t string_find_substr(const std::string & str, const std::string & pattern, long pos) {
+	if (pos >= str.size()) {
+		throw std::invalid_argument("Invalid parameter(s) for string-find.");
+	}
+
+	size_t p = str.find(pattern, pos);
+	
+	return p != str.npos ? p : -1;
+}

clib/ml_string.h

@@ -0,0 +1,22 @@
+
+#pragma once
+
+#include <string>
+#include <regex>
+
+void replace_substring(std::string &src, const std::string &substr, const std::string &replacement);
+
+// Returns true if where contains regex
+bool regexp_search(const std::string &where, const std::string &regex_str);
+
+std::vector<std::string> regexp_strsplit(const std::string &string_to_split, const std::string &rgx_str);
+
+std::string string_lucase(std::string s, const std::string &strcase);
+
+std::string string_trim(std::string s, const std::string &chars_to_trim, const std::string &rltrim);
+
+std::string string_padd(const std::string & str, int pad_len, char fill_char, bool from_right);
+
+std::string string_substring(const std::string & str, long pos, long count);
+
+size_t string_find_substr(const std::string & str, const std::string & pattern, long pos);

lexer.cpp

@@ -155,6 +155,10 @@ namespace usql {
 		    return TokenType::keyword_order;
 	    if (token == "by")
 		    return TokenType::keyword_by;
+	    if (token == "offset")
+		    return TokenType::keyword_offset;
+	    if (token == "limit")
+		    return TokenType::keyword_limit;
 	    if (token == "asc")
 		    return TokenType::keyword_asc;
 	    if (token == "desc")
@@ -324,6 +328,12 @@ namespace usql {
 		    case TokenType::keyword_by:
 			    txt = "by";
 			    break;
+		    case TokenType::keyword_offset:
+			    txt = "offset";
+			    break;
+		    case TokenType::keyword_limit:
+			    txt = "limit";
+			    break;
 		    case TokenType::keyword_asc:
 			    txt = "asc";
 			    break;

lexer.h

@@ -27,6 +27,8 @@ namespace usql {
 	keyword_where,
 	keyword_order,
 	keyword_by,
+	keyword_offset,
+	keyword_limit,
 	keyword_asc,
 	keyword_desc,
 	keyword_delete,

main.cpp

@@ -17,12 +17,12 @@ int main(int argc, char *argv[]) {
 		    "insert into a (i, s) values(5, 'five')",
 		    "insert into a (i, s) values(to_date('20.12.1973', '%d.%m.%Y'), 'six')",
 		    "save table a into '/tmp/a.csv'",
-		    "select i, s from a where i > 2 order by 1 desc"
+		    "select i, s from a where i > 2 order by 1 desc offset 1 limit 1",
 //		    "select i, s from a where i = 1",
 //		    "select i, s from a where s = 'two'",
 //		    "select i, s from a where i <= 3 and s = 'one'",
 //		    "select i, s from a where i > 0",
-//		    "delete from a where i = 4",
+		    "delete from a where i = 4",
 //		    "select i, s from a where i > 0",
 //		    "update a set f = 9.99 where i = 3",
 //		    "select i, s, f from a where i = 3",
@@ -37,10 +37,10 @@ int main(int argc, char *argv[]) {
 //		    "drop table x",
 //		    "select i, s, f from a where i > 300",
 //		    "select i, to_string(i, '%d.%m.%Y'), s, f from a where i > 300",
-//		    "create table prices (datetime integer, symbol varchar(8), prev_close float, open float, price float, change float, change_prct varchar(16))",
+		    "create table prices (datetime integer, symbol varchar(8), prev_close float, open float, price float, change float, change_prct varchar(16))",
-//		    "load prices from '/Users/vaclavt/Library/Mobile Documents/com~apple~CloudDocs/Development/usql/prices.csv'",
+		    "load prices from '/Users/vaclavt/Library/Mobile Documents/com~apple~CloudDocs/Development/usql/prices.csv'",
 //		    "insert into prices (datetime, symbol, prev_close, open, price, change, change_prct) values (1626979443, 'MPC', 54.08, 53.82, 53.63, -0.832101, '-0.83 %')",
-//		    "select to_string(datetime, '%d.%m.%Y %H:%M:%S'), symbol, prev_close, open, price, change, change_prct from prices where symbol = 'SYF'"
+		    "select to_string(datetime, '%d.%m.%Y %H:%M:%S'), symbol, prev_close, open, price, change, change_prct, datetime from prices where symbol = 'SYF' order by 8 desc limit 10"
 	};
 
 
@@ -52,7 +52,7 @@ int main(int argc, char *argv[]) {
 		time_point<high_resolution_clock> end_time = high_resolution_clock::now();
 
 		std::cout << command << std::endl;
-		std::cout << "run time: " << duration_cast<milliseconds>(end_time - start_time).count() << " ms " << std::endl;
+		std::cout << "run time: " << duration_cast<milliseconds>(end_time - start_time).count() << " ms " << std::endl <<std::endl;
 
 		result->print();
 	}

parser.cpp

@@ -235,12 +235,12 @@ namespace usql {
 
 	std::unique_ptr<Node> where_node = parse_where_clause();
 
-	std::vector<ColOrderNode> orderby_node = parse_orderby_clause();
+	std::vector<ColOrderNode> orderby_node = parse_order_by_clause();
 
-//    if (m_lexer.tokenType() == TokenType::keyword_offset) {}
+	OffsetLimitNode offsetlimit_node = parse_offset_limit_clause();
-//    if (m_lexer.tokenType() == TokenType::keyword_limit) {}
 
-	return std::make_unique<SelectFromTableNode>(table_name, std::move(cols), std::move(where_node), orderby_node);
+
+	return std::make_unique<SelectFromTableNode>(table_name, std::move(cols), std::move(where_node), orderby_node, offsetlimit_node);
     }
 
     std::unique_ptr<Node> Parser::parse_delete_from_table() {
@@ -312,7 +312,7 @@ namespace usql {
     }
 
 
-    std::vector<ColOrderNode>  Parser::parse_orderby_clause() {
+    std::vector<ColOrderNode>  Parser::parse_order_by_clause() {
 	std::vector<ColOrderNode> order_cols;
 	
 	if (m_lexer.tokenType() == TokenType::keyword_order) {
@@ -330,7 +330,7 @@ namespace usql {
 				col_index = std::stoi(tokenString);
 				break;
 			default:
-				throw Exception("column index alloved in order by clause at this moment");
+				throw Exception("column index allowed in order by clause at this moment");
 			}
 
 			if (m_lexer.tokenType() == TokenType::keyword_asc) {
@@ -344,12 +344,37 @@ namespace usql {
 
 			m_lexer.skipTokenOptional(TokenType::comma);
 
-		} while (m_lexer.tokenType() != TokenType::eof);	// && m_lexer.tokenType() != TokenType::keyword_offset && m_lexer.tokenType() != TokenType::keyword_limit);
+		} while (m_lexer.tokenType() != TokenType::eof && m_lexer.tokenType() != TokenType::keyword_offset && m_lexer.tokenType() != TokenType::keyword_limit);
     	}
 	    
 	return order_cols;
     }
 
+    OffsetLimitNode Parser::parse_offset_limit_clause() {
+	int offset = 0;
+	int limit = 999999999;
+
+	if (m_lexer.tokenType() == TokenType::keyword_offset) {
+		m_lexer.skipToken(TokenType::keyword_offset);
+
+		if (m_lexer.tokenType() != TokenType::int_number)
+			throw Exception("expecting integer in offset clause");
+
+		offset = std::stoi(m_lexer.consumeCurrentToken().token_string);
+	}
+
+	if (m_lexer.tokenType() == TokenType::keyword_limit) {
+		m_lexer.skipToken(TokenType::keyword_limit);
+
+		if (m_lexer.tokenType() != TokenType::int_number)
+			throw Exception("expecting integer in limit clause");
+
+		limit = std::stoi(m_lexer.consumeCurrentToken().token_string);
+	}
+
+	return OffsetLimitNode{offset, limit};
+    }
+
     std::unique_ptr<Node> Parser::parse_relational_expression() {
 	    auto left = parse_operand_node();
 	    auto operation = parse_relational_operator();

parser.h

@@ -34,6 +34,7 @@ namespace usql {
 	save_table,
 	drop_table,
 	column_name,
+	offset_limit,
 	column_order,
 	column_value,
 	function,
@@ -62,6 +63,16 @@ namespace usql {
 	ColOrderNode(int index, bool asc) : Node(NodeType::column_name), col_name(""), col_index(index), ascending(asc) {}
     };
 
+
+   
+   struct OffsetLimitNode : Node {
+	int	offset;
+	int	limit;
+
+	OffsetLimitNode(int off, int lim) : Node(NodeType::offset_limit), offset(off), limit(lim) {}
+   };
+
+
     struct SelectColNode : Node {
         std::unique_ptr<Node>	value;
         std::string name;
@@ -227,9 +238,10 @@ namespace usql {
 	std::unique_ptr<std::vector<SelectColNode>> cols_names;
 	std::unique_ptr<Node> where;
 	std::vector<ColOrderNode> order_by;
+	OffsetLimitNode	offset_limit;
 
-	SelectFromTableNode(std::string name, std::unique_ptr<std::vector<SelectColNode>> names, std::unique_ptr<Node> where_clause, std::vector<ColOrderNode> orderby) :
+	SelectFromTableNode(std::string name, std::unique_ptr<std::vector<SelectColNode>> names, std::unique_ptr<Node> where_clause, std::vector<ColOrderNode> orderby, OffsetLimitNode offlim) :
-	Node(NodeType::select_from), table_name(name), cols_names(std::move(names)), where(std::move(where_clause)), order_by(orderby) {}
+	Node(NodeType::select_from), table_name(name), cols_names(std::move(names)), where(std::move(where_clause)), order_by(orderby), offset_limit(offlim) {}
     };
 
     struct CreateTableAsSelectNode : Node {
@@ -303,7 +315,8 @@ namespace usql {
 	std::unique_ptr<Node> parse_update_table();
 
 	std::unique_ptr<Node> parse_where_clause();
-	std::vector<ColOrderNode>  parse_orderby_clause();
+	std::vector<ColOrderNode>  parse_order_by_clause();
+	OffsetLimitNode parse_offset_limit_clause();
 
 	std::unique_ptr<Node> parse_operand_node();
 	std::unique_ptr<Node> parse_value();

row.cpp

@@ -77,13 +77,17 @@ namespace usql {
     	}
     }
 
-    void Row::print() {
+    void Row::print(const std::vector<int> & col_char_sizes) {
+	    std::string out{"| "};
+
 	    for (int ci = 0; ci < m_columns.size(); ci++) {
-		    if (ci > 0) std::cout << ",";
+		auto value = m_columns[ci]->getStringValue();
-		    auto v = m_columns[ci]->getStringValue();
+
-		    std::cout << v;
+	 	// TODO use string functions
+		out.append(value + std::string(col_char_sizes[ci] - value.size(), ' ') + " | ");
 	    }
-	    std::cout << std::endl;
+
+	    std::cout << out << std::endl;
     }
 
 }

row.h

@@ -86,7 +86,7 @@ namespace usql {
 		return m_columns[i].get();
 	}
 
-	void print();
+	void print(const std::vector<int> & col_char_sizes);
 
     private:
 	std::vector<std::unique_ptr<ColValue>> m_columns;

table.cpp

@@ -1,6 +1,7 @@
 
 #include "table.h"
 #include "csvreader.h"
+#include "ml_string.h"
 
 
 namespace usql {
@@ -8,7 +9,16 @@ namespace usql {
 Table::Table(const std::string name, const std::vector<ColDefNode> columns) {
 	m_name = name;
 	m_col_defs = columns;
-	m_rows.clear();
+	m_rows.reserve(16);
+}
+
+Table::Table(const Table &other) {
+	m_name = other.m_name;
+	m_col_defs = other.m_col_defs;
+	m_rows.reserve(other.m_rows.size());
+	for(const Row& orig_row : other.m_rows) {
+		add_copy_of_row(orig_row);
+	}
 }
 
 ColDefNode Table::get_column_def(const std::string &col_name) {
@@ -96,18 +106,26 @@ int Table::load_csv_string(const std::string &content) {
 }
 
 void Table::print() {
-	std::cout << "** " << m_name << " **" << std::endl;
+	std::string out{"| "};
-	for (auto row : m_rows) {
+	std::string out2{"+-"};
-		row.print();
+	std::vector<int> col_char_sizes{};
-	}
-}
 
-Table::Table(const Table &other) {
+	for(auto col_def : m_col_defs) {
-	m_name = other.m_name;
+		int col_size = col_def.type == ColumnType::varchar_type ? col_def.length : 10;
-	m_col_defs = other.m_col_defs;
+		col_char_sizes.push_back(col_size);
-	for(const Row& orig_row : other.m_rows) {
+
-		add_copy_of_row(orig_row);
+		out.append(string_padd(col_def.name, col_size, ' ', true) + " | ");
+		out2.append(string_padd("-", col_size, '-', true) + "-+ ");
 	}
+
+	// std::cout << "** " << m_name << " **" << std::endl;
+	std::cout << out << std::endl;
+	std::cout << out2 << std::endl;
+
+	for(auto row : m_rows) {
+		row.print(col_char_sizes);
+	}
+	std::cout << std::endl;
 }
 
 void Table::add_row(const Row &row) {

table.h

@@ -4,7 +4,6 @@
 #include "row.h"
 
 #include <vector>
-#include <list>
 
 namespace usql {
 
@@ -17,6 +16,7 @@ namespace usql {
 	ColDefNode get_column_def(int col_index);
 
 	int columns_count() const { return m_col_defs.size(); };
+	int rows_count() const { return m_rows.size(); };
 
 	Row create_empty_row();   // TODO this means unnecessary copying
 	void add_row(const Row &row);
@@ -33,7 +33,7 @@ namespace usql {
 
 	std::string m_name;
 	std::vector<ColDefNode> m_col_defs;
-	std::list<Row> m_rows;
+	std::vector<Row> m_rows;
     };
 
 }

usql.cpp

@@ -106,7 +106,7 @@ std::unique_ptr<Table> USql::execute_save(SaveTableNode &node) {
 	file << csv_string;
 	file.close();
 
-	return create_stmt_result_table(0, "save succeeded", 0);
+	return create_stmt_result_table(0, "save succeeded", table_def->rows_count());
 }
 
 std::unique_ptr<Table> USql::execute_drop(DropTableNode &node) {
@@ -191,18 +191,20 @@ std::unique_ptr<Table> USql::execute_select(SelectFromTableNode &node) {
 	}
 
 	// order by
-	execute_order_by(node, table, result);
+	execute_order_by(node, table, result.get());
 
+	// offset & limit
+	execute_offset_limit(node.offset_limit, result.get());
 
 	return std::move(result);
 }
 
-void USql::execute_order_by(SelectFromTableNode &node, Table *table, std::__unique_if<Table>::__unique_single &result) const {
+void USql::execute_order_by(SelectFromTableNode &node, Table *table, Table *result) const {
 	if (node.order_by.size() == 0) return;
 
-	auto compare_rows = [&node, &table, this](const Row &a, const Row &b) {
+	auto compare_rows = [&node, &result, this](const Row &a, const Row &b) {
 		for(auto order_by_col_def : node.order_by) {
-			ColDefNode col_def = table->get_column_def(order_by_col_def.col_index - 1);	 // TODO validate index
+			ColDefNode col_def = result->get_column_def(order_by_col_def.col_index - 1);	 // TODO validate index
 			ColValue *a_val = a.ith_column(col_def.order);
 			ColValue *b_val = b.ith_column(col_def.order);
 
@@ -217,7 +219,16 @@ void USql::execute_order_by(SelectFromTableNode &node, Table *table, std::__uniq
 	     return false;
 	};
 
-	result->m_rows.sort(compare_rows);
+	std::sort(result->m_rows.begin(), result->m_rows.end(), compare_rows);
+}
+
+void USql::execute_offset_limit(OffsetLimitNode &node, Table *result) const {
+	if (node.offset > 0)
+		result->m_rows.erase(result->m_rows.begin(),
+				     result->rows_count() > node.offset ? result->m_rows.begin() + node.offset : result->m_rows.end());
+
+	if (node.limit > 0 && node.limit < result->rows_count())
+		result->m_rows.erase(result->m_rows.begin() + node.limit, result->m_rows.end());
 }
 
 int USql::compare_col_values(const ColDefNode &col_def, ColValue *a_val, ColValue *b_val) const {
@@ -247,7 +258,7 @@ std::tuple<int, ColDefNode> USql::get_column_definition(Table *table, SelectColN
 		auto node = static_cast<FunctionNode *>(select_col_node->value.get());
 
 		if (node->function == "to_string") {
-			ColDefNode cdef = ColDefNode{new_col_name, ColumnType::varchar_type, col_order, 64, true};
+			ColDefNode cdef = ColDefNode{new_col_name, ColumnType::varchar_type, col_order, 32, true};
 			return std::make_tuple(-1, cdef);
 		} else if (node->function == "to_date") {
 			ColDefNode cdef = ColDefNode{new_col_name, ColumnType::integer_type, col_order, 1, true};
@@ -264,16 +275,14 @@ std::unique_ptr<Table> USql::execute_delete(DeleteFromTableNode &node) {
 	Table *table = find_table(node.table_name);
 
 	// execute access plan
-	int affected_rows = 0;
+	int affected_rows = table->rows_count();
-	auto it = table->m_rows.begin();
+
-	for (; it != table->m_rows.end();) {
+	table->m_rows.erase(
-		if (eval_where(node.where.get(), table, *it)) {
+		std::remove_if(table->m_rows.begin(), table->m_rows.end(), 
-			it = table->m_rows.erase(it);
+				[&node, table, this](Row &row){return eval_where(node.where.get(), table, row);}),
-			affected_rows++;
+		table->m_rows.end());
-		} else {
+
-			++it;
+	affected_rows -= table->rows_count();
-		}
-	}
 
 	return create_stmt_result_table(0, "delete succeeded", affected_rows);
 }
@@ -520,7 +529,7 @@ std::unique_ptr<ValueNode> USql::eval_arithmetic_operator(ColumnType outType, Ar
 std::unique_ptr<Table> USql::create_stmt_result_table(long code, const std::string &text, long affected_rows) {
 	std::vector<ColDefNode> result_tbl_col_defs{};
 	result_tbl_col_defs.push_back(ColDefNode("code", ColumnType::integer_type, 0, 1, false));
-	result_tbl_col_defs.push_back(ColDefNode("desc", ColumnType::varchar_type, 1, 255, false));
+	result_tbl_col_defs.push_back(ColDefNode("desc", ColumnType::varchar_type, 1, 48, false));
 	result_tbl_col_defs.push_back(ColDefNode("affected_rows", ColumnType::integer_type, 0, 1, true));
 
 	auto table_def = std::make_unique<Table>("result", result_tbl_col_defs);

usql.h

@@ -57,8 +57,9 @@ private:
 
     int compare_col_values(const ColDefNode &col_def, ColValue *a_val, ColValue *b_val) const;
 
-    void
+    void execute_order_by(SelectFromTableNode &node, Table *table, Table *result) const;
-    execute_order_by(SelectFromTableNode &node, Table *table, std::__unique_if<Table>::__unique_single &result) const;
+
+    void execute_offset_limit(OffsetLimitNode &node, Table *result) const;
 };
 
 }	// namespace