WIP on indexes cat

.gitignore

@@ -7,3 +7,4 @@ localhost.session.sql
 CMakeFiles
 cmake-build-debug-coverage
 .idea
+.cache

Readme.md

@@ -1,22 +1,23 @@
 ### WIP
-compare in row.cpp shoud take into account m_visible
 
 ### TODO
 - create local_install.sh
 
+- change float type keyword to double and in code functions too
+
+- add functions:
+	- coalesce
+	- now
+	- rtrim, ltrim, rpad, lpad
+	- round
+
 - set xxx - without value to reset to default value
-- escape " in save csv
+
-- is null | is not null
-- coalesce, date functions now, add_date; string functions rtrim, ltrim, rpad, lpad; math function round
 - add pipe | concatenation
 - add support for 1_000_000 numbers
 - expand_asterix_char should support multiple and everywhere *
 
+- support for reusing space of deleted rows
 - support for uniqueue indexes (primary key)
-- support for indexes
 - add drop m_index
 - support for joining
-
-- add const wherever should be
-- use static methods where posible
-- use references where pointer cannot be nullptr

debug.h

@@ -14,10 +14,15 @@ std::vector<std::string> k_debug_sql_commands {
 		 "set 'DATE_FORMAT' = '%Y-%m-%d'",
 		 "create index sf1_symbol on sf1(symbol)",
 		 "load into sf1 '/srv/SHARADAR_SF1.csv'",
-		 "set 'USE_INDEXSCAN' = 'false'",
+		// "set 'USE_INDEXSCAN' = 'true'",
-		 "select dimension, to_string(calendar_date, '%d.%m.%Y'), pp(eps, \"%.2f\"), pp(shareswadil), pp(revenue), pp(netinc), pp(cashneq), pp(assets), pp(debt), pp(ncfdebt), pp(roe*100), pp(intangibles), calendar_date from sf1 where symbol = 'MU' and dimension = 'ARQ' order by dimension, calendar_date desc limit 5",
+		 "select distinct dimension from sf1 where symbol = 'ALLY'",
-		 "set 'USE_INDEXSCAN' = 'true'",
+		 "select calendar_date, calendar_date, dimension from sf1 where symbol = 'ALLY' and dimension = 'MRQ' and calendar_date <= 1653487723 order by calendar_date desc limit 10",
-		 "select dimension, to_string(calendar_date, '%d.%m.%Y'), pp(eps, \"%.2f\"), pp(shareswadil), pp(revenue), pp(netinc), pp(cashneq), pp(assets), pp(debt), pp(ncfdebt), pp(roe*100), pp(intangibles), calendar_date from sf1 where symbol = 'MU' and dimension = 'ARQ' order by dimension, calendar_date desc limit 5"
+		//  "set 'USE_INDEXSCAN' = 'false'",
+		//  "select dimension, to_char(calendar_date, '%d.%m.%Y'), pp(eps, \"%.2f\"), pp(shareswadil), pp(revenue), pp(netinc), pp(cashneq), pp(assets), pp(debt), pp(ncfdebt), pp(roe*100), pp(intangibles), calendar_date from sf1 where symbol = 'MU' and dimension = 'ARQ' order by dimension, calendar_date desc limit 5",
+		//  "set 'USE_INDEXSCAN' = 'true'",
+		//  "select dimension, to_char(calendar_date, '%d.%m.%Y'), pp(eps, \"%.2f\"), pp(shareswadil), pp(revenue), pp(netinc), pp(cashneq), pp(assets), pp(debt), pp(ncfdebt), pp(roe*100), pp(intangibles), calendar_date from sf1 where symbol = 'MU' and dimension = 'ARQ' order by dimension, calendar_date desc limit 5",
+		//  "select max(calendar_date), calendar_date from sf1 where symbol = 'MU' and dimension = 'ARQ'",
+		//  "select max(calendar_date), min(calendar_date) from sf1 where symbol = 'MU' and dimension = 'ARQ'"
 
 		// "create table a (i integer not null, s varchar(64))",
 		// "create index a_i on a(i)",
@@ -26,6 +31,7 @@ std::vector<std::string> k_debug_sql_commands {
 		// "insert into a (i, s) values(2, 'second two')",
 		// "insert into a (i, s) values(3, 'three')",
 		// "insert into a (i, s) values(4, 'four')",
+		// "insert into a (i) values(5)",
 		// "save a into '/tmp/a.csv'",
 		// "set 'USE_INDEXSCAN' = 'true'",
 		// "select * from a where 1 = i",
@@ -33,8 +39,25 @@ std::vector<std::string> k_debug_sql_commands {
 		// "select * from a where i = 2",
 		// "update a set i = 5 where i = 2",
 		// "select * from a where i = 5",
-		// "select max(i) from a where s = 'two'"
+		// "select max(i) from a where s = 'two'",
 		// "select min(i), max(i), count(i) from a",
-		// "select i + 1000 from a"
+		// "select i + 1000 from a",
+		// "select i, s from a where i = 5",
+		// "select i, s, coalesce(s, i) from a where i = 5",
+
+		// "create table a (i float, j float)",
+		// "insert into a (i, j) values(null, 123456789.12345)",
+		// "select pp(coalesce(i, j)) from a",
+
+		// "create table d (datetime date)",
+		// "insert into d (datetime) values(1648634993)",
+		// "select to_char(datetime, '%Y%m%d') from d",
+		// "select to_int(datetime) from d",
+		// "select max(to_char(datetime, '%Y%m%d')) from d",
+		// "select max(to_int(to_float(to_char(datetime, '%Y%m%d')))) from d",
+
+		// "select * from usql_tables",
+		// "select * from usql_columns",
+		// "select * from usql_indexes",
 };
 

lexer.cpp

@@ -162,7 +162,7 @@ TokenType Lexer::type(const std::string &token) {
 	if (token == "not") return TokenType::keyword_not;
 	if (token == "null") return TokenType::keyword_null;
 	if (token == "integer") return TokenType::keyword_integer;
-	if (token == "float") return TokenType::keyword_float;
+	if (token == "float" || token == "double") return TokenType::keyword_float;
 	if (token == "varchar") return TokenType::keyword_varchar;
 	if (token == "date") return TokenType::keyword_date;
 	if (token == "boolean") return TokenType::keyword_bool;

parser.cpp

@@ -5,6 +5,17 @@ namespace usql {
 
 	// TOOD handle premature eof
 
+	std::string column_type_name(const ColumnType type) {
+		if (type == ColumnType::integer_type)	return "integer_type";
+		if (type == ColumnType::float_type)	return "float_type";
+		if (type == ColumnType::varchar_type)	return "varchar_type";
+		if (type == ColumnType::date_type)	return "date_type";
+		if (type == ColumnType::bool_type)	return "bool_type";
+
+		throw Exception("invalid column type: " + std::to_string((int)type));	
+	};
+
+
     Parser::Parser() {
 	    m_lexer = Lexer{};
     }
@@ -383,8 +394,7 @@ namespace usql {
 		std::unique_ptr<Node> left = parse_expression();
 		do {
 			left = parse_expression(std::move(left));
-
+		} while (m_lexer.tokenType() != TokenType::eof && m_lexer.tokenType() != TokenType::keyword_order && m_lexer.tokenType() != TokenType::keyword_offset && m_lexer.tokenType() != TokenType::keyword_limit && m_lexer.tokenType() != TokenType::semicolon);
-	} while (m_lexer.tokenType() != TokenType::eof && m_lexer.tokenType() != TokenType::keyword_order && m_lexer.tokenType() != TokenType::keyword_offset && m_lexer.tokenType() != TokenType::keyword_limit);
 	
 		return left;
     }
@@ -529,4 +539,3 @@ namespace usql {
     }
 
 }	// namespace
-

parser.h

@@ -21,6 +21,9 @@ enum class ColumnType {
 	bool_type
 };
 
+std::string column_type_name(const ColumnType type);
+
+
 enum class NodeType {
 	true_node,
 	null_value,
@@ -51,6 +54,7 @@ enum class NodeType {
 	error
 };
 
+
 struct Node {
 	NodeType node_type;
 
@@ -58,7 +62,7 @@ struct Node {
 	virtual ~Node() = default;
 
 	virtual void dump() const {
-		std::cout << "type: Node" << std::endl;
+		std::cout << "type: Node" << (int)node_type << std::endl;
 	}
 };
 
@@ -115,20 +119,86 @@ struct ColDefNode : Node {
 			null(nullable) {}
 
 	void dump() const override {
-		std::cout << "type: ColDefNode, name: " << name << ", type: " << (int)type << " TODO add more" << std::endl;
+		std::cout << "type: ColDefNode, name: " << name << ", type: " << column_type_name(type) << ", order: " << order << ", length: " << length << ", null: " << null << std::endl;
 	}
 };
 
 struct FunctionNode : Node {
-	std::string				function;	// TODO use enum
+
-	std::vector<std::unique_ptr<Node>>	params;
+	enum class Type {
+		to_char,
+		to_date,
+		to_int,
+		to_float,
+		date_add,
+		pp,
+		lower,
+		upper,
+		coalesce,
+		min,
+		max,
+		count
+	};
+
+	static Type get_function(const std::string &str) {
+		if (str=="to_char")		return Type::to_char;
+		if (str=="to_date")		return Type::to_date;
+		if (str=="to_int")		return Type::to_int;
+		if (str=="to_float")	return Type::to_float;
+		if (str=="date_add")	return Type::date_add;
+		if (str=="pp")			return Type::pp;
+		if (str=="lower")		return Type::lower;
+		if (str=="upper")		return Type::upper;
+		if (str=="coalesce")	return Type::coalesce;
+		if (str=="min")			return Type::min;
+		if (str=="max")			return Type::max;
+		if (str=="count")		return Type::count;
+
+		throw Exception("invalid function: " + str);	
+	};
+
+	static std::string function_name(const Type type) {
+		if (type == Type::to_char)		return "to_char";
+		if (type == Type::to_date)		return "to_date";
+		if (type == Type::to_int)		return "to_int";
+		if (type == Type::to_float)		return "to_float";
+		if (type == Type::date_add)		return "date_add";
+		if (type == Type::pp)			return "pp";
+		if (type == Type::lower)		return "lower";
+		if (type == Type::upper)		return "upper";
+		if (type == Type::coalesce)		return "coalesce";
+		if (type == Type::min)			return "min";
+		if (type == Type::max)			return "max";
+		if (type == Type::count)		return "count";
+
+		throw Exception("invalid function: " + std::to_string((int)type));	
+	};
+
 
 	FunctionNode(std::string  func_name, std::vector<std::unique_ptr<Node>> pars) :
-	Node(NodeType::function), function(std::move(func_name)), params(std::move(pars)) {}
+	Node(NodeType::function), function(get_function(func_name)), params(std::move(pars)) {}
+
+	bool is_agg_function() {
+		return (function == Type::count || function == Type::min || function == Type::max);
+	}
+
+
+	friend std::ostream &operator<<(std::ostream &output, const Type &t ) {
+		output << function_name(t);
+		return output;
+	}
 
 	void dump() const override {
-		std::cout << "type: FunctionNode, function: " << function << " TODO add more" << std::endl;
+		std::cout << "type: FunctionNode, function: " << function_name(function) << "(";
+		for(int i = 0; i < params.size(); i++){
+			if (i > 0) std::cout << ",";
+			params[i]->dump();
 		}			
+		std::cout << ")" << std::endl;
+	}
+
+	Type								function;
+	std::vector<std::unique_ptr<Node>>	params;
 };
 
 struct TrueNode : Node {
@@ -325,7 +395,12 @@ struct CreateTableNode : Node {
 			Node(NodeType::create_table), table_name(std::move(name)), cols_defs(std::move(defs)) {}
 
 	void dump() const override {
-		std::cout << "type: CreateTableNode, table_name: " << table_name << "TODO complete me" << std::endl;
+		std::cout << "type: CreateTableNode, table_name: " << table_name << "(";
+		for(int i = 0; i < cols_defs.size(); i++) {
+			if (i > 0) std::cout << ",";
+			cols_defs[i].dump();
+		}	
+		std::cout << ")" << std::endl;
 	}
 };
 
@@ -338,7 +413,17 @@ struct InsertIntoTableNode : Node {
 			Node(NodeType::insert_into), table_name(std::move(name)), cols_names(std::move(names)), cols_values(std::move(values)) {}
 
 	void dump() const override {
-		std::cout << "type: InsertIntoTableNode, table_name: " << table_name << "TODO complete me" << std::endl;
+		std::cout << "type: InsertIntoTableNode, table_name: " << table_name << "(";
+		for(int i = 0; i < cols_names.size(); i++) {
+			if (i > 0) std::cout << ",";
+			cols_names[i].dump();
+		}	
+		std::cout << ") values (";
+		for(int i = 0; i < cols_values.size(); i++) {
+			if (i > 0) std::cout << ",";
+			cols_values[i]->dump();
+		}	
+		std::cout << ")" << std::endl;
 	}
 };
 
@@ -354,8 +439,22 @@ struct SelectFromTableNode : Node {
 	Node(NodeType::select_from), table_name(std::move(name)), cols_names(std::move(names)), where(std::move(where_clause)), order_by(std::move(orderby)), offset_limit(std::move(offlim)), distinct(distinct_) {}
 
 	void dump() const override {
-		std::cout << "type: SelectFromTableNode, table_name: " << table_name << "TODO complete me" << std::endl;
+		std::cout << "type: SelectFromTableNode, table_name: " << table_name;
+		std::cout << "colums: ";	
+		for(int i = 0; i < cols_names->size(); i++) {
+			if (i > 0) std::cout << ",";
+			cols_names->operator[](i).dump();
+		}
+		std::cout << "where: ";	
 		where->dump();
+		std::cout << "offset,limit: ";
+		for(int i = 0; i < order_by.size(); i++) {
+			if (i > 0) std::cout << ",";
+			order_by[i].dump();
+		}	
+		std::cout << "offset,limit: ";
+		offset_limit.dump();
+		std::cout << std::endl;
 	}
 };
 
@@ -384,8 +483,16 @@ struct UpdateTableNode : Node {
 			where(std::move(where_clause)) {}
 
 	void dump() const override {
-		std::cout << "type: UpdateTableNode, table_name: " << table_name << "TODO complete me" << std::endl;
+		std::cout << "type: UpdateTableNode, table_name: " << table_name << " set ";
+		for(int i = 0; i < cols_names.size(); i++) {
+			if (i > 0) std::cout << ",";
+			cols_names[i].dump();
+			std::cout << " = ";
+			values[i]->dump();
+		}	
+		std::cout << " where: ";
 		where->dump();
+		std::cout << std::endl;
 	}
 };
 
@@ -431,8 +538,10 @@ struct DeleteFromTableNode : Node {
 			Node(NodeType::delete_from), table_name(std::move(name)), where(std::move(where_clause)) {}
 
 	void dump() const override {
-		std::cout << "type: DeleteFromTableNode, table_name: " << table_name << std::endl;
+		std::cout << "type: DeleteFromTableNode, table_name: " << table_name;
+		std::cout << "where: ";	
 		where->dump();
+		std::cout << std::endl;
 	}
 };
 
@@ -472,7 +581,6 @@ struct CreateIndexNode : Node {
 };
 
 class Parser {
-private:
 public:
 	Parser();
 

row.h

@@ -171,7 +171,7 @@ public:
 			case 5:
 				return (ColValue &) *std::get_if<ColBooleanValue>(&m_columns[i]);
 			default:
-				throw Exception("should not happen");
+				throw Exception("ColValue &operator[](int i), type index invalid :" + std::to_string(type_index));
 		}
 	}
 
@@ -182,7 +182,7 @@ public:
 
 	[[nodiscard]] bool is_visible() const { return m_visible; };
 	void set_visible() { m_visible = true; };
-	void set_deleted() { m_visible = true; };
+	void set_deleted() { m_visible = false; };
 
 private:
 	bool m_visible;

settings.cpp

@@ -12,7 +12,7 @@ std::vector<std::pair<std::string, std::string>> Settings::m_settings =
 	      std::make_pair("BOOL_TRUE_LITERAL", "Y"),
 	      std::make_pair("BOOL_FALSE_LITERAL", "N"),
           std::make_pair("DOUBLE_FORMAT", "%.2f"),
-	      std::make_pair("USE_INDEXSCAN", "N"),
+	      std::make_pair("USE_INDEXSCAN", "Y"),
 	      std::make_pair("MAX_PARALLELISM", "1") };		// values "AUTO" or number of workers; when number negative means std::thread::hardware_concurrency() - number 
 
 

usql.cpp

@@ -6,9 +6,37 @@
 
 namespace usql {
 
+USql::USql() {
+	// create catalogue tables first
+
+	std::vector<std::string> k_debug_sql_commands {
+		"create table usql_tables(name varchar(32) not null, modified boolean not null)",
+		"create table usql_columns(table_name varchar(32) not null, column_name varchar(32) not null, column_type varchar(16) not null, column_length integer not null, nullable boolean not null, column_order integer not null)",
+		"create table usql_indexes(index_name varchar(32) not null, table_name varchar(32), column_name varchar(32) not null)"
+	};
+
+	// create cataloque tables
+	for (const auto &command : k_debug_sql_commands) {
+		std::unique_ptr<Node> create_table_node = m_parser.parse(command);
+		const CreateTableNode &node = static_cast<CreateTableNode &>(*create_table_node); 
+
+		Table table{node.table_name, node.cols_defs};
+		m_tables.push_back(table);
+
+	}
+	// insert data into cataloque tables
+	for (const auto &command : k_debug_sql_commands) {
+		std::unique_ptr<Node> create_table_node = m_parser.parse(command);
+		const CreateTableNode &node = static_cast<CreateTableNode &>(*create_table_node); 
+
+		execute_create_table_sys_catalogue(node);
+	}
+}
+
 std::unique_ptr<Table> USql::execute(const std::string &command) {
 	try {
 		std::unique_ptr<Node> node = m_parser.parse(command);
+		// node->dump();
 		return execute(*node);
 
 	} catch (const std::exception &e) {
@@ -173,21 +201,25 @@ std::unique_ptr<ValueNode> USql::eval_function_value_node(Table *table, Row &row
 		evaluatedPars.push_back(eval_value_node(table, row, param.get(), nullptr, nullptr));
 	}
 
+	// coalesce function can have first parameter null, so must be calles before following "return null"
+	if (fnc->function == FunctionNode::Type::coalesce) return coalesce_function(evaluatedPars);
+	
 	if (evaluatedPars.empty() || evaluatedPars[0]->isNull())
 		return std::make_unique<NullValueNode>();
 
-	// TODO use some enum
+	if (fnc->function == FunctionNode::Type::lower) return lower_function(evaluatedPars);
-	if (fnc->function == "lower") return lower_function(evaluatedPars);
+	if (fnc->function == FunctionNode::Type::upper) return upper_function(evaluatedPars);
-	if (fnc->function == "upper") return upper_function(evaluatedPars);
+	if (fnc->function == FunctionNode::Type::to_date) return to_date_function(evaluatedPars);
-	if (fnc->function == "to_date") return to_date_function(evaluatedPars);
+	if (fnc->function == FunctionNode::Type::to_char) return to_char_function(evaluatedPars);
-	if (fnc->function == "to_string") return to_string_function(evaluatedPars);
+	if (fnc->function == FunctionNode::Type::to_int) return to_int_function(evaluatedPars);
-	if (fnc->function == "date_add") return date_add_function(evaluatedPars);
+	if (fnc->function == FunctionNode::Type::to_float) return to_float_function(evaluatedPars);
-	if (fnc->function == "pp") return pp_function(evaluatedPars);
+	if (fnc->function == FunctionNode::Type::date_add) return date_add_function(evaluatedPars);
-	if (fnc->function == "count") return count_function(agg_func_value, evaluatedPars);
+	if (fnc->function == FunctionNode::Type::pp) return pp_function(evaluatedPars);
-	if (fnc->function == "max") return max_function(evaluatedPars, col_def_node, agg_func_value);
+	if (fnc->function == FunctionNode::Type::count) return count_function(agg_func_value, evaluatedPars);
-	if (fnc->function == "min") return min_function(evaluatedPars, col_def_node, agg_func_value);
+	if (fnc->function == FunctionNode::Type::max) return max_function(evaluatedPars, col_def_node, agg_func_value);
+	if (fnc->function == FunctionNode::Type::min) return min_function(evaluatedPars, col_def_node, agg_func_value);
 
-	throw Exception("invalid function: " + fnc->function);
+	throw Exception("invalid function: " + FunctionNode::function_name(fnc->function));
 }
 
 

usql.h

@@ -13,15 +13,16 @@ namespace usql {
 class USql {
 	
 public:
-    USql() = default;
+	USql();
-
     std::unique_ptr<Table> execute(const std::string &command);
 
 private:
     std::unique_ptr<Table> execute(Node &node);
 
     std::unique_ptr<Table> execute_create_table(const CreateTableNode &node);
+	bool execute_create_table_sys_catalogue(const CreateTableNode &node);
     std::unique_ptr<Table> execute_create_index(const CreateIndexNode &node);
+	bool execute_create_index_sys_catalogue(const CreateIndexNode &node);
     std::unique_ptr<Table> execute_create_table_as_table(const CreateTableAsSelectNode &node);
     std::unique_ptr<Table> execute_load(const LoadIntoTableNode &node);
     std::unique_ptr<Table> execute_save(const SaveTableNode &node);
@@ -59,9 +60,6 @@ private:
 	void check_index_not_exists(const std::string &index_name);
 
 private:
-    Parser m_parser;
-    std::list<Table> m_tables;
-
     static void execute_distinct(SelectFromTableNode &node, Table *result);
     static void execute_order_by(SelectFromTableNode &node, Table *result);
     static void execute_offset_limit(OffsetLimitNode &node, Table *result);
@@ -73,8 +71,11 @@ private:
 
     static std::unique_ptr<ValueNode> lower_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars);
     static std::unique_ptr<ValueNode> upper_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars);
+    static std::unique_ptr<ValueNode> coalesce_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars);
     static std::unique_ptr<ValueNode> to_date_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars);
-    static std::unique_ptr<ValueNode> to_string_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars);
+    static std::unique_ptr<ValueNode> to_char_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars);
+    static std::unique_ptr<ValueNode> to_int_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars);
+    static std::unique_ptr<ValueNode> to_float_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars);
     static std::unique_ptr<ValueNode> date_add_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars);
     static std::unique_ptr<ValueNode> pp_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars);
 
@@ -94,6 +95,11 @@ private:
 	bool normalize_where(const Node *node) const;
 
     Table::rows_scanner get_iterator(Table *table, const Node *where) const;
+
+
+private:
+    Parser m_parser;
+    std::list<Table> m_tables;
 };
 
 

usql_ddl.cpp

@@ -8,17 +8,41 @@
 namespace usql {
 
 
-
 std::unique_ptr<Table> USql::execute_create_table(const CreateTableNode &node) {
 	check_table_not_exists(node.table_name);
 
 	Table table{node.table_name, node.cols_defs};
 	m_tables.push_back(table);
 
+	execute_create_table_sys_catalogue(node);
+
 	return create_stmt_result_table(0, "table created", 0);
 }
 
 
+bool USql::execute_create_table_sys_catalogue(const CreateTableNode &node) {
+	// usql_tables 
+	auto r = execute("insert into usql_tables(name, modified) values('" + node.table_name + "', false)");
+
+	// usql_columns 
+	for(const ColDefNode & col_def : node.cols_defs) {
+		std::string i {"insert into usql_columns(table_name, column_name, column_type, column_length, nullable, column_order) values("};
+					i += "'" + node.table_name + "', ";
+					i += "'" + col_def.name + "', ";
+					i += "'" + column_type_name(col_def.type) + "', ";
+					i += std::to_string(col_def.length) + ", ";
+					i += (col_def.null ?  "true, " : "false, ");
+					i += std::to_string(col_def.order);
+					i += ")";
+
+		auto r = execute(i);
+		// r->print();
+	}
+
+	return true;
+}
+
+
 std::unique_ptr<Table> USql::execute_create_index(const CreateIndexNode &node) {
 	Table *table_def = find_table(node.table_name);				// throws exception if not found
 	ColDefNode col_def = table_def->get_column_def(node.column_name);	// throws exception if not found
@@ -31,10 +55,25 @@ std::unique_ptr<Table> USql::execute_create_index(const CreateIndexNode &node) {
 
 	table_def->index_rows(node.index_name);
 
+	execute_create_index_sys_catalogue(node);
+
 	return create_stmt_result_table(0, "index created", 0);
 }
 
 
+bool USql::execute_create_index_sys_catalogue(const CreateIndexNode &node) {
+	std::string i {"insert into usql_indexes(index_name, table_name, column_name) values("};
+				i += "'" + node.index_name + "', ";
+				i += "'" + node.table_name + "', ";
+				i += "'" + node.column_name + "')";
+
+	auto r = execute(i);
+	// r->print();
+
+	return true;
+}
+
+
 std::unique_ptr<Table> USql::execute_create_table_as_table(const CreateTableAsSelectNode &node) {
 	check_table_not_exists(node.table_name);
 
@@ -57,7 +96,6 @@ std::unique_ptr<Table> USql::execute_create_table_as_table(const CreateTableAsSe
 }
 
 
-
 std::unique_ptr<Table> USql::execute_drop(const DropTableNode &node) {
 	auto name_cmp = [node](const Table& t) { return t.m_name == node.table_name; };
 
@@ -70,11 +108,13 @@ std::unique_ptr<Table> USql::execute_drop(const DropTableNode &node) {
 	throw Exception("table not found (" + node.table_name + ")");
 }
 
+
 std::unique_ptr<Table> USql::execute_set(const SetNode &node) {
 	Settings::set_setting(node.name, node.value);
 	return create_stmt_result_table(0, "set succeeded", 1);
 }
 
+
 std::unique_ptr<Table> USql::execute_show(const ShowNode &node) {
 	std::string value = Settings::get_setting(node.name);
 	return create_stmt_result_table(0, "show succeeded: " + value, 1);

usql_dml.cpp

@@ -35,7 +35,7 @@ std::pair<bool, std::vector<rowid_t>> USql::look_for_usable_index(const Node *wh
 					if (used_index != nullptr) {
 						std::vector<rowid_t> rowids = used_index->search((ValueNode *)ron->right.get());
 #ifndef NDEBUG
-						std::cout << "using index " << table->m_name << "(" << used_index->get_column_name() << "), " << rowids.size() << "/" << table->rows_count()  << std::endl;
+						std::cerr << "using index " << table->m_name << "(" << used_index->get_column_name() << "), " << rowids.size() << "/" << table->rows_count()  << std::endl;
 #endif
 						return std::make_pair(true, rowids);
 					}
@@ -108,24 +108,21 @@ void USql::select_row(SelectFromTableNode &where_node,
 			rslt_row->setColumnValue((ColDefNode *) &rslt_tbl_col_defs[idx], col_value);
 		}
 	}
-
 	// for aggregate is validated more than needed
 	rslt_table->commit_row(*rslt_row);
 }
 
 bool USql::check_for_aggregate_only_functions(SelectFromTableNode &node, size_t result_cols_cnt) {
 	size_t aggregate_funcs = 0;
+
 	for (size_t i = 0; i < node.cols_names->size(); i++) {
 		SelectColNode * col_node = &node.cols_names->operator[](i);
-		if (col_node->value->node_type == NodeType::function) {
+		if (col_node->value->node_type == NodeType::function && ((FunctionNode *)col_node->value.get())->is_agg_function())
-			auto func_node = static_cast<FunctionNode *>(col_node->value.get());
-			if (func_node->function == "count" || func_node->function == "min" || func_node->function == "max")
 			aggregate_funcs++;
 	}
-	}
 	// check whether aggregates are not present or all columns are aggregates
 	if (aggregate_funcs > 0 && aggregate_funcs != result_cols_cnt) {
-		throw Exception("aggregate functions with no aggregates");
+		throw Exception("aggregate functions mixed with no aggregate functions in select clause");
 	}
 
 	return aggregate_funcs > 0;
@@ -158,8 +155,7 @@ void USql::setup_order_columns(std::vector<ColOrderNode> &node, Table *table) {
 void USql::execute_distinct(SelectFromTableNode &node, Table *result) {
 	if (!node.distinct) return;
 
-	auto compare_rows = [](const Row &a, const Row &b) { return a.compare(b) >= 0; };
+	std::sort(result->m_rows.begin(), result->m_rows.end(), [](const Row &a, const Row &b) { return a.compare(b) > 0; });
-	std::sort(result->m_rows.begin(), result->m_rows.end(), compare_rows);
 
 	result->m_rows.erase(std::unique(result->m_rows.begin(), result->m_rows.end()), result->m_rows.end());
 }
@@ -229,33 +225,54 @@ std::tuple<int, ColDefNode> USql::get_node_definition(Table *table, Node * node,
 	} else if (node->node_type == NodeType::function) {
 		auto func_node = static_cast<FunctionNode *>(node);
 
-		if (func_node->function == "to_string") {
+		if (func_node->function == FunctionNode::Type::to_char) {
 			ColDefNode col_def = ColDefNode{col_name, ColumnType::varchar_type, col_order, 32, true};
-			return std::make_tuple(-1, col_def);
+			return std::make_tuple(FUNCTION_CALL, col_def);
-		} else if (func_node->function == "to_date") {
+		} else if (func_node->function == FunctionNode::Type::to_int) {
 			ColDefNode col_def = ColDefNode{col_name, ColumnType::integer_type, col_order, 1, true};
-			return std::make_tuple(-1, col_def);
+			return std::make_tuple(FUNCTION_CALL, col_def);
-		} else if (func_node->function == "pp") {
+		} else if (func_node->function == FunctionNode::Type::to_float) {
+			ColDefNode col_def = ColDefNode{col_name, ColumnType::float_type, col_order, 1, true};
+			return std::make_tuple(FUNCTION_CALL, col_def);
+		} else if (func_node->function == FunctionNode::Type::to_date) {
+			ColDefNode col_def = ColDefNode{col_name, ColumnType::integer_type, col_order, 1, true};
+			return std::make_tuple(FUNCTION_CALL, col_def);
+		} else if (func_node->function == FunctionNode::Type::pp) {
 			ColDefNode col_def = ColDefNode{col_name, ColumnType::varchar_type, col_order, 10, true};
-			return std::make_tuple(-1, col_def);
+			return std::make_tuple(FUNCTION_CALL, col_def);
-		} else if (func_node->function == "lower" || func_node->function == "upper") {
+		} else if (func_node->function == FunctionNode::Type::lower || func_node->function == FunctionNode::Type::upper) {
 			// TODO get length, use get_db_column_definition
 			ColDefNode col_def = ColDefNode{col_name, ColumnType::varchar_type, col_order, 256, true};
-			return std::make_tuple(-1, col_def);
+			return std::make_tuple(FUNCTION_CALL, col_def);
-		} else if (func_node->function == "min" || func_node->function == "max") {
+		} else if (func_node->function == FunctionNode::Type::coalesce) {
+			// TODO handle cases here
+			if (func_node->params.empty()) throw Exception("Coalesce without parameters");
+			if (func_node->params[0]->node_type != NodeType::database_value) throw Exception("Coalesce first parameter must be database column");
+
+			ColDefNode tbl_col_def = get_db_column_definition(table, func_node->params[0].get());
+			ColDefNode col_def = ColDefNode{col_name, tbl_col_def.type, col_order, tbl_col_def.length, true};
+			return std::make_tuple(FUNCTION_CALL, col_def);
+		} else if (func_node->function == FunctionNode::Type::min || func_node->function == FunctionNode::Type::max) {
 			auto col_type = ColumnType::float_type;
 			size_t col_len = 1;
 			auto &v = func_node->params[0];
+
 			if (v->node_type == NodeType::database_value) {
 				ColDefNode src_col_def = get_db_column_definition(table, v.get());
 				col_type = src_col_def.type;
 				col_len = src_col_def.length;
+
+			} else if (node->node_type == NodeType::function) {
+				std::tuple<int, ColDefNode> fun_type = get_node_definition(table, static_cast<Node*>(& *v), "", 0);
+				col_type = std::get<1>(fun_type).type;
+				col_len = std::get<1>(fun_type).length;
 			}
+
 			ColDefNode col_def = ColDefNode{col_name, col_type, col_order, col_len, true};
-			return std::make_tuple(-1, col_def);
+			return std::make_tuple(FUNCTION_CALL, col_def);
-		} else if (func_node->function == "count") {
+		} else if (func_node->function == FunctionNode::Type::count) {
 			ColDefNode col_def = ColDefNode{col_name, ColumnType::integer_type, col_order, 1, true};
-			return std::make_tuple(-1, col_def);
+			return std::make_tuple(FUNCTION_CALL, col_def);
 		}
 		throw Exception("Unsupported function");
 

usql_function.cpp

@@ -7,17 +7,31 @@
 
 namespace usql {
 
-std::unique_ptr<ValueNode> USql::to_string_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars) {
+std::unique_ptr<ValueNode> USql::to_char_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars) {
+	if (evaluatedPars[0]->node_type == NodeType::int_value && evaluatedPars.size()==2) {		// TODO when propper date is introduced
 		long date = evaluatedPars[0]->getDateValue();
 		std::string format = evaluatedPars[1]->getStringValue();
 		std::string formatted_date = date_to_string(date, format);
+	
 		return std::make_unique<StringValueNode>(formatted_date);
+	} else {
+		return std::make_unique<StringValueNode>(evaluatedPars[0]->getStringValue());
+	}
+}
+
+std::unique_ptr<ValueNode> USql::to_int_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars) {
+	return std::make_unique<IntValueNode>(evaluatedPars[0]->getIntegerValue());
+}
+
+std::unique_ptr<ValueNode> USql::to_float_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars) {
+	return std::make_unique<DoubleValueNode>(evaluatedPars[0]->getDoubleValue());
 }
 
 std::unique_ptr<ValueNode> USql::to_date_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars) {
 	std::string date = evaluatedPars[0]->getStringValue();
 	std::string format = evaluatedPars[1]->getStringValue();
 	long epoch_time = string_to_date(date, format);
+	
 	return std::make_unique<IntValueNode>(epoch_time);        // No DateValueNode for now
 }
 
@@ -27,58 +41,82 @@ std::unique_ptr<ValueNode> USql::date_add_function(const std::vector<std::unique
 	std::string part = evaluatedPars[2]->getStringValue();
 
 	long new_date = add_to_date(datetime, quantity, part);
+	
 	return std::make_unique<IntValueNode>(new_date);        // No DateValueNode for now
 }
 
 
-std::unique_ptr<ValueNode> USql::upper_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars) {
-	std::string str = evaluatedPars[0]->getStringValue();
-	std::transform(str.begin(), str.end(), str.begin(), [](unsigned char c) -> unsigned char { return toupper(c); });
-	return std::make_unique<StringValueNode>(str);
-}
-
 std::unique_ptr<ValueNode> USql::lower_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars) {
 	std::string str = evaluatedPars[0]->getStringValue();
 	std::transform(str.begin(), str.end(), str.begin(), [](unsigned char c) -> unsigned char { return tolower(c); });
+	
 	return std::make_unique<StringValueNode>(str);
 }
 
+std::unique_ptr<ValueNode> USql::upper_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars) {
+	std::string str = evaluatedPars[0]->getStringValue();
+	std::transform(str.begin(), str.end(), str.begin(), [](unsigned char c) -> unsigned char { return toupper(c); });
+	
+	return std::make_unique<StringValueNode>(str);
+}
+
+std::unique_ptr<ValueNode> USql::coalesce_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars) {
+	for(const auto & par : evaluatedPars) {
+		if (!par->isNull()) {
+			// TODO implement rest and take it out as a function
+		 	if (par->node_type == NodeType::int_value)
+				return std::make_unique<IntValueNode>(par->getIntegerValue());
+		 	if (par->node_type == NodeType::float_value)
+				return std::make_unique<DoubleValueNode>(par->getDoubleValue());
+			if (par->node_type == NodeType::string_value)
+				return std::make_unique<StringValueNode>(par->getStringValue());
+			if (par->node_type == NodeType::bool_value)
+				return std::make_unique<BooleanValueNode>(par->getBooleanValue());
+		}
+	}
+	return std::make_unique<NullValueNode>();
+}
+
 std::unique_ptr<ValueNode> USql::pp_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars) {
+	constexpr auto k_num_format_rpad = 10;
+	constexpr auto k_num_format_maxlen = 20;
+
 	auto &parsed_value = evaluatedPars[0];
 
 	if (parsed_value->node_type == NodeType::int_value || parsed_value->node_type == NodeType::float_value) {
 		std::string format = evaluatedPars.size() > 1 ? evaluatedPars[1]->getStringValue() : "";
-		char buf[20] {0};	// TODO constant here
+		char buf[k_num_format_maxlen] {0};
 		double value = parsed_value->getDoubleValue();
 
 		if (format == "100%")
-			std::snprintf(buf, 20, "%.2f%%", value);
+			std::snprintf(buf, k_num_format_maxlen, "%.2f%%", value);
 		else if (format == "%.2f")
-			std::snprintf(buf, 20, "%.2f", value);
+			std::snprintf(buf, k_num_format_maxlen, "%.2f", value);
 		else if (value >= 1000000000000)
-			std::snprintf(buf, 20, "%7.2fT", value/1000000000000);
+			std::snprintf(buf, k_num_format_maxlen, "%7.2fT", value/1000000000000);
 		else if (value >= 1000000000)
-			std::sprintf(buf, "%7.2fB", value/1000000000);
+			std::snprintf(buf, k_num_format_maxlen, "%7.2fB", value/1000000000);
 		else if (value >= 1000000)
-			std::snprintf(buf, 20, "%7.2fM", value/1000000);
+			std::snprintf(buf, k_num_format_maxlen, "%7.2fM", value/1000000);
 		else if (value >= 100000)
-			std::snprintf(buf, 20, "%7.2fM", value/100000);	// 0.12M
+			std::snprintf(buf, k_num_format_maxlen, "%7.2fM", value/100000);	// 0.12M
 		else if (value <= -1000000000000)
-			std::snprintf(buf, 20, "%7.2fT", value/1000000000000);
+			std::snprintf(buf, k_num_format_maxlen, "%7.2fT", value/1000000000000);
 		else if (value <= -1000000000)
-			std::snprintf(buf, 20, "%7.2fB", value/1000000000);
+			std::snprintf(buf, k_num_format_maxlen, "%7.2fB", value/1000000000);
 		else if (value <= -1000000)
-			std::snprintf(buf, 20, "%7.2fM", value/1000000);
+			std::snprintf(buf, k_num_format_maxlen, "%7.2fM", value/1000000);
 		else if (value <= -100000)
-			std::snprintf(buf, 20, "%7.2fM", value/100000);	// 0.12M
+			std::snprintf(buf, k_num_format_maxlen, "%7.2fM", value/100000);	// 0.12M
 		else if (value == 0)
 			buf[0]='0';
 		else
-			return std::make_unique<StringValueNode>(parsed_value->getStringValue().substr(0, 10));
+			return std::make_unique<StringValueNode>(parsed_value->getStringValue().substr(0, k_num_format_rpad));
-		// TODO introduce constant for 10
+
 		std::string s {buf};
-		return std::make_unique<StringValueNode>(string_padd(s.erase(s.find_last_not_of(' ')+1), 10, ' ', false));
+		return std::make_unique<StringValueNode>(string_padd(s.erase(s.find_last_not_of(' ') + 1), k_num_format_rpad, ' ', false));
 	}
+
 	return std::make_unique<StringValueNode>(parsed_value->getStringValue());
 }