WIP on indexes cat

Readme.md

@@ -1,10 +1,8 @@
 ### WIP
-compare in row.cpp shoud take into account m_visible
 
 ### TODO
 - create local_install.sh
 
-
 - change float type keyword to double and in code functions too
 
 - add functions:

debug.h

@@ -10,10 +10,13 @@ std::vector<std::string> k_debug_sql_commands {
 		// "delete from history_earnings_dates where symbol='BABA' and datetime=to_date('2021-11-04', '%Y-%m-%d')",
 		// "select * from history_earnings_dates"
 
-		//  "create table sf1 (symbol varchar(8) not null, dimension varchar(3), calendar_date date, date_key date, report_period date, last_updated date, accoci float, assets float, assetsavg float, assetsc float, assetsnc float, assetturnover float, bvps float, capex float, cashneq float, cashnequsd float, cor float, consolinc float, currentratio float, de float, debt float, debtc float, debtnc float, debtusd float, deferredrev float, depamor float, deposits float, divyield float, dps float, ebit float, ebitda float, ebitdamargin float, ebitdausd float, ebitusd float, ebt float, eps float, epsdil float, epsusd float, equity float, equityavg float, equityusd float, ev float, evebit float, evebitda float, fcf float, fcfps float, fxusd float, gp float, grossmargin float, intangibles float, intexp float, invcap float, invcapavg float, inventory float, investments float, investmentsc float, investmentsnc float, liabilities float, liabilitiesc float, liabilitiesnc float, marketcap float, ncf float, ncfbus float, ncfcommon float, ncfdebt float, ncfdiv float, ncff float, ncfi float, ncfinv float, ncfo float, ncfx float, netinc float, netinccmn float, netinccmnusd float, netincdis float, netincnci float, netmargin float, opex float, opinc float, payables float, payoutratio float, pb float, pe float, pe1 float, ppnenet float, prefdivis float, price float, ps float, ps1 float, receivables float, retearn float, revenue float, revenueusd float, rnd float, roa float, roe float, roic float, ros float, sbcomp float, sgna float, sharefactor float, sharesbas float, shareswa float, shareswadil float, sps float, tangibles float, taxassets float, taxexp float, taxliabilities float, tbvps float, workingcapital float)",
+		 "create table sf1 (symbol varchar(8) not null, dimension varchar(3), calendar_date date, date_key date, report_period date, last_updated date, accoci float, assets float, assetsavg float, assetsc float, assetsnc float, assetturnover float, bvps float, capex float, cashneq float, cashnequsd float, cor float, consolinc float, currentratio float, de float, debt float, debtc float, debtnc float, debtusd float, deferredrev float, depamor float, deposits float, divyield float, dps float, ebit float, ebitda float, ebitdamargin float, ebitdausd float, ebitusd float, ebt float, eps float, epsdil float, epsusd float, equity float, equityavg float, equityusd float, ev float, evebit float, evebitda float, fcf float, fcfps float, fxusd float, gp float, grossmargin float, intangibles float, intexp float, invcap float, invcapavg float, inventory float, investments float, investmentsc float, investmentsnc float, liabilities float, liabilitiesc float, liabilitiesnc float, marketcap float, ncf float, ncfbus float, ncfcommon float, ncfdebt float, ncfdiv float, ncff float, ncfi float, ncfinv float, ncfo float, ncfx float, netinc float, netinccmn float, netinccmnusd float, netincdis float, netincnci float, netmargin float, opex float, opinc float, payables float, payoutratio float, pb float, pe float, pe1 float, ppnenet float, prefdivis float, price float, ps float, ps1 float, receivables float, retearn float, revenue float, revenueusd float, rnd float, roa float, roe float, roic float, ros float, sbcomp float, sgna float, sharefactor float, sharesbas float, shareswa float, shareswadil float, sps float, tangibles float, taxassets float, taxexp float, taxliabilities float, tbvps float, workingcapital float)",
-		//  "set 'DATE_FORMAT' = '%Y-%m-%d'",
+		 "set 'DATE_FORMAT' = '%Y-%m-%d'",
-		//  "create index sf1_symbol on sf1(symbol)",
+		 "create index sf1_symbol on sf1(symbol)",
-		//  "load into sf1 '/srv/SHARADAR_SF1.csv'",
+		 "load into sf1 '/srv/SHARADAR_SF1.csv'",
+		// "set 'USE_INDEXSCAN' = 'true'",
+		 "select distinct dimension from sf1 where symbol = 'ALLY'",
+		 "select calendar_date, calendar_date, dimension from sf1 where symbol = 'ALLY' and dimension = 'MRQ' and calendar_date <= 1653487723 order by calendar_date desc limit 10",
 		//  "set 'USE_INDEXSCAN' = 'false'",
 		//  "select dimension, to_char(calendar_date, '%d.%m.%Y'), pp(eps, \"%.2f\"), pp(shareswadil), pp(revenue), pp(netinc), pp(cashneq), pp(assets), pp(debt), pp(ncfdebt), pp(roe*100), pp(intangibles), calendar_date from sf1 where symbol = 'MU' and dimension = 'ARQ' order by dimension, calendar_date desc limit 5",
 		//  "set 'USE_INDEXSCAN' = 'true'",
@@ -46,14 +49,15 @@ std::vector<std::string> k_debug_sql_commands {
 		// "insert into a (i, j) values(null, 123456789.12345)",
 		// "select pp(coalesce(i, j)) from a",
 
-		"create table d (datetime date)",
+		// "create table d (datetime date)",
-		"insert into d (datetime) values(1648634993)",
+		// "insert into d (datetime) values(1648634993)",
-		"select to_char(datetime, '%Y%m%d') from d",
+		// "select to_char(datetime, '%Y%m%d') from d",
-		"select to_int(datetime) from d",
+		// "select to_int(datetime) from d",
-		"select max(to_char(datetime, '%Y%m%d')) from d",
+		// "select max(to_char(datetime, '%Y%m%d')) from d",
-		"select max(to_int(to_float(to_char(datetime, '%Y%m%d')))) from d",
+		// "select max(to_int(to_float(to_char(datetime, '%Y%m%d')))) from d",
 
-		"select * from usql_tables",
+		// "select * from usql_tables",
-		"select * from usql_columns"
+		// "select * from usql_columns",
+		// "select * from usql_indexes",
 };
 

lexer.cpp

@@ -162,7 +162,7 @@ TokenType Lexer::type(const std::string &token) {
 	if (token == "not") return TokenType::keyword_not;
 	if (token == "null") return TokenType::keyword_null;
 	if (token == "integer") return TokenType::keyword_integer;
-	if (token == "float") return TokenType::keyword_float;
+	if (token == "float" || token == "double") return TokenType::keyword_float;
 	if (token == "varchar") return TokenType::keyword_varchar;
 	if (token == "date") return TokenType::keyword_date;
 	if (token == "boolean") return TokenType::keyword_bool;

parser.cpp

@@ -385,19 +385,18 @@ namespace usql {
 
     
     std::unique_ptr<Node> Parser::parse_where_clause() {
-	if (m_lexer.tokenType() != TokenType::keyword_where) {
+		if (m_lexer.tokenType() != TokenType::keyword_where) {
-		return std::make_unique<TrueNode>();
+			return std::make_unique<TrueNode>();
-	}
+		}
 
-	m_lexer.skipToken(TokenType::keyword_where);
+		m_lexer.skipToken(TokenType::keyword_where);
 
-	std::unique_ptr<Node> left = parse_expression();
+		std::unique_ptr<Node> left = parse_expression();
-	do {
+		do {
-		left = parse_expression(std::move(left));
+			left = parse_expression(std::move(left));
-
+		} while (m_lexer.tokenType() != TokenType::eof && m_lexer.tokenType() != TokenType::keyword_order && m_lexer.tokenType() != TokenType::keyword_offset && m_lexer.tokenType() != TokenType::keyword_limit && m_lexer.tokenType() != TokenType::semicolon);
-	} while (m_lexer.tokenType() != TokenType::eof && m_lexer.tokenType() != TokenType::keyword_order && m_lexer.tokenType() != TokenType::keyword_offset && m_lexer.tokenType() != TokenType::keyword_limit);
 	
-	return left;
+		return left;
     }
 
     std::unique_ptr<Node> Parser::parse_expression() {

parser.h

@@ -175,9 +175,6 @@ struct FunctionNode : Node {
 	};
 
 
-	Type								function;
-	std::vector<std::unique_ptr<Node>>	params;
-
 	FunctionNode(std::string  func_name, std::vector<std::unique_ptr<Node>> pars) :
 	Node(NodeType::function), function(get_function(func_name)), params(std::move(pars)) {}
 
@@ -199,6 +196,9 @@ struct FunctionNode : Node {
 		}			
 		std::cout << ")" << std::endl;
 	}
+
+	Type								function;
+	std::vector<std::unique_ptr<Node>>	params;
 };
 
 struct TrueNode : Node {
@@ -581,7 +581,6 @@ struct CreateIndexNode : Node {
 };
 
 class Parser {
-private:
 public:
 	Parser();
 

row.h

@@ -171,7 +171,7 @@ public:
 			case 5:
 				return (ColValue &) *std::get_if<ColBooleanValue>(&m_columns[i]);
 			default:
-				throw Exception("should not happen");
+				throw Exception("ColValue &operator[](int i), type index invalid :" + std::to_string(type_index));
 		}
 	}
 

settings.cpp

@@ -12,7 +12,7 @@ std::vector<std::pair<std::string, std::string>> Settings::m_settings =
 	      std::make_pair("BOOL_TRUE_LITERAL", "Y"),
 	      std::make_pair("BOOL_FALSE_LITERAL", "N"),
           std::make_pair("DOUBLE_FORMAT", "%.2f"),
-	      std::make_pair("USE_INDEXSCAN", "N"),
+	      std::make_pair("USE_INDEXSCAN", "Y"),
 	      std::make_pair("MAX_PARALLELISM", "1") };		// values "AUTO" or number of workers; when number negative means std::thread::hardware_concurrency() - number 
 
 

usql.cpp

@@ -11,7 +11,8 @@ USql::USql() {
 
 	std::vector<std::string> k_debug_sql_commands {
 		"create table usql_tables(name varchar(32) not null, modified boolean not null)",
-		"create table usql_columns(table_name varchar(32) not null, column_name varchar(32) not null, column_type varchar(16) not null, column_length integer not null, nullable boolean not null, column_order integer not null)"
+		"create table usql_columns(table_name varchar(32) not null, column_name varchar(32) not null, column_type varchar(16) not null, column_length integer not null, nullable boolean not null, column_order integer not null)",
+		"create table usql_indexes(index_name varchar(32) not null, table_name varchar(32), column_name varchar(32) not null)"
 	};
 
 	// create cataloque tables

usql.h

@@ -22,6 +22,7 @@ private:
     std::unique_ptr<Table> execute_create_table(const CreateTableNode &node);
 	bool execute_create_table_sys_catalogue(const CreateTableNode &node);
     std::unique_ptr<Table> execute_create_index(const CreateIndexNode &node);
+	bool execute_create_index_sys_catalogue(const CreateIndexNode &node);
     std::unique_ptr<Table> execute_create_table_as_table(const CreateTableAsSelectNode &node);
     std::unique_ptr<Table> execute_load(const LoadIntoTableNode &node);
     std::unique_ptr<Table> execute_save(const SaveTableNode &node);

usql_ddl.cpp

@@ -8,7 +8,6 @@
 namespace usql {
 
 
-
 std::unique_ptr<Table> USql::execute_create_table(const CreateTableNode &node) {
 	check_table_not_exists(node.table_name);
 
@@ -56,10 +55,25 @@ std::unique_ptr<Table> USql::execute_create_index(const CreateIndexNode &node) {
 
 	table_def->index_rows(node.index_name);
 
+	execute_create_index_sys_catalogue(node);
+
 	return create_stmt_result_table(0, "index created", 0);
 }
 
 
+bool USql::execute_create_index_sys_catalogue(const CreateIndexNode &node) {
+	std::string i {"insert into usql_indexes(index_name, table_name, column_name) values("};
+				i += "'" + node.index_name + "', ";
+				i += "'" + node.table_name + "', ";
+				i += "'" + node.column_name + "')";
+
+	auto r = execute(i);
+	// r->print();
+
+	return true;
+}
+
+
 std::unique_ptr<Table> USql::execute_create_table_as_table(const CreateTableAsSelectNode &node) {
 	check_table_not_exists(node.table_name);
 

usql_dml.cpp

@@ -35,7 +35,7 @@ std::pair<bool, std::vector<rowid_t>> USql::look_for_usable_index(const Node *wh
 					if (used_index != nullptr) {
 						std::vector<rowid_t> rowids = used_index->search((ValueNode *)ron->right.get());
 #ifndef NDEBUG
-						std::cout << "using index " << table->m_name << "(" << used_index->get_column_name() << "), " << rowids.size() << "/" << table->rows_count()  << std::endl;
+						std::cerr << "using index " << table->m_name << "(" << used_index->get_column_name() << "), " << rowids.size() << "/" << table->rows_count()  << std::endl;
 #endif
 						return std::make_pair(true, rowids);
 					}
@@ -155,8 +155,7 @@ void USql::setup_order_columns(std::vector<ColOrderNode> &node, Table *table) {
 void USql::execute_distinct(SelectFromTableNode &node, Table *result) {
 	if (!node.distinct) return;
 
-	auto compare_rows = [](const Row &a, const Row &b) { return a.compare(b) >= 0; };
+	std::sort(result->m_rows.begin(), result->m_rows.end(), [](const Row &a, const Row &b) { return a.compare(b) > 0; });
-	std::sort(result->m_rows.begin(), result->m_rows.end(), compare_rows);
 
 	result->m_rows.erase(std::unique(result->m_rows.begin(), result->m_rows.end()), result->m_rows.end());
 }