From 7e8d750f631634d02e7ca212864df3465714ff48 Mon Sep 17 00:00:00 2001 From: VaclavT Date: Sat, 18 Dec 2021 10:53:43 +0100 Subject: [PATCH] some code refactoring --- debug.h | 37 +++++++++++++++---------------- index.h | 31 +++++++++++++++++--------- main.cpp | 2 +- table.cpp | 8 ++----- usql.h | 10 ++++----- usql_ddl.cpp | 10 +-------- usql_dml.cpp | 61 +++++++++++++++++++++++----------------------------- 7 files changed, 76 insertions(+), 83 deletions(-) diff --git a/debug.h b/debug.h index 0e05578..84059ae 100644 --- a/debug.h +++ b/debug.h @@ -1,7 +1,7 @@ #pragma once -std::vector c_sql_commands { +std::vector k_debug_sql_commands { // "create table history_earnings_dates (datetime date, symbol varchar(8), time varchar(18), title varchar(256))", // "set 'DATE_FORMAT' = '%Y-%m-%d'", // "load into history_earnings_dates '/Users/vaclavt/Development/mlisp_fin/data/history_earnings_dates.csv'" @@ -10,27 +10,28 @@ std::vector c_sql_commands { // "delete from history_earnings_dates where symbol='BABA' and datetime=to_date('2021-11-04', '%Y-%m-%d')", // "select * from history_earnings_dates" - "create table sf1 (symbol varchar(8) not null, dimension varchar(3), calendar_date date, date_key date, report_period date, last_updated date, accoci float, assets float, assetsavg float, assetsc float, assetsnc float, assetturnover float, bvps float, capex float, cashneq float, cashnequsd float, cor float, consolinc float, currentratio float, de float, debt float, debtc float, debtnc float, debtusd float, deferredrev float, depamor float, deposits float, divyield float, dps float, ebit float, ebitda float, ebitdamargin float, ebitdausd float, ebitusd float, ebt float, eps float, epsdil float, epsusd float, equity float, equityavg float, equityusd float, ev float, evebit float, evebitda float, fcf float, fcfps float, fxusd float, gp float, grossmargin float, intangibles float, intexp float, invcap float, invcapavg float, inventory float, investments float, investmentsc float, investmentsnc float, liabilities float, liabilitiesc float, liabilitiesnc float, marketcap float, ncf float, ncfbus float, ncfcommon float, ncfdebt float, ncfdiv float, ncff float, ncfi float, ncfinv float, ncfo float, ncfx float, netinc float, netinccmn float, netinccmnusd float, netincdis float, netincnci float, netmargin float, opex float, opinc float, payables float, payoutratio float, pb float, pe float, pe1 float, ppnenet float, prefdivis float, price float, ps float, ps1 float, receivables float, retearn float, revenue float, revenueusd float, rnd float, roa float, roe float, roic float, ros float, sbcomp float, sgna float, sharefactor float, sharesbas float, shareswa float, shareswadil float, sps float, tangibles float, taxassets float, taxexp float, taxliabilities float, tbvps float, workingcapital float)", - "set 'DATE_FORMAT' = '%Y-%m-%d'", - "create index sf1_symbol on sf1(symbol)", - "load into sf1 '/srv/SHARADAR_SF1.csv'", - "set 'USE_INDEXSCAN' = 'false'", - "select dimension, to_string(calendar_date, '%d.%m.%Y'), pp(eps, \"%.2f\"), pp(shareswadil), pp(revenue), pp(netinc), pp(cashneq), pp(assets), pp(debt), pp(ncfdebt), pp(roe*100), pp(intangibles), calendar_date from sf1 where symbol = 'MU' and dimension = 'ARQ' order by dimension, calendar_date desc limit 5", - "set 'USE_INDEXSCAN' = 'true'", - "select dimension, to_string(calendar_date, '%d.%m.%Y'), pp(eps, \"%.2f\"), pp(shareswadil), pp(revenue), pp(netinc), pp(cashneq), pp(assets), pp(debt), pp(ncfdebt), pp(roe*100), pp(intangibles), calendar_date from sf1 where symbol = 'MU' and dimension = 'ARQ' order by dimension, calendar_date desc limit 5" + // "create table sf1 (symbol varchar(8) not null, dimension varchar(3), calendar_date date, date_key date, report_period date, last_updated date, accoci float, assets float, assetsavg float, assetsc float, assetsnc float, assetturnover float, bvps float, capex float, cashneq float, cashnequsd float, cor float, consolinc float, currentratio float, de float, debt float, debtc float, debtnc float, debtusd float, deferredrev float, depamor float, deposits float, divyield float, dps float, ebit float, ebitda float, ebitdamargin float, ebitdausd float, ebitusd float, ebt float, eps float, epsdil float, epsusd float, equity float, equityavg float, equityusd float, ev float, evebit float, evebitda float, fcf float, fcfps float, fxusd float, gp float, grossmargin float, intangibles float, intexp float, invcap float, invcapavg float, inventory float, investments float, investmentsc float, investmentsnc float, liabilities float, liabilitiesc float, liabilitiesnc float, marketcap float, ncf float, ncfbus float, ncfcommon float, ncfdebt float, ncfdiv float, ncff float, ncfi float, ncfinv float, ncfo float, ncfx float, netinc float, netinccmn float, netinccmnusd float, netincdis float, netincnci float, netmargin float, opex float, opinc float, payables float, payoutratio float, pb float, pe float, pe1 float, ppnenet float, prefdivis float, price float, ps float, ps1 float, receivables float, retearn float, revenue float, revenueusd float, rnd float, roa float, roe float, roic float, ros float, sbcomp float, sgna float, sharefactor float, sharesbas float, shareswa float, shareswadil float, sps float, tangibles float, taxassets float, taxexp float, taxliabilities float, tbvps float, workingcapital float)", + // "set 'DATE_FORMAT' = '%Y-%m-%d'", + // "create index sf1_symbol on sf1(symbol)", + // "load into sf1 '/srv/SHARADAR_SF1.csv'", + // "set 'USE_INDEXSCAN' = 'false'", + // "select dimension, to_string(calendar_date, '%d.%m.%Y'), pp(eps, \"%.2f\"), pp(shareswadil), pp(revenue), pp(netinc), pp(cashneq), pp(assets), pp(debt), pp(ncfdebt), pp(roe*100), pp(intangibles), calendar_date from sf1 where symbol = 'MU' and dimension = 'ARQ' order by dimension, calendar_date desc limit 5", + // "set 'USE_INDEXSCAN' = 'true'", + // "select dimension, to_string(calendar_date, '%d.%m.%Y'), pp(eps, \"%.2f\"), pp(shareswadil), pp(revenue), pp(netinc), pp(cashneq), pp(assets), pp(debt), pp(ncfdebt), pp(roe*100), pp(intangibles), calendar_date from sf1 where symbol = 'MU' and dimension = 'ARQ' order by dimension, calendar_date desc limit 5" -// "create table a (i integer not null, s varchar(64))", -// "create index a_i on a(i)", -// "insert into a (i, s) values(1, 'one')", -// "insert into a (i, s) values(2, 'two')", -// "insert into a (i, s) values(2, 'second two')", -// "insert into a (i, s) values(3, 'three')", -// "set 'USE_INDEXSCAN' = 'true'", + "create table a (i integer not null, s varchar(64))", + "create index a_i on a(i)", + "insert into a (i, s) values(1, 'one')", + "insert into a (i, s) values(2, 'two')", + "insert into a (i, s) values(2, 'second two')", + "insert into a (i, s) values(3, 'three')", + "set 'USE_INDEXSCAN' = 'true'", // "select * from a where 1 = i", // "delete from a where i = 2 and s ='two'", // "select * from a where i = 2", -// "update a set i = 5 where i = 2", -// "select * from a where i = 5", + "update a set i = 5 where i = 2", + "select * from a where i = 5", // "select max(i) from a where s = 'two'" + "select min(i), max(i) from a" }; diff --git a/index.h b/index.h index 3dca108..d54a5ca 100644 --- a/index.h +++ b/index.h @@ -1,6 +1,7 @@ #pragma once #include "exception.h" +#include "parser.h" #include #include @@ -10,19 +11,19 @@ namespace usql { -enum class IndexedDataType { - integer, - string -}; - using rowid_t = size_t; // int is now enough but size_t is correct +static const int k_default_rowids_size = 16; + template class Index { public: - Index(std::string index_name, std::string col_name, IndexedDataType type) : + Index(std::string index_name, std::string col_name, ColumnType type) : m_index_name(std::move(index_name)), m_column_name(std::move(col_name)), - m_data_type(type), m_uniq(false) {} + m_data_type(type), m_uniq(false) { + if (type != ColumnType::integer_type && type != ColumnType::varchar_type) + throw Exception("creating index on unsupported type"); + } void insert(K key, rowid_t rowid) { auto search = m_index.find(key); @@ -34,7 +35,7 @@ public: } else { std::vector rowids{rowid}; if (!m_uniq) - rowids.reserve(8); + rowids.reserve(k_default_rowids_size); m_index[key] = rowids; } } @@ -57,6 +58,16 @@ public: } } + std::vector search(ValueNode * key) { + if (m_data_type == ColumnType::integer_type) + return search(key->getIntegerValue()); + else if (m_data_type == ColumnType::varchar_type) + return search(key->getStringValue()); + else + throw Exception("using index on unsupported type"); + } + + void truncate() { m_index.clear(); } @@ -78,7 +89,7 @@ public: return m_index_name; } - [[nodiscard]] IndexedDataType get_data_type() const { + [[nodiscard]] ColumnType get_data_type() const { return m_data_type; } @@ -86,7 +97,7 @@ private: bool m_uniq; std::string m_index_name; std::string m_column_name; - IndexedDataType m_data_type; + ColumnType m_data_type; std::map > m_index; }; diff --git a/main.cpp b/main.cpp index 7fcb8c8..36fa181 100644 --- a/main.cpp +++ b/main.cpp @@ -129,7 +129,7 @@ void repl() { void debug() { usql::USql uSql{}; - for (const auto &command : c_sql_commands) { + for (const auto &command : k_debug_sql_commands) { time_point start_time = high_resolution_clock::now(); auto result = uSql.execute(command); time_point end_time = high_resolution_clock::now(); diff --git a/table.cpp b/table.cpp index bcf3b1a..45586d0 100644 --- a/table.cpp +++ b/table.cpp @@ -356,9 +356,7 @@ Index * Table::get_index(const std::string &index_name) { return idx.get_index_name() == index_name; }); - if (it != m_indexes.end()) return &(*it); - - return nullptr; + return (it != m_indexes.end()) ? &(*it) : nullptr; } Index * Table::get_index_for_column(const std::string &col_name) { @@ -367,9 +365,7 @@ Index * Table::get_index_for_column(const std::string &col_name) { return idx.get_column_name() == col_name; }); - if (it != m_indexes.end()) return &(*it); - - return nullptr; + return (it != m_indexes.end()) ? &(*it) : nullptr; } diff --git a/usql.h b/usql.h index 5cccea2..1e13978 100644 --- a/usql.h +++ b/usql.h @@ -83,11 +83,11 @@ private: static std::unique_ptr count_function(ColValue *agg_func_value, const std::vector> &evaluatedPars); - static void eval_where_on_row(SelectFromTableNode &where_node, - Table *src_table, Row *src_row, - Table *rslt_table, Row *rslt_row, - const std::vector &rslt_tbl_col_defs, const std::vector &src_table_col_index, - bool is_aggregated) ; + static void select_row(SelectFromTableNode &where_node, + Table *src_table, Row *src_row, + Table *rslt_table, + const std::vector &rslt_tbl_col_defs, const std::vector &src_table_col_index, + bool is_aggregated) ; std::pair> probe_index_scan(const Node *where, Table *table) const; std::pair> look_for_usable_index(const Node *where, Table *table) const; diff --git a/usql_ddl.cpp b/usql_ddl.cpp index 54d4664..6066623 100644 --- a/usql_ddl.cpp +++ b/usql_ddl.cpp @@ -27,15 +27,7 @@ std::unique_ptr USql::execute_create_index(const CreateIndexNode &node) { if (col_def.null) throw Exception("index on not null supported only"); if (table_def->get_index_for_column(node.column_name) != nullptr) throw Exception("column is already indexed"); - IndexedDataType type; - if (col_def.type == ColumnType::integer_type) - type = IndexedDataType::integer; - else if (col_def.type == ColumnType::varchar_type) - type = IndexedDataType::string; - else - throw Exception("creating index on unsupported type"); - - table_def->create_index({node.index_name, node.column_name, type}); + table_def->create_index({node.index_name, node.column_name, col_def.type}); table_def->index_rows(node.index_name); diff --git a/usql_dml.cpp b/usql_dml.cpp index 9bdc7be..c8d060d 100644 --- a/usql_dml.cpp +++ b/usql_dml.cpp @@ -22,22 +22,19 @@ std::pair> USql::probe_index_scan(const Node *where, std::pair> USql::look_for_usable_index(const Node *where, Table *table) const { if (where->node_type == NodeType::relational_operator) { auto * ron = (RelationalOperatorNode *)where; + // TODO impllemen > & < https://en.cppreference.com/w/cpp/container/map/upper_bound if (ron->op == RelationalOperatorType::equal) { if (ron->left->node_type == NodeType::database_value && ((ron->right->node_type == NodeType::int_value) || (ron->right->node_type == NodeType::string_value)) - ) { + ) { auto col_name = ((DatabaseValueNode *)ron->left.get())->col_name; Index * used_index = table->get_index_for_column(col_name); if (used_index != nullptr) { - std::vector rowids; - - if (used_index->get_data_type() == IndexedDataType::integer) - rowids = used_index->search(((ValueNode *) ron->right.get())->getIntegerValue()); - else if (used_index->get_data_type() == IndexedDataType::string) - rowids = used_index->search(((ValueNode *) ron->right.get())->getStringValue()); - + std::vector rowids = used_index->search((ValueNode *)ron->right.get()); +#ifndef NDEBUG std::cout << "using index " << table->m_name << "(" << used_index->get_column_name() << "), " << rowids.size() << "/" << table->rows_count() << std::endl; +#endif return std::make_pair(true, rowids); } } @@ -80,19 +77,21 @@ bool USql::normalize_where(const Node *node) const { return true; } -void USql::eval_where_on_row(SelectFromTableNode &where_node, - Table *src_table, Row *src_row, - Table *rslt_table, Row *rslt_row, - const std::vector &rslt_tbl_col_defs, - const std::vector &src_table_col_index, - bool is_aggregated) { +void USql::select_row(SelectFromTableNode &where_node, + Table *src_table, Row *src_row, + Table *rslt_table, + const std::vector &rslt_tbl_col_defs, + const std::vector &src_table_col_index, + bool is_aggregated) { - if (eval_where(where_node.where.get(), src_table, *src_row)) { - // prepare empty src_row and copy column values - // when aggregate functions in rslt_table only one src_row for src_table - if (!is_aggregated || rslt_table->rows_count() == 0) { + Row *rslt_row = nullptr; + + // when aggregate functions in rslt_table only one row exists + // TODO add function to get rows count + if (is_aggregated && !rslt_table->m_rows.empty()) + rslt_row = &rslt_table->m_rows[0]; + else rslt_row = &rslt_table->create_empty_row(); - } for (auto idx = 0; idx < rslt_table->columns_count(); idx++) { auto src_table_col_idx = src_table_col_index[idx]; @@ -109,11 +108,8 @@ void USql::eval_where_on_row(SelectFromTableNode &where_node, } } - // add src_row to rslt_table - if (!is_aggregated) { - rslt_table->commit_row(*rslt_row); - } - } + // for aggregate is validated more than needed + rslt_table->commit_row(*rslt_row); } bool USql::check_for_aggregate_only_functions(SelectFromTableNode &node, size_t result_cols_cnt) { @@ -155,12 +151,12 @@ void USql::setup_order_columns(std::vector &node, Table *table) { if (order_node.col_index < 0 || order_node.col_index >= table->columns_count()) throw Exception("unknown column in order by clause (" + order_node.col_name + ")"); - } + } } void USql::execute_distinct(SelectFromTableNode &node, Table *result) { if (!node.distinct) return; - + auto compare_rows = [](const Row &a, const Row &b) { return a.compare(b) >= 0; }; std::sort(result->m_rows.begin(), result->m_rows.end(), compare_rows); @@ -334,8 +330,7 @@ std::unique_ptr
USql::execute_delete(const DeleteFromTableNode &node) { // execute access plan Table::rows_scanner i = get_iterator(table, node.where.get()); while(Row *row = i.next()) { - bool to_delete = eval_where(node.where.get(), table, *row); - if (to_delete) { + if (eval_where(node.where.get(), table, *row)) { row->set_deleted(); table->unindex_row(*row); @@ -356,8 +351,7 @@ std::unique_ptr
USql::execute_update(const UpdateTableNode &node) { // execute access plan Table::rows_scanner i = get_iterator(table, node.where.get()); while(Row *row = i.next()) { - bool to_update = eval_where(node.where.get(), table, *row); - if (to_update) { + if (eval_where(node.where.get(), table, *row)) { Row old_row = * row; int col_idx = 0; @@ -410,12 +404,11 @@ std::unique_ptr
USql::execute_select(SelectFromTableNode &node) const { setup_order_columns(node.order_by, result.get()); // execute access plan - Row *new_row = nullptr; - - Table::rows_scanner i = get_iterator(table, node.where.get()); while(Row *row = i.next()) { - eval_where_on_row(node, table, row, result.get(), new_row, result_tbl_col_defs, source_table_col_index, is_aggregated); + if (eval_where(node.where.get(), table, *row)) { // put it into row_scanner.next + select_row(node, table, row, result.get(), result_tbl_col_defs, source_table_col_index, is_aggregated); + } } execute_distinct(node, result.get());