vaclavt/usql
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

some code refactoring

Browse Source
This commit is contained in:
VaclavT
2021-12-18 10:53:43 +01:00
parent d645471c15
commit 7e8d750f63
7 changed files with 76 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
debug.h
View File

@@ -1,7 +1,7 @@
#pragma once
std::vector<std::string> c_sql_commands {
std::vector<std::string> k_debug_sql_commands {
// "create table history_earnings_dates (datetime date, symbol varchar(8), time varchar(18), title varchar(256))",
// "set 'DATE_FORMAT' = '%Y-%m-%d'",
// "load into history_earnings_dates '/Users/vaclavt/Development/mlisp_fin/data/history_earnings_dates.csv'"
@@ -10,27 +10,28 @@ std::vector<std::string> c_sql_commands {
// "delete from history_earnings_dates where symbol='BABA' and datetime=to_date('2021-11-04', '%Y-%m-%d')",
// "select * from history_earnings_dates"
"create table sf1 (symbol varchar(8) not null, dimension varchar(3), calendar_date date, date_key date, report_period date, last_updated date, accoci float, assets float, assetsavg float, assetsc float, assetsnc float, assetturnover float, bvps float, capex float, cashneq float, cashnequsd float, cor float, consolinc float, currentratio float, de float, debt float, debtc float, debtnc float, debtusd float, deferredrev float, depamor float, deposits float, divyield float, dps float, ebit float, ebitda float, ebitdamargin float, ebitdausd float, ebitusd float, ebt float, eps float, epsdil float, epsusd float, equity float, equityavg float, equityusd float, ev float, evebit float, evebitda float, fcf float, fcfps float, fxusd float, gp float, grossmargin float, intangibles float, intexp float, invcap float, invcapavg float, inventory float, investments float, investmentsc float, investmentsnc float, liabilities float, liabilitiesc float, liabilitiesnc float, marketcap float, ncf float, ncfbus float, ncfcommon float, ncfdebt float, ncfdiv float, ncff float, ncfi float, ncfinv float, ncfo float, ncfx float, netinc float, netinccmn float, netinccmnusd float, netincdis float, netincnci float, netmargin float, opex float, opinc float, payables float, payoutratio float, pb float, pe float, pe1 float, ppnenet float, prefdivis float, price float, ps float, ps1 float, receivables float, retearn float, revenue float, revenueusd float, rnd float, roa float, roe float, roic float, ros float, sbcomp float, sgna float, sharefactor float, sharesbas float, shareswa float, shareswadil float, sps float, tangibles float, taxassets float, taxexp float, taxliabilities float, tbvps float, workingcapital float)",
"set 'DATE_FORMAT' = '%Y-%m-%d'",
"create index sf1_symbol on sf1(symbol)",
"load into sf1 '/srv/SHARADAR_SF1.csv'",
"set 'USE_INDEXSCAN' = 'false'",
"select dimension, to_string(calendar_date, '%d.%m.%Y'), pp(eps, \"%.2f\"), pp(shareswadil), pp(revenue), pp(netinc), pp(cashneq), pp(assets), pp(debt), pp(ncfdebt), pp(roe*100), pp(intangibles), calendar_date from sf1 where symbol = 'MU' and dimension = 'ARQ' order by dimension, calendar_date desc limit 5",
"set 'USE_INDEXSCAN' = 'true'",
"select dimension, to_string(calendar_date, '%d.%m.%Y'), pp(eps, \"%.2f\"), pp(shareswadil), pp(revenue), pp(netinc), pp(cashneq), pp(assets), pp(debt), pp(ncfdebt), pp(roe*100), pp(intangibles), calendar_date from sf1 where symbol = 'MU' and dimension = 'ARQ' order by dimension, calendar_date desc limit 5"
// "create table sf1 (symbol varchar(8) not null, dimension varchar(3), calendar_date date, date_key date, report_period date, last_updated date, accoci float, assets float, assetsavg float, assetsc float, assetsnc float, assetturnover float, bvps float, capex float, cashneq float, cashnequsd float, cor float, consolinc float, currentratio float, de float, debt float, debtc float, debtnc float, debtusd float, deferredrev float, depamor float, deposits float, divyield float, dps float, ebit float, ebitda float, ebitdamargin float, ebitdausd float, ebitusd float, ebt float, eps float, epsdil float, epsusd float, equity float, equityavg float, equityusd float, ev float, evebit float, evebitda float, fcf float, fcfps float, fxusd float, gp float, grossmargin float, intangibles float, intexp float, invcap float, invcapavg float, inventory float, investments float, investmentsc float, investmentsnc float, liabilities float, liabilitiesc float, liabilitiesnc float, marketcap float, ncf float, ncfbus float, ncfcommon float, ncfdebt float, ncfdiv float, ncff float, ncfi float, ncfinv float, ncfo float, ncfx float, netinc float, netinccmn float, netinccmnusd float, netincdis float, netincnci float, netmargin float, opex float, opinc float, payables float, payoutratio float, pb float, pe float, pe1 float, ppnenet float, prefdivis float, price float, ps float, ps1 float, receivables float, retearn float, revenue float, revenueusd float, rnd float, roa float, roe float, roic float, ros float, sbcomp float, sgna float, sharefactor float, sharesbas float, shareswa float, shareswadil float, sps float, tangibles float, taxassets float, taxexp float, taxliabilities float, tbvps float, workingcapital float)",
// "set 'DATE_FORMAT' = '%Y-%m-%d'",
// "create index sf1_symbol on sf1(symbol)",
// "load into sf1 '/srv/SHARADAR_SF1.csv'",
// "set 'USE_INDEXSCAN' = 'false'",
// "select dimension, to_string(calendar_date, '%d.%m.%Y'), pp(eps, \"%.2f\"), pp(shareswadil), pp(revenue), pp(netinc), pp(cashneq), pp(assets), pp(debt), pp(ncfdebt), pp(roe*100), pp(intangibles), calendar_date from sf1 where symbol = 'MU' and dimension = 'ARQ' order by dimension, calendar_date desc limit 5",
// "set 'USE_INDEXSCAN' = 'true'",
// "select dimension, to_string(calendar_date, '%d.%m.%Y'), pp(eps, \"%.2f\"), pp(shareswadil), pp(revenue), pp(netinc), pp(cashneq), pp(assets), pp(debt), pp(ncfdebt), pp(roe*100), pp(intangibles), calendar_date from sf1 where symbol = 'MU' and dimension = 'ARQ' order by dimension, calendar_date desc limit 5"
// "create table a (i integer not null, s varchar(64))",
// "create index a_i on a(i)",
// "insert into a (i, s) values(1, 'one')",
// "insert into a (i, s) values(2, 'two')",
// "insert into a (i, s) values(2, 'second two')",
// "insert into a (i, s) values(3, 'three')",
// "set 'USE_INDEXSCAN' = 'true'",
"create table a (i integer not null, s varchar(64))",
"create index a_i on a(i)",
"insert into a (i, s) values(1, 'one')",
"insert into a (i, s) values(2, 'two')",
"insert into a (i, s) values(2, 'second two')",
"insert into a (i, s) values(3, 'three')",
"set 'USE_INDEXSCAN' = 'true'",
// "select * from a where 1 = i",
// "delete from a where i = 2 and s ='two'",
// "select * from a where i = 2",
// "update a set i = 5 where i = 2",
// "select * from a where i = 5",
"update a set i = 5 where i = 2",
"select * from a where i = 5",
// "select max(i) from a where s = 'two'"
"select min(i), max(i) from a"
};

31
index.h
View File

@@ -1,6 +1,7 @@
#pragma once
#include "exception.h"
#include "parser.h"
#include <iostream>
#include <utility>
@@ -10,19 +11,19 @@
namespace usql {
enum class IndexedDataType {
integer,
string
};
using rowid_t = size_t; // int is now enough but size_t is correct
static const int k_default_rowids_size = 16;
template<typename K>
class Index {
public:
Index(std::string index_name, std::string col_name, IndexedDataType type) :
Index(std::string index_name, std::string col_name, ColumnType type) :
m_index_name(std::move(index_name)), m_column_name(std::move(col_name)),
m_data_type(type), m_uniq(false) {}
m_data_type(type), m_uniq(false) {
if (type != ColumnType::integer_type && type != ColumnType::varchar_type)
throw Exception("creating index on unsupported type");
}
void insert(K key, rowid_t rowid) {
auto search = m_index.find(key);
@@ -34,7 +35,7 @@ public:
} else {
std::vector<rowid_t> rowids{rowid};
if (!m_uniq)
rowids.reserve(8);
rowids.reserve(k_default_rowids_size);
m_index[key] = rowids;
}
}
@@ -57,6 +58,16 @@ public:
}
}
std::vector<rowid_t> search(ValueNode * key) {
if (m_data_type == ColumnType::integer_type)
return search(key->getIntegerValue());
else if (m_data_type == ColumnType::varchar_type)
return search(key->getStringValue());
else
throw Exception("using index on unsupported type");
}
void truncate() {
m_index.clear();
}
@@ -78,7 +89,7 @@ public:
return m_index_name;
}
[[nodiscard]] IndexedDataType get_data_type() const {
[[nodiscard]] ColumnType get_data_type() const {
return m_data_type;
}
@@ -86,7 +97,7 @@ private:
bool m_uniq;
std::string m_index_name;
std::string m_column_name;
IndexedDataType m_data_type;
ColumnType m_data_type;
std::map<K, std::vector<rowid_t> > m_index;
};

2
main.cpp
View File

@@ -129,7 +129,7 @@ void repl() {
void debug() {
usql::USql uSql{};
for (const auto &command : c_sql_commands) {
for (const auto &command : k_debug_sql_commands) {
time_point<high_resolution_clock> start_time = high_resolution_clock::now();
auto result = uSql.execute(command);
time_point<high_resolution_clock> end_time = high_resolution_clock::now();

8
table.cpp
View File

@@ -356,9 +356,7 @@ Index<IndexValue> * Table::get_index(const std::string &index_name) {
return idx.get_index_name() == index_name;
});
if (it != m_indexes.end()) return &(*it);
return nullptr;
return (it != m_indexes.end()) ? &(*it) : nullptr;
}
Index<IndexValue> * Table::get_index_for_column(const std::string &col_name) {
@@ -367,9 +365,7 @@ Index<IndexValue> * Table::get_index_for_column(const std::string &col_name) {
return idx.get_column_name() == col_name;
});
if (it != m_indexes.end()) return &(*it);
return nullptr;
return (it != m_indexes.end()) ? &(*it) : nullptr;
}

10
usql.h
View File

@@ -83,11 +83,11 @@ private:
static std::unique_ptr<ValueNode> count_function(ColValue *agg_func_value, const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars);
static void eval_where_on_row(SelectFromTableNode &where_node,
Table *src_table, Row *src_row,
Table *rslt_table, Row *rslt_row,
const std::vector<ColDefNode> &rslt_tbl_col_defs, const std::vector<int> &src_table_col_index,
bool is_aggregated) ;
static void select_row(SelectFromTableNode &where_node,
Table *src_table, Row *src_row,
Table *rslt_table,
const std::vector<ColDefNode> &rslt_tbl_col_defs, const std::vector<int> &src_table_col_index,
bool is_aggregated) ;
std::pair<bool, std::vector<rowid_t>> probe_index_scan(const Node *where, Table *table) const;
std::pair<bool, std::vector<rowid_t>> look_for_usable_index(const Node *where, Table *table) const;

10
usql_ddl.cpp
View File

@@ -27,15 +27,7 @@ std::unique_ptr<Table> USql::execute_create_index(const CreateIndexNode &node) {
if (col_def.null) throw Exception("index on not null supported only");
if (table_def->get_index_for_column(node.column_name) != nullptr) throw Exception("column is already indexed");
IndexedDataType type;
if (col_def.type == ColumnType::integer_type)
type = IndexedDataType::integer;
else if (col_def.type == ColumnType::varchar_type)
type = IndexedDataType::string;
else
throw Exception("creating index on unsupported type");
table_def->create_index({node.index_name, node.column_name, type});
table_def->create_index({node.index_name, node.column_name, col_def.type});
table_def->index_rows(node.index_name);

61
usql_dml.cpp
View File

@@ -22,22 +22,19 @@ std::pair<bool, std::vector<rowid_t>> USql::probe_index_scan(const Node *where,
std::pair<bool, std::vector<rowid_t>> USql::look_for_usable_index(const Node *where, Table *table) const {
if (where->node_type == NodeType::relational_operator) {
auto * ron = (RelationalOperatorNode *)where;
// TODO impllemen > & < https://en.cppreference.com/w/cpp/container/map/upper_bound
if (ron->op == RelationalOperatorType::equal) {
if (ron->left->node_type == NodeType::database_value &&
((ron->right->node_type == NodeType::int_value) || (ron->right->node_type == NodeType::string_value))
) {
) {
auto col_name = ((DatabaseValueNode *)ron->left.get())->col_name;
Index<IndexValue> * used_index = table->get_index_for_column(col_name);
if (used_index != nullptr) {
std::vector<rowid_t> rowids;
if (used_index->get_data_type() == IndexedDataType::integer)
rowids = used_index->search(((ValueNode *) ron->right.get())->getIntegerValue());
else if (used_index->get_data_type() == IndexedDataType::string)
rowids = used_index->search(((ValueNode *) ron->right.get())->getStringValue());
std::vector<rowid_t> rowids = used_index->search((ValueNode *)ron->right.get());
#ifndef NDEBUG
std::cout << "using index " << table->m_name << "(" << used_index->get_column_name() << "), " << rowids.size() << "/" << table->rows_count() << std::endl;
#endif
return std::make_pair(true, rowids);
}
}
@@ -80,19 +77,21 @@ bool USql::normalize_where(const Node *node) const {
return true;
}
void USql::eval_where_on_row(SelectFromTableNode &where_node,
Table *src_table, Row *src_row,
Table *rslt_table, Row *rslt_row,
const std::vector<ColDefNode> &rslt_tbl_col_defs,
const std::vector<int> &src_table_col_index,
bool is_aggregated) {
void USql::select_row(SelectFromTableNode &where_node,
Table *src_table, Row *src_row,
Table *rslt_table,
const std::vector<ColDefNode> &rslt_tbl_col_defs,
const std::vector<int> &src_table_col_index,
bool is_aggregated) {
if (eval_where(where_node.where.get(), src_table, *src_row)) {
// prepare empty src_row and copy column values
// when aggregate functions in rslt_table only one src_row for src_table
if (!is_aggregated || rslt_table->rows_count() == 0) {
Row *rslt_row = nullptr;
// when aggregate functions in rslt_table only one row exists
// TODO add function to get rows count
if (is_aggregated && !rslt_table->m_rows.empty())
rslt_row = &rslt_table->m_rows[0];
else
rslt_row = &rslt_table->create_empty_row();
}
for (auto idx = 0; idx < rslt_table->columns_count(); idx++) {
auto src_table_col_idx = src_table_col_index[idx];
@@ -109,11 +108,8 @@ void USql::eval_where_on_row(SelectFromTableNode &where_node,
}
}
// add src_row to rslt_table
if (!is_aggregated) {
rslt_table->commit_row(*rslt_row);
}
}
// for aggregate is validated more than needed
rslt_table->commit_row(*rslt_row);
}
bool USql::check_for_aggregate_only_functions(SelectFromTableNode &node, size_t result_cols_cnt) {
@@ -155,12 +151,12 @@ void USql::setup_order_columns(std::vector<ColOrderNode> &node, Table *table) {
if (order_node.col_index < 0 || order_node.col_index >= table->columns_count())
throw Exception("unknown column in order by clause (" + order_node.col_name + ")");
}
}
}
void USql::execute_distinct(SelectFromTableNode &node, Table *result) {
if (!node.distinct) return;
auto compare_rows = [](const Row &a, const Row &b) { return a.compare(b) >= 0; };
std::sort(result->m_rows.begin(), result->m_rows.end(), compare_rows);
@@ -334,8 +330,7 @@ std::unique_ptr<Table> USql::execute_delete(const DeleteFromTableNode &node) {
// execute access plan
Table::rows_scanner i = get_iterator(table, node.where.get());
while(Row *row = i.next()) {
bool to_delete = eval_where(node.where.get(), table, *row);
if (to_delete) {
if (eval_where(node.where.get(), table, *row)) {
row->set_deleted();
table->unindex_row(*row);
@@ -356,8 +351,7 @@ std::unique_ptr<Table> USql::execute_update(const UpdateTableNode &node) {
// execute access plan
Table::rows_scanner i = get_iterator(table, node.where.get());
while(Row *row = i.next()) {
bool to_update = eval_where(node.where.get(), table, *row);
if (to_update) {
if (eval_where(node.where.get(), table, *row)) {
Row old_row = * row;
int col_idx = 0;
@@ -410,12 +404,11 @@ std::unique_ptr<Table> USql::execute_select(SelectFromTableNode &node) const {
setup_order_columns(node.order_by, result.get());
// execute access plan
Row *new_row = nullptr;
Table::rows_scanner i = get_iterator(table, node.where.get());
while(Row *row = i.next()) {
eval_where_on_row(node, table, row, result.get(), new_row, result_tbl_col_defs, source_table_col_index, is_aggregated);
if (eval_where(node.where.get(), table, *row)) { // put it into row_scanner.next
select_row(node, table, row, result.get(), result_tbl_col_defs, source_table_col_index, is_aggregated);
}
}
execute_distinct(node, result.get());