better support for booleans

lexer.cpp

@@ -203,6 +203,10 @@ namespace usql {
 		    return TokenType::keyword_date;
 	    if (token == "boolean")
 		    return TokenType::keyword_bool;
+	    if (token == "true")
+		    return TokenType::keyword_true;
+	    if (token == "false")
+		    return TokenType::keyword_false;
 	    if (token == "distinct")
 		    return TokenType::keyword_distinct;
 	    if (token == "show")
@@ -395,6 +399,12 @@ namespace usql {
 		    case TokenType::keyword_bool:
 			    txt = "boolean";
 			    break;
+		    case TokenType::keyword_true:
+			    txt = "true";
+			    break;
+		    case TokenType::keyword_false:
+			    txt = "false";
+			    break;
 		    case TokenType::keyword_distinct:
 			    txt = "distinct";
 			    break;

lexer.h

@@ -49,6 +49,8 @@ namespace usql {
 	keyword_varchar,
 	keyword_date,
 	keyword_bool,
+	keyword_true,
+	keyword_false,
 	keyword_distinct,
 	keyword_show,
 	int_number,

main.cpp

@@ -127,19 +127,28 @@ void repl() {
 
 void debug() {
 	std::vector<std::string> sql_commands{
+//"create table history_dividends (symbol varchar(8), ex_date date, pay_date date, div_rate float)",
+//"set 'DATE_FORMAT' = '%m/%d/%Y' ",
+//"insert into history_dividends (symbol,ex_date,pay_date,div_rate) values ('symbol', 'ex-date', 'pay-date', 0.1)",
+//zpusobi crash "insert into history_dividends (symbol,ex_date,pay_date,div_rate) values ('symbol', ex-date, pay-date)"
 //		    "create table ticker ( tablee varchar(5) not null, permaticker integer, ticker varchar(10) not null, name varchar(256) not null, exchange varchar(32), isdelisted boolean, category varchar(32), cusips varchar(256), siccode integer, sicsector varchar(256), sicindustry varchar(256), famasector varchar(256), famaindustry varchar(256), sector varchar(128), industry varchar(128), scalemarketcap varchar(64), scalerevenue varchar(64), relatedtickers varchar(128), currency varchar(3), location varchar(64), lastupdated date, firstadded date, firstpricedate date, lastpricedate date, firstquarter date, lastquarter date, secfilings varchar(256), companysite varchar(256))",
 //		    "load ticker from '/Users/vaclavt/Library/Mobile Documents/com~apple~CloudDocs/Development/usql/tickers.csv')",
 //		    "select * from ticker where ticker = 'WFC' and tablee = 'SF1'",
 //		    "set 'DATE_FORMAT' = '%Y-%m-%d'",
 //		    "show 'DATE_FORMAT'",
-		    "create table sf1 ( ticker varchar(8), dimension varchar(3), calendar_date date, date_key date, report_period date, last_updated date, accoci float, assets float, assetsavg float, assetsc float, assetsnc float, assetturnover float, bvps float, capex float, cashneq float, cashnequsd float, cor float, consolinc float, currentratio float, de float, debt float, debtc float, debtnc float, debtusd float, deferredrev float, depamor float, deposits float, divyield float, dps float, ebit float, ebitda float, ebitdamargin float, ebitdausd float, ebitusd float, ebt float, eps float, epsdil float, epsusd float, equity float, equityavg float, equityusd float, ev float, evebit float, evebitda float, fcf float, fcfps float, fxusd float, gp float, grossmargin float, intangibles float, intexp float, invcap float, invcapavg float, inventory float, investments float, investmentsc float, investmentsnc float, liabilities float, liabilitiesc float, liabilitiesnc float, marketcap float, ncf float, ncfbus float, ncfcommon float, ncfdebt float, ncfdiv float, ncff float, ncfi float, ncfinv float, ncfo float, ncfx float, netinc float, netinccmn float, netinccmnusd float, netincdis float, netincnci float, netmargin float, opex float, opinc float, payables float, payoutratio float, pb float, pe float, pe1 float, ppnenet float, prefdivis float, price float, ps float, ps1 float, receivables float, retearn float, revenue float, revenueusd float, rnd float, roa float, roe float, roic float, ros float, sbcomp float, sgna float, sharefactor float, sharesbas float, shareswa float, shareswadil float, sps float, tangibles float, taxassets float, taxexp float, taxliabilities float, tbvps float, workingcapital float)",
-		    "load sf1 from '/tmp/sf1.csv'",
+//		    "create table prices (datetime date, symbol varchar(8), prev_close float, open float, price float, change float, change_prct varchar(16))",
+//		    "load into prices from \"/Users/vaclavt/Development/mlisp_fin/tmp/prices.csv\"",
+//		    "select * from prices where symbol = 'AIG' limit 5",
+//		    "create table sf1 ( ticker varchar(8), dimension varchar(3), calendar_date date, date_key date, report_period date, last_updated date, accoci float, assets float, assetsavg float, assetsc float, assetsnc float, assetturnover float, bvps float, capex float, cashneq float, cashnequsd float, cor float, consolinc float, currentratio float, de float, debt float, debtc float, debtnc float, debtusd float, deferredrev float, depamor float, deposits float, divyield float, dps float, ebit float, ebitda float, ebitdamargin float, ebitdausd float, ebitusd float, ebt float, eps float, epsdil float, epsusd float, equity float, equityavg float, equityusd float, ev float, evebit float, evebitda float, fcf float, fcfps float, fxusd float, gp float, grossmargin float, intangibles float, intexp float, invcap float, invcapavg float, inventory float, investments float, investmentsc float, investmentsnc float, liabilities float, liabilitiesc float, liabilitiesnc float, marketcap float, ncf float, ncfbus float, ncfcommon float, ncfdebt float, ncfdiv float, ncff float, ncfi float, ncfinv float, ncfo float, ncfx float, netinc float, netinccmn float, netinccmnusd float, netincdis float, netincnci float, netmargin float, opex float, opinc float, payables float, payoutratio float, pb float, pe float, pe1 float, ppnenet float, prefdivis float, price float, ps float, ps1 float, receivables float, retearn float, revenue float, revenueusd float, rnd float, roa float, roe float, roic float, ros float, sbcomp float, sgna float, sharefactor float, sharesbas float, shareswa float, shareswadil float, sps float, tangibles float, taxassets float, taxexp float, taxliabilities float, tbvps float, workingcapital float)",
+//		    "load sf1 from '/tmp/sf1.csv'",
 //		    "select ticker, calendar_date from sf1 where calendar_date > to_date('2019-01-01', '%Y-%m-%d') limit 1",
 //		    "select ticker, dimension, calendar_date, eps, dps, roa*100 as roa, roe*100 as roe, pp(revenue), netinc from sf1 where (ticker = 'AIG' or ticker = 'WFC') and dimension = 'MRY' and calendar_date > to_date('2019-01-01', '%Y-%m-%d') order by 3 desc
-		    "select ticker, dimension, calendar_date, eps, dps, roa*100 as roa, roe*100 as roe, pp(revenue) as revenue, pp(netinc) as netinc from sf1 where (ticker = 'AIG' or ticker = 'WFC') and dimension = 'MRY' and calendar_date > to_date('2019-01-01', '%Y-%m-%d') order by 3 desc",
-//		    "create table a (i integer not null, s varchar(64), f float null, d date null,  b boolean)",
-//		    "insert into a (i, s, b) values(1, upper('one'), 'Y')",
-//		    "insert into a (i, s, b, f) values(1 + 10000, upper('one'), 'Y', 3.1415)",
+//		    "select ticker, dimension, calendar_date, eps, dps, roa*100 as roa, roe*100 as roe, pp(revenue) as revenue, pp(netinc) as netinc from sf1 where (ticker = 'AIG' or ticker = 'WFC') and dimension = 'MRY' and calendar_date > to_date('2019-01-01', '%Y-%m-%d') order by 3 desc",
+		    "create table a (i integer not null, s varchar(64), f float null, d date null,  b boolean)",
+		    "insert into a (i, s, b) values(1, upper('one'), 'Y')",
+		    "insert into a (i, s, b, f) values(1 + 10000, upper('one'), 'N', 3.1415)",
+		    "select * from a where d is null",
+//		    "select * from a where b='N'",
 //		    "update a set i = i * 100, f = f + 0.01 where i > 1",
 //		    "select to_string(i, '%d.%m.%Y %H:%M:%S'), i, s from a where i < to_date('20.12.2019', '%d.%m.%Y')",
 //		    "select i + 2 as first, i, s, b, f from a where i >=1 order by 1 desc offset 0 limit 1",
@@ -199,8 +208,8 @@ void debug() {
 
 int main(int argc, char *argv[]) {
 
-	// debug();
-	repl();
+	debug();
+	// repl();
 
 	return 0;
 }

parser.cpp

@@ -450,6 +450,10 @@ namespace usql {
 	    if (token_type == TokenType::keyword_null)
 		return std::make_unique<NullValueNode>();
 
+	    // true / false
+	    if (token_type == TokenType::keyword_true || token_type == TokenType::keyword_false)
+		return std::make_unique<BooleanValueNode>(token_type == TokenType::keyword_true);
+
 	    throw Exception("Unknown operand node " + tokenString);
     }
 

parser.h

@@ -141,7 +141,7 @@ namespace usql {
 
 	long getIntegerValue() override { return value; };
 	double getDoubleValue() override { return (double) value; };
-	std::string getStringValue() override { return std::to_string(value); }
+	std::string getStringValue() override { return Settings::int_to_string(value); }
 	long getDateValue() override { return value; };
 	bool getBooleanValue() override { return value != 0; };
     };
@@ -153,7 +153,7 @@ namespace usql {
 
 	long getIntegerValue() override { return (long) value; };
 	double getDoubleValue() override { return value; };
-	std::string getStringValue() override { return std::to_string(value); }
+	std::string getStringValue() override { return Settings::double_to_string(value); }
 	long getDateValue() override { return (long) value; };
 	bool getBooleanValue() override { return value != 0.0; };
     };
@@ -163,11 +163,11 @@ namespace usql {
 
 	explicit StringValueNode(const std::string &value) : ValueNode(NodeType::string_value), value(value) {}
 
-	long getIntegerValue() override { return std::stoi(value); };
-	double getDoubleValue() override { return std::stod(value); };
+	long getIntegerValue() override { return Settings::string_to_int(value); };
+	double getDoubleValue() override { return Settings::string_to_double(value); };
 	std::string getStringValue() override { return value; };
 	long getDateValue() override { return Settings::string_to_date(value); };
-	bool getBooleanValue() override { return value == "true"; };
+	bool getBooleanValue() override { return Settings::string_to_bool(value); };
     };
 
     struct BooleanValueNode : ValueNode {
@@ -177,7 +177,7 @@ namespace usql {
 
         long getIntegerValue() override { return (long) value; };
         double getDoubleValue() override { return (double) value; };
-        std::string getStringValue() override { return value ? "true" : "false"; }
+        std::string getStringValue() override { return Settings::bool_to_string(value); }
         long getDateValue() override { return (long) value; };
         bool getBooleanValue() override { return value; };
     };

settings.cpp

@@ -12,16 +12,20 @@ std::vector<std::pair<std::string, std::string>> Settings::m_settings =
 	      std::make_pair("DOUBLE_FORMAT", "%.2f") };
 
 
-long Settings::string_to_date(const std::string &datestr) {
-	return ::string_to_date(datestr, get_setting("DATE_FORMAT"));
+
+long Settings::string_to_int(const std::string &intstr) {
+	return std::stoi(intstr); 
+}
+
+std::string Settings::int_to_string(long intval) {
+	return std::to_string(intval);
 }
 
 
-std::string Settings::date_to_string(long date) {
-	return ::date_to_string(date, get_setting("DATE_FORMAT"));
+double Settings::string_to_double(const std::string &doublestr) {
+	return std::stod(doublestr);	// TODO use fast parsing 
 }
-
-
+ 
 std::string Settings::double_to_string(double d) {
 	char buffer[32];
 	int r, buf_size = 32;
@@ -29,9 +33,37 @@ std::string Settings::double_to_string(double d) {
 	r = snprintf(buffer, buf_size, get_setting("DOUBLE_FORMAT").c_str(), d);
 	if (r > 0 && r < buf_size) return std::string(buffer);
 
+	// ?? return std::to_string(d);
 	return "ERROR";
 }
 
+
+long Settings::string_to_date(const std::string &datestr) {
+	return ::string_to_date(datestr, get_setting("DATE_FORMAT"));
+}
+
+std::string Settings::date_to_string(long date) {
+	return ::date_to_string(date, get_setting("DATE_FORMAT"));
+}
+
+
+bool Settings::string_to_bool(const std::string &boolstr) {
+	if (boolstr=="true" || boolstr == get_setting("BOOL_TRUE_LITERAL"))
+		return true;
+
+	if (boolstr=="false" || boolstr == get_setting("BOOL_FALSE_LITERAL"))
+		return false;
+
+	throw Exception("string_to_bool, unrecognized value: " + boolstr);
+}
+
+std::string Settings::bool_to_string(bool boolval) {
+	return boolval ? "true" : "false"; 
+}
+
+
+
+
 std::string Settings::get_setting(const std::string &name) {
 	for(const auto& pair : m_settings) {
 		if (pair.first == name) return pair.second;

settings.h

@@ -11,10 +11,17 @@ public:
     static void set_setting(const std::string &name, const std::string &value);
     static std::string get_setting(const std::string &name);
 
+    static long string_to_int(const std::string &intstr);
+    static std::string int_to_string(long intval);
+
+    static double string_to_double(const std::string &doublestr);
+    static std::string double_to_string(double doubleval);
+
     static long string_to_date(const std::string &datestr);
-    static std::string date_to_string(long date);
-    static std::string double_to_string(double d);
-    // TODO add bool_to_string and use it in ColBooleanValue
+    static std::string date_to_string(long dateval);
+
+    static bool string_to_bool(const std::string &boolstr);
+    static std::string bool_to_string(bool boolval);
 
 private:
     static std::vector<std::pair<std::string, std::string>> m_settings;

usql.cpp

@@ -404,12 +404,12 @@ bool USql::eval_relational_operator(const RelationalOperatorNode &filter, Table
 		(left_value->node_type == NodeType::float_value && right_value->node_type == NodeType::int_value) ||
 		(left_value->node_type == NodeType::float_value && right_value->node_type == NodeType::float_value)) {
 		comparator = left_value->getDoubleValue() - right_value->getDoubleValue();
-	} else if (left_value->node_type == NodeType::string_value || right_value->node_type == NodeType::string_value) {
-		comparator = left_value->getStringValue().compare(right_value->getStringValue());
-	} else if (left_value->node_type == NodeType::bool_value && right_value->node_type == NodeType::bool_value) {
+	} else if (left_value->node_type == NodeType::bool_value || right_value->node_type == NodeType::bool_value) {
 		bool bl = left_value->getBooleanValue();
 		bool br = right_value->getBooleanValue();
 		comparator = bl == br ? 0 : 1;
+	} else if (left_value->node_type == NodeType::string_value || right_value->node_type == NodeType::string_value) {
+		comparator = left_value->getStringValue().compare(right_value->getStringValue());
 	// date values are essentially int values so handled above
 	} else {
 		throw Exception("Undefined combination of types");