faster double and date parsing

This commit is contained in:
2021-08-31 18:56:57 +02:00
parent 4e54c6d134
commit 412e2fd455
4 changed files with 1332 additions and 9 deletions

1270
clib/fast_double_parser.h Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -23,19 +23,59 @@ std::string date_to_string(const long datetime, const std::string format) {
return "invalid argument";
}
time_t time_to_epoch ( const struct tm *ltm, int utcdiff ) {
// const int mon_days [] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31};
const int mon_days [] = {31, 31+28, 31+28+31,
31+28+31+30, 31+28+31+30+31, 31+28+31+30+31+30,
31+28+31+30+31+30+31, 31+28+31+30+31+30+31+31, 31+28+31+30+31+30+31+31+30,
31+28+31+30+31+30+31+31+30+31, 31+28+31+30+31+30+31+31+30+31+30, 31+28+31+30+31+30+31+31+30+31+30+31};
long tyears, tdays, leaps, utc_hrs;
int i;
tyears = ltm->tm_year - 70 ; // tm->tm_year is from 1900.
leaps = (tyears + 2) / 4; // no of next two lines until year 2100.
//i = (ltm->tm_year 100) / 100;
//leaps -= ( (i/4)*3 + i%4 );
// tdays = 0;
// for (i=0; i < ltm->tm_mon; i++) tdays += mon_days[i];
tdays = mon_days[ltm->tm_mon-1];
tdays += ltm->tm_mday-1; // days of month passed.
tdays = tdays + (tyears * 365) + leaps;
utc_hrs = ltm->tm_hour + utcdiff/3600; // for your time zone.
return (tdays * 86400) + (utc_hrs * 3600) + (ltm->tm_min * 60) + ltm->tm_sec;
}
std::istringstream in_ss;
long string_to_date(const std::string &datestr, const std::string &format) {
// format for example "%d.%m.%Y";
in_ss.clear();
in_ss.str(datestr);
struct tm tm;
memset(&tm, 0, sizeof(tm));
if (strptime(datestr.c_str(), format.c_str(), &tm)) {
time_t curTime;
struct tm * timeinfo;
date::sys_seconds tp;
date::from_stream(in_ss, format.c_str(), tp);
return tp.time_since_epoch().count();
time(&curTime );
timeinfo = localtime(&curTime);
timeinfo->tm_year = tm.tm_year;
timeinfo->tm_mon = tm.tm_mon;
timeinfo->tm_mday = tm.tm_mday;
timeinfo->tm_hour = tm.tm_hour;
timeinfo->tm_min = tm.tm_min;
timeinfo->tm_sec = tm.tm_sec;
return time_to_epoch(timeinfo, -1 * timeinfo->tm_gmtoff);
}
// throw Exception("invalid date string or format");
return -1;
}
long add_to_date(const long datetime, const long quantity, const std::string &part) {
// part is one of 'year', 'month', 'day', 'hour', 'minute' or 'second'

10
row.h
View File

@@ -35,6 +35,8 @@ namespace usql {
bool getBoolValue() override { throw Exception("getDateValue not supported on ColNullValue"); };
int compare(ColValue &other) override;
virtual ~ColNullValue() = default;
};
@@ -52,6 +54,8 @@ namespace usql {
int compare(ColValue &other) override;
long m_integer;
virtual ~ColIntegerValue() = default;
};
@@ -68,6 +72,8 @@ namespace usql {
int compare(ColValue &other) override;
virtual ~ColDoubleValue() = default;
double m_double;
};
@@ -103,6 +109,8 @@ namespace usql {
int compare(ColValue &other) override;
virtual ~ColDateValue() = default;
long m_date; // seconds since epoch for now
};
@@ -119,6 +127,8 @@ namespace usql {
int compare(ColValue &other) override;
virtual ~ColBooleanValue() = default;
bool m_bool;
};

View File

@@ -1,7 +1,9 @@
#include "table.h"
#include "csvreader.h"
#include "ml_string.h"
#include "fast_double_parser.h"
#include <charconv>
#include <fstream>
#include <algorithm>
@@ -138,11 +140,12 @@ void Table::create_row_from_vector(const std::vector<ColDefNode> &colDefs, const
}
double Table::string_to_double(const std::string &s) {
try {
return std::stod(s);
} catch (std::invalid_argument &e) {
double result;
const char * endptr = fast_double_parser::parse_number(s.c_str(), &result);
if (endptr == nullptr) {
throw Exception("error parsing as double: " + s);
}
return result;
}
long Table::string_to_long(const std::string &s) {