small refactoring

This commit is contained in:
2021-12-19 12:58:17 +01:00
parent 04c0ed3f03
commit 35cba3b0c4
8 changed files with 753 additions and 746 deletions

View File

@@ -1,4 +1,4 @@
#include <errno.h> #include <cerrno>
#include "exception.h" #include "exception.h"
#include "csvreader.h" #include "csvreader.h"
@@ -41,7 +41,7 @@ int CsvReader::parseCSV(const std::string &filename, std::vector<ColDefNode> &co
size_t len = 0; size_t len = 0;
int read_chars; long read_chars;
while ((read_chars = getline(&line_str, &len, fp)) != -1) { while ((read_chars = getline(&line_str, &len, fp)) != -1) {
if (skip_header && !header_skiped) { if (skip_header && !header_skiped) {
header_skiped = true; header_skiped = true;
@@ -59,7 +59,7 @@ int CsvReader::parseCSV(const std::string &filename, std::vector<ColDefNode> &co
if (*aChar == quote_character) { if (*aChar == quote_character) {
inQuote = !inQuote; inQuote = !inQuote;
} else if (*aChar == field_separator) { } else if (*aChar == field_separator) {
if (inQuote == true) { if (inQuote) {
field += *aChar; field += *aChar;
} else { } else {
line.push_back(field); line.push_back(field);
@@ -80,9 +80,6 @@ int CsvReader::parseCSV(const std::string &filename, std::vector<ColDefNode> &co
field.clear(); field.clear();
line.clear(); line.clear();
// DEBUG
// if (row_cnt > 50000) break;
//
} }
fclose(fp); fclose(fp);
@@ -109,17 +106,17 @@ int CsvReader::parseCSV(const std::string &filename, std::vector<ColDefNode> &co
inQuote = !inQuote; inQuote = !inQuote;
} else if (*aChar == field_separator) { } else if (*aChar == field_separator) {
newLine = false; newLine = false;
if (inQuote == true) { if (inQuote) {
field += *aChar; field += *aChar;
} else { } else {
line.push_back(field); line.push_back(field);
field.clear(); field.clear();
} }
} else if (*aChar == line_separator || *aChar == line_separator2) { } else if (*aChar == line_separator || *aChar == line_separator2) {
if (inQuote == true) { if (inQuote) {
field += *aChar; field += *aChar;
} else { } else {
if (newLine == false) { if (!newLine) {
line.push_back(field); line.push_back(field);
if (header_skiped) { if (header_skiped) {
table.create_row_from_vector(cols_def, line); table.create_row_from_vector(cols_def, line);

View File

@@ -24,7 +24,7 @@ namespace usql {
bool header_skiped; bool header_skiped;
public: public:
CsvReader(bool skip_hdr = true, char field_sep = ',', char quote_ch = '"', char line_sep = '\r', char line_sep2 = '\n'); explicit CsvReader(bool skip_hdr = true, char field_sep = ',', char quote_ch = '"', char line_sep = '\r', char line_sep2 = '\n');
int parseCSV2(const std::string &csvSource, std::vector<ColDefNode> &cols_def, Table& table); int parseCSV2(const std::string &csvSource, std::vector<ColDefNode> &cols_def, Table& table);

View File

@@ -25,6 +25,8 @@ std::vector<std::string> k_debug_sql_commands {
"insert into a (i, s) values(2, 'two')", "insert into a (i, s) values(2, 'two')",
"insert into a (i, s) values(2, 'second two')", "insert into a (i, s) values(2, 'second two')",
"insert into a (i, s) values(3, 'three')", "insert into a (i, s) values(3, 'three')",
"insert into a (i, s) values(4, 'four')",
"save a into '/tmp/a.csv'",
"set 'USE_INDEXSCAN' = 'true'", "set 'USE_INDEXSCAN' = 'true'",
// "select * from a where 1 = i", // "select * from a where 1 = i",
// "delete from a where i = 2 and s ='two'", // "delete from a where i = 2 and s ='two'",

View File

@@ -293,4 +293,4 @@ namespace usql {
} }
} }
} } // namespace usql

12
row.cpp
View File

@@ -29,6 +29,18 @@ int ColStringValue::compare(ColValue &other) const {
return other.isNull() ? 1 : m_string->compare(other.getStringValue()); // null goes to end return other.isNull() ? 1 : m_string->compare(other.getStringValue()); // null goes to end
} }
std::string ColStringValue::getCsvStringValue() const {
auto src_str = getStringValue();
std::string toSearch{"\""}, replaceStr{"\\\""};
size_t pos = src_str.find(toSearch);
while(pos != std::string::npos) {
src_str.replace(pos, toSearch.size(), replaceStr);
pos =src_str.find(toSearch, pos + replaceStr.size());
}
return src_str;
}
int ColDateValue::compare(ColValue &other) const { int ColDateValue::compare(ColValue &other) const {
long r = m_date - other.getIntegerValue(); long r = m_date - other.getIntegerValue();
return other.isNull() ? 1 : r > 0 ? 1 : r == 0 ? 0 : -1; return other.isNull() ? 1 : r > 0 ? 1 : r == 0 ? 0 : -1;

6
row.h
View File

@@ -88,11 +88,7 @@ struct ColStringValue : ColValue {
long getIntegerValue() const override { return std::stoi(*m_string); }; long getIntegerValue() const override { return std::stoi(*m_string); };
double getDoubleValue() const override { return std::stod(*m_string); }; double getDoubleValue() const override { return std::stod(*m_string); };
std::string getStringValue() const override { return *m_string; }; std::string getStringValue() const override { return *m_string; };
std::string getCsvStringValue() const override { std::string getCsvStringValue() const override;;
// TODO handle correctly CSV string
// ?? return std::regex_replace(getStringValue(), std::regex( "\"" ), "\\\"" );
return getStringValue();
};
long getDateValue() const override { return std::stoi(*m_string); }; long getDateValue() const override { return std::stoi(*m_string); };
bool getBoolValue() const override { throw Exception("Not supported on ColStringValue"); }; bool getBoolValue() const override { throw Exception("Not supported on ColStringValue"); };

2
utils/cp_to_mlisp.sh Normal file → Executable file
View File

@@ -1,4 +1,4 @@
#!/usr/bin/env bash
cp \ cp \
csvreader.h \ csvreader.h \