code changes in load

This commit is contained in:
2021-08-13 10:24:04 +02:00
parent cd92e27270
commit e53d062ff5
10 changed files with 287 additions and 141 deletions

View File

@@ -2,6 +2,9 @@
#include "csvreader.h"
#include "parser.h"
#include <fstream>
namespace usql {
CsvReader::CsvReader(bool skip_hdr, char field_sep, char quote_ch, char line_sep, char line_sep2) {
@@ -14,8 +17,67 @@ namespace usql {
header_skiped = !skip_hdr;
}
int CsvReader::parseCSV2(const std::string &csvSource, std::vector<ColDefNode> &cols_def,
void (Table::*function)(const std::vector<ColDefNode>&, const std::vector<std::string>&), Table& a) {
int CsvReader::parseCSV(const std::string &filename, std::vector<ColDefNode> &cols_def, Table& table) {
std::vector<ColDefNode> cdefs;
cdefs.reserve(cols_def.size());
for(auto & cd : cols_def) {
cdefs.emplace_back(table.get_column_def(cd.name));
}
int row_cnt = 0;
bool inQuote(false);
std::string field;
std::string csvSource;
std::vector<std::string> line;
line.reserve(32);
std::fstream newfile;
newfile.open(filename, std::ios::in);
/// if (newfile.is_open()){ //checking whether the file is open
while(getline(newfile, csvSource)){
if (skip_header && !header_skiped) {
header_skiped = true;
continue;
}
std::string::const_iterator aChar = csvSource.begin();
while (aChar != csvSource.end()) {
if (*aChar == quote_character) {
inQuote = !inQuote;
} else if (*aChar == field_separator) {
if (inQuote == true) {
field += *aChar;
} else {
line.push_back(field);
field.clear();
}
} else {
field.push_back(*aChar);
}
aChar++;
}
if (!field.empty())
line.push_back(field);
table.create_row_from_vector(cols_def, line);
row_cnt++;
field.clear();
line.clear();
}
newfile.close();
return row_cnt;
}
int CsvReader::parseCSV2(const std::string &csvSource, std::vector<ColDefNode> &cols_def, Table& table) {
int row_cnt = 0;
bool inQuote(false);
bool newLine(false);
@@ -44,7 +106,7 @@ namespace usql {
if (newLine == false) {
line.push_back(field);
if (header_skiped) {
(a.*function)(cols_def, line);
table.create_row_from_vector(cols_def, line);
row_cnt++;
}
header_skiped = true;
@@ -64,7 +126,7 @@ namespace usql {
if (!field.empty()) line.push_back(field);
if (header_skiped) {
(a.*function)(cols_def, line);
table.create_row_from_vector(cols_def, line);
row_cnt++;
header_skiped = true;
}