vaclavt
/
usql
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

insert rows as you go through csv

This commit is contained in:
VaclavT 2021-08-11 23:40:18 +02:00
parent 710531c455
commit cd92e27270
2 changed files with 63 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
csvreader.cpp
View File

@ -1,5 +1,6 @@
#include "csvreader.h"
#include "parser.h"
namespace usql {
@ -10,78 +11,65 @@ namespace usql {
line_separator = line_sep;
line_separator2 = line_sep2;
header_skiped = false;
header_skiped = !skip_hdr;
}
std::vector<std::vector<std::string>> CsvReader::parseCSV(const std::string &csvSource) {
int linesRead = 0;
bool inQuote(false);
bool newLine(false);
std::string field;
int CsvReader::parseCSV2(const std::string &csvSource, std::vector<ColDefNode> &cols_def,
void (Table::*function)(const std::vector<ColDefNode>&, const std::vector<std::string>&), Table& a) {
int row_cnt = 0;
bool inQuote(false);
bool newLine(false);
std::string field;
std::vector<std::vector<std::string>> parsed_data;
parsed_data.reserve(256);
std::vector<std::string> line;
line.reserve(32);
std::vector<std::string> line;
line.reserve(32);
std::string::const_iterator aChar = csvSource.begin();
while (aChar != csvSource.end()) {
if (*aChar == quote_character) {
newLine = false;
inQuote = !inQuote;
} else if (*aChar == field_separator) {
newLine = false;
if (inQuote == true) {
field += *aChar;
} else {
line.push_back(field);
field.clear();
}
} else if (*aChar == line_separator || *aChar == line_separator2) {
if (inQuote == true) {
field += *aChar;
} else {
if (newLine == false) {
line.push_back(field);
if (header_skiped) {
(a.*function)(cols_def, line);
row_cnt++;
}
header_skiped = true;
field.clear();
line.clear();
newLine = true;
}
}
} else {
newLine = false;
field.push_back(*aChar);
}
std::string::const_iterator aChar = csvSource.begin();
while (aChar != csvSource.end()) {
if (*aChar == quote_character) {
newLine = false;
inQuote = !inQuote;
} else if (*aChar == field_separator) {
newLine = false;
if (inQuote == true) {
field += *aChar;
} else {
line.push_back(field);
field.clear();
}
} else if (*aChar == line_separator || *aChar == line_separator2) {
if (inQuote == true) {
field += *aChar;
} else {
if (newLine == false) {
line.push_back(field);
add_line(line, parsed_data);
field.clear();
line.clear();
linesRead++;
if (linesRead == 16) {
int linesEstimation =
csvSource.size() /
(std::distance(csvSource.begin(), aChar) / linesRead);
if (linesEstimation > parsed_data.capacity())
parsed_data.reserve(linesEstimation);
}
newLine = true;
}
}
} else {
newLine = false;
field.push_back(*aChar);
}
aChar++;
}
aChar++;
}
if (!field.empty()) line.push_back(field);
if (field.size())
line.push_back(field);
add_line(line, parsed_data);
return parsed_data;
}
void CsvReader::add_line(const std::vector<std::string> &line, std::vector<std::vector<std::string>> &lines) {
if (skip_header && !header_skiped) {
header_skiped = true;
} else {
if (line.size())
lines.push_back(line);
}
}
if (header_skiped) {
(a.*function)(cols_def, line);
row_cnt++;
header_skiped = true;
}
return row_cnt;
}
} // namespace

15
csvreader.h
View File

@ -5,6 +5,10 @@
#include <string>
#include <vector>
#include <regex>
#include <functional>
#include "parser.h"
#include "table.h"
namespace usql {
@ -20,12 +24,11 @@ namespace usql {
bool header_skiped;
public:
CsvReader(bool skip_hdr = false, char field_sep = ',', char quote_ch = '"', char line_sep = '\r',
char line_sep2 = '\n');
CsvReader(bool skip_hdr = true, char field_sep = ',', char quote_ch = '"', char line_sep = '\r', char line_sep2 = '\n');
std::vector<std::vector<std::string>> parseCSV(const std::string &csvSource);
int parseCSV2(const std::string &csvSource, std::vector<ColDefNode> &cols_def,
void (Table::*function)(const std::vector<ColDefNode>&, const std::vector<std::string>&), Table& a);
private:
void add_line(const std::vector<std::string> &line, std::vector<std::vector<std::string>> &lines);
};
}
} // namespace