thread pool changed
This commit is contained in:
parent
43d45e5bca
commit
471f218bdf
|
|
@ -5,10 +5,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
|||
set(CMAKE_CXX_EXTENSIONS OFF)
|
||||
|
||||
# set(CMAKE_CXX_FLAGS "-O3 -Wall -Wextra")
|
||||
|
||||
set(CMAKE_OSX_DEPLOYMENT_TARGET "10.14")
|
||||
|
||||
set(CMAKE_CXX_FLAGS_DEBUG "-Wall")
|
||||
# set(CMAKE_CXX_FLAGS_DEBUG "-Wall -d")
|
||||
|
||||
project(usql)
|
||||
|
||||
|
|
|
|||
206
csvreader.cpp
206
csvreader.cpp
|
|
@ -8,138 +8,136 @@
|
|||
|
||||
namespace usql {
|
||||
|
||||
CsvReader::CsvReader(bool skip_hdr, char field_sep, char quote_ch, char line_sep, char line_sep2) {
|
||||
skip_header = skip_hdr;
|
||||
field_separator = field_sep;
|
||||
quote_character = quote_ch;
|
||||
line_separator = line_sep;
|
||||
line_separator2 = line_sep2;
|
||||
CsvReader::CsvReader(bool skip_hdr, char field_sep, char quote_ch, char line_sep, char line_sep2) {
|
||||
skip_header = skip_hdr;
|
||||
field_separator = field_sep;
|
||||
quote_character = quote_ch;
|
||||
line_separator = line_sep;
|
||||
line_separator2 = line_sep2;
|
||||
|
||||
header_skiped = !skip_hdr;
|
||||
}
|
||||
header_skiped = !skip_hdr;
|
||||
}
|
||||
|
||||
|
||||
size_t CsvReader::parseCSVFile(const std::string &filename, std::vector<ColDefNode> &cols_def, Table &table) {
|
||||
size_t row_cnt = 0;
|
||||
size_t CsvReader::parseCSVFile(const std::string &filename, std::vector<ColDefNode> &cols_def, Table &table) {
|
||||
size_t row_cnt = 0;
|
||||
|
||||
errno = 0;
|
||||
FILE* fp = fopen(filename.c_str(), "r");
|
||||
if (fp == NULL)
|
||||
throw Exception("load from csv, cannot read from file(" + std::to_string(errno) + ")");
|
||||
errno = 0;
|
||||
FILE *fp = fopen(filename.c_str(), "r");
|
||||
if (fp == NULL)
|
||||
throw Exception("load from csv, cannot read from file(" + std::to_string(errno) + ")");
|
||||
|
||||
char* line_str = NULL;
|
||||
size_t len = 0;
|
||||
char *line_str = NULL;
|
||||
size_t len = 0;
|
||||
|
||||
try {
|
||||
// TODO handle it by settings
|
||||
const std::size_t hw_concurrency = std::max(0, (int)(std::thread::hardware_concurrency() - 2));
|
||||
// std::cout << "pool size: " << hw_concurrency << "\n";
|
||||
const bool use_threadpool = hw_concurrency > 1;
|
||||
|
||||
thread_pool tp{hw_concurrency};
|
||||
std::mutex row_cnt_mutex;
|
||||
|
||||
long read_chars;
|
||||
while ((read_chars = getline(&line_str, &len, fp)) != -1) {
|
||||
if (skip_header && !header_skiped) {
|
||||
header_skiped = true;
|
||||
continue;
|
||||
}
|
||||
if (read_chars > 0 && line_str[read_chars - 1] == '\n') {
|
||||
line_str[read_chars - 1] = '\0';
|
||||
--read_chars;
|
||||
|
||||
try {
|
||||
long read_chars;
|
||||
while ((read_chars = getline(&line_str, &len, fp)) != -1) {
|
||||
if (skip_header && !header_skiped) {
|
||||
header_skiped = true;
|
||||
continue;
|
||||
}
|
||||
if (read_chars > 0 && line_str[read_chars - 1] == '\n') {
|
||||
line_str[read_chars - 1] = '\0';
|
||||
--read_chars;
|
||||
}
|
||||
|
||||
if (use_threadpool) {
|
||||
std::string str{line_str};
|
||||
dispatch(tp, std::function<void()>
|
||||
([this, str, &cols_def, &table, &row_cnt, &row_cnt_mutex]() {
|
||||
auto parsed = parseCSVString(str, cols_def, table);
|
||||
{
|
||||
std::unique_lock<std::mutex> lock(row_cnt_mutex);
|
||||
row_cnt += parsed;
|
||||
}
|
||||
}
|
||||
));
|
||||
} else
|
||||
row_cnt += parseCSVString(line_str, cols_def, table);
|
||||
}
|
||||
|
||||
if (use_threadpool) {
|
||||
dispatch(tp, std::function<void()>
|
||||
([this, line_str, &cols_def, &table, &row_cnt, &row_cnt_mutex]() {
|
||||
// std::cout << "thread: " << std::this_thread::get_id() << " rownum " << row_cnt << "\n";
|
||||
auto parsed = parseCSVString(line_str, cols_def, table);
|
||||
{
|
||||
std::unique_lock<std::mutex> lock(row_cnt_mutex);
|
||||
row_cnt += parsed;
|
||||
}
|
||||
}
|
||||
));
|
||||
} else
|
||||
row_cnt += parseCSVString(line_str, cols_def, table);
|
||||
fclose(fp);
|
||||
tp.join();
|
||||
|
||||
} catch (const std::exception &e) {
|
||||
if (line_str)
|
||||
free(line_str);
|
||||
|
||||
throw e;
|
||||
}
|
||||
|
||||
if (use_threadpool) tp.finish();
|
||||
|
||||
fclose(fp);
|
||||
|
||||
} catch (const std::exception &e) {
|
||||
if (line_str)
|
||||
free(line_str);
|
||||
|
||||
throw e;
|
||||
return row_cnt;
|
||||
}
|
||||
|
||||
if (line_str)
|
||||
free(line_str);
|
||||
size_t CsvReader::parseCSVString(const std::string &csvSource, std::vector<ColDefNode> &cols_def, Table &table) {
|
||||
size_t row_cnt = 0;
|
||||
bool inQuote(false);
|
||||
bool newLine(false);
|
||||
|
||||
return row_cnt;
|
||||
}
|
||||
std::vector<std::string> columns;
|
||||
std::string field;
|
||||
columns.reserve(256);
|
||||
field.reserve(64);
|
||||
|
||||
size_t CsvReader::parseCSVString(const std::string csvSource, std::vector<ColDefNode> &cols_def, Table& table) {
|
||||
size_t row_cnt = 0;
|
||||
bool inQuote(false);
|
||||
bool newLine(false);
|
||||
|
||||
std::vector<std::string> columns;
|
||||
std::string field;
|
||||
columns.reserve(256);
|
||||
field.reserve(64);
|
||||
|
||||
std::string::const_iterator aChar = csvSource.begin();
|
||||
std::string::const_iterator aEnd = csvSource.end();
|
||||
while (aChar != aEnd) {
|
||||
if (*aChar == quote_character) {
|
||||
newLine = false;
|
||||
inQuote = !inQuote;
|
||||
} else if (*aChar == field_separator) {
|
||||
newLine = false;
|
||||
if (inQuote) {
|
||||
field += *aChar;
|
||||
} else {
|
||||
columns.push_back(field);
|
||||
field.clear();
|
||||
}
|
||||
} else if (*aChar == line_separator || *aChar == line_separator2) {
|
||||
if (inQuote) {
|
||||
field += *aChar;
|
||||
} else {
|
||||
if (!newLine) {
|
||||
std::string::const_iterator aChar = csvSource.begin();
|
||||
std::string::const_iterator aEnd = csvSource.end();
|
||||
while (aChar != aEnd) {
|
||||
if (*aChar == quote_character) {
|
||||
newLine = false;
|
||||
inQuote = !inQuote;
|
||||
} else if (*aChar == field_separator) {
|
||||
newLine = false;
|
||||
if (inQuote) {
|
||||
field += *aChar;
|
||||
} else {
|
||||
columns.push_back(field);
|
||||
if (header_skiped) {
|
||||
table.create_row_from_vector(cols_def, columns);
|
||||
row_cnt++;
|
||||
}
|
||||
header_skiped = true;
|
||||
field.clear();
|
||||
columns.clear();
|
||||
newLine = true;
|
||||
}
|
||||
} else if (*aChar == line_separator || *aChar == line_separator2) {
|
||||
if (inQuote) {
|
||||
field += *aChar;
|
||||
} else {
|
||||
if (!newLine) {
|
||||
columns.push_back(field);
|
||||
if (header_skiped) {
|
||||
table.create_row_from_vector(cols_def, columns);
|
||||
row_cnt++;
|
||||
}
|
||||
header_skiped = true;
|
||||
field.clear();
|
||||
columns.clear();
|
||||
newLine = true;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
newLine = false;
|
||||
field.push_back(*aChar);
|
||||
}
|
||||
} else {
|
||||
newLine = false;
|
||||
field.push_back(*aChar);
|
||||
|
||||
aChar++;
|
||||
}
|
||||
|
||||
aChar++;
|
||||
if (!field.empty())
|
||||
columns.push_back(field);
|
||||
|
||||
if (header_skiped) {
|
||||
table.create_row_from_vector(cols_def, columns);
|
||||
row_cnt++;
|
||||
} else
|
||||
header_skiped = true;
|
||||
|
||||
return row_cnt;
|
||||
}
|
||||
|
||||
if (!field.empty())
|
||||
columns.push_back(field);
|
||||
|
||||
if (header_skiped) {
|
||||
table.create_row_from_vector(cols_def, columns);
|
||||
row_cnt++;
|
||||
} else
|
||||
header_skiped = true;
|
||||
|
||||
return row_cnt;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ public:
|
|||
explicit CsvReader(bool skip_hdr = true, char field_sep = ',', char quote_ch = '"', char line_sep = '\r', char line_sep2 = '\n');
|
||||
|
||||
size_t parseCSVFile(const std::string &filename, std::vector<ColDefNode> &cols_def, Table& table);
|
||||
size_t parseCSVString(const std::string csvSource, std::vector<ColDefNode> &cols_def, Table& table);
|
||||
size_t parseCSVString(const std::string &csvSource, std::vector<ColDefNode> &cols_def, Table& table);
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
|
|
|||
125
threadpool.h
125
threadpool.h
|
|
@ -1,104 +1,79 @@
|
|||
#include <iostream>
|
||||
|
||||
|
||||
#include <thread>
|
||||
#include <mutex>
|
||||
#include <functional>
|
||||
#include <future>
|
||||
#include <queue>
|
||||
#include <condition_variable>
|
||||
#include <vector>
|
||||
|
||||
struct thread_pool {
|
||||
explicit thread_pool(std::size_t size) {
|
||||
start(size);
|
||||
finished.reserve(1024);
|
||||
class thread_pool {
|
||||
public:
|
||||
thread_pool(std::size_t size) : stop(false) {
|
||||
for (std::size_t i = 0; i < size; ++i) {
|
||||
workers.emplace_back([this] { spawn(); });
|
||||
}
|
||||
}
|
||||
|
||||
std::mutex mutex;
|
||||
std::condition_variable condition;
|
||||
std::deque<std::packaged_task<void()>> work;
|
||||
virtual ~thread_pool() {
|
||||
if (!stop) join();
|
||||
}
|
||||
|
||||
std::vector<std::future<void>> finished;
|
||||
|
||||
// queue( lambda ) will enqueue the lambda into the tasks for the threads
|
||||
template<class F, class R=std::result_of_t<F &()>>
|
||||
std::future<R> queue(F &&f) {
|
||||
// wrap the function object into a packaged task, splitting
|
||||
// execution from the return value:
|
||||
std::packaged_task<R()> p(std::forward<F>(f));
|
||||
|
||||
auto r = p.get_future();
|
||||
void post(std::function<void()> f) {
|
||||
{
|
||||
std::unique_lock<std::mutex> l(mutex);
|
||||
work.emplace_back(std::move(p));
|
||||
std::unique_lock<std::mutex> lock(mutex);
|
||||
tasks.push(f);
|
||||
}
|
||||
|
||||
condition.notify_one();
|
||||
|
||||
return r; // return the future result of the task
|
||||
}
|
||||
|
||||
// start threads_num threads in the thread pool.
|
||||
void start(std::size_t threads_num = 1) {
|
||||
for (std::size_t i = 0; i < threads_num; ++i) {
|
||||
finished.push_back(
|
||||
std::async(
|
||||
std::launch::async,
|
||||
[this] { thread_task(); }
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// abort() cancels all non-started tasks, and tells every working thread
|
||||
// stop running, and waits for them to finish up.
|
||||
void abort() {
|
||||
cancel_pending();
|
||||
finish();
|
||||
}
|
||||
|
||||
// cancel_pending() merely cancels all non-started tasks:
|
||||
void cancel_pending() {
|
||||
std::unique_lock<std::mutex> l(mutex);
|
||||
work.clear();
|
||||
}
|
||||
|
||||
// finish enques a "stop the thread" message for every thread, then waits for them:
|
||||
void finish() {
|
||||
void join() {
|
||||
{
|
||||
std::unique_lock<std::mutex> l(mutex);
|
||||
for (auto &&unused:finished) {
|
||||
work.emplace_back();
|
||||
}
|
||||
std::unique_lock<std::mutex> lock(mutex);
|
||||
stop = true;
|
||||
}
|
||||
condition.notify_all();
|
||||
finished.clear();
|
||||
}
|
||||
|
||||
~thread_pool() {
|
||||
finish();
|
||||
condition.notify_all();
|
||||
|
||||
for (std::size_t i = 0; i < workers.size(); ++i) {
|
||||
workers[i].join();
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
void thread_task() {
|
||||
while (true) {
|
||||
std::packaged_task<void()> f;
|
||||
void spawn() {
|
||||
std::function<void()> task;
|
||||
|
||||
bool task_queue_empty = tasks.empty();
|
||||
while (!stop || !task_queue_empty) {
|
||||
bool task_valid = false;
|
||||
{
|
||||
std::unique_lock<std::mutex> l(mutex);
|
||||
if (work.empty()) {
|
||||
condition.wait(l, [&] { return !work.empty(); });
|
||||
}
|
||||
f = std::move(work.front());
|
||||
work.pop_front();
|
||||
std::unique_lock<std::mutex> lock(mutex);
|
||||
condition.wait(lock, [this]() {
|
||||
return (!tasks.empty()) || (tasks.empty() && stop);
|
||||
});
|
||||
|
||||
if (!tasks.empty()) {
|
||||
task = std::move(tasks.front());
|
||||
tasks.pop();
|
||||
task_valid = true;
|
||||
}
|
||||
|
||||
task_queue_empty = tasks.empty();
|
||||
}
|
||||
// if the task is invalid, it means we are asked to abort:
|
||||
if (!f.valid()) return;
|
||||
|
||||
f();
|
||||
|
||||
if (task_valid) task();
|
||||
}
|
||||
}
|
||||
|
||||
public:
|
||||
std::vector<std::thread> workers;
|
||||
std::queue<std::function<void()>> tasks;
|
||||
|
||||
std::mutex mutex;
|
||||
std::condition_variable condition;
|
||||
bool stop;
|
||||
};
|
||||
|
||||
inline void dispatch(thread_pool &pool, std::function<void()> f) {
|
||||
pool.queue(f);
|
||||
pool.post(f);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue