thread pool changed

CMakeLists.txt

@@ -5,10 +5,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS        OFF)
 
 # set(CMAKE_CXX_FLAGS "-O3 -Wall -Wextra")
-
+# set(CMAKE_CXX_FLAGS_DEBUG "-Wall -d")
-set(CMAKE_OSX_DEPLOYMENT_TARGET "10.14")
-
-set(CMAKE_CXX_FLAGS_DEBUG "-Wall")
 
 project(usql)
 

csvreader.cpp

@@ -30,15 +30,14 @@ size_t CsvReader::parseCSVFile(const std::string &filename, std::vector<ColDefNo
 		char *line_str = NULL;
 		size_t len = 0;
 
-	try {
 		// TODO handle it by settings
 		const std::size_t hw_concurrency = std::max(0, (int)(std::thread::hardware_concurrency() - 2));
-		// std::cout << "pool size: " << hw_concurrency << "\n";
 		const bool use_threadpool = hw_concurrency > 1;
 
 		thread_pool tp{hw_concurrency};
 		std::mutex row_cnt_mutex;
 
+		try {
 			long read_chars;
 			while ((read_chars = getline(&line_str, &len, fp)) != -1) {
 				if (skip_header && !header_skiped) {
@@ -51,10 +50,10 @@ size_t CsvReader::parseCSVFile(const std::string &filename, std::vector<ColDefNo
 				}
 
 				if (use_threadpool) {
+					std::string str{line_str};
 					dispatch(tp, std::function<void()>
-						([this, line_str, &cols_def, &table, &row_cnt, &row_cnt_mutex]() {
+							([this, str, &cols_def, &table, &row_cnt, &row_cnt_mutex]() {
-							 // std::cout << "thread: " << std::this_thread::get_id() << " rownum " << row_cnt << "\n";
+								 auto parsed = parseCSVString(str, cols_def, table);
-							 auto parsed = parseCSVString(line_str, cols_def, table);
 								 {
 									std::unique_lock<std::mutex> lock(row_cnt_mutex);
 									row_cnt += parsed;
@@ -65,9 +64,8 @@ size_t CsvReader::parseCSVFile(const std::string &filename, std::vector<ColDefNo
 					row_cnt += parseCSVString(line_str, cols_def, table);
 			}
 
-		if (use_threadpool) tp.finish();
-
 			fclose(fp);
+			tp.join();
 
 		} catch (const std::exception &e) {
 			if (line_str)
@@ -82,7 +80,7 @@ size_t CsvReader::parseCSVFile(const std::string &filename, std::vector<ColDefNo
 		return row_cnt;
 	}
 
-size_t CsvReader::parseCSVString(const std::string csvSource, std::vector<ColDefNode> &cols_def, Table& table) {
+	size_t CsvReader::parseCSVString(const std::string &csvSource, std::vector<ColDefNode> &cols_def, Table &table) {
 		size_t row_cnt = 0;
 		bool inQuote(false);
 		bool newLine(false);

csvreader.h

@@ -27,7 +27,7 @@ public:
 	explicit CsvReader(bool skip_hdr = true, char field_sep = ',', char quote_ch = '"', char line_sep = '\r', char line_sep2 = '\n');
 
 	size_t parseCSVFile(const std::string &filename, std::vector<ColDefNode> &cols_def, Table& table);
-	size_t parseCSVString(const std::string csvSource, std::vector<ColDefNode> &cols_def, Table& table);
+	size_t parseCSVString(const std::string &csvSource, std::vector<ColDefNode> &cols_def, Table& table);
 };
 
 }	// namespace

threadpool.h

@@ -1,104 +1,79 @@
-#include <iostream>
-
-
 #include <thread>
 #include <mutex>
 #include <functional>
-#include <future>
 #include <queue>
 #include <condition_variable>
 #include <vector>
 
-struct thread_pool {
+class thread_pool {
-	explicit thread_pool(std::size_t size) {
+public:
-		start(size);
+	thread_pool(std::size_t size) : stop(false) {
-		finished.reserve(1024);
+		for (std::size_t i = 0; i < size; ++i) {
+			workers.emplace_back([this] { spawn(); });
+		}
 	}
 
-	std::mutex mutex;
+	virtual ~thread_pool() {
-	std::condition_variable condition;
+		if (!stop) join();
-	std::deque<std::packaged_task<void()>> work;
+	}
 
-	std::vector<std::future<void>> finished;
+	void post(std::function<void()> f) {
-
-	// queue( lambda ) will enqueue the lambda into the tasks for the threads
-	template<class F, class R=std::result_of_t<F &()>>
-	std::future<R> queue(F &&f) {
-		// wrap the function object into a packaged task, splitting
-		// execution from the return value:
-		std::packaged_task<R()> p(std::forward<F>(f));
-
-		auto r = p.get_future();
 		{
-			std::unique_lock<std::mutex> l(mutex);
+			std::unique_lock<std::mutex> lock(mutex);
-			work.emplace_back(std::move(p));
+			tasks.push(f);
 		}
+
 		condition.notify_one();
-
-		return r; // return the future result of the task
 	}
 
-	// start threads_num threads in the thread pool.
+	void join() {
-	void start(std::size_t threads_num = 1) {
-		for (std::size_t i = 0; i < threads_num; ++i) {
-			finished.push_back(
-					std::async(
-							std::launch::async,
-							[this] { thread_task(); }
-					)
-			);
-		}
-	}
-
-	// abort() cancels all non-started tasks, and tells every working thread
-	// stop running, and waits for them to finish up.
-	void abort() {
-		cancel_pending();
-		finish();
-	}
-
-	// cancel_pending() merely cancels all non-started tasks:
-	void cancel_pending() {
-		std::unique_lock<std::mutex> l(mutex);
-		work.clear();
-	}
-
-	// finish enques a "stop the thread" message for every thread, then waits for them:
-	void finish() {
 		{
-			std::unique_lock<std::mutex> l(mutex);
+			std::unique_lock<std::mutex> lock(mutex);
-			for (auto &&unused:finished) {
+			stop = true;
-				work.emplace_back();
-			}
-		}
-		condition.notify_all();
-		finished.clear();
 		}
 
-	~thread_pool() {
+		condition.notify_all();
-		finish();
+
+		for (std::size_t i = 0; i < workers.size(); ++i) {
+			workers[i].join();
+		}
 	}
 
 private:
-	void thread_task() {
+	void spawn() {
-		while (true) {
+		std::function<void()> task;
-			std::packaged_task<void()> f;
-			{
-				std::unique_lock<std::mutex> l(mutex);
-				if (work.empty()) {
-					condition.wait(l, [&] { return !work.empty(); });
-				}
-				f = std::move(work.front());
-				work.pop_front();
-			}
-			// if the task is invalid, it means we are asked to abort:
-			if (!f.valid()) return;
 		
-			f();
+		bool task_queue_empty = tasks.empty();
+		while (!stop || !task_queue_empty) {
+			bool task_valid = false;
+			{
+				std::unique_lock<std::mutex> lock(mutex);
+				condition.wait(lock, [this]() {
+					return (!tasks.empty()) || (tasks.empty() && stop);
+				});
+
+				if (!tasks.empty()) {
+					task = std::move(tasks.front());
+					tasks.pop();
+					task_valid = true;
+				} 
+				
+				task_queue_empty = tasks.empty();
+			}
+
+			if (task_valid) task();
 		}
 	}
+
+public:
+	std::vector<std::thread> workers;
+	std::queue<std::function<void()>> tasks;
+
+	std::mutex mutex;
+	std::condition_variable condition;
+	bool stop;
 };
 
 inline void dispatch(thread_pool &pool, std::function<void()> f) {
-	pool.queue(f);
+	pool.post(f);
 }