a bit od examples in doc

.gitignore

@@ -6,3 +6,6 @@ ml
 CMakeFiles
 Testing
 .cache
+Makefile
+cmake_install.cmake
+CMakeCache.txt

.vscode/c_cpp_properties.json

@@ -7,10 +7,6 @@
                 "${workspaceFolder}/clib"
             ],
             "defines": [],
-            "macFrameworkPath": [
-                "/Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/System/Library/Frameworks"
-            ],
-            "compilerPath": "/usr/bin/clang",
             "cStandard": "c17",
             "intelliSenseMode": "macos-clang-x64",
             "configurationProvider": "ms-vscode.cmake-tools"

.vscode/launch.json

@@ -16,6 +16,18 @@
             "externalConsole": false,
             "MIMode": "gdb"
         }, 
+        {
+            "name": "ml -f debug.lsp",
+            "type": "cppdbg",
+            "request": "launch",
+            "program": "${workspaceFolder}/build/ml",
+            "args": ["-f", "${workspaceFolder}/debug.lsp"],
+            "stopAtEntry": false,
+            "cwd": "${workspaceFolder}",
+            "environment": [],
+            "console": "externalTerminal",
+            "MIMode": "gdb"
+        },
         {
             "name": "ml -d -f debug.lsp",
             "type": "cppdbg",

.vscode/settings.json

@@ -73,6 +73,8 @@
         "variant": "cpp",
         "__functional_base_03": "cpp",
         "charconv": "cpp",
-        "cinttypes": "cpp"
-    }
+        "cinttypes": "cpp",
+        "filesystem": "cpp"
+    },
+    "cmake.configureOnOpen": true
 }

CMakeLists.txt

@@ -1,10 +1,9 @@
 cmake_minimum_required(VERSION 3.0.0)
-project(ml VERSION 0.1.0)
+project(ml VERSION 0.5.0)
 
 set(CMAKE_CXX_STANDARD          17)
 # set(CMAKE_CXX_STANDARD_REQUIRED ON)
 # set(CMAKE_CXX_EXTENSIONS        OFF)
-# set(CMAKE_OSX_DEPLOYMENT_TARGET "10.14")
 
 set(CPACK_PROJECT_NAME ${PROJECT_NAME})
 set(CPACK_PROJECT_VERSION ${PROJECT_VERSION})
@@ -16,9 +15,9 @@ set(CPACK_PROJECT_VERSION ${PROJECT_VERSION})
 # set(CMAKE_CXX_FLAGS "-Wall -Wextra")
 set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG")
 
-include_directories(/usr/local/opt/openssl/include ${CMAKE_SOURCE_DIR}/clib ${CMAKE_SOURCE_DIR})
+include_directories(/opt/homebrew/opt/openssl@1.1/include ${CMAKE_SOURCE_DIR}/clib ${CMAKE_SOURCE_DIR})
 
-link_directories(/usr/local/lib /usr/local/opt/openssl/lib)
+link_directories(/opt/homebrew/opt/openssl@1.1/lib)
 
 
 project(ml)
@@ -62,3 +61,6 @@ INSTALL(TARGETS ml
 
 INSTALL(DIRECTORY stdlib/ DESTINATION var/mlisp
         FILES_MATCHING	PATTERN "*.lsp")
+
+INSTALL(DIRECTORY doc/ DESTINATION var/mlisp
+        FILES_MATCHING	PATTERN "*.md")

Readme.md

@@ -40,11 +40,10 @@ see /doc/Doc.md
 
 #### Compile
 ```
-mkdir -p build
 gcc -std=c99 -c -O2 -o linenoise.o clib/linenoise.c
-c++ -c -O2  -I/usr/local/opt/openssl/include -Iclib --std=c++17  ml.cpp ml_io.cpp ml_date.cpp ml_string.cpp ml_util.cpp ml_profiler.cpp ml_usql.cpp clib/json11.cpp clib/csvparser.cpp clib/sslclient.cpp clib/printf.cpp usql/exception.cpp usql/lexer.cpp usql/parser.cpp usql/usql.cpp usql/table.cpp usql/table.h usql/row.cpp usql/csvreader.cpp usql/usql.cpp usql/usql_ddl.cpp usql/usql_dml.cpp usql/settings.cpp
+c++ -c -O2  -I/usr/local/opt/openssl/include -Iclib -I./ --std=c++17  ml.cpp ml_io.cpp ml_date.cpp ml_string.cpp ml_util.cpp ml_profiler.cpp ml_usql.cpp clib/json11.cpp clib/csvparser.cpp clib/sslclient.cpp clib/printf.cpp clib/sslclient.cpp clib/tcpnet.cpp usql/exception.cpp usql/lexer.cpp usql/parser.cpp usql/usql.cpp usql/table.cpp usql/table.h usql/row.cpp usql/csvreader.cpp usql/usql.cpp usql/usql_ddl.cpp usql/usql_dml.cpp usql/usql_function.cpp  usql/settings.cpp
 
-// on linux c++ -o build/ml -O2 -L/usr/local/lib -L/usr/local/opt/openssl/lib -lm -lstdc++ -lcrypto -lssl *.o
+// on linux c++ -o ml -O2 -L/usr/local/lib -L/usr/local/opt/openssl/lib -lm -lstdc++ -lcrypto -lssl *.o
 c++ -o build/ml -O2 -L/usr/local/lib -L/usr/local/opt/openssl/lib -lm -lstdc++ -lcrypto -lssl -Wl,-stack_size -Wl,0x1000000 *.o
 ```
 or use 
@@ -57,6 +56,7 @@ make
 ```
 cp build/ml /usr/local/bin/ml
 cp stdlib/*.lsp /usr/local/var/mlisp/
+cp doc/*.md /usr/local/var/mlisp/
 ```
 or
 ```
@@ -65,26 +65,21 @@ utils/local_install.sh
 
 
 ### KNOWNN BUGS
-- in (include "file.lsp") are not available stdlib.lsp finctions like (string-trim ..) etc
 - (read-url "https://api.nasdaq.com/api/calendar/dividends/") ; hangs in sslclient.cpp line 132
 
 ### TODO
-- add functions from stdlib into doc
-- string-case, save-csv fix doc
-- add test for >>> (range 1.5 (inc 5.5)) => (1.500000 2.500000 3.500000 4.500000 5.500000)
-- better formating of help
+- order of arguments in functions like member and filter - filter swap lambda and list
+- add possibility to specify filename for input2 history file
+- better output of !e in repl (resp !e !ee)
 - unify -f and -run options
 - add debug support, at least call stack
-- multiline editing (see kilocpp editor)
+- add mem stats to benchmark
 - execute system command should capture stderr
-- add some mem stats to benchmark
-- order of arguments in functions like member and filter - filter swap lambda and list
-- better output of !e in repl (resp !e !ee)
+- multiline editing (see kilocpp editor)
 
 #### Doc
-- fix man entry on (member) example and retval
+- add functions from stdlib into doc
 - doc for set!
-- doc for doc::??? functions
 
 #### Code
 - tcpnet should use RAII
@@ -93,13 +88,7 @@ utils/local_install.sh
 
 #### Language
 - string functions
-    - compare - needed for sorting, cmp ignore case
     - regexp match, regexp tokens    
-- date support
-  - decode-universal-time (http://www.lispworks.com/documentation/HyperSpec/Body/f_dec_un.htm)
-- env functions 
-  - get-env, set-env; set-env cannot be implemented in stdlib.lsp, because popen is in fact subshell
-- macros and then loop, let etc
 
 #### Performance
 - push_back - repeatedly without reserving size

clib/csvparser.cpp

@@ -14,16 +14,20 @@ CsvParser::CsvParser(bool skip_hdr, char field_sep, char quote_ch, char line_sep
 }
 
 MlValue CsvParser::parseCSV(const std::string &csvSource) {
-    int linesRead = 0;
+	constexpr size_t INITIAL_PARSED_ROWS_SIZE = 128;
+	constexpr size_t INITIAL_COLUMNS_SIZE = 32;
+	constexpr size_t ROWS_READ_FOR_SIZE_ESTIMATION = 16;
+
+    size_t linesRead = 0;
 	bool inQuote(false);
 	bool newLine(false);
 	std::string field;
 
-	std::vector<MlValue> parsed_data;
-	parsed_data.reserve(128);	// TODO introduce constant here
+	std::vector<MlValue> parsed_rows;
+	parsed_rows.reserve(INITIAL_PARSED_ROWS_SIZE);
 
 	std::vector<MlValue> line;
-	line.reserve(32);		// TODO introduce constant here
+	line.reserve(INITIAL_COLUMNS_SIZE);
 
 	std::string::const_iterator aChar = csvSource.begin();
 	std::string::const_iterator aEnd = csvSource.end();
@@ -45,14 +49,14 @@ MlValue CsvParser::parseCSV(const std::string &csvSource) {
 			} else {
 				if (!newLine) {
 					line.push_back(ivalualize(field));
-					add_line(line, parsed_data);
+					add_row(line, parsed_rows);
 					field.clear();
 					line.clear();
 					linesRead++;
-					if (linesRead == 16) {
+					if (linesRead == ROWS_READ_FOR_SIZE_ESTIMATION) {
 					    size_t linesEstimation = csvSource.size() / (std::distance(csvSource.begin(), aChar) / linesRead);
-						if (linesEstimation > parsed_data.capacity())
-					    	parsed_data.reserve(linesEstimation);
+						if (linesEstimation > parsed_rows.capacity())
+					    	parsed_rows.reserve(linesEstimation);
 					}
 					newLine = true;
 				}
@@ -68,18 +72,18 @@ MlValue CsvParser::parseCSV(const std::string &csvSource) {
 	if (!field.empty())
 		line.push_back(ivalualize(field));
 
-	add_line(line, parsed_data);
+	add_row(line, parsed_rows);
 
-	return parsed_data;
+	return parsed_rows;
 }
 
 
-void CsvParser::add_line(const std::vector<MlValue> &line, std::vector<MlValue>  &lines) {
+void CsvParser::add_row(const std::vector<MlValue> &columns, std::vector<MlValue>  &rows) {
 	if (skip_header && !header_skiped) {
 		header_skiped = true;
 	} else {
-		if (!line.empty())
-			lines.emplace_back(line);
+		if (!columns.empty())
+			rows.emplace_back(columns);
 	}
 }
 

clib/csvparser.h

@@ -26,7 +26,7 @@ public:
     MlValue parseCSV(const std::string &csvSource);
 
 private:
-    void add_line(const std::vector<MlValue> &line, std::vector<MlValue>  &lines);
+    void add_row(const std::vector<MlValue> &columns, std::vector<MlValue>  &rows);
 
     static MlValue ivalualize(const std::string &value) ;
 

clib/printf.cpp

@@ -6,11 +6,11 @@
 #include <vector>
 
 
-std::string
-mini_sprintf_format(bool left_align, bool sign, bool space_on_left, bool padding_by_zero, int width, int precision,
+std::string mini_sprintf_format(bool left_align, bool sign, bool space_on_left, bool padding_by_zero,
+								int width, int precision,
 								int length, char specifier, const MlValue &value) {
 	std::string s;
-	std::ostringstream stream_str;        // PERF simpler solution..without string stream
+	std::ostringstream stream_str;        // PERF string append should be faster
 	bool is_positive = false;
 
 	if (specifier == 's') {
@@ -47,7 +47,7 @@ mini_sprintf_format(bool left_align, bool sign, bool space_on_left, bool padding
 			stream_str << std::setprecision(precision);
 
 		stream_str << dval;
-		s = stream_str.str(); // TODO ??
+		s = stream_str.str();
 	}
 
 	if (width > -1 && s.size() < width) {

clib/sslclient.cpp

@@ -5,7 +5,6 @@
 #include <netinet/in.h>
 #include <sys/socket.h>
 
-#include <cstdio>
 #include <iostream>
 #include <regex>
 #include <sstream>
@@ -16,8 +15,6 @@
 // https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/GET
 
 
-// TODO user streams instead of printf's
-
 std::pair<int, std::string> HttpClient::doRequest(const std::string &method, const std::string &url, const std::map<std::string, std::string> &headers, const std::string &request_body) {
 	// split url to parts
 	parseURL(url);
@@ -39,14 +36,12 @@ std::pair<int, std::string> HttpClient::doRequest(const std::string &method, con
 		return std::make_pair(403, "");
 	}
 
-
 	// get headers
-	std::string::size_type position = ssl_read_packet.find("\r\n\r\n");
-	if (position == std::string::npos)
+	std::string::size_type end_of_headers = ssl_read_packet.find("\r\n\r\n");
+	if (end_of_headers == std::string::npos)
 		throw std::runtime_error("invalid reply, end of headers not found");
 	
-
-	const std::string response = ssl_read_packet.substr(0, position);
+	const std::string response = ssl_read_packet.substr(0, end_of_headers);
 	auto status_pos = response.find("\r\n");
 
 	const std::string response_headers = response.substr(status_pos + 2, response.length() - 2 - status_pos);
@@ -58,8 +53,20 @@ std::pair<int, std::string> HttpClient::doRequest(const std::string &method, con
 	// parse headers
 	responseHeaders(response_headers);
 
+	// get content length if given
+	int content_len = 0;
+	auto cl_it = headers_map.find("Content-Length");
+	if (cl_it != headers_map.end()) {
+		content_len = std::stoi(cl_it->second);
+	}
+
+	// and fetch the rest if not read completely
+	while (content_len > 0 && content_len > ssl_read_packet.length() - 4 - end_of_headers) {
+		/* auto read_bytes = */ sslRecvPacket();
+	}
+
 	// get body
-	const std::string body = ssl_read_packet.substr(position + 4, ssl_read_packet.length() - 4 - position);
+	const std::string body = ssl_read_packet.substr(end_of_headers + 4, ssl_read_packet.length() - 4 - end_of_headers);
 
 
 	return std::make_pair(resp_status, body);
@@ -147,17 +154,22 @@ std::string HttpClient::inetAddress(const std::string &hostname) {
 }
 
 int HttpClient::sslRecvPacket() {
-	ssl_read_packet.resize(4096);
-	ssl_read_packet.clear();
+	int buf_len = 16384;
+	unsigned char buf[buf_len + 1];
 
-	int len = 16384;
-	char buf[len + 1];
-	memset(buf, 0, sizeof(buf));
+	int len = 0;
+	int read_bytes = 0;
 	do {
-		len = SSL_read(ssl, buf, len);
-		if (len >= 0) {
-			buf[len] = 0;
-			ssl_read_packet.append((const char *) buf);
+		memset(buf, 0, sizeof(buf));
+		len = SSL_read(ssl, buf, buf_len);
+		if (len > 0) {
+			read_bytes += len;
+			buf[len] = 0;	// actually redundant
+
+			std::vector<unsigned char> data(buf, buf + len);
+			std::string chunk(data.begin(), data.end());
+
+			ssl_read_packet.append(chunk);
 		}
 	} while (len > 0);
 
@@ -171,7 +183,7 @@ int HttpClient::sslRecvPacket() {
 			return -1;
 	}
 
-	return (int) ssl_read_packet.length();
+	return read_bytes;
 }
 
 int HttpClient::sslSendPacket(const std::string &buf) {
@@ -198,7 +210,7 @@ int HttpClient::sslRequest(const std::string &server_name, const std::string &re
 	int s;
 	s = socket(AF_INET, SOCK_STREAM, 0);
 	if (!s) {
-		printf("sslRequest, error creating socket.\n");
+		std::cerr << "HttpClient::sslRequest, error creating socket" << std::endl;
 		return -1;
 	}
 
@@ -213,7 +225,7 @@ int HttpClient::sslRequest(const std::string &server_name, const std::string &re
 
 	// connect to server
 	if (connect(s, (struct sockaddr *) &sa, socklen)) {
-		printf("sslRequest, error connecting to server.\n");
+		std::cerr << "HttpClient::sslRequest, error connecting to server"  << std::endl;
 		return -1;
 	}
 	
@@ -224,7 +236,7 @@ int HttpClient::sslRequest(const std::string &server_name, const std::string &re
 	SSL_CTX *ctx = SSL_CTX_new(meth);
 	ssl = SSL_new(ctx);
 	if (!ssl) {
-		printf("sslRequest, error creating SSL.\n");
+		std::cerr << "HttpClient::sslRequest, error creating SSL" << std::endl;
 		logSSL();
 		return -1;
 	}
@@ -235,27 +247,28 @@ int HttpClient::sslRequest(const std::string &server_name, const std::string &re
 
 	int err = SSL_connect(ssl);
 	if (err <= 0) {
-		printf("sslRequest, error creating SSL connection.  err=%x\n", err);
+		std::cerr << "HttpClient::sslRequest, error creating SSL connection. " << err << std::endl;
 		logSSL();
 		fflush(stdout);
 		return -1;
 	}
 
 	// log cipher
-	// printf ("SSL connection using %s\n", SSL_get_cipher (ssl));
+	// std::cerr << "HttpClient::sslRequest, SSL connection using" << SSL_get_cipher (ssl) << std::endl;
 	// showCerts(ssl);
 
 
 	// send request
-	// printf ("SSL sending request %s\n", request.c_str());
+	// std::cerr << "HttpClient::sslRequest, SSL sending request: " << request << std::endl;
 
 	int written_bytes = sslSendPacket(request);
 	if (written_bytes != request.length()) {
-		printf("sslRequest, error sending request\n");
+		std::cerr << "HttpClient::sslRequest, error sending request" << std::endl;
 		return -1;
 	}
 
 	// read response and return its length
+	ssl_read_packet.clear();
 	return sslRecvPacket();
 }
 
@@ -275,15 +288,15 @@ void HttpClient::showCerts(SSL* ssl) {
 
     cert = SSL_get_peer_certificate(ssl); /* get the server's certificate */
     if ( cert != NULL ) {
-		printf("Server certificates:\n");
+		std::cerr << "HttpClient::showCerts, Server certificates: " << std::endl;
 		line = X509_NAME_oneline(X509_get_subject_name(cert), 0, 0);
-		printf("Subject: %s\n", line);
+		std::cerr << "HttpClient::showCerts, Subject: " << line << std::endl;
 		free(line);       /* free the malloc'ed string */
 		line = X509_NAME_oneline(X509_get_issuer_name(cert), 0, 0);
-		printf("Issuer: %s\n", line);
+		std::cerr << "HttpClient::showCerts, Issuer: " << line << std::endl;
 		free(line);       /* free the malloc'ed string */
 		X509_free(cert);     /* free the malloc'ed certificate copy */
 	} else {
-    	printf("No certificates.\n");
+		std::cerr << "HttpClient::showCerts, No certificates." << std::endl;
     }
 }

clib/sslclient.h

@@ -9,7 +9,6 @@
 
 
 class HttpClient {
-    // TODO at this moment only https is implemented
 
 private:
     SSL *ssl;

clib/tcpnet.cpp

@@ -71,10 +71,7 @@ int TcpNet::server(int portno, const std::function<std::pair<bool, std::string>(
 			shutdown = response.first;
 			std::string response_str = response.second;
 
-			auto response_len = response_str.size();
-			auto n = write(newsockfd, response_str.c_str(), response_len);
-			if (n < 0 || response_len != n)
-				error("ERROR writing to socket");
+			write_to_socket(newsockfd, response_str);
 
 			requests_processed++;
 		}
@@ -134,11 +131,26 @@ std::string TcpNet::client(const std::string &address, int portno, const std::st
 	return response[0];
 }
 
+
+
+
+
+
 std::string TcpNet::read_from_socket(int sockfd) {
 	char buffer[TCPNET_BUFFER_SIZE];
 	std::string request;
 
+	// read length header
+	unsigned long long readdatalen = 0;
+	if (USE_LENGTH_HEADER) {
+		long n = read(sockfd, &readdatalen, sizeof(readdatalen));
+		if (n != 0 && n != sizeof(readdatalen))
+			error("ERROR reading length header failed");
+	}
+
+	// read data
 	long n;
+	long readed = 0;
 	do {
 		memset(buffer, 0, TCPNET_BUFFER_SIZE);
 		n = read(sockfd, buffer, TCPNET_BUFFER_SIZE - 1);
@@ -153,15 +165,26 @@ std::string TcpNet::read_from_socket(int sockfd) {
 
 		std::string part{buffer};
 		request.append(part);
-	} while (n == TCPNET_BUFFER_SIZE - 1);    // TODO what if data exactly of this size
+		readed += n;
+	} while ((USE_LENGTH_HEADER && readed < readdatalen) || n == TCPNET_BUFFER_SIZE - 1);    // TODO what if data exactly of this size
 
 	return request;
 }
 
 void TcpNet::write_to_socket(int sockfd, const std::string &str) {
 	const char *buffer = str.c_str();
-	int pos = 0;
 	long n;
+
+	// write length header
+	unsigned long long writedatalen = str.length(); 
+	if (USE_LENGTH_HEADER) {
+		n = write(sockfd, &writedatalen, (int) sizeof(writedatalen));
+		if (n < 0)
+			error("ERROR writing size number to socket");
+	}
+	
+	// write data
+	int pos = 0;
 	do {
 		n = write(sockfd, buffer + pos, (int) (str.length() - pos));
 		if (n < 0)

clib/tcpnet.h

@@ -17,6 +17,7 @@ public:
 	[[nodiscard]] std::vector<std::string> client(const std::string &address, int portno, const std::vector<std::string> &requests) const;
 
 private:
+	static constexpr bool USE_LENGTH_HEADER = true;
 	static std::string read_from_socket(int sockfd) ;
 	static void write_to_socket(int sockfd, const std::string &str) ;
 };

debug.lsp

@@ -1,3 +1,22 @@
-(def l '(nil 1 2 3))
-(print (filter (lambda (e) (eval e)) l))
-(print (filter (lambda (eval e) (print e)) l))
+(dotimes i 4 (print i) )
+
+(def acc 0)
+(print (dotimes i 4 (do (set! acc (+ acc i)) (print acc))) )
+
+(defmacro until (cnd body)
+	(list 'while '(not (eval cnd)) 
+		body
+    ))  
+
+
+
+(def stop nil)
+
+
+(until stop
+(do
+    (def line (input2 "=> "))
+	(if (= line "!q")
+		(set! stop #t)
+		(print line))
+))

debug_cron_wip.lsp

@@ -0,0 +1,121 @@
+
+; TODO read jovs specification from filesystem
+
+(include "/usr/local/var/mlisp/terminal.lsp")
+
+
+
+(defn cron::sleep-to-next-minute ()
+	(* 1000 (- 60 (int (date-to-str (get-universal-time) "%S")))))
+
+
+(defn cron::eval-time-spec (val time_spec)
+  	(cond
+  	    ((nil? time_spec)
+			#t)	; no time_spec -> it's true
+	    ((= time_spec "*")
+	    	#t)
+	    ((string-regex? time_spec "^[0-9]{1,4}$")
+			(= val (int time_spec)))
+	    ((string-regex? time_spec "^[0-9]{1,2}-[0-9]{1,2}$")
+			(def interval (string-split time_spec "-"))
+			(and (>= val (int (first interval))) (<= val (int (second interval)))))
+	    (#t
+			nil)	; TODO exception
+	)
+)
+
+
+(defn cron::add-job (name time_spec job)
+  	; TODO if already there replace it
+  	(set! cron::entries (push cron::entries (list name time_spec job)))
+)
+
+
+(defn cron::time-to-run (now time_spec)
+	(do
+		; second, minute, hour, day, month, year, day-of-the-week
+		(def time (decode-universal-time (get-universal-time)))
+
+		(def time_minute (second time))
+		(def time_hour (third time))
+		(def time_day (fourth time))
+		(def time_month (fifth time))
+		(def time_year (sixth time))
+		(def time_weekday (seventh time))
+
+		(def  bool	     (cron::eval-time-spec time_minute  (find-val-in-list time_spec "minute")))
+		(set! bool (and bool (cron::eval-time-spec time_hour    (find-val-in-list time_spec "hour"))))
+		(set! bool (and bool (cron::eval-time-spec time_weekday (find-val-in-list time_spec "weekday"))))
+		(set! bool (and bool (cron::eval-time-spec time_day     (find-val-in-list time_spec "day"))))
+		(set! bool (and bool (cron::eval-time-spec time_month   (find-val-in-list time_spec "month"))))
+		(set! bool (and bool (cron::eval-time-spec time_year    (find-val-in-list time_spec "year"))))
+		; (print "time-to-run test" bool)
+		bool
+))
+
+(defn cron::run-job (name code)
+	(do
+		;(print "running job:" name "->" code)
+		(def run_code (+ "(thread-create " (display code) ")"))
+		(eval (parse run_code))
+	)
+)
+
+(defn cron::start ()
+	(do
+		(def wait_time (cron::sleep-to-next-minute))
+		(print "cron daemon started, waiting" wait_time "miliseconds for first run")
+		; (thread-sleep wait_time)
+		(while (not cron::stop)
+		    (def now (get-universal-time))
+		    (for e cron::entries
+			    (def job_name (first e))
+			    (def time_spec (second e))
+			    (def job_code (third e))
+
+				; (print "\n\n")
+				(def is_for_run (cron::time-to-run now time_spec))
+			    ; (print "DEBUG" job_name "checked" is_for_run)
+			    (if is_for_run
+					(cron::run-job job_name job_code)
+					; (print "not running" job_name)
+			    )
+			)
+
+			(thread-sleep (cron::sleep-to-next-minute))
+		)
+		(threads-join)
+))
+
+
+(defn cron::stop ()
+	(set! cron::stop #t))
+
+
+(def cron::entries '())	; must be here
+(def cron::stop nil)	; must be here
+
+
+
+
+(cron::add-job "every_min_print" 
+	       '(("minute" "*")) 
+	       '(print "executing: every_min_print"))
+;	       '(do (sleep 3) (print "executing: every_min_print")))
+
+(cron::add-job "every_min_between_00_and_05" 
+	       '(("minute" "00-05")) 
+	       '(print "executing: every_min_between_00_and_05"))
+
+(cron::add-job "13:20_working_day" 
+	       '(("weekday" "1-5")("hour" "13")("minute" "20")) 
+	       '(print "executing: 13:20_working_day"))
+
+(cron::add-job "15:20_working_day" 
+	       '(("weekday" "1-5")("hour" "15")("minute" "20")) 
+	       '(print "executing: 15:20_working_day"))
+
+
+(cron::start)
+

doc/Doc.md

@@ -2,8 +2,8 @@
 |option|Description|
 |:-|-|
 |-h|Prints help|
-|-b|Skips loadin of std lib|
-|-c code|Runs given code|
+|-b|Skips loading of std lib|
+|-c code|Runs passed code|
 |-f source_file ..|Executes code in files|
 |-run source_file ..|Executes code in file, if first line of file is sheebang, it is skipped|
 |-i|Runs REPL|
@@ -23,11 +23,12 @@
 ## Syntax and Special Forms
 |Special Form|Argument Evaluations|Purpose|Section|
 |:-|-|-|-|
-|`(if cond a b)`|`if` only evaluates its `cond` argument. If `cond` is truthy (non-zero), then `a` is evaluated. Otherwise, `b` is evaluated.|This special form is the main method of control flow.|Language|
-|`(cond (test1 action1) (test2 action2) ... (testn actionn))`|The first clause whose test evaluates to non-nil is selected; all other clauses are ignored, and the consequents of the selected clause are evaluated in order. If none of the test conditions are evaluated to be true, then the cond statement returns nil.|This special form is method of control flow.|Language|
+|`(if cond a b)`|`if` only evaluates its `cond` argument. If `cond` is truthy, then `a` is evaluated. Otherwise, `b` is evaluated.|This special form is the main method of control flow.|Language|
+|`(cond (test1 action1 ..) (test2 action2 ..) ... (testn ..))`|The first clause whose test evaluates to non-nil is selected; all other clauses are ignored, and the consequents of the selected clause are evaluated in order. If none of the test conditions are evaluated to be true, then the cond statement returns nil.|This special form is method of control flow.|Language|
 |`(do a b c ...)`|`do` takes a list of s-expressions and evaluates them in the order they were given (in the current scope), and then returns the result of the last s-expression.|This special form allows lambda functions to have multi-step bodies.|Language|
 |`(scope a b c ...)`|`scope` takes a list of s-expressions and evaluates them in the order they were given _in a new scope_, and then returns the result of the last s-expression.|This special form allows the user to evaluate blocks of code in new scopes.|Language|
 |`(defn name params body)`|`defn` evaluates none of its arguments.|This special form allows the user to conveniently define functions.|Language|
+|`(defmacro name params body)`|`defmacro` evaluates none of its arguments.|This special form allows the user to conveniently define macros.|Language|
 |`(def name value)`|`def` evaluates the `value` argument, which is then assigned to `name` in the current scope.|This special form allows the user to bind atoms to values in a scope.|Language|
 |`(lambda params body)`|`lambda` evaluates none of its arguments.|This special form allows the user to define anonymous functions.|Language|
 |`(quote x)`|`quote` evaluates none of its arguments.|This is equivalent to the `'expr` syntactic sugar.|Language|
@@ -36,9 +37,17 @@
 |`(and ...)`|`and` evaluates logical and on its list argument.|`and` evaluates only until its true, it stops when becomes false.|Language|
 |`(or ...)`|`or` evaluates logical or on its list argument.|`or` evaluates while its false, it stops when becomes true.|Language|
 |`(benchmark msg_string ...)`|`benchmark` takes a list of s-expressions and evaluates them in the order they were given (in the current scope), and then returns the result of the last s-expression and prints msg_string and timing info.||Language|
+|`(set! x)`|`set!` ...|....|Language|
+
+## Macros
+|Signature|Description|Example|Section|
+|:-|-|-|-|
+|`(unles cond body)`|`unless` only evaluates its `cond` argument. If `cond` is not truthy, then `body` is evaluated.|`>>> (unless #t (print "it is #f")) => nil`|Language|
+|`(dotimes v n body)`|Iterates over `v` from 0 below `n` and evaluated `body`. Returns the last value of v|`>>> (dotimes i 5 i) => 4`|Language|
+|`(until cond body)`|Evaluated `body` as long as `cond` is not truthy|`>>> (def i 0)(until (> i 3) (set! i (inc i))) => 4`|Language|
 
 ## Library
-|Signature|Description|Return values|Section|
+|Signature|Description|Example|Section|
 |:-|-|-|-|
 |`(= a b)`|Test whether two values are equal|`1` when equal otherwise `nil`|Language|
 |`(!= a b)`|Test whether two values are not equal|`1` when not equal otherwise `nil`|Language|
@@ -54,60 +63,69 @@
 |`(list ..)`|Create a list of values||List manipulation|
 |`(insert list index element)`|Insert an element into a list. Indexed from 0|new list with value inserted|List manipulation|
 |`(index list index)`|Return element at index in list. First element is at index 0|Element at index|List manipulation|
-|`(remove list index)`|Remove a value at an index from a list|List with element removed|List manipulation|
-|`(len list)`|Get the length of a list|list length|List manipulation|
-|`(push list element)`|Add an item to the end of a list|new list with element added|List manipulation|
-|`(pop list)`|Returns last element of list|Last element|List manipulation|
+|`(remove list index)`|Remove a value at an index from a list|`>>> (remove '(1 2 3 4) 1) => (1 3 4)`|List manipulation|
+|`(len list)`|Get the length of a list|`>>> (len '(1 2 3 4)) => 4`|List manipulation|
+|`(push list element)`|Add an item to the end of a list|`>>> (push '(1 2 3 4) 5) => (1 2 3 4 5)`|List manipulation|
+|`(pop list)`|Returns last element of list|`>>> (pop '(1 2 3 4)) => 4`|List manipulation|
 |`(head list)`|Returns first element of a list|`>>> (head '(1 2 3)) => 1`|List manipulation|
 |`(tail list)`|Return all elements of list except first one|`>>> (tail '(1 2 3)) => (2 3)`|List manipulation|
 |`(first list)`|Returns first element of a list|`>>> (first '(1 2 3)) => 1`|List manipulation|
 |`(last list)`|Returns last element of list|`>>> (last '(1 2 3)) => 2`|List manipulation|
 |`(range low high)`|Returns list with elements in range low .. high|`(range 1 5) => (1 2 3 4)`|List manipulation|
-|`(member lst item)`|Returns #t when item is inluded in lst otherwise nil||List manipulation|
+|`(member lst item)`|Returns #t when item is inluded in lst otherwise nil|`>>> (member '(1 2 3) 1) => #t`|List manipulation|
 |`(uniq list)`|Filter out any duplicates from list|`>>> (uniq '(1 2 2 3 4 5 2 2)) => (1 2 3 4 5)`|List manipulation|
 |`(flatten list)`|Flattens a list to single level|`>>> (flatten '(1 (2 2) 3)) => (1 2 2 3)`|List manipulation|
 |`(take list count)`|Returns sublist with count element of list|`>>> (take '(1 2 3 4) 3) => (1 2 3)`|List manipulation|
 |`(make-list len)`|Return list with len nil elements|`(make-list 3) => (nil nil nil)`|List manipulation|
-|`(make-list-of len value)`|Return list with len value elements||List manipulation|
+|`(make-list-of size value)`|Makes list with size elements of values|`>>> (make-list-of 5 0) => (0 0 0 0 0)`|List manipulation|
+|`(concat-lists seq1 seq2)`|Appends all elements from seq2 to seq1|`>>> (concat-lists '(1 2 3) '(4 5 6)) => (1 2 3 4 5 6)`|List manipulation|
+|`(take lst size)`|Makes sublist of a lst with size length|`>>> (take '(1 2 3 4 5 6) 3) => (1 2 3)`|List manipulation|
+|`(find-val-in-list lst name)`|Returns tail od sublist which first element is name|`>>> (find-val-in-list '(("a" ("av" "avv")) ("b" "bv") (31 32 33) (41 42 43)) "b") => "bv" >>> `|List manipulation|
 |`(map ..)`|Returns list that is the result of executing lambda on its elements|`(map (lambda (x) (+ x 10)) '(1 2 3 4 5 6)) => (11 12 13 14 15 16)`|List manipulation|
 |`(filter lambda list)`|Returns list with elements for which passed lambda returns true|`(filter (lambda (x) (> x 2)) '(1 2 3 4 5)) => (3 4 5)`|List manipulation|
-|`(reduce lambda acumulator list)`|Reduces list|`>>> (reduce (lambda (x y) (+ (* x 10) y)) 0 '(1 2 3 4)) => 1234`|List manipulation|
+|`(reduce lambda acumulator list)`|Reduces list|`>>> (reduce (lambda (acc e) (+ (* acc 10) e)) 0 '(1 2 3 4)) => 1234`|List manipulation|
 |`(exit code)`|Exit the program with an integer code||System|
 |`(quit code)`|Same as (exit ..)||System|
-|`(print ..)`|Print several values and return the last one||IO|
-|`(input [prompt])`|Get user input with an optional prompt||IO|
-|`(random low high)`|Get a random number between two numbers inclusively||System|
+|`(print ..)`|Print one or several values separated by space and return the last one|`>>> (print "pi" "is" 3.14)\npi is 3.140000 => 3.140000`|IO|
+|`(random low high)`|Get a random number between two numbers inclusively|`>>> (random 1 6) => 5`|System|
 |`(include file)`|Read a file and execute its code||IO|
-|`(read-file filename)`|Get the contents of a file||IO|
+|`(input [prompt])`|Get user input with an optional prompt||IO|
+|`(input2 [prompt])`|Get user input using libnoise with an optional prompt||IO|
+|`(read)`|Reads in the printed representation of a Lisp object from input-stream, builds a corresponding Lisp object, and returns the object||IO|
+|`(read-file filename)`|Get the contents of a file|`>>> (read-file "/tmp/a.txt") => "test"`|IO|
 |`(read-file-lines filename lambda)`|Reads file and for each line call lambda with passing the line as a parameter|`(read-file-lines "/tmp/f.txt" (lambda (ln) (print ln))`|IO|
-|`(write-file filename content-str)`|Write a string to a file||IO|
-|`(read-url url [headers] [body] [method])`|Reads URL|Returns list (status-code content)|IO|
-|`(system-cmd command_str)`|Execute system command||System|
-|`(ls-dir dir)`|List a dir|List of directory entries|
-|`(is-file? filename)`|Returns true if passed filename is a file||IO|
-|`(is-dir? filename)`|Returns true if passed filename is a directory||IO|
+|`(write-file filename content-str)`|Write a string to a file|`>>> (write-file "/tmp/a.txt" "test") => #t`|IO|
+|`(read-url url [headers] [body] [method])`|Reads URL. Returns list (status-code content)||IO|
+|`(system-cmd command_str)`|Execute system command|`>>> (system-cmd-fork "ml"  "-c" "(print 123) (sleep 1) (print \"aaa\")") => (0 "") >>> 123\n aaa`|System|
+|`(system-cmd-fork cmd [par ..])`|Execute system command as an independent process|`>>> (system-cmd "date") => (0 "Fri Feb 25 12:35:28 CET 2022\n")`|System|
+|`(ls-dir dir)`|List a dir|List of directory entries|`>>> (ls-dir "/tmp") => ("." ".." "vscode-ipc-cccbe1dd-8c71-4028-a863-df975ad5887b.sock" "vscode-ipc-1163bb52-d088-41dc-80a5-81b6a7c7fa36.sock"  "vscode-ipc-630f21df-26b5-43d4-8b2e-5175d53ce317.sock")`|IO|
+|`(is-file? filename)`|Returns true if passed filename is a file|`>>> (is-file? "/tmp") => nil`|IO|
+|`(is-dir? filename)`|Returns true if passed filename is a directory|`>>> (is-dir? "/tmp") => #t`|IO|
+|`(mk-dir dirname)`|Creates directory wirh dirname. Does not create missing parent directories. Returns true if creation successfull|`>>> (mk-dir "/tmp/testdir") => #t`|IO|
+|`(rm-dir dirname)`|Removes directory wirh dirname. Does not create missing parent directories. Returns true if removal successfull|`>>> (rm-dir "/tmp/testdir") => #t`|IO|
 |`(tcp-server port handler)`|Starts listening on port and when request comes calls passed lambda and writes its returned value back to client. Lambda must return either string or two element list where first element is boolean and second string.When first element is true it closes listening socker.|(`tcp-server 7777 (lambda (str) (list #t (string-upcase str))))`|IO|
 |`(tcp-client address port data)`|Opens connection to server on port, writes there data and returns response.|`(print (tcp-client "127.0.0.1" 7777 "abcd"))`|IO|
-|`(is-dir? filename)`|Returns true if passed filename is a directory||IO|
-|`(parse-csv string)`|Parse CSV string||String manipulation|
-|`(parse-json json_string)`|Parse JSON string||String manipulation|
-|`(save-csv ..)`|Save list as csv||IO|
-|`(get-universal-time)`|Get current time as secs from epoch|`>>> (get-universal-time) => 1642152733`|Date and time|
-|`(get-localtime-offset)`|Offset in seconds between local time and gmt time||Date and time|
+|`(parse-csv string)`|Parse CSV string|`>>> (parse-csv "A,B\n1,1\n2,2") => ((1 1) (2 2))`|String manipulation|
+|`(parse-json json_string)`|Parse JSON string|`>>> (parse-json "{\"k1\":\"v1\", \"k2\":42, \"k3\":[\"a\",123,true,false,null]}") => (("k1" "v1") ("k2" 42) ("k3" ("a" 123 #t nil nil)))`|String manipulation|
+|`(make-csv ..)`|Returns list as csv string|`>>> (make-csv '(("A" "B")(1 1)(2 2))) => "A,B\n1,1\n2,2"`|String manipulation|
+|`(get-universal-time)`|Get current time as number of seconds from epoch|`>>> (get-universal-time) => 1642152733`|Date and time|
+|`(get-universal-time-ms)`|Get current time as number of miliseconds from epoch|`>>> (get-universal-time-ms) => 1653575251097`|Date and time|
+|`(get-localtime-offset)`|Offset in seconds between local time and gmt time|`>>> (get-localtime-offset) => 3600`|Date and time|
 |`(date-to-str date format)`|Converts date to formated string. Format is strftime format (https://www.tutorialspoint.com/c_standard_library/c_function_strftime.htm)|`>>> (date-to-str (get-universal-time) "%Y-%m-%d %H:%M:%S") => "2021-03-13 19:53:01"`|Date and time|
 |`(str-to-date string format)`|Converst string to time of secs since epoch. |`>>> (str-to-date "2021-03-13 19:53:01" "%Y-%m-%d %H:%M:%S") => 1615665181`|Date and time|
 |`(date-add date amount unit)`|Add number of units to date. A unit is one of 'year', 'month', 'day', 'hour', 'minute' or 'second'|`>>> (date-to-str (date-add (str-to-date "2021-03-13 19:53:01" "%Y-%m-%d %H:%M:%S") 10 "day") "%Y-%m-%d %H:%M:%S") => "2021-03-23 20:53:01"`|Date and time|
-|`start-of-day datetime`|Returns epoch time of start of a day|`>>> (start-of-day (str-to-date "2021-05-13 10:32:12" "%Y-%m-%d %H:%M:%S")) => 1620864000`|Date and time|
-|`end-of-day datetime`|Returns epoch time of end of day|`>>> (end-of-day (str-to-date "2021-05-13 10:32:12" "%Y-%m-%d %H:%M:%S")) => 1620950399`|Date and time|
-|`start-of-month datetime`|Returns epoch time of start of month|`>>> (start-of-month (str-to-date "2021-05-13 10:32:12" "%Y-%m-%d %H:%M:%S")) => 1619827200`|Date and time|
-|`start-of-next-month datetime`|Returns epoch time of start of following month|`>>> (start-of-next-month (str-to-date "2021-05-13 10:32:12" "%Y-%m-%d %H:%M:%S")) => 1622505599`|Date and time|
-|`end-of-next-month datetime`|Returns epoch time of end of following month|`>>> (end-of-next-month datetime (str-to-date "2021-05-13 10:32:12" "%Y-%m-%d %H:%M:%S")) => 1622505600`|Date and time|
-|`end-of-month datetime`|Returns epoch time of end of month|`>>> (end-of-month (str-to-date "2021-05-13 10:32:12" "%Y-%m-%d %H:%M:%S")) => 1625097599`|Date and time|
-|`start-of-prev-month datetime`|Returns epoch time of start of previous month|`>>> (start-of-prev-month (str-to-date "2021-05-13 10:32:12" "%Y-%m-%d %H:%M:%S")) => 1617235200`|Date and time|
-|`end-of-prev-month datetime`|Returns epoch time of end of previous month|`>>> (end-of-prev-month (str-to-date "2021-05-13 10:32:12" "%Y-%m-%d %H:%M:%S")) => 1619827199`|Date and time|
-|`start-of-year datetime`|Returns epoch time of start of year|`>>> (start-of-year (str-to-date "2021-05-13 10:32:12" "%Y-%m-%d %H:%M:%S")) => 1609459200`|Date and time|
-|`end-of-year datetime`|Returns epoch time of end of year|`>>> (end-of-year (str-to-date "2021-05-13 10:32:12" "%Y-%m-%d %H:%M:%S")) => 1640995199`|Date and time|
-|`(debug ..)`|TODO add me|TODO add me|IO|
+|`(start-of-day datetime)`|Returns epoch time of start of a day|`>>> (start-of-day (str-to-date "2021-05-13 10:32:12" "%Y-%m-%d %H:%M:%S")) => 1620864000`|Date and time|
+|`(end-of-day datetime)`|Returns epoch time of end of day|`>>> (end-of-day (str-to-date "2021-05-13 10:32:12" "%Y-%m-%d %H:%M:%S")) => 1620950399`|Date and time|
+|`(start-of-month datetime)`|Returns epoch time of start of month|`>>> (start-of-month (str-to-date "2021-05-13 10:32:12" "%Y-%m-%d %H:%M:%S")) => 1619827200`|Date and time|
+|`(start-of-next-month datetime)`|Returns epoch time of start of following month|`>>> (start-of-next-month (str-to-date "2021-05-13 10:32:12" "%Y-%m-%d %H:%M:%S")) => 1622505599`|Date and time|
+|`(end-of-next-month datetime)`|Returns epoch time of end of following month|`>>> (end-of-next-month datetime (str-to-date "2021-05-13 10:32:12" "%Y-%m-%d %H:%M:%S")) => 1622505600`|Date and time|
+|`(end-of-month datetime)`|Returns epoch time of end of month|`>>> (end-of-month (str-to-date "2021-05-13 10:32:12" "%Y-%m-%d %H:%M:%S")) => 1625097599`|Date and time|
+|`(start-of-prev-month datetime)`|Returns epoch time of start of previous month|`>>> (start-of-prev-month (str-to-date "2021-05-13 10:32:12" "%Y-%m-%d %H:%M:%S")) => 1617235200`|Date and time|
+|`(end-of-prev-month datetime)`|Returns epoch time of end of previous month|`>>> (end-of-prev-month (str-to-date "2021-05-13 10:32:12" "%Y-%m-%d %H:%M:%S")) => 1619827199`|Date and time|
+|`(start-of-year datetime)`|Returns epoch time of start of year|`>>> (start-of-year (str-to-date "2021-05-13 10:32:12" "%Y-%m-%d %H:%M:%S")) => 1609459200`|Date and time|
+|`(end-of-year datetime)`|Returns epoch time of end of year|`>>> (end-of-year (str-to-date "2021-05-13 10:32:12" "%Y-%m-%d %H:%M:%S")) => 1640995199`|Date and time|
+|`(decode-universal-time datetime)`|Returns the decoded time represented by the given universal time. Returns list with (second, minute, hour, day, month, year, day-of-the-week)|`>>> (decode-universal-time 0) => (0 0 0 1 1 1970 4)`|Date and time|
+|`(debug ..)`|Returns string representation of passed params|`>>> (debug '(1 2 3)) => "(1 2 3)"`|IO|
 |`(display ..)`|Displays passed parameters|`>>> (display '(1 2 3)) => "(1 2 3)"`|IO|
 |`(string-replace source substr replacement)`|Replace a substring with a replacement string in a source string|`>>> (string-replace "abcdefg" "de" "DE") => "abcDEfg"`|String manipulation|
 |`(string-replace-re source substr replacement)`|Replace a substring regex with a replacement string in a source string|`>>> (string-replace-re "there is a subsequence in the string" "\\b(sub)([^ ]*)" "sub-$2") => "there is a sub-sequence in the string"`|String manipulation|
@@ -116,38 +134,52 @@
 |`(string-pad str len char rpad_lpad)`|Pad string from start or to end with char to length||String manipulation|
 |`(string-lpad str len char)`|Pad string from start with char to length len|`>>> (string-lpad "0" 10 "x")  => "xxxxxxxxx0"`|String manipulation|
 |`(string-rpad str len char)`|Pad string from righ with char to length len|`>>> (string-rpad "0" 10 "x")  => "0xxxxxxxxx"`|String manipulation|
-|`(string-split str separator)`|Splits string into list by regexp|`>>> (string-split "split me by space" "\s+") => ("split" "me" "by" "space")`|
+|`(string-split str separator)`|Splits string into list by regexp|`>>> (string-split "split me by space" "\s+") => ("split" "me" "by" "space")`|String manipulation|
 |`(string-rltrim str len RKRRKR)`|Removes " \n\r\t" from the begininfg or end or both ends of str||Regex|
-|`(string-ltrim str)`|Removes " \n\r\t" from the begininfg of str||String manipulation|
-|`(string-rtrim str)`|Removes " \n\r\t" from the end of str||String manipulation|
-|`(string-trim str)`|Removes " \n\r\t" from the both strart and end of str||String manipulation|
-|`(string-case str RKRRKR)`|Returns up or down cased string||String manipulation|
+|`(string-ltrim str)`|Removes " \n\r\t" from the begininfg of str|`>>> (string-ltrim "   ***   ") => "***   "`|String manipulation|
+|`(string-rtrim str)`|Removes " \n\r\t" from the end of str|`>>> (string-rtrim "   ***   ") => "   ***"`|String manipulation|
+|`(string-trim str)`|Removes " \n\r\t" from the both strart and end of str|`>>> (string-trim "   ***   ") => "***"`|String manipulation|
+|`(string-case str UPLO)`|Returns up or down cased string based on second parameter (upper/lower)|`>>> (string-case "abc" "upper") => "ABC"`|String manipulation|
 |`(string-upcase str)`|Returns up cased string|`>>> (string-upcase "abcdefghchijklmn")  => "ABCDEFGHCHIJKLMN"`|String manipulation|
 |`(string-downcase str)`|Returns down cased string |`>>> (string-downcase "ABCDefghchijklmn") => "abcdefghchijklmn"`|String manipulation|
 |`(string-join lst sep)`|Returns string created as elements of concatenation of lst elements separated by sep|`>>> (string-join ("A" "B" "C" "D") ",") => "A,B,C,D"`|String manipulation|
 |`(string-len str)`|Returns string length|`>>> (string-len "abcdef") => 6`|String manipulation|
 |`(string-substr str pos len)`|Returns substring from str starting at pos with len. If pos is negative returns substring from the end of string. First character is on pos 0|`>>> (string-substr "ABCD" -2 2) => "CD"`|String manipulation|
 |`(string-find str lookup pos)`|Returns position of lookup in str starting on position. First char index is 0. If not found returns nil|`>>> (string-find " long long int;" "long" 2) => 6`|String manipulation|
+|`(string-cmp str1 str2)`|Compares two strings. Returns 0 where strangs are equal; < 0 when the value of the first character that does not match is lower in the compared string, or all compared characters match but the compared string is shorter.; >0 when the value of the first character that does not match is greater in the compared string, or all compared characters match but the compared string is longer.|`>>> (string-cmp "aaa" "xaa") => -23`|String manipulation|
+|`(string-cmp-ic str1 str2)`|Compares two strings ignoring case. Returns 0, < 0, > 0.|`>>> (string-cmp-ic "aaa" "AaA") => 0`|String manipulation|
 |`(int value)`|Cast an item to an int|`>>> (int 3.41) => 3`|Type casting|
 |`(float value)`|Cast item to a float|`>>> (int 3.41) => 3.14`|Type casting|
 |`(string value)`|Cast int or float item to a string|`>>> (string 3.14) => "3.14"`|Type casting|
 |`(eval <exp>)`|Eval returns the value of the second evaluation|`>>> (eval '(+ 1 2)) => 3`|Language|
 |`(type e)`|Returns data type of e|`>>> (type (+ 1 2)) => "int"`|Type casting|
+|`(list? e)`|Returns true if type of e is list|`>>> (list? 12) => nil >>> (list? '(1 2)) => #t`|Type casting|
+|`(empty-list? e)`|Returns true if type of e is empty list|`>>> (empty-list? '(1 2)) => nil >>> (empty-list? '()) => #t`|Type casting|
+|`(string? e)`|Returns true if type of e is string|`>>> (string? "str") => #t`|Type casting|
+|`(int? e)`|Returns true if type of e is integer|`>>> (int? "str") => nil >>> (int? 9) => #t`|Type casting|
+|`(float? e)`|Returns true if type of e is float|`>>> (float? 9) => nil >>> (float? 9.0) => #t`|Type casting|
+|`(nil? e)`|Returns true if type of e is nil|`>>> (nil? 9.0) => nil >>> (nil? nil) => #t >>> (nil? '()) => nil`|Type casting|
+|`(true? e)`|Returns true if type of e is #t|`>>> (true? 1) => nil >>> (true? #t) => #t`|Type casting|
 |`(parse ..)`|Parses string to be evaluated|`>>> (eval (first (parse "(+ 1 2)"))) => 3`|Language|
-|`(make-list-of size value)`|Makes list with size elements of values|`>>> (make-list-of 5 0) => (0 0 0 0 0)`|List manipulation|
-|`(make-list size)`|Makes list of nil values with size length|`>>> (make-list 5) => (nil nil nil nil nil)`|List manipulation|
-|`(empty-list? lst)`|Return true is lst is list with zero elements|`>>> (empty-list? '()) => 1`|Type casting|
 |`(quick-sort-by list cmp)`|Returns list sorted by comparsion function||Language|
 |`(quick-sort list)`|Return sorted list|`>>> (quick-sort '(2 4 6 1 7 3 3 9 5)) => (1 2 3 3 4 5 6 7 9)`|Language|
 |`(quick-sort-reverse list)`|Return reverse sorted list|`>>> (quick-sort-reverse '(2 4 6 1 7 3 3 9 5)) => (9 7 6 5 4 3 3 2 1)`|Language|
-|`(not c)`|Logical NOT of c|`>>> (not 1) => nil`|Logical|
+|`(not c)`|Logical NOT of c|`>>> (not 1) => nil`|Language|
 |`(neg n)`|Negates number|`>>> (neg -5) => 5`|Language|
-|`(is-pos? n)`|Returns true if n is positive number||Language|
-|`(is-neg? n)`|Returns true if n is negative number||Language|
+|`(is-pos? n)`|Returns true if n is positive number|`>>> (is-pos? -1) => nil`|Language|
+|`(is-neg? n)`|Returns true if n is negative number|`>>> (is-neg? -1) => #t`|Language|
 |`(dec n)`|Return n decremented by 1|`>>> (dec 5) => 4`|Language|
 |`(inc n)`|Return n incremented by 1|`>>> (inc 5) => 6`|Language|
-|`(sleep time)`|Pauses execution for time interval of seconds||System|
+|`(sleep time)`|Pauses execution for time interval of seconds. Calls system sleep command|`>>> (sleep 1) => (0 "")`|System|
+|`(function? e)`|Returns true if type of e is lambda, macro or builtin|`>>> (function? index) => #t`|Language|
+|`(unless test v)`|If test is not truthy then v is evaluated. This is macro.|`>>> (unless (> 1 2) "1 < 2") => "1 < 2"`|Language|
+|`(dotimes v n body)`|Evaluates body n-times. For each eveluation v is set to value between 0 and n minus one|`(dotimes i 10 (print "i :" i))`|Language|
+|`(doc::man "func_name")`|Prints short help for func_name||Language|
+|`(doc::appropos "string")`|Looks for functions to be related passed string. String can be more words separated by space||Language|
+|`(doc::look "string")`|Alias for appropos||Language|
+|`(doc::lookup "string")`|Alias for appropos||Language|
 |`(get-env var)`|Return environment variable var|`>>> (get-env "HOME") => "/Users/vaclavt"`|System|
+|`(set-env var value)`|Sets environment variable var to value|`>>> (set-env "XXYYZZ" "haha") => "haha"`|System|
 |`(second list)`|Returns second element of list|`>>> (second '(1 2 3 4 5 6 7)) => 2`|List manipulation|
 |`(third list)`|Returns third element of list|`>>> (third '(1 2 3 4 5 6 7)) => 3`|List manipulation|
 |`(fourth list)`|Returns fourth element of list|`>>> (fourth '(1 2 3 4 5 6 7)) => 4`|List manipulation|
@@ -157,8 +189,7 @@
 |`(sprintf ..)`|Writes string pointed by format to the standard output|`>>>  (sprintf "%s, %d, %.2f" (list "string" 1000 3.14)) => "string, 1000, 3.14"`|String manipulation|
 |`(thread-create code..)`|Creates new thread, starts evalueating code and returns immediatelly thread id||Threading|
 |`(thread-under-lock lockname code)`|Acquire lock with lockname and eval code. "ilock" currently is only allowed lockname||Threading|
-|`(thread-sleep milisecons)`|Sleeps thread for given amount of miliseconds||Threading|
+|`(thread-sleep milisecons)`|Sleeps thread for given amount of miliseconds|`>>> (thread-sleep 100) => 100`|Threading|
 |`(threads-join)`|Wait for all running threads to finish||Threading|
 |`(try block catch_block [finally_block])`|Evaluates block and if an exception is thrown, evaluates catch_block.Eventually in both cases evals finally_block. Return evaluated last expression from block if no exception or last expression from catch_block, if exception is thrown from block. Variable ml-exception in catch block is available with astring describing exception||Exceptions|
-|`(throw-exception exp_desc)`|Throws an exception with exp_desc describing what happened ||Exceptions|
-|`(xx ..)`|Desc|example|section|
+|`(throw exp_desc)`|Throws an exception with exp_desc describing what happened. exp_desc will be evaluated a returned as string|`>>> (throw (+ 1 2))\n3`|Exceptions|

doc/readme.txt

@@ -0,0 +1,22 @@
+to convert Doc.md to Doc.html use https://markdowntohtml.com/ with custom ccs style
+
+body {
+   color: black;
+}
+table {
+  border-collapse: collapse;
+}
+table, th, td {
+  border: 1px solid;
+}
+th {
+  text-align: left;
+}
+td {
+  height: 50px;
+  vertical-align: top;
+}
+code{
+    background: #0003;
+    color: #a31515;
+}

docker/Dockerfile

@@ -0,0 +1,31 @@
+# docker build -t ml:latest .
+# docker run --rm -it ml ml -v
+# docker run --rm -it --entrypoint sh ml
+
+
+FROM alpine:3.15.4 AS builder
+
+# Install all dependencies required for compiling ml
+RUN apk add --verbose build-base musl-dev openssl-dev make git cmake 
+
+RUN git clone http://gitea.stocksriddle.one/vaclavt/mlisp.git
+
+# Compile 
+RUN cd /mlisp \
+  && rm -f CMakeCache.txt \
+  && cmake -DCMAKE_BUILD_TYPE=Release . \
+  && cmake --build ./ --target clean -j 4 -- \
+  && cmake --build ./ --target all -j 4 --
+
+
+# Create image and copy compiled installation into it
+FROM alpine:3.15.4
+
+RUN apk add --no-cache openssl libstdc++
+
+CMD mkdir -p /usr/local/var/mlisp/
+COPY --from=builder /mlisp/stdlib/*.lsp /usr/local/var/mlisp/
+COPY --from=builder /mlisp/doc/*.md /usr/local/var/mlisp/
+
+CMD mkdir -p /usr/local/bin/
+COPY --from=builder /mlisp/ml /usr/local/bin/

ml.cpp

@@ -21,10 +21,12 @@
 #include <sstream>
 #include <cstdlib>
 #include <iostream>
+#include <fstream>
 #include <ctime>
 #include <chrono>
 #include <thread>
 #include <mutex>
+#include <numeric>
 
 
 #define TOO_FEW_ARGS "too few arguments to function"
@@ -47,6 +49,8 @@
 #define INDEX_OUT_OF_RANGE "index out of range"
 #define MALFORMED_PROGRAM "malformed program"
 #define NOT_IMPLEMENTED_YET_ERROR "not implemented yet"
+#define DIVISION_BY_ZERO "division by zero"
+#define INVALID_HEADER_FORMAT "invalid header format"
 
 
 #define STRING_TYPE "string"
@@ -123,7 +127,7 @@ MlValue MlValue::nil() {
 }
 
 // Construct a lambda function
-MlValue::MlValue(const std::vector<MlValue> &params, MlValue ret, MlEnvironment const &env) : type(LAMBDA) {
+MlValue::MlValue(const std::vector<MlValue> &params, MlValue ret, MlEnvironment const &env, const Type ftype) : type(ftype) {
 	// We store the params and the result in the list member
 	// instead of having dedicated members. This is to save memory.
 	list.emplace_back(params);
@@ -162,7 +166,8 @@ std::vector<std::string> MlValue::get_used_atoms() {
 			result.push_back(as_atom());
 			return result;
 		case LAMBDA:
-			// If this is a lambda, get the list of used atoms in the body
+		case MACRO:
+			// If this is a lambda or macro, get the list of used atoms in the body
 			// of the expression.
 			return list[1].get_used_atoms();
 		case LIST:
@@ -180,10 +185,18 @@ std::vector<std::string> MlValue::get_used_atoms() {
 	}
 }
 
+bool MlValue::is_nil() const {
+	return type == NIL;
+}
+
 bool MlValue::is_builtin() const {
 	return type == BUILTIN;
 }
 
+bool MlValue::is_macro() const {
+	return type == MACRO;
+}
+
 bool MlValue::is_number() const {
 	return type == INT || type == FLOAT;
 }
@@ -318,8 +331,9 @@ bool MlValue::operator==(MlValue other) const {
 			// data in the str member.
 			return str == other.str;
 		case LAMBDA:
+		case MACRO:
 		case LIST:
-			// Both lambdas and lists store their
+			// All lambdas, macros and lists store their
 			// data in the list member.
 			return list == other.list;
 		case QUOTE:
@@ -526,9 +540,10 @@ std::string MlValue::get_type_name() const {
 			return STRING_TYPE;
 		case BUILTIN:
 		case LAMBDA:
+		case MACRO:
 			// Instead of differentiating between
-			// lambda and builtin types, we group them together.
-			// This is because they are both callable.
+			// lambda, macro and builtin types, we group them together.
+			// This is because they are all callable.
 			return FUNCTION_TYPE;
 		case NIL:
 			return NIL_TYPE;
@@ -541,7 +556,6 @@ std::string MlValue::get_type_name() const {
 }
 
 std::string MlValue::display() const {
-	std::string result;
 	switch (type) {
 		case QUOTE:
 			return "'" + list[0].debug();
@@ -554,17 +568,14 @@ std::string MlValue::display() const {
 		case STRING:
 			return str;
 		case LAMBDA:
-			for (size_t i = 0; i < list.size(); i++) {
-				result += list[i].debug();
-				if (i < list.size() - 1) result += " ";
-			}
-			return "(lambda " + result + ")";
+			return "(lambda " + std::accumulate(list.begin(), list.end(), std::string(), [](const std::string &a, const MlValue &b) -> std::string
+							   {return a + (a.empty() ? "" : " ") + b.debug(); }) + ")";
+		case MACRO:
+			return "(macro " + std::accumulate(list.begin(), list.end(), std::string(), [](const std::string &a, const MlValue &b) -> std::string
+							   {return a + (a.empty() ? "" : " ") + b.debug(); }) + ")";
 		case LIST:
-			for (size_t i = 0; i < list.size(); i++) {
-				result += list[i].debug();
-				if (i < list.size() - 1) result += " ";
-			}
-			return "(" + result + ")";
+			return "(" + std::accumulate(list.begin(), list.end(), std::string(), [](const std::string &a, const MlValue &b) -> std::string
+							   {return a + (a.empty() ? "" : " ") + b.debug(); }) + ")";
 		case BUILTIN:
 			return "<" + str + " at " + std::to_string(long(stack_data.b)) + ">";
 		case NIL:
@@ -580,41 +591,15 @@ std::string MlValue::display() const {
 std::string MlValue::debug() const {
 	std::string result;
 	switch (type) {
-		case QUOTE:
-			return "'" + list[0].debug();
-		case ATOM:
-			return str;
-		case INT:
-			return std::to_string(stack_data.i);
-		case FLOAT:
-			return std::to_string(stack_data.f);
 		case STRING:
+			result.reserve(str.size());
 			for (char c : str) {
 				if (c == '"') result += "\\\"";
 				else result.push_back(c);
 			}
 			return "\"" + result + "\"";
-		case LAMBDA:
-			for (size_t i = 0; i < list.size(); i++) {
-				result += list[i].debug();
-				if (i < list.size() - 1) result += " ";
-			}
-			return "(lambda " + result + ")";
-		case LIST:
-			for (size_t i = 0; i < list.size(); i++) {
-				result += list[i].debug();
-				if (i < list.size() - 1) result += " ";
-			}
-			return "(" + result + ")";
-		case BUILTIN:
-			return "<" + str + " at " + std::to_string(long(stack_data.b)) + ">";
-		case NIL:
-			return "nil";
-		case TRUE:
-			return "#t";
 		default:
-			// This should never be reached.
-			throw MlError(*this, MlEnvironment(), INTERNAL_ERROR);
+			return display();
 	}
 }
 
@@ -646,19 +631,15 @@ const char * MlError::what() const noexcept {
 }
 
 void MlEnvironment::combine(MlEnvironment const &other) {
-	// Normally, I would use the `insert` method of the `map` class,
-	// but it doesn't overwrite previously declared values for keys.
-	auto itr = other.defs.begin();
-	for (; itr != other.defs.end(); itr++) {
-		// Iterate through the keys and assign each value.
+	// We need to overwrite previously declared values for keys.
+	for (auto itr = other.defs.begin(); itr != other.defs.end(); itr++) {
 		defs[itr->first] = itr->second;
 	}
 }
 
 std::ostream &operator<<(std::ostream &os, MlEnvironment const &e) {
-	auto itr = e.defs.begin();
 	os << "{ ";
-	for (; itr != e.defs.end(); itr++) {
+	for (auto itr = e.defs.begin(); itr != e.defs.end(); itr++) {
 		os << '\'' << itr->first << "' : " << itr->second.debug() << ", ";
 	}
 	return os << "}";
@@ -687,8 +668,11 @@ void MlEnvironment::setX(const std::string &name, const MlValue& value) {
 MlValue MlValue::apply(std::vector<MlValue> args, MlEnvironment &env) {
 	MlEnvironment e;
 	std::vector<MlValue> params;
+	MlValue macro_eval;
+
 	switch (type) {
 		case LAMBDA:
+		case MACRO:
 			// Get the list of parameter atoms
 			params = list[0].list;
 			if (params.size() != args.size())
@@ -708,8 +692,17 @@ MlValue MlValue::apply(std::vector<MlValue> args, MlEnvironment &env) {
 				e.set(params[i].str, args[i]);
 			}
 
+			if (type == LAMBDA) {
 				// Evaluate the function body with the function scope
 				return list[1].eval(e);
+			} else {
+				// macro evals twice
+				macro_eval = list[1].eval(e);
+				if (MlPerfMon::instance().isDebugOn())
+					std::cout << std::endl << "DEBUG macro 1st eval: " << macro_eval.debug() << std::endl << std::endl;
+
+				return macro_eval.eval(e);
+			}
 		case BUILTIN:
 			// Here, we call the builtin function with the current scope.
 			// This allows us to write special forms without syntactic sugar.
@@ -740,11 +733,11 @@ MlValue MlValue::eval(MlEnvironment &env) {
 			args = std::vector<MlValue>(list.begin() + 1, list.end());
 
 			function = list[0].eval(env);
-			if (function.type == BUILTIN || function.type == LAMBDA) {
+			if (function.type == BUILTIN || function.type == LAMBDA || function.type == MACRO) {
 				// Only evaluate our arguments if it's not builtin!
 				// Builtin functions can be special forms, so we
 				// leave them to evaluate their arguments.
-				if (!function.is_builtin())
+				if (!function.is_builtin() && !function.is_macro())
 					for (auto & arg : args)
 						arg = arg.eval(env);
 
@@ -785,7 +778,6 @@ MlValue parse(std::string &s, int &ptr) {
 	if (s[ptr] == ';')
 		throw std::runtime_error(INTERNAL_ERROR);
 
-
 	if (s.empty()) {
 		return MlValue();
 
@@ -906,8 +898,7 @@ MlValue run(const std::string &code, MlEnvironment &env) {
 	if (parsed.empty())
 		return MlValue::nil();
 
-	// Iterate over the expressions and evaluate them
-	// in this environment.
+	// Iterate over the expressions and evaluate them in this environment.
 	for (size_t i = 0; i < parsed.size() - 1; i++)
 		parsed[i].eval(env);
 
@@ -936,7 +927,7 @@ MlValue lambda(std::vector<MlValue> args, MlEnvironment &env) {
 	if (args[0].get_type_name() != LIST_TYPE)
 		throw MlError(MlValue("lambda", lambda), env, INVALID_LAMBDA);
 
-	return MlValue(args[0].as_list(), args[1], env);
+	return MlValue(args[0].as_list(), args[1], env, MlValue::Type::LAMBDA);
 }
 
 // if-else (SPECIAL FORM)
@@ -992,7 +983,21 @@ MlValue defun(std::vector<MlValue> args, MlEnvironment &env) {
 	if (args[1].get_type_name() != LIST_TYPE)
 		throw MlError(MlValue("defn", defun), env, INVALID_LAMBDA);
 
-	MlValue f = MlValue(args[1].as_list(), args[2], env);
+	MlValue f = MlValue(args[1].as_list(), args[2], env, MlValue::Type::LAMBDA);
+	env.set(args[0].display(), f);
+	return f;
+}
+
+// Define a macro with parameters and a result expression (SPECIAL FORM)
+MlValue defmacro(std::vector<MlValue> args, MlEnvironment &env)
+{
+	if (args.size() != 3)
+		throw MlError(MlValue("defmacro", defmacro), env, args.size() > 3 ? TOO_MANY_ARGS : TOO_FEW_ARGS);
+
+	if (args[1].get_type_name() != LIST_TYPE)
+		throw MlError(MlValue("defmacro", defmacro), env, INVALID_LAMBDA);
+
+	MlValue f = MlValue(args[1].as_list(), args[2], env, MlValue::Type::MACRO);
 	env.set(args[0].display(), f);
 	return f;
 }
@@ -1130,6 +1135,24 @@ MlValue input(std::vector<MlValue> args, MlEnvironment &env) {
 	return MlValue::string(s);
 }
 
+// Get user input with an optional prompt using line noise
+MlValue input2(std::vector<MlValue> args, MlEnvironment &env) {
+	// TODO add setup etc
+	eval_args(args, env);
+
+	if (args.size() > 1)
+		throw MlError(MlValue("input2", input2), env, TOO_MANY_ARGS);
+
+	char *line = linenoise(args.empty() ? ">>> " : args[0].as_string().c_str());
+	if (line == nullptr) throw std::runtime_error("date_to_string, cannot initialise linenoise");
+
+	std::string input{line};
+	linenoise_add_to_history(input);
+
+
+	return MlValue::string(input);
+}
+
 // Get a random number between two numbers inclusively
 MlValue random(std::vector<MlValue> args, MlEnvironment &env) {
 	eval_args(args, env);
@@ -1153,6 +1176,18 @@ MlValue parse_csv(std::vector<MlValue> args, MlEnvironment &env) {
 	return csv.parseCSV(args[0].as_string());
 }
 
+// Reads in the printed representation of a Lisp object from input-stream, builds a corresponding Lisp object, and returns the object
+MlValue read(std::vector<MlValue> args, MlEnvironment &env) {
+	eval_args(args, env);
+
+	if (args.size() != 0)
+		throw MlError(MlValue("read", read), env, TOO_MANY_ARGS);
+
+	std::string line;
+	std::getline(std::cin, line);
+	return run(line, env);
+}
+
 // Get the contents of a file
 MlValue read_file(std::vector<MlValue> args, MlEnvironment &env) {
 	eval_args(args, env);
@@ -1213,8 +1248,10 @@ MlValue read_url(std::vector<MlValue> args, MlEnvironment &env) {
 	// headers
 	if (args.size() > 1) {
 		for (const auto &hdr_val_pair: args[1].as_list()) {
-			// TODO check its 2 string elements list
 			const auto &pair = hdr_val_pair.as_list();
+			if (pair.size() != 2)
+				throw MlError(MlValue("read_url", read_url), env, INVALID_HEADER_FORMAT);
+
 			headers[pair[0].as_string()] = pair[1].as_string();
 		}
 	}
@@ -1237,7 +1274,6 @@ MlValue read_url(std::vector<MlValue> args, MlEnvironment &env) {
 MlValue parse_json(std::vector<MlValue> args, MlEnvironment &env) {
 	eval_args(args, env);
 
-	// TODO add support for more params specifying options
 	if (args.size() != 1)
 		throw MlError(MlValue("parse-json", parse_json), env, args.size() > 1 ? TOO_MANY_ARGS : TOO_FEW_ARGS);
 
@@ -1245,10 +1281,9 @@ MlValue parse_json(std::vector<MlValue> args, MlEnvironment &env) {
 	std::string err;
 	auto json = json11::Json::parse(str, err);
 
-	if (!err.empty()) {
-		// TODO handle error
+	if (!err.empty())
 		return MlValue::string("ERROR json parsing: " + err);
-	}
+
 
 	return json.ivalualize();
 }
@@ -1263,6 +1298,16 @@ MlValue get_universal_time(std::vector<MlValue> args, MlEnvironment &env) {
 	return MlValue(now());
 }
 
+// Get current time as miliseconds from epoch
+MlValue get_universal_time_ms(std::vector<MlValue> args, MlEnvironment &env) {
+	eval_args(args, env);
+
+	if (!args.empty())
+		throw MlError(MlValue("get-universal-time-ms", get_universal_time_ms), env, TOO_MANY_ARGS);
+
+	return MlValue(now_ms());
+}
+
 // Get offsets in secs between local timezone and gmt
 MlValue get_localtime_offset(std::vector<MlValue> args, MlEnvironment &env) {
 	eval_args(args, env);
@@ -1315,6 +1360,22 @@ MlValue system_cmd(std::vector<MlValue> args, MlEnvironment &env) {
 	return exec_system_cmd(args[0].as_string());
 }
 
+// Execute system command as forked process so its independent from its parrent
+MlValue system_cmd_fork(std::vector<MlValue> args, MlEnvironment &env) {
+	eval_args(args, env);
+
+	// TODO add support for more params constructing options as one string
+	if (args.size() < 1)
+		throw MlError(MlValue("system-cmd-fork", system_cmd_fork), env, TOO_FEW_ARGS);
+
+	std::vector<std::string> exec_args;
+	exec_args.reserve(args.size());
+	for (auto const& a : args)
+			exec_args.emplace_back(a.as_string());
+
+	return exec_system_cmd_fork(exec_args);
+}
+
 
 // list directory
 MlValue ls_dir(std::vector<MlValue> args, MlEnvironment &env) {
@@ -1346,6 +1407,25 @@ MlValue is_dir(std::vector<MlValue> args, MlEnvironment &env) {
 	return MlValue(is_path_dir(args[0].as_string()));
 }
 
+MlValue mk_dir(std::vector<MlValue> args, MlEnvironment &env) {
+	eval_args(args, env);
+
+	if (args.size() != 1)
+		throw MlError(MlValue("mk-dir", mk_dir), env, args.size() > 1 ? TOO_MANY_ARGS : TOO_FEW_ARGS);
+
+	return MlValue((bool)mk_path_dir(args[0].as_string()));
+}
+
+MlValue rm_dir(std::vector<MlValue> args, MlEnvironment &env) {
+	eval_args(args, env);
+
+	if (args.size() != 1)
+		throw MlError(MlValue("rm-dir", rm_dir), env, args.size() > 1 ? TOO_MANY_ARGS : TOO_FEW_ARGS);
+
+	return MlValue((bool)rm_path_dir(args[0].as_string()));
+}
+
+// starts tcp listening server
 MlValue tcp_server(std::vector<MlValue> args, MlEnvironment &env) {
 	eval_args(args, env);
 
@@ -1369,6 +1449,7 @@ MlValue tcp_server(std::vector<MlValue> args, MlEnvironment &env) {
 	return MlValue((long)r);
 }
 
+// tcp client
 MlValue tcp_client(std::vector<MlValue> args, MlEnvironment &env) {
 	eval_args(args, env);
 
@@ -1389,18 +1470,40 @@ MlValue tcp_client(std::vector<MlValue> args, MlEnvironment &env) {
 	}
 }
 
+// get environment variable
+MlValue get_env(std::vector<MlValue> args, MlEnvironment &env) {
+	eval_args(args, env);
+
+	if (args.size() != 1)
+		throw MlError(MlValue("get-env", get_env), env, args.size() > 1 ? TOO_MANY_ARGS : TOO_FEW_ARGS);
+
+	if (const char* env_p = std::getenv(args[0].as_string().c_str()))
+		return MlValue::string(env_p);
+	else
+		// TODO maybe better to return MlValue::nil();
+		return MlValue::string("");
+}
+
+// set environment variable
+MlValue set_env(std::vector<MlValue> args, MlEnvironment &env) {
+	eval_args(args, env);
+
+	if (args.size() != 2)
+		throw MlError(MlValue("set-env", set_env), env, args.size() > 2 ? TOO_MANY_ARGS : TOO_FEW_ARGS);
+	
+	setenv(args[0].as_string().c_str(), args[1].as_string().c_str(), true);
+
+	return args[1];
+}
+
 // Read a file and execute its code
 MlValue include(std::vector<MlValue> args, MlEnvironment &env) {
-	// Import is technically not a special form, it's more of a macro.
-	// We can evaluate our arguments.
 	eval_args(args, env);
 
 	if (args.size() != 1)
 		throw MlError(MlValue("include", include), env, args.size() > 1 ? TOO_MANY_ARGS : TOO_FEW_ARGS);
 
-	MlEnvironment e;
-	MlValue result = run(read_file_contents(args[0].as_string()), e);
-	env.combine(e);
+	MlValue result = run(read_file_contents(args[0].as_string()), env);
 	return result;
 }
 
@@ -1466,7 +1569,7 @@ MlValue divide(std::vector<MlValue> args, MlEnvironment &env) {
 
 	if ((args[1].get_type_name() == "int" && args[1] == 0l) ||
 	    (args[1].get_type_name() == "float" && args[1] == 0.0))
-		throw std::invalid_argument("divide by zero.");
+		throw std::invalid_argument(DIVISION_BY_ZERO);
 
 	return args[0] / args[1];
 }
@@ -1821,6 +1924,19 @@ MlValue string_find(std::vector<MlValue> args, MlEnvironment &env) {
 	return pos == -1 ? MlValue::nil() : MlValue((long) pos);
 }
 
+MlValue string_cmp(std::vector<MlValue> args, MlEnvironment &env) {
+	eval_args(args, env);
+
+	if (args.size() != 2)
+		throw MlError(MlValue("string-cmp", string_cmp), env, args.size() > 2 ? TOO_MANY_ARGS : TOO_FEW_ARGS);
+
+	if ( (args[0].is_nil() || (args[0].is_list() && args[0].as_list().empty())) ||
+		 (args[1].is_nil() || (args[1].is_list() && args[1].as_list().empty())) )
+		 	return MlValue::nil();
+
+
+	return MlValue((long)args[0].as_string().compare(args[1].as_string()));
+}
 
 // trims characters " \n\r\t" from left or right or both ends of a string
 MlValue string_rltrim(std::vector<MlValue> args, MlEnvironment &env) {
@@ -1952,7 +2068,6 @@ MlValue range(std::vector<MlValue> args, MlEnvironment &env) {
 
 // Benchmarks a block of expressions in the current environment (SPECIAL FORM)
 MlValue benchmark(std::vector<MlValue> args, MlEnvironment &env) {
-	// TODO add some memory stats
 	using namespace std::chrono;
 	high_resolution_clock::time_point t1 = high_resolution_clock::now();
 
@@ -1976,9 +2091,10 @@ MlValue thread_create(std::vector<MlValue> args, MlEnvironment &env) {
 			    MlValue acc = arg.eval(env);
 		} catch (const MlError &e) {
 			std::cerr << "thread_create exception: " << e.description() << std::endl;
+		    throw;
 		} catch (const std::exception &e) {
 		    std::cerr << "thread_create exception: " << e.what() << std::endl;
-		    throw e;
+		    throw;
 	    }
 	};
 
@@ -2061,10 +2177,12 @@ MlValue try_block(std::vector<MlValue> args, MlEnvironment &env) {
 }
 
 MlValue throw_exception(std::vector<MlValue> args, MlEnvironment &env) {
+	eval_args(args, env);
+
 	if (args.size() != 1)
 		throw MlError(MlValue("throw", throw_exception), env, args.size() > 1 ? TOO_MANY_ARGS : TOO_FEW_ARGS);
 
-	throw std::runtime_error(args[0].as_string());
+	throw std::runtime_error(args[0].cast_to_string().as_string());
 }
 
 MlValue usql(std::vector<MlValue> args, MlEnvironment &env) {
@@ -2078,9 +2196,6 @@ MlValue usql(std::vector<MlValue> args, MlEnvironment &env) {
 
 }	// namespace builtin
 
-void load_std_lib(MlEnvironment &env) {
-	run(STDLIB_LOADER, env);
-}
 
 // Does this environment, or its parent environment, have a variable?
 bool MlEnvironment::has(const std::string &name) const {
@@ -2097,7 +2212,7 @@ bool MlEnvironment::has(const std::string &name) const {
 
 
 
-std::map <const std::string, Builtin> builtin_funcs
+std::map<const std::string, Builtin> builtin_funcs
 {
 	// Special forms
 	std::make_pair("def", builtin::define),
@@ -2110,6 +2225,7 @@ std::map <const std::string, Builtin> builtin_funcs
 	std::make_pair("scope", builtin::scope),
 	std::make_pair("quote", builtin::quote),
 	std::make_pair("defn", builtin::defun),
+	std::make_pair("defmacro", builtin::defmacro),
 	std::make_pair("and", builtin::do_and),
 	std::make_pair("or", builtin::do_or),
 	std::make_pair("set!", builtin::setx),
@@ -2158,19 +2274,26 @@ std::map <const std::string, Builtin> builtin_funcs
 	std::make_pair("exit", builtin::exit),
 	std::make_pair("quit", builtin::exit),
 	std::make_pair("print", builtin::print),
-	std::make_pair("input", builtin::input),
 	std::make_pair("random", builtin::random),
 	std::make_pair("include", builtin::include),
+	std::make_pair("input", builtin::input),
+	std::make_pair("input2", builtin::input2),
+	std::make_pair("read", builtin::read),
 	std::make_pair("read-file", builtin::read_file),
 	std::make_pair("read-file-lines", builtin::read_file_lines),
 	std::make_pair("write-file", builtin::write_file),
 	std::make_pair("read-url", builtin::read_url),
 	std::make_pair("system-cmd", builtin::system_cmd),
+	std::make_pair("system-cmd-fork", builtin::system_cmd_fork),
 	std::make_pair("ls-dir", builtin::ls_dir),
 	std::make_pair("is-file?", builtin::is_file),
 	std::make_pair("is-dir?", builtin::is_dir),
+	std::make_pair("mk-dir", builtin::mk_dir),
+	std::make_pair("rm-dir", builtin::rm_dir),
 	std::make_pair("tcp-server", builtin::tcp_server),
 	std::make_pair("tcp-client", builtin::tcp_client),
+	std::make_pair("get-env", builtin::get_env),
+	std::make_pair("set-env", builtin::set_env),
 
 	// parsing operations
 	std::make_pair("parse-csv", builtin::parse_csv),
@@ -2178,6 +2301,7 @@ std::map <const std::string, Builtin> builtin_funcs
 
 	// Datetime operations
 	std::make_pair("get-universal-time", builtin::get_universal_time),
+	std::make_pair("get-universal-time-ms", builtin::get_universal_time_ms),
 	std::make_pair("get-localtime-offset", builtin::get_localtime_offset),
 	std::make_pair("date-to-str", builtin::date_to_str),
 	std::make_pair("str-to-date", builtin::str_to_date),
@@ -2198,6 +2322,7 @@ std::map <const std::string, Builtin> builtin_funcs
 	std::make_pair("string-len", builtin::string_len),
 	std::make_pair("string-substr", builtin::string_substr),
 	std::make_pair("string-find", builtin::string_find),
+	std::make_pair("string-cmp", builtin::string_cmp),
 
 	// Casting operations
 	std::make_pair("int", builtin::cast_to_int),
@@ -2245,14 +2370,13 @@ MlValue MlEnvironment::get(const std::string &name) const {
 }
 
 // Get vector of executables in this scope
-std::vector<std::string> MlEnvironment::get_lambdas_list() const {
-	std::vector<std::string> lambdas{128};
+std::vector<std::string> MlEnvironment::get_lambdas_list(const std::string &token) const {
+	std::vector<std::string> lambdas{};
+	lambdas.reserve(16);
 
-	for (auto it = defs.begin(); it != defs.end(); it++) {
-		if (it->second.get_type_name() == FUNCTION_TYPE) {
+	for (auto it = defs.begin(); it != defs.end(); it++)
+		if (it->second.get_type_name() == FUNCTION_TYPE && (token.empty() || it->first.find(token) == 0))
 			lambdas.push_back(it->first);
-		}
-	}
 
 	return lambdas;
 }
@@ -2264,15 +2388,14 @@ void repl(MlEnvironment &env) {
 	MlValue tmp;
 	std::vector<MlValue> parsed;
 
-	setup_linenoise(env);
+	setup_linenoise_repl(env, builtin_funcs);
 
 	while (true) {
 		char *line = linenoise(">>> ");
 		if (line == nullptr) break;
 
-		linenoise_line_read(line);
-
 		input = std::string(line);
+		linenoise_add_to_history(input);
 
 		if (input == "!quit" || input == "!q")
 			break;
@@ -2296,8 +2419,6 @@ void repl(MlEnvironment &env) {
 			}
 		}
 	}
-
-	close_linenoise();
 }
 
 
@@ -2316,6 +2437,12 @@ std::vector<std::string> getCmdOption(char *argv[], int argc, const std::string
 	return tokens;
 }
 
+
+void load_std_lib(MlEnvironment &env) {
+	run(STDLIB_LOADER, env);
+}
+
+
 int main(int argc, char *argv[]) {
 	MlEnvironment env;
 	std::vector<MlValue> args;
@@ -2328,11 +2455,13 @@ int main(int argc, char *argv[]) {
 	srand(time(NULL));
 	try {
 		// performance monitor on
-		if (cmdOptionExists(argv, argv + argc, "-p"))
+		if (cmdOptionExists(argv, argv + argc, "-p")) {
 			MlPerfMon::instance().turnOn();
+		}
 		// better stacktrace
-		if (cmdOptionExists(argv, argv + argc, "-d"))
+		if (cmdOptionExists(argv, argv + argc, "-d")) {
 			MlPerfMon::instance().debugOn();
+		}
 		// skip loading std lib
 		if (!cmdOptionExists(argv, argv + argc, "-b")) {
 			load_std_lib(env);
@@ -2347,6 +2476,9 @@ int main(int argc, char *argv[]) {
 			std::cout << VERSION << std::endl;
 			return 0;
 		}
+
+		setup_linenoise();
+		
 		// passed code
 		if (cmdOptionExists(argv, argv + argc, "-c")) {
 			std::vector<std::string> codes = getCmdOption(argv, argc, "-c");
@@ -2358,10 +2490,10 @@ int main(int argc, char *argv[]) {
 				run(read_file_contents(file), env);
 		// sheebang
 		} else if (cmdOptionExists(argv, argv + argc, "-run")) {
-			for (auto & file : getCmdOption(argv, argc, "-run")) {	// TODO check only one file is specified ??
+			for (auto & file : getCmdOption(argv, argc, "-run")) {
 				std::string file_content = read_file_contents(file);
 				if (file_content.find("#!") == 0)        // shebang ?
-					file_content.erase(0, file_content.find('\n') + 1);   // TODO mac osx newline??
+					file_content.erase(0, file_content.find('\n') + 1);
 
 				run(file_content, env);
 			}
@@ -2370,6 +2502,8 @@ int main(int argc, char *argv[]) {
 			repl(env);
 		}
 
+		close_linenoise();
+
 		MlPerfMon::instance().print_results();
 
 		return 0;
@@ -2378,7 +2512,6 @@ int main(int argc, char *argv[]) {
 		std::cerr << e.description() << std::endl;
 	} catch (const std::exception &e) {
 		std::cerr << MlPerfMon::instance().callstack() << e.what() << std::endl;
-		MlPerfMon::instance().clear_callstack();
 	}
 
 	return 1;

ml.h

@@ -7,9 +7,9 @@
 #include <exception>
 
 #ifdef NDEBUG
-const std::string VERSION = "ml 0.5 (" __DATE__ " " __TIME__ "), Release";
+const std::string VERSION = "ml 0.5.1 (" __DATE__ " " __TIME__ "), Release";
 #else
-const std::string VERSION = "ml 0.5 (" __DATE__ " " __TIME__ "), Debug";
+const std::string VERSION = "ml 0.5.1 (" __DATE__ " " __TIME__ "), Debug";
 #endif
 
 const std::string STDLIB_LOADER =
@@ -50,7 +50,7 @@ public:
     void setX(const std::string &name, const MlValue& value);
 
     // Get vector of executables in this scope
-    std::vector<std::string> get_lambdas_list() const;
+    std::vector<std::string> get_lambdas_list(const std::string &token) const;
 
     void combine(MlEnvironment const &other);
 
@@ -102,6 +102,20 @@ typedef MlValue (*Builtin)(std::vector<MlValue>, MlEnvironment &);
 class MlPerfMon;
 
 class MlValue {
+public:
+    enum Type {
+        QUOTE,
+        ATOM,
+        INT,
+        FLOAT,
+        LIST,
+        STRING,
+        LAMBDA,
+        MACRO,
+        BUILTIN,
+        NIL,
+        TRUE
+    } type;
 
 public:
     MlValue();  // Constructs a nil value
@@ -116,7 +130,7 @@ public:
     static MlValue string(const std::string &s);
     static MlValue nil();
 
-    MlValue(const std::vector<MlValue> &params, MlValue ret, MlEnvironment const &env); // Construct a lambda function
+    MlValue(const std::vector<MlValue> &params, MlValue ret, MlEnvironment const &env, const Type ftype); // Construct a lambda or macro function
     MlValue(const std::string &name, Builtin b);    // Construct a builtin function
 
     std::vector<std::string> get_used_atoms();
@@ -127,7 +141,9 @@ public:
     // Evaluate this value as lisp code.
     MlValue eval(MlEnvironment &env);
 
+    bool is_nil() const;
     bool is_builtin() const;
+    bool is_macro() const;
     bool is_number() const;
     bool is_string() const;
     bool is_list() const;
@@ -175,19 +191,6 @@ public:
     friend std::ostream &operator<<(std::ostream &os, MlValue const &v);
 
 private:
-    enum {
-        QUOTE,
-        ATOM,
-        INT,
-        FLOAT,
-        LIST,
-        STRING,
-        LAMBDA,
-        BUILTIN,
-        NIL,
-        TRUE
-    } type;
-
     union {
         long i;
         double f;

ml_date.cpp

@@ -2,6 +2,7 @@
 #include "ml_date.h"
 #include <cstring>
 #include <stdexcept>
+#include <chrono>
 
 long now() {	// get-universal-time
 	time_t t = std::time(nullptr);
@@ -10,6 +11,13 @@ long now() {	// get-universal-time
 	return now;
 }
 
+long now_ms() {	// get-universal-time-ms
+	using namespace std::chrono;
+	milliseconds ms = duration_cast< milliseconds >(system_clock::now().time_since_epoch());
+
+	return ms.count();
+}
+
 long get_gmt_localtime_offset() {
 	std::time_t current_time;
 	std::time(&current_time);
@@ -33,7 +41,7 @@ std::string date_to_string(const long datetime, const std::string& format) {
 }
 
 time_t time_to_epoch ( const struct tm *ltm, int utcdiff ) {
-	const int mon_days [] = {0, 31, 31+28, 31+28+31,
+	constexpr int mon_days [] = {0, 31, 31+28, 31+28+31,
 				31+28+31+30, 31+28+31+30+31, 31+28+31+30+31+30,
 				31+28+31+30+31+30+31, 31+28+31+30+31+30+31+31, 31+28+31+30+31+30+31+31+30,
 				31+28+31+30+31+30+31+31+30+31, 31+28+31+30+31+30+31+31+30+31+30, 31+28+31+30+31+30+31+31+30+31+30+31};

ml_date.h

@@ -6,6 +6,7 @@
 
 
 long now();
+long now_ms();
 
 long get_gmt_localtime_offset();
 

ml_io.cpp

@@ -1,10 +1,14 @@
 
 #include "ml_io.h"
 
+#include <cstring>
 #include <dirent.h>
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include <iostream>
+#include <fstream>
+
 
 std::string read_file_contents(const std::string &filename) {
 	std::ifstream f;
@@ -50,8 +54,13 @@ MlValue list_dir(const std::string &path) {
 }
 
 bool is_path_file(const std::string &path) {
-	std::ifstream ifile(path.c_str());
-	return (bool) ifile;
+	struct stat buf{};
+	stat(path.c_str(), &buf);
+	return (bool) S_ISREG(buf.st_mode) ||
+			(bool) S_ISBLK(buf.st_mode) ||
+			(bool) S_ISCHR(buf.st_mode) ||
+			(bool) S_ISFIFO(buf.st_mode) ||
+			(bool) S_ISSOCK(buf.st_mode);
 }
 
 bool is_path_dir(const std::string &path) {
@@ -60,12 +69,22 @@ bool is_path_dir(const std::string &path) {
 	return (bool) S_ISDIR(buf.st_mode);
 }
 
-int mk_path_dir(const std::string &path) {
-	return ::mkdir(path.c_str(), 0755);
+bool mk_path_dir(const std::string &path) {
+	auto r = ::mkdir(path.c_str(), 0755);
+	if (r == -1) {
+		std::string err_msg = std::strerror(errno);
+		std::cerr << "mkdir failed: " << std::strerror(errno) << std::endl;
+	}
+	return r == 0;
 }
 
-int rm_path_dir(const std::string &path) {
-	return ::rmdir(path.c_str());
+bool rm_path_dir(const std::string &path) {
+	auto r = ::rmdir(path.c_str());
+	if (r == -1) {
+		std::string err_msg = std::strerror(errno);
+		std::cerr << "rmdir failed: " << std::strerror(errno) << std::endl;
+	}
+	return r == 0;
 }
 
 MlValue exec_system_cmd(const std::string &cmd) {
@@ -93,3 +112,32 @@ MlValue exec_system_cmd(const std::string &cmd) {
 
 	return std::vector<MlValue> { MlValue((long)cmd_retval), MlValue::string(cmd_output) };
 }
+
+
+MlValue exec_system_cmd_fork(const std::vector<std::string> &args) {
+	int pid = fork();
+	if (pid == 0) {
+	    // Child process, execute the command
+
+    	std::vector<char*> exec_args;
+		exec_args.reserve(args.size());
+
+	    for (auto const& a : args) 
+	        exec_args.emplace_back(const_cast<char*>(a.c_str()));
+
+    	exec_args.push_back(nullptr);	// exec must end with null
+
+    	execvp(args[0].c_str(), exec_args.data());
+
+		std::cerr << "execvp error: " << errno << ", " << strerror(errno) <<  std::endl;
+    	exit(1);	// indicate error, if here
+
+	} else if (pid > 0) {
+	    // Parent process, child process is now independent
+		return std::vector<MlValue> { MlValue((long)0), MlValue::string("") };
+
+	} else {
+	    // Fork error, still in parent process (there are no child process at this point)
+		return std::vector<MlValue> { MlValue((long)errno), MlValue::string(strerror(errno) ) };
+	}
+}

ml_io.h

@@ -4,8 +4,7 @@
 
 #include <string>
 #include <vector>
-#include <iostream>
-#include <fstream>
+
 
 std::string read_file_contents(const std::string &filename);
 
@@ -14,11 +13,10 @@ bool write_file_contents(const std::string &filename, const std::string &content
 MlValue list_dir(const std::string &path);
 
 bool is_path_file(const std::string &path);
-
 bool is_path_dir(const std::string &path);
 
-int mk_path_dir(const std::string &path);
-
-int rm_path_dir(const std::string &path);
+bool mk_path_dir(const std::string &path);
+bool rm_path_dir(const std::string &path);
 
 MlValue exec_system_cmd(const std::string &path);
+MlValue exec_system_cmd_fork(const std::vector<std::string> &args);

ml_profiler.cpp

@@ -79,7 +79,7 @@ void MlPerfMon::restore_callstack_position(size_t to_position) {
 }
 
 void MlPerfMon::clear_callstack() {
-    bool e = call_stack.empty();
+    auto r = call_stack.empty();
 }
 
 void MlPerfMon::print_results() const {

ml_profiler.h

@@ -31,6 +31,7 @@ public:
 
     void turnOn();
     void debugOn();
+    bool isDebugOn() { return debugStacktraceOn; }
 
     void add_method_call(const MlValue &function, const std::vector<MlValue> &args);
     void end_method_call();

ml_util.cpp

@@ -1,24 +1,27 @@
 
 #include "ml_util.h"
+#include "string.h"
 
-#include <string>
 #include <vector>
+#include <algorithm>
+#include <iostream>
+// #include <filesystem>
 
-const std::vector<std::string> commands {
-	"eval", "type", "parse", "do", "if", "for", "while", "scope", "quote", "defn", 
-	"def", "lambda", "benchmark", "=", "!=", ">", "<", ">=", "<=", "+", "-", "*", "/", "%", 
-	"list", "insert", "index", "remove", "len", "push", "pop", "head", "tail", "first", "last", 
-	"range", "map", "filter", "reduce", "exit", "quit", "print", "input", "random", "include", 
-	"read-file", "write-file", "read-url", "system-cmd", "ls-dir", "is-file?", "is-dir?", 
-	"parse-csv", "parse-json", "get-universal-time", "date-to-str", "str-to-date", "date-add", "debug", 
-	"sprintf", "display", "string-replace", "string-regex?", "string-pad", "int", "float", "string",
-	"benchmark", "thread-create", "thread-under-lock", "thread-sleep", "threads-join", "try", "throw",
-	"usql"
-	};
+std::vector<std::string> commands {};
+std::string history_file;
 
+// kryplove z applu a jejich problemy s filesystemem
+/*
 std::string get_history_file_dir() {
-	// TODO not portable and in function
-	std::string file{"/.ml_history.txt"};
+	std::string filename{".ml_history.txt"};
+	const char *t = std::getenv("HOME");
+
+	if (t == nullptr) return std::filesystem::temp_directory_path() / filename;
+	else return std::filesystem::path(std::string{t}) /filename;
+}	
+ */
+
+std::string get_history_filepath(const std::string &file) {
 	const char *t = std::getenv("HOME");
 
 	if (t == nullptr) return "/tmp/" + file;
@@ -27,29 +30,37 @@ std::string get_history_file_dir() {
 
 MlEnvironment * repl_env = nullptr;
 
-void setup_linenoise(const MlEnvironment &env) {
-    repl_env = (MlEnvironment*) &env;
-
-	std::string history_file = get_history_file_dir();
+void setup_linenoise() {
+	history_file = get_history_filepath("/.ml_history.txt");
+	linenoiseHistoryLoad(history_file.c_str()); 
 
 	linenoiseHistorySetMaxLen(500);
-	linenoiseSetCompletionCallback(completion);
-	linenoiseSetHintsCallback(hints);
 	linenoiseSetMultiLine(1);
-	linenoiseHistoryLoad(history_file.c_str()); 
 }
 
-void linenoise_line_read(char *line) {
-    linenoiseHistoryAdd(line);
+void setup_linenoise_repl(const MlEnvironment &env, const std::map<const std::string, Builtin> &builtins) {
+	// basic linenoise setup must be already set
+
+    repl_env = (MlEnvironment*) &env;
+
+	linenoiseSetCompletionCallback(completion);
+	linenoiseSetHintsCallback(hints);
+
+    commands.reserve(builtins.size());
+    for(auto it = builtins.begin(); it != builtins.end(); ++it)
+        commands.push_back(it->first);
+}
+
+void linenoise_add_to_history(const std::string &line) {
+	if (!line.empty())
+    	linenoiseHistoryAdd(line.c_str());
 }
 
 void close_linenoise() {
-    std::string history_file = get_history_file_dir();
-
     linenoiseHistorySave(history_file.c_str());
 }
 
-size_t last_token_index( std::string str ) {
+size_t last_token_index(std::string str) {
     // remove trailing white space
     while( !str.empty() && std::isspace( str.back() ) ) str.pop_back() ;
 
@@ -68,15 +79,14 @@ void completion(const char *buf, linenoiseCompletions *lc) {
         std::string token = str.substr(pos+1);
         std::string begining = str.substr(0, pos+1);
 
-        // TODO optimize not to get all lambdas, but those beginning with token
-        std::vector<std::string> lambdas = repl_env->get_lambdas_list();
-    	lambdas.insert(end(lambdas), begin(commands), end(commands));
+        auto suggestions = repl_env->get_lambdas_list(token);
+        std::copy_if(commands.begin(), commands.end(), std::back_inserter(suggestions), 
+                        [&token] (const std::string &cmd) { return token.empty() || cmd.find(token)==0; });
 
-        for (const auto & lambda : lambdas) {
-            if(lambda.find(token) == 0) {
-                std::string completion_string = begining + lambda;
-                linenoiseAddCompletion(lc, completion_string.c_str());
-            }
+        std::sort(suggestions.begin(), suggestions.end());
+        for (const auto &suggestion : suggestions) {
+            std::string completion = begining + suggestion;
+            linenoiseAddCompletion(lc, completion.c_str());
         }
     }
 }

ml_util.h

@@ -1,11 +1,13 @@
 #pragma once
 
 #include "ml.h"
-
 #include "linenoise.h"
 
-void setup_linenoise(const MlEnvironment &env);
-void linenoise_line_read(char *line);
+#include <string>
+
+void setup_linenoise();
+void setup_linenoise_repl(const MlEnvironment &env, const std::map<const std::string, Builtin> &builtins);
+void linenoise_add_to_history(const std::string &line);
 void close_linenoise();
 
 

stdlib/doc.lsp

@@ -28,9 +28,9 @@
 
 (defn doc::print (entry)
 (do	(print (term-green (doc::strip-backticks (second entry))) "-" (third entry))
-	; (print (last entry) "\n")	; doc section
-	(if (> (string-len (fourth entry)) 2)
-		(do	(def examp (doc::strip-backticks (fourth entry)))
+	(if (> (string-len (fourth entry)) 2)	; example
+		(for example (string-regex-list (fourth entry) "(`.*?`)" "token") 
+			(def examp (doc::strip-backticks (first example)))
 			(def pos (string-find examp "=>" 0))
 			(if (and pos (> pos 0))
 				(print (term-magenta
@@ -43,6 +43,12 @@
 	nil
 ))
 
+(defn doc::print-entry (entry)
+(do (def call (doc::strip-backticks (second entry)))
+	(def desc (doc::strip-backticks (third entry)))
+	(print (term-red (first entry)) "-" (term-green (doc::strip-backticks (second entry))) "-" (third entry))
+))
+
 (defn doc::man (what)
 (do	(def man (filter (lambda (x) (= (first x) what)) doc::doc_entries))
 	(if man
@@ -75,26 +81,54 @@
 	
 	(def sorted (quick-sort-by scores (lambda (a b) (< (first a) (first b)))))
 	(for e (take sorted 10)
-		(def entry (second e))
-		(def call (doc::strip-backticks (second entry)))
-		(def desc (doc::strip-backticks (third entry)))
-		(print (term-red (first entry)) "-" (term-green (doc::strip-backticks (second entry))) "-" (third entry))
-	)
+		(doc::print-entry (second e)))
+
 	(if (> (len sorted) 10) (print "..."))
 ))
 
+(defn doc::section (what)
+(do
+	(def entries '())
+	(for entry doc::doc_entries
+		; ("throw-exception" "`(throw-exception exp_desc)`" "Throws an exception with exp_desc describing what happened " "" "Exceptions"
+		; section matches
+		(if (= (string-downcase (fifth entry)) (string-downcase what))
+			(set! entries (push entries entry)))
+	)
+	
+	(for e (quick-sort-by entries (lambda (a b) (> (string-cmp (first a) (first b)) 0)))
+		(doc::print-entry e))
+))
+
+(defn doc::all ()
+	(for e (quick-sort-by doc::doc_entries (lambda (a b) (> (string-cmp (first a) (first b)) 0)))
+		(doc::print-entry e))
+)
+
 (defn doc::appropos (which)
 	(doc::look which))
 
-;(defn doc::section (which)
-;	(print (term-red "implement me!")))
+(defn doc::lookup (which)
+	(doc::look which))
+
+(defn doc::doc ()
+(do
+	(print "Usage:")
+	(print "\t(doc::doc)         - shows this help")
+	(print "\t(doc::man func)    - func must be a string, ie (doc::man \"for\")")
+	(print "\t(doc::look str)    - str must be a string, ie (doc::look \"length\")")
+	(print "\t(doc::lookup)      - alias for doc::look")
+	(print "\t(doc::appropos)    - alias for doc::look")
+	(print "\t(doc::all)         - show short info about all functions")
+	(print "\t(doc::section sec) - show help for section, sec is string one of:")
+	(print "\t\t\t\t\"List manipulation\" \"Language\" \"System\"")
+	(print "\t\t\t\t\"String manipulation\" \"Date and time\" \"IO\" \"Regex\"")
+	(print "\t\t\t\t\"Type casting\" \"Threading\" \"Exceptions\"")
+))
+
 
 (def doc::doc_entries '())	; must be here
 
 ; read doc into memory
 (doc::read-doc-file "/usr/local/var/mlisp/Doc.md")
 
-;;example
-; (doc::man "first")
-; (doc::look "string pad")
-; (doc::look "list flat")

stdlib/stdlib.lsp

@@ -1,8 +1,8 @@
 ; not a bool
 (defn not (x) (if x nil #t))
 
-(defn is-pos? (n) (> n nil))
-(defn is-neg? (n) (< n nil))
+(defn is-pos? (n) (> n 0))
+(defn is-neg? (n) (< n 0))
 
 (defn neg (n) (- 0 n))
 (defn dec (n) (- n 1))
@@ -26,6 +26,8 @@
 (defn string-downcase (str) 
     (string-case str "lower"))
 
+(defn string-cmp-ic (a b) 
+    (string-cmp (string-upcase a) (string-upcase b)))
 
 (defn string-join (lst sep)
 (do
@@ -46,9 +48,6 @@
 (defn sleep (time) 
   (system-cmd (+ "sleep " (string time))))
 
-(defn get-env (var) 
-    (second (system-cmd (+ "echo ${" var "} | tr -d \"\n\""))))
-
 
 (defn third (l) (index l 2))
 (defn fourth (l) (index l 3))
@@ -60,10 +59,36 @@
 (defn tenth (l) (index l 9))
 (defn nth (i l) (index l (- i 1)))
 
+
+(defn list? (e) (= (type e) "list"))
+(defn empty-list? (e) (and (list? e) (= (len e) 0)))
+(defn string? (e) (= (type e) "string"))
+(defn int? (e) (= (type e) "int"))
+(defn float? (e) (= (type e) "float"))
+(defn nil? (e) (= (type e) "nil"))
+(defn true? (e) (= (type e) "#t"))
+(defn function? (e) (= (type e) "function"))
+
+
+(defmacro unless (test v)
+	(list 'if (list 'not test) v))
+
+(defmacro dotimes (v n body)
+	(list 'for v '(range 0 (eval n))
+		body
+    ))    
+
+(defmacro until (cnd body)
+	(list 'while '(not (eval cnd)) 
+		body
+    ))  
+
+
 ; return 1 when list contains item otherwise nil
 (defn member (lst itm) 
     (do
-        ; TODO check if is empty list
+        (if (list? lst)
+            (do
                 (def found_index -1)
                 (def i 0)
                 (def lst_len (len lst))
@@ -77,6 +102,8 @@
                 (if (!= -1 found_index)
                     #t
                     nil)
+            )
+            nil)
     ))
 
 (defn make-list-of (size val)
@@ -92,8 +119,6 @@
 (defn make-list (size)
     (make-list-of size nil))
 
-(defn empty-list? (lst) (and (= (type lst) "list") (= (len lst) 0)))
-
 (defn uniq (lst)
     (do
         (def rslt '())    
@@ -127,6 +152,12 @@
 		(map (lambda (i) (index lst i)) (range 0 n))
 		lst))
 
+(defn concat-lists (seq1 seq2)
+	(do	(def l seq1)
+		(dotimes i (len seq2) 
+			(set! l (push l (index seq2 i))))
+	))
+    
 (defn quick-sort-by (l cmp)
     (if (<= (len l) 1)
         l
@@ -156,7 +187,7 @@
 (defn end-of-prev-month (datetime) (date-add (end-of-month datetime) -1 "month"))
 (defn start-of-year (datetime) (str-to-date (+ (date-to-str datetime "%Y") "-01-01 00:00:00") "%Y-%m-%d %H:%M:%S"))
 (defn end-of-year (datetime) (str-to-date (+ (date-to-str datetime "%Y") "-12-31 23:59:59") "%Y-%m-%d %H:%M:%S"))
-
+(defn decode-universal-time (time) (map (lambda (x) (int x)) (string-split (date-to-str time "%S %M %H %d %m %Y %w") "\s+")))
 
 ; from list of lists creates csv string
 ; (print (make-csv '(("r1c1" "r1c2") ("r2c1" "r2c2"))))
@@ -169,7 +200,7 @@
             (def c 0)
             (for col row
                 (if (= c 0)
-                    (def cols_str col)
+                    (def cols_str (display col))
                     (def cols_str (+ cols_str "," (display col)))) 
                 (def c (inc c)))
             (if (= r 0)
@@ -253,5 +284,6 @@
         )
     ))
 
-; load and init do system
+
+; load and init doc system
 (include "/usr/local/var/mlisp/doc.lsp")

tests/cmd_fork.lsp

@@ -0,0 +1 @@
+(system-cmd-fork "ml"  "-c" "(print 123) (sleep 1) (print \"aaa\")")

tests/test.lsp

@@ -1,6 +1,6 @@
 (include "/usr/local/var/mlisp/ut.lsp")
 
-(def local_time_offset 3600)
+(def local_time_offset 7200)
 
 ;;  prepare some code to be used in tests
 (defn fact (n)
@@ -9,11 +9,11 @@
       (* n (fact (- n 1)))
     ))
 
-; for scitej 4000 stack must be 16MB, otherwise 1000 is ok
-(defn scitej (n)
+; for adder 4000 stack must be 16MB, otherwise 1000 is ok
+(defn adder (n)
    (if (<= n 1)
       1
-      (+ n (scitej (- n 1)))
+      (+ n (adder (- n 1)))
     ))
 
 (write-file "/tmp/f.txt" "line 1\nline 2\nline3")
@@ -29,6 +29,11 @@
 
 (thread-create (tcp-server 7778 (lambda (str) (list #t (+ "(print \"" (string-upcase str) "\")")))))
 
+
+(ut::define-test "result of (unless #t (print \"#f\"))" '(ut::assert-false (unless #t (print "#f"))))
+(ut::define-test "result of (dotimes i 5 i)" '(ut::assert-equal 4 (dotimes i 5 i)))
+(ut::define-test "result of (def i 0)(until (> i 3) (set! i (inc i)))" '(ut::assert-equal 4 (do (def i 0)(until (> i 3) (set! i (inc i))))))
+
 (ut::define-test "result of (and (> 2 1) (> 2 1))" '(ut::assert-true (and (> 2 1) (> 2 1))))
 (ut::define-test "result of (or  (> 2 1) (> 2 1))" '(ut::assert-true (or  (> 2 1) (> 2 1))))
 (ut::define-test "retult of (and (> 2 1) (> 1 2))" '(ut::assert-false (and (> 2 1) (> 1 2))))
@@ -43,18 +48,21 @@
 (ut::define-test "result of (def a 20) (cond ((> a 30)" '(ut::assert-equal "a <= 20" (cond ((> a 30) "a > 30") ((> a 20) "a > 20")(#t "a <= 20"))))
 (ut::define-test "result of (def b 30) (cond ((> b 30)" '(ut::assert-equal "b > 20" (cond ((> b 30) "b > 30") ((> b 20) "b > 20")(#t "b <= 20"))))
 
-(ut::define-test "result of (member '(1 2 3) 1" '(ut::assert-true (member '(1 2 3) 1)))
-(ut::define-test "result of (member '(1 2 3) 3" '(ut::assert-true (member '(1 2 3) 3)))
-(ut::define-test "result of (member '(1 2 3) 4" '(ut::assert-false (member '(1 2 3) 4)))
+(ut::define-test "result of (member '(1 2 3) 1)" '(ut::assert-true (member '(1 2 3) 1)))
+(ut::define-test "result of (member '(1 2 3) 3)" '(ut::assert-true (member '(1 2 3) 3)))
+(ut::define-test "result of (member '(1 2 3) 4)" '(ut::assert-false (member '(1 2 3) 4)))
 
-(ut::define-test "result of (take '(1 2 3 4) 3" '(ut::assert-equal '(1 2 3) (take '(1 2 3 4) 3)))
+(ut::define-test "result of (reduce (lambda (acc e) (+ acc (string (+ \"<th>\" e \"</th>\")))) \"\" '(\"h1\" \"h2\" \"h3\"))" '(ut::assert-equal "<th>h1</th><th>h2</th><th>h3</th>" (reduce (lambda (acc e) (+ acc (string (+ "<th>" e "</th>")))) "" '("h1" "h2" "h3"))))
+
+(ut::define-test "result of (take '(1 2 3 4) 3)" '(ut::assert-equal '(1 2 3) (take '(1 2 3 4) 3)))
 
 (ut::define-test "result of (make-list 3)" '(ut::assert-equal '(nil nil nil) (make-list 3)))
 (ut::define-test "result of (make-list-of 3 999)" '(ut::assert-equal '(999 999 999) (make-list-of 3 999)))
 
+(ut::define-test "result of (range 1.5 5.5)" '(ut::assert-equal '(1.500000 2.500000 3.500000 4.500000) (range 1.5  5.5)))
 
 (ut::define-test "result of (fact 5)" '(ut::assert-equal 120 (fact 5)))
-(ut::define-test "result of (scitej 1000)" '(ut::assert-equal 500500 (scitej 1000)))
+(ut::define-test "result of (adder 1000)" '(ut::assert-equal 500500 (adder 1000)))
 
 (ut::define-test "result of (len json_list)" '(ut::assert-equal 3 (len json_list)))
 
@@ -91,11 +99,23 @@
 (ut::define-test "result of (string-substr \"ABCDEF\"4 42)" '(ut::assert-equal "EF" (string-substr "ABCDEF" 4 42)))
 (ut::define-test "result of (string-substr \"ABCDEF\" -2 2)" '(ut::assert-equal "EF" (string-substr "ABCDEF" -2 2)))
 
+(ut::define-test "result of (string-ltrim \"   ***   \")" '(ut::assert-equal "***   " (string-ltrim "   ***   ")))
+(ut::define-test "result of (string-rtrim \"   ***   \")" '(ut::assert-equal "   ***" (string-rtrim "   ***   ")))
+(ut::define-test "result of (string-trim \"   ***   \")" '(ut::assert-equal "***" (string-trim "   ***   ")))
+
 (ut::define-test "result of (string-find \" long long int;\" \"long\")" '(ut::assert-equal 1 (string-find " long long int;" "long")))
 (ut::define-test "result of (string-find \" long long int;\" \"long\" 2)" '(ut::assert-equal 6 (string-find " long long int;" "long" 2)))
 (ut::define-test "result of (string-find \" long long int;\" \" \")" '(ut::assert-equal 0 (string-find " long long int;" " ")))
 (ut::define-test "result of (string-find \" long long int;\" \"o\")" '(ut::assert-equal 2 (string-find " long long int;" "o")))
 (ut::define-test "result of (string-find \" long long int;\" \"float\")" '(ut::assert-nil (string-find " long long int;" "float")))
+(ut::define-test "result of (string-cmp \"aaa\" \"aaa\")" '(ut::assert-equal 0 (string-cmp "aaa" "aaa")))
+(ut::define-test "result of (string-cmp-ic \"aaa\" \"AaA\")" '(ut::assert-equal 0 (string-cmp-ic "aaa" "AaA")))
+
+(ut::define-test "result of (is-pos? -1)" '(ut::assert-false (is-pos? -1)))
+(ut::define-test "result of (is-neg? -1)" '(ut::assert-true (is-neg? -1)))
+(ut::define-test "result of (neg -5)" '(ut::assert-equal 5 (neg -5)))
+(ut::define-test "result of (inc 4)" '(ut::assert-equal 5 (inc 4)))
+(ut::define-test "result of (dec 4)" '(ut::assert-equal 3 (dec 4)))
 
 (ut::define-test "result of (itok 65)" '(ut::assert-equal "A" (itok 65)))
 (ut::define-test "result of (itok 48)" '(ut::assert-equal "0" (itok 48)))
@@ -107,6 +127,9 @@
 (ut::define-test "result of (is-file? \"/tmp/file_whichnotex_ists\")" '(ut::assert-false (is-file? "/tmp/file_whichnotex_ists")))
 (ut::define-test "result of (is-dir? \"/tmp/file_whichnotex_ists\")" '(ut::assert-false (is-dir? "/tmp/file_whichnotex_ists")))
 (ut::define-test "result of (is-dir? \"/tmp\"" '(ut::assert-true (is-dir? "/tmp")))
+(ut::define-test "result of (is-file? \"/tmp\"" '(ut::assert-false (is-file? "/tmp")))
+(ut::define-test "result of (mk-dir \"/tmp/testdir\"" '(ut::assert-true (mk-dir "/tmp/testdir")))
+(ut::define-test "result of (rm-dir \"/tmp/testdir\"" '(ut::assert-true (rm-dir "/tmp/testdir")))
 
 (ut::define-test "result of (uniq '(1 2 2 2 3 4 5 4 4 1 2 2 2 3 4 5 4 4 61 2 2 2 3 4 5 4 4 66))" '(ut::assert-equal '(1 2 3 4 5 61 66) (uniq '(1 2 2 2 3 4 5 4 4 1 2 2 2 3 4 5 4 4 61 2 2 2 3 4 5 4 4 66))))
 (ut::define-test "result of (flatten '(1 2 (3 3 (4)) 5 6))" '(ut::assert-equal '(1 2 3 3 4 5 6) (flatten (1 2 (3 3 (4)) 5 6))))
@@ -115,6 +138,8 @@
 (ut::define-test "result of (sprintf \"%.2f\" '(1.23456))" '(ut::assert-equal "1.23" (sprintf "%.2f" '(1.23456))))
 (ut::define-test "result of (sprintf \"%.d\" '(10000000))" '(ut::assert-equal "10000000" (sprintf "%.d" '(10000000))))
 
+(ut::define-test "result of (set-env \"XXYYZZ\" \"haha\")" '(ut::assert-equal "haha" (set-env "XXYYZZ" "haha")))
+(ut::define-test "result of (get-env \"XXYYZZ\")" '(ut::assert-equal "haha" (get-env "XXYYZZ")))
 (ut::define-test "result of (get-env \"HOME\")" '(ut::assert-equal "/Users/vaclavt" (get-env "HOME")))
 
 (ut::define-test "result of (!= nil nil)" '(ut::assert-false (!= nil nil)))
@@ -136,7 +161,9 @@
 (ut::define-test "result of (end-of-prev-month)" '(ut::assert-equal 1619827199 (end-of-prev-month (str-to-date "2021-05-13 10:32:12" "%Y-%m-%d %H:%M:%S"))))
 (ut::define-test "result of (start-of-year)" '(ut::assert-equal 1609459200 (start-of-year (str-to-date "2021-05-13 10:32:12" "%Y-%m-%d %H:%M:%S"))))
 (ut::define-test "result of (end-of-year)" '(ut::assert-equal 1640995199 (end-of-year (str-to-date "2021-05-13 10:32:12" "%Y-%m-%d %H:%M:%S"))))
+(ut::define-test "result of (decode-universal-time 0)" '(ut::assert-equal '(0 0 0 1 1 1970 4) (decode-universal-time 0)))
 
+;(ut::define-test "result of (print ( + 15.0 (read)))" '(ut::assert-equal 30 (print ( + 15.0 (read)))))
 (ut::define-test "result of (read-file-lines \"/tmp/f.txt\" counter)" '(ut::assert-equal 3 (read-file-lines "/tmp/f.txt" counter)))
 
 (ut::define-test "result of create table" '(ut::assert-equal ((0 "table created" 0)) (usql "create table a (i integer not null, s varchar(64), f float null, d date null,  b boolean)")))

tests/test_tcp.lsp

@@ -0,0 +1,19 @@
+(thread-create
+ 	(tcp-server 7777 (lambda (str) (list #f (+ "(print \"" (string-upcase str) "\")"))))
+)
+
+(thread-sleep 1)
+
+(thread-create
+	(def code (tcp-client "127.0.0.1" 7777 ("abcd" "xyz")))
+	(for c code
+		(print "executing code:" c)
+ 		(eval (parse c))
+	)
+
+	(def code (tcp-client "127.0.0.1" 7777 "abcd"))
+	(print "executing code:" code)
+ 	(eval (parse code))
+)
+(threads-join)
+(print "ok")

usql/lexer.cpp

@@ -162,7 +162,7 @@ TokenType Lexer::type(const std::string &token) {
 	if (token == "not") return TokenType::keyword_not;
 	if (token == "null") return TokenType::keyword_null;
 	if (token == "integer") return TokenType::keyword_integer;
-	if (token == "float") return TokenType::keyword_float;
+	if (token == "float" || token == "double") return TokenType::keyword_float;
 	if (token == "varchar") return TokenType::keyword_varchar;
 	if (token == "date") return TokenType::keyword_date;
 	if (token == "boolean") return TokenType::keyword_bool;

usql/parser.cpp

@@ -3,7 +3,18 @@
 
 namespace usql {
 
-// TOOD handle premature eof
+	// TOOD handle premature eof
+
+	std::string column_type_name(const ColumnType type) {
+		if (type == ColumnType::integer_type)	return "integer_type";
+		if (type == ColumnType::float_type)	return "float_type";
+		if (type == ColumnType::varchar_type)	return "varchar_type";
+		if (type == ColumnType::date_type)	return "date_type";
+		if (type == ColumnType::bool_type)	return "bool_type";
+
+		throw Exception("invalid column type: " + std::to_string((int)type));	
+	};
+
 
     Parser::Parser() {
 	    m_lexer = Lexer{};
@@ -383,8 +394,7 @@ namespace usql {
 		std::unique_ptr<Node> left = parse_expression();
 		do {
 			left = parse_expression(std::move(left));
-
-	} while (m_lexer.tokenType() != TokenType::eof && m_lexer.tokenType() != TokenType::keyword_order && m_lexer.tokenType() != TokenType::keyword_offset && m_lexer.tokenType() != TokenType::keyword_limit);
+		} while (m_lexer.tokenType() != TokenType::eof && m_lexer.tokenType() != TokenType::keyword_order && m_lexer.tokenType() != TokenType::keyword_offset && m_lexer.tokenType() != TokenType::keyword_limit && m_lexer.tokenType() != TokenType::semicolon);
 	
 		return left;
     }
@@ -529,4 +539,3 @@ namespace usql {
     }
 
 }	// namespace
-

usql/parser.h

@@ -21,6 +21,9 @@ enum class ColumnType {
 	bool_type
 };
 
+std::string column_type_name(const ColumnType type);
+
+
 enum class NodeType {
 	true_node,
 	null_value,
@@ -51,6 +54,7 @@ enum class NodeType {
 	error
 };
 
+
 struct Node {
 	NodeType node_type;
 
@@ -58,7 +62,7 @@ struct Node {
 	virtual ~Node() = default;
 
 	virtual void dump() const {
-		std::cout << "type: Node" << std::endl;
+		std::cout << "type: Node" << (int)node_type << std::endl;
 	}
 };
 
@@ -115,20 +119,86 @@ struct ColDefNode : Node {
 			null(nullable) {}
 
 	void dump() const override {
-		std::cout << "type: ColDefNode, name: " << name << ", type: " << (int)type << " TODO add more" << std::endl;
+		std::cout << "type: ColDefNode, name: " << name << ", type: " << column_type_name(type) << ", order: " << order << ", length: " << length << ", null: " << null << std::endl;
 	}
 };
 
 struct FunctionNode : Node {
-	std::string				function;	// TODO use enum
-	std::vector<std::unique_ptr<Node>>	params;
+
+	enum class Type {
+		to_char,
+		to_date,
+		to_int,
+		to_float,
+		date_add,
+		pp,
+		lower,
+		upper,
+		coalesce,
+		min,
+		max,
+		count
+	};
+
+	static Type get_function(const std::string &str) {
+		if (str=="to_char")		return Type::to_char;
+		if (str=="to_date")		return Type::to_date;
+		if (str=="to_int")		return Type::to_int;
+		if (str=="to_float")	return Type::to_float;
+		if (str=="date_add")	return Type::date_add;
+		if (str=="pp")			return Type::pp;
+		if (str=="lower")		return Type::lower;
+		if (str=="upper")		return Type::upper;
+		if (str=="coalesce")	return Type::coalesce;
+		if (str=="min")			return Type::min;
+		if (str=="max")			return Type::max;
+		if (str=="count")		return Type::count;
+
+		throw Exception("invalid function: " + str);	
+	};
+
+	static std::string function_name(const Type type) {
+		if (type == Type::to_char)		return "to_char";
+		if (type == Type::to_date)		return "to_date";
+		if (type == Type::to_int)		return "to_int";
+		if (type == Type::to_float)		return "to_float";
+		if (type == Type::date_add)		return "date_add";
+		if (type == Type::pp)			return "pp";
+		if (type == Type::lower)		return "lower";
+		if (type == Type::upper)		return "upper";
+		if (type == Type::coalesce)		return "coalesce";
+		if (type == Type::min)			return "min";
+		if (type == Type::max)			return "max";
+		if (type == Type::count)		return "count";
+
+		throw Exception("invalid function: " + std::to_string((int)type));	
+	};
+
 
 	FunctionNode(std::string  func_name, std::vector<std::unique_ptr<Node>> pars) :
-	Node(NodeType::function), function(std::move(func_name)), params(std::move(pars)) {}
+	Node(NodeType::function), function(get_function(func_name)), params(std::move(pars)) {}
+
+	bool is_agg_function() {
+		return (function == Type::count || function == Type::min || function == Type::max);
+	}
+
+
+	friend std::ostream &operator<<(std::ostream &output, const Type &t ) {
+		output << function_name(t);
+		return output;
+	}
 
 	void dump() const override {
-		std::cout << "type: FunctionNode, function: " << function << " TODO add more" << std::endl;
+		std::cout << "type: FunctionNode, function: " << function_name(function) << "(";
+		for(int i = 0; i < params.size(); i++){
+			if (i > 0) std::cout << ",";
+			params[i]->dump();
 		}			
+		std::cout << ")" << std::endl;
+	}
+
+	Type								function;
+	std::vector<std::unique_ptr<Node>>	params;
 };
 
 struct TrueNode : Node {
@@ -325,7 +395,12 @@ struct CreateTableNode : Node {
 			Node(NodeType::create_table), table_name(std::move(name)), cols_defs(std::move(defs)) {}
 
 	void dump() const override {
-		std::cout << "type: CreateTableNode, table_name: " << table_name << "TODO complete me" << std::endl;
+		std::cout << "type: CreateTableNode, table_name: " << table_name << "(";
+		for(int i = 0; i < cols_defs.size(); i++) {
+			if (i > 0) std::cout << ",";
+			cols_defs[i].dump();
+		}	
+		std::cout << ")" << std::endl;
 	}
 };
 
@@ -338,7 +413,17 @@ struct InsertIntoTableNode : Node {
 			Node(NodeType::insert_into), table_name(std::move(name)), cols_names(std::move(names)), cols_values(std::move(values)) {}
 
 	void dump() const override {
-		std::cout << "type: InsertIntoTableNode, table_name: " << table_name << "TODO complete me" << std::endl;
+		std::cout << "type: InsertIntoTableNode, table_name: " << table_name << "(";
+		for(int i = 0; i < cols_names.size(); i++) {
+			if (i > 0) std::cout << ",";
+			cols_names[i].dump();
+		}	
+		std::cout << ") values (";
+		for(int i = 0; i < cols_values.size(); i++) {
+			if (i > 0) std::cout << ",";
+			cols_values[i]->dump();
+		}	
+		std::cout << ")" << std::endl;
 	}
 };
 
@@ -354,8 +439,22 @@ struct SelectFromTableNode : Node {
 	Node(NodeType::select_from), table_name(std::move(name)), cols_names(std::move(names)), where(std::move(where_clause)), order_by(std::move(orderby)), offset_limit(std::move(offlim)), distinct(distinct_) {}
 
 	void dump() const override {
-		std::cout << "type: SelectFromTableNode, table_name: " << table_name << "TODO complete me" << std::endl;
+		std::cout << "type: SelectFromTableNode, table_name: " << table_name;
+		std::cout << "colums: ";	
+		for(int i = 0; i < cols_names->size(); i++) {
+			if (i > 0) std::cout << ",";
+			cols_names->operator[](i).dump();
+		}
+		std::cout << "where: ";	
 		where->dump();
+		std::cout << "offset,limit: ";
+		for(int i = 0; i < order_by.size(); i++) {
+			if (i > 0) std::cout << ",";
+			order_by[i].dump();
+		}	
+		std::cout << "offset,limit: ";
+		offset_limit.dump();
+		std::cout << std::endl;
 	}
 };
 
@@ -384,8 +483,16 @@ struct UpdateTableNode : Node {
 			where(std::move(where_clause)) {}
 
 	void dump() const override {
-		std::cout << "type: UpdateTableNode, table_name: " << table_name << "TODO complete me" << std::endl;
+		std::cout << "type: UpdateTableNode, table_name: " << table_name << " set ";
+		for(int i = 0; i < cols_names.size(); i++) {
+			if (i > 0) std::cout << ",";
+			cols_names[i].dump();
+			std::cout << " = ";
+			values[i]->dump();
+		}	
+		std::cout << " where: ";
 		where->dump();
+		std::cout << std::endl;
 	}
 };
 
@@ -431,8 +538,10 @@ struct DeleteFromTableNode : Node {
 			Node(NodeType::delete_from), table_name(std::move(name)), where(std::move(where_clause)) {}
 
 	void dump() const override {
-		std::cout << "type: DeleteFromTableNode, table_name: " << table_name << std::endl;
+		std::cout << "type: DeleteFromTableNode, table_name: " << table_name;
+		std::cout << "where: ";	
 		where->dump();
+		std::cout << std::endl;
 	}
 };
 
@@ -472,7 +581,6 @@ struct CreateIndexNode : Node {
 };
 
 class Parser {
-private:
 public:
 	Parser();
 

usql/row.h

@@ -171,7 +171,7 @@ public:
 			case 5:
 				return (ColValue &) *std::get_if<ColBooleanValue>(&m_columns[i]);
 			default:
-				throw Exception("should not happen");
+				throw Exception("ColValue &operator[](int i), type index invalid :" + std::to_string(type_index));
 		}
 	}
 
@@ -182,7 +182,7 @@ public:
 
 	[[nodiscard]] bool is_visible() const { return m_visible; };
 	void set_visible() { m_visible = true; };
-	void set_deleted() { m_visible = true; };
+	void set_deleted() { m_visible = false; };
 
 private:
 	bool m_visible;

usql/settings.cpp

@@ -12,7 +12,7 @@ std::vector<std::pair<std::string, std::string>> Settings::m_settings =
 	      std::make_pair("BOOL_TRUE_LITERAL", "Y"),
 	      std::make_pair("BOOL_FALSE_LITERAL", "N"),
           std::make_pair("DOUBLE_FORMAT", "%.2f"),
-	      std::make_pair("USE_INDEXSCAN", "N"),
+	      std::make_pair("USE_INDEXSCAN", "Y"),
 	      std::make_pair("MAX_PARALLELISM", "1") };		// values "AUTO" or number of workers; when number negative means std::thread::hardware_concurrency() - number 
 
 

usql/table.h

@@ -6,6 +6,7 @@
 
 #include <utility>
 #include <vector>
+#include <mutex>
 
 
 namespace usql {

usql/usql.cpp

@@ -6,9 +6,36 @@
 
 namespace usql {
 
+USql::USql() {
+	// create catalogue tables first
+
+	std::vector<std::string> k_debug_sql_commands {
+		"create table usql_tables(name varchar(32) not null, modified boolean not null)",
+		"create table usql_columns(table_name varchar(32) not null, column_name varchar(32) not null, column_type varchar(16) not null, column_length integer not null, nullable boolean not null, column_order integer not null)"
+	};
+
+	// create cataloque tables
+	for (const auto &command : k_debug_sql_commands) {
+		std::unique_ptr<Node> create_table_node = m_parser.parse(command);
+		const CreateTableNode &node = static_cast<CreateTableNode &>(*create_table_node); 
+
+		Table table{node.table_name, node.cols_defs};
+		m_tables.push_back(table);
+
+	}
+	// insert data into cataloque tables
+	for (const auto &command : k_debug_sql_commands) {
+		std::unique_ptr<Node> create_table_node = m_parser.parse(command);
+		const CreateTableNode &node = static_cast<CreateTableNode &>(*create_table_node); 
+
+		execute_create_table_sys_catalogue(node);
+	}
+}
+
 std::unique_ptr<Table> USql::execute(const std::string &command) {
 	try {
 		std::unique_ptr<Node> node = m_parser.parse(command);
+		// node->dump();
 		return execute(*node);
 
 	} catch (const std::exception &e) {
@@ -121,7 +148,7 @@ std::unique_ptr<ValueNode> USql::eval_value_node(Table *table, Row &row, Node *n
 
 std::unique_ptr<ValueNode> USql::eval_database_value_node(Table *table, Row &row, Node *node) {
 	auto *dvl = static_cast<DatabaseValueNode *>(node);
-	ColDefNode col_def = table->get_column_def( dvl->col_name); // TODO optimize it to just get this def once
+	ColDefNode col_def = table->get_column_def(dvl->col_name); // TODO optimize it to just get this def once
 	ColValue &db_value = row[col_def.order];
 
 	if (db_value.isNull())
@@ -173,21 +200,25 @@ std::unique_ptr<ValueNode> USql::eval_function_value_node(Table *table, Row &row
 		evaluatedPars.push_back(eval_value_node(table, row, param.get(), nullptr, nullptr));
 	}
 
+	// coalesce function can have first parameter null, so must be calles before following "return null"
+	if (fnc->function == FunctionNode::Type::coalesce) return coalesce_function(evaluatedPars);
+	
 	if (evaluatedPars.empty() || evaluatedPars[0]->isNull())
 		return std::make_unique<NullValueNode>();
 
-	// TODO use some enum
-	if (fnc->function == "lower") return lower_function(evaluatedPars);
-	if (fnc->function == "upper") return upper_function(evaluatedPars);
-	if (fnc->function == "to_date") return to_date_function(evaluatedPars);
-	if (fnc->function == "to_string") return to_string_function(evaluatedPars);
-	if (fnc->function == "date_add") return date_add_function(evaluatedPars);
-	if (fnc->function == "pp") return pp_function(evaluatedPars);
-	if (fnc->function == "count") return count_function(agg_func_value, evaluatedPars);
-	if (fnc->function == "max") return max_function(evaluatedPars, col_def_node, agg_func_value);
-	if (fnc->function == "min") return min_function(evaluatedPars, col_def_node, agg_func_value);
+	if (fnc->function == FunctionNode::Type::lower) return lower_function(evaluatedPars);
+	if (fnc->function == FunctionNode::Type::upper) return upper_function(evaluatedPars);
+	if (fnc->function == FunctionNode::Type::to_date) return to_date_function(evaluatedPars);
+	if (fnc->function == FunctionNode::Type::to_char) return to_char_function(evaluatedPars);
+	if (fnc->function == FunctionNode::Type::to_int) return to_int_function(evaluatedPars);
+	if (fnc->function == FunctionNode::Type::to_float) return to_float_function(evaluatedPars);
+	if (fnc->function == FunctionNode::Type::date_add) return date_add_function(evaluatedPars);
+	if (fnc->function == FunctionNode::Type::pp) return pp_function(evaluatedPars);
+	if (fnc->function == FunctionNode::Type::count) return count_function(agg_func_value, evaluatedPars);
+	if (fnc->function == FunctionNode::Type::max) return max_function(evaluatedPars, col_def_node, agg_func_value);
+	if (fnc->function == FunctionNode::Type::min) return min_function(evaluatedPars, col_def_node, agg_func_value);
 
-	throw Exception("invalid function: " + fnc->function);
+	throw Exception("invalid function: " + FunctionNode::function_name(fnc->function));
 }
 
 

usql/usql.h

@@ -13,14 +13,14 @@ namespace usql {
 class USql {
 	
 public:
-    USql() = default;
-
+	USql();
     std::unique_ptr<Table> execute(const std::string &command);
 
 private:
     std::unique_ptr<Table> execute(Node &node);
 
     std::unique_ptr<Table> execute_create_table(const CreateTableNode &node);
+	bool execute_create_table_sys_catalogue(const CreateTableNode &node);
     std::unique_ptr<Table> execute_create_index(const CreateIndexNode &node);
     std::unique_ptr<Table> execute_create_table_as_table(const CreateTableAsSelectNode &node);
     std::unique_ptr<Table> execute_load(const LoadIntoTableNode &node);
@@ -59,9 +59,6 @@ private:
 	void check_index_not_exists(const std::string &index_name);
 
 private:
-    Parser m_parser;
-    std::list<Table> m_tables;
-
     static void execute_distinct(SelectFromTableNode &node, Table *result);
     static void execute_order_by(SelectFromTableNode &node, Table *result);
     static void execute_offset_limit(OffsetLimitNode &node, Table *result);
@@ -73,8 +70,11 @@ private:
 
     static std::unique_ptr<ValueNode> lower_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars);
     static std::unique_ptr<ValueNode> upper_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars);
+    static std::unique_ptr<ValueNode> coalesce_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars);
     static std::unique_ptr<ValueNode> to_date_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars);
-    static std::unique_ptr<ValueNode> to_string_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars);
+    static std::unique_ptr<ValueNode> to_char_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars);
+    static std::unique_ptr<ValueNode> to_int_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars);
+    static std::unique_ptr<ValueNode> to_float_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars);
     static std::unique_ptr<ValueNode> date_add_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars);
     static std::unique_ptr<ValueNode> pp_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars);
 
@@ -94,6 +94,11 @@ private:
 	bool normalize_where(const Node *node) const;
 
     Table::rows_scanner get_iterator(Table *table, const Node *where) const;
+
+
+private:
+    Parser m_parser;
+    std::list<Table> m_tables;
 };
 
 

usql/usql_ddl.cpp

@@ -15,10 +15,35 @@ std::unique_ptr<Table> USql::execute_create_table(const CreateTableNode &node) {
 	Table table{node.table_name, node.cols_defs};
 	m_tables.push_back(table);
 
+	execute_create_table_sys_catalogue(node);
+
 	return create_stmt_result_table(0, "table created", 0);
 }
 
 
+bool USql::execute_create_table_sys_catalogue(const CreateTableNode &node) {
+	// usql_tables 
+	auto r = execute("insert into usql_tables(name, modified) values('" + node.table_name + "', false)");
+
+	// usql_columns 
+	for(const ColDefNode & col_def : node.cols_defs) {
+		std::string i {"insert into usql_columns(table_name, column_name, column_type, column_length, nullable, column_order) values("};
+					i += "'" + node.table_name + "', ";
+					i += "'" + col_def.name + "', ";
+					i += "'" + column_type_name(col_def.type) + "', ";
+					i += std::to_string(col_def.length) + ", ";
+					i += (col_def.null ?  "true, " : "false, ");
+					i += std::to_string(col_def.order);
+					i += ")";
+
+		auto r = execute(i);
+		// r->print();
+	}
+
+	return true;
+}
+
+
 std::unique_ptr<Table> USql::execute_create_index(const CreateIndexNode &node) {
 	Table *table_def = find_table(node.table_name);				// throws exception if not found
 	ColDefNode col_def = table_def->get_column_def(node.column_name);	// throws exception if not found
@@ -57,7 +82,6 @@ std::unique_ptr<Table> USql::execute_create_table_as_table(const CreateTableAsSe
 }
 
 
-
 std::unique_ptr<Table> USql::execute_drop(const DropTableNode &node) {
 	auto name_cmp = [node](const Table& t) { return t.m_name == node.table_name; };
 
@@ -70,11 +94,13 @@ std::unique_ptr<Table> USql::execute_drop(const DropTableNode &node) {
 	throw Exception("table not found (" + node.table_name + ")");
 }
 
+
 std::unique_ptr<Table> USql::execute_set(const SetNode &node) {
 	Settings::set_setting(node.name, node.value);
 	return create_stmt_result_table(0, "set succeeded", 1);
 }
 
+
 std::unique_ptr<Table> USql::execute_show(const ShowNode &node) {
 	std::string value = Settings::get_setting(node.name);
 	return create_stmt_result_table(0, "show succeeded: " + value, 1);

usql/usql_dml.cpp

@@ -35,7 +35,7 @@ std::pair<bool, std::vector<rowid_t>> USql::look_for_usable_index(const Node *wh
 					if (used_index != nullptr) {
 						std::vector<rowid_t> rowids = used_index->search((ValueNode *)ron->right.get());
 #ifndef NDEBUG
-						std::cout << "using index " << table->m_name << "(" << used_index->get_column_name() << "), " << rowids.size() << "/" << table->rows_count()  << std::endl;
+						std::cerr << "using index " << table->m_name << "(" << used_index->get_column_name() << "), " << rowids.size() << "/" << table->rows_count()  << std::endl;
 #endif
 						return std::make_pair(true, rowids);
 					}
@@ -108,24 +108,21 @@ void USql::select_row(SelectFromTableNode &where_node,
 			rslt_row->setColumnValue((ColDefNode *) &rslt_tbl_col_defs[idx], col_value);
 		}
 	}
-
 	// for aggregate is validated more than needed
 	rslt_table->commit_row(*rslt_row);
 }
 
 bool USql::check_for_aggregate_only_functions(SelectFromTableNode &node, size_t result_cols_cnt) {
 	size_t aggregate_funcs = 0;
+
 	for (size_t i = 0; i < node.cols_names->size(); i++) {
 		SelectColNode * col_node = &node.cols_names->operator[](i);
-		if (col_node->value->node_type == NodeType::function) {
-			auto func_node = static_cast<FunctionNode *>(col_node->value.get());
-			if (func_node->function == "count" || func_node->function == "min" || func_node->function == "max")
+		if (col_node->value->node_type == NodeType::function && ((FunctionNode *)col_node->value.get())->is_agg_function())
 			aggregate_funcs++;
 	}
-	}
 	// check whether aggregates are not present or all columns are aggregates
 	if (aggregate_funcs > 0 && aggregate_funcs != result_cols_cnt) {
-		throw Exception("aggregate functions with no aggregates");
+		throw Exception("aggregate functions mixed with no aggregate functions in select clause");
 	}
 
 	return aggregate_funcs > 0;
@@ -158,8 +155,7 @@ void USql::setup_order_columns(std::vector<ColOrderNode> &node, Table *table) {
 void USql::execute_distinct(SelectFromTableNode &node, Table *result) {
 	if (!node.distinct) return;
 
-	auto compare_rows = [](const Row &a, const Row &b) { return a.compare(b) >= 0; };
-	std::sort(result->m_rows.begin(), result->m_rows.end(), compare_rows);
+	std::sort(result->m_rows.begin(), result->m_rows.end(), [](const Row &a, const Row &b) { return a.compare(b) > 0; });
 
 	result->m_rows.erase(std::unique(result->m_rows.begin(), result->m_rows.end()), result->m_rows.end());
 }
@@ -220,7 +216,7 @@ ColDefNode USql::get_db_column_definition(Table *table, Node *node) {
 	throw Exception("Undefined table node - get_db_column_definition");
 }
 
-std::tuple<int, ColDefNode> USql::get_node_definition(Table *table, Node * node, const std::string & col_name, int col_order ) {
+std::tuple<int, ColDefNode> USql::get_node_definition(Table *table, Node * node, const std::string &col_name, int col_order ) {
 	if (node->node_type == NodeType::database_value) {
 		ColDefNode src_col_def = get_db_column_definition(table, node);
 		ColDefNode col_def = ColDefNode{col_name, src_col_def.type, col_order, src_col_def.length, src_col_def.null};
@@ -229,33 +225,54 @@ std::tuple<int, ColDefNode> USql::get_node_definition(Table *table, Node * node,
 	} else if (node->node_type == NodeType::function) {
 		auto func_node = static_cast<FunctionNode *>(node);
 
-		if (func_node->function == "to_string") {
+		if (func_node->function == FunctionNode::Type::to_char) {
 			ColDefNode col_def = ColDefNode{col_name, ColumnType::varchar_type, col_order, 32, true};
-			return std::make_tuple(-1, col_def);
-		} else if (func_node->function == "to_date") {
+			return std::make_tuple(FUNCTION_CALL, col_def);
+		} else if (func_node->function == FunctionNode::Type::to_int) {
 			ColDefNode col_def = ColDefNode{col_name, ColumnType::integer_type, col_order, 1, true};
-			return std::make_tuple(-1, col_def);
-		} else if (func_node->function == "pp") {
+			return std::make_tuple(FUNCTION_CALL, col_def);
+		} else if (func_node->function == FunctionNode::Type::to_float) {
+			ColDefNode col_def = ColDefNode{col_name, ColumnType::float_type, col_order, 1, true};
+			return std::make_tuple(FUNCTION_CALL, col_def);
+		} else if (func_node->function == FunctionNode::Type::to_date) {
+			ColDefNode col_def = ColDefNode{col_name, ColumnType::integer_type, col_order, 1, true};
+			return std::make_tuple(FUNCTION_CALL, col_def);
+		} else if (func_node->function == FunctionNode::Type::pp) {
 			ColDefNode col_def = ColDefNode{col_name, ColumnType::varchar_type, col_order, 10, true};
-			return std::make_tuple(-1, col_def);
-		} else if (func_node->function == "lower" || func_node->function == "upper") {
+			return std::make_tuple(FUNCTION_CALL, col_def);
+		} else if (func_node->function == FunctionNode::Type::lower || func_node->function == FunctionNode::Type::upper) {
 			// TODO get length, use get_db_column_definition
 			ColDefNode col_def = ColDefNode{col_name, ColumnType::varchar_type, col_order, 256, true};
-			return std::make_tuple(-1, col_def);
-		} else if (func_node->function == "min" || func_node->function == "max") {
-			auto col_type= ColumnType::float_type;
+			return std::make_tuple(FUNCTION_CALL, col_def);
+		} else if (func_node->function == FunctionNode::Type::coalesce) {
+			// TODO handle cases here
+			if (func_node->params.empty()) throw Exception("Coalesce without parameters");
+			if (func_node->params[0]->node_type != NodeType::database_value) throw Exception("Coalesce first parameter must be database column");
+
+			ColDefNode tbl_col_def = get_db_column_definition(table, func_node->params[0].get());
+			ColDefNode col_def = ColDefNode{col_name, tbl_col_def.type, col_order, tbl_col_def.length, true};
+			return std::make_tuple(FUNCTION_CALL, col_def);
+		} else if (func_node->function == FunctionNode::Type::min || func_node->function == FunctionNode::Type::max) {
+			auto col_type = ColumnType::float_type;
 			size_t col_len = 1;
-			auto & v = func_node->params[0];
+			auto &v = func_node->params[0];
+
 			if (v->node_type == NodeType::database_value) {
 				ColDefNode src_col_def = get_db_column_definition(table, v.get());
 				col_type = src_col_def.type;
 				col_len = src_col_def.length;
+
+			} else if (node->node_type == NodeType::function) {
+				std::tuple<int, ColDefNode> fun_type = get_node_definition(table, static_cast<Node*>(& *v), "", 0);
+				col_type = std::get<1>(fun_type).type;
+				col_len = std::get<1>(fun_type).length;
 			}
+
 			ColDefNode col_def = ColDefNode{col_name, col_type, col_order, col_len, true};
-			return std::make_tuple(-1, col_def);
-		} else if (func_node->function == "count") {
+			return std::make_tuple(FUNCTION_CALL, col_def);
+		} else if (func_node->function == FunctionNode::Type::count) {
 			ColDefNode col_def = ColDefNode{col_name, ColumnType::integer_type, col_order, 1, true};
-			return std::make_tuple(-1, col_def);
+			return std::make_tuple(FUNCTION_CALL, col_def);
 		}
 		throw Exception("Unsupported function");
 

usql/usql_function.cpp

@@ -7,17 +7,31 @@
 
 namespace usql {
 
-std::unique_ptr<ValueNode> USql::to_string_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars) {
+std::unique_ptr<ValueNode> USql::to_char_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars) {
+	if (evaluatedPars[0]->node_type == NodeType::int_value && evaluatedPars.size()==2) {		// TODO when propper date is introduced
 		long date = evaluatedPars[0]->getDateValue();
 		std::string format = evaluatedPars[1]->getStringValue();
 		std::string formatted_date = date_to_string(date, format);
+	
 		return std::make_unique<StringValueNode>(formatted_date);
+	} else {
+		return std::make_unique<StringValueNode>(evaluatedPars[0]->getStringValue());
+	}
+}
+
+std::unique_ptr<ValueNode> USql::to_int_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars) {
+	return std::make_unique<IntValueNode>(evaluatedPars[0]->getIntegerValue());
+}
+
+std::unique_ptr<ValueNode> USql::to_float_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars) {
+	return std::make_unique<DoubleValueNode>(evaluatedPars[0]->getDoubleValue());
 }
 
 std::unique_ptr<ValueNode> USql::to_date_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars) {
 	std::string date = evaluatedPars[0]->getStringValue();
 	std::string format = evaluatedPars[1]->getStringValue();
 	long epoch_time = string_to_date(date, format);
+	
 	return std::make_unique<IntValueNode>(epoch_time);        // No DateValueNode for now
 }
 
@@ -27,58 +41,82 @@ std::unique_ptr<ValueNode> USql::date_add_function(const std::vector<std::unique
 	std::string part = evaluatedPars[2]->getStringValue();
 
 	long new_date = add_to_date(datetime, quantity, part);
+	
 	return std::make_unique<IntValueNode>(new_date);        // No DateValueNode for now
 }
 
 
-std::unique_ptr<ValueNode> USql::upper_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars) {
-	std::string str = evaluatedPars[0]->getStringValue();
-	std::transform(str.begin(), str.end(), str.begin(), [](unsigned char c) -> unsigned char { return toupper(c); });
-	return std::make_unique<StringValueNode>(str);
-}
-
 std::unique_ptr<ValueNode> USql::lower_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars) {
 	std::string str = evaluatedPars[0]->getStringValue();
 	std::transform(str.begin(), str.end(), str.begin(), [](unsigned char c) -> unsigned char { return tolower(c); });
+	
 	return std::make_unique<StringValueNode>(str);
 }
 
+std::unique_ptr<ValueNode> USql::upper_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars) {
+	std::string str = evaluatedPars[0]->getStringValue();
+	std::transform(str.begin(), str.end(), str.begin(), [](unsigned char c) -> unsigned char { return toupper(c); });
+	
+	return std::make_unique<StringValueNode>(str);
+}
+
+std::unique_ptr<ValueNode> USql::coalesce_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars) {
+	for(const auto & par : evaluatedPars) {
+		if (!par->isNull()) {
+			// TODO implement rest and take it out as a function
+		 	if (par->node_type == NodeType::int_value)
+				return std::make_unique<IntValueNode>(par->getIntegerValue());
+		 	if (par->node_type == NodeType::float_value)
+				return std::make_unique<DoubleValueNode>(par->getDoubleValue());
+			if (par->node_type == NodeType::string_value)
+				return std::make_unique<StringValueNode>(par->getStringValue());
+			if (par->node_type == NodeType::bool_value)
+				return std::make_unique<BooleanValueNode>(par->getBooleanValue());
+		}
+	}
+	return std::make_unique<NullValueNode>();
+}
+
 std::unique_ptr<ValueNode> USql::pp_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars) {
+	constexpr auto k_num_format_rpad = 10;
+	constexpr auto k_num_format_maxlen = 20;
+
 	auto &parsed_value = evaluatedPars[0];
 
 	if (parsed_value->node_type == NodeType::int_value || parsed_value->node_type == NodeType::float_value) {
 		std::string format = evaluatedPars.size() > 1 ? evaluatedPars[1]->getStringValue() : "";
-		char buf[20] {0};	// TODO constant here
+		char buf[k_num_format_maxlen] {0};
 		double value = parsed_value->getDoubleValue();
 
 		if (format == "100%")
-			std::snprintf(buf, 20, "%.2f%%", value);
+			std::snprintf(buf, k_num_format_maxlen, "%.2f%%", value);
 		else if (format == "%.2f")
-			std::snprintf(buf, 20, "%.2f", value);
+			std::snprintf(buf, k_num_format_maxlen, "%.2f", value);
 		else if (value >= 1000000000000)
-			std::snprintf(buf, 20, "%7.2fT", value/1000000000000);
+			std::snprintf(buf, k_num_format_maxlen, "%7.2fT", value/1000000000000);
 		else if (value >= 1000000000)
-			std::sprintf(buf, "%7.2fB", value/1000000000);
+			std::snprintf(buf, k_num_format_maxlen, "%7.2fB", value/1000000000);
 		else if (value >= 1000000)
-			std::snprintf(buf, 20, "%7.2fM", value/1000000);
+			std::snprintf(buf, k_num_format_maxlen, "%7.2fM", value/1000000);
 		else if (value >= 100000)
-			std::snprintf(buf, 20, "%7.2fM", value/100000);	// 0.12M
+			std::snprintf(buf, k_num_format_maxlen, "%7.2fM", value/100000);	// 0.12M
 		else if (value <= -1000000000000)
-			std::snprintf(buf, 20, "%7.2fT", value/1000000000000);
+			std::snprintf(buf, k_num_format_maxlen, "%7.2fT", value/1000000000000);
 		else if (value <= -1000000000)
-			std::snprintf(buf, 20, "%7.2fB", value/1000000000);
+			std::snprintf(buf, k_num_format_maxlen, "%7.2fB", value/1000000000);
 		else if (value <= -1000000)
-			std::snprintf(buf, 20, "%7.2fM", value/1000000);
+			std::snprintf(buf, k_num_format_maxlen, "%7.2fM", value/1000000);
 		else if (value <= -100000)
-			std::snprintf(buf, 20, "%7.2fM", value/100000);	// 0.12M
+			std::snprintf(buf, k_num_format_maxlen, "%7.2fM", value/100000);	// 0.12M
 		else if (value == 0)
 			buf[0]='0';
 		else
-			return std::make_unique<StringValueNode>(parsed_value->getStringValue().substr(0, 10));
-		// TODO introduce constant for 10
+			return std::make_unique<StringValueNode>(parsed_value->getStringValue().substr(0, k_num_format_rpad));
+
 		std::string s {buf};
-		return std::make_unique<StringValueNode>(string_padd(s.erase(s.find_last_not_of(' ')+1), 10, ' ', false));
+		return std::make_unique<StringValueNode>(string_padd(s.erase(s.find_last_not_of(' ') + 1), k_num_format_rpad, ' ', false));
 	}
+
 	return std::make_unique<StringValueNode>(parsed_value->getStringValue());
 }
 

utils/Lisp.tmLanguage

@@ -51,33 +51,10 @@
 				</dict>
 			</dict>
 			<key>match</key>
-			<string>(?:\()((?i:def|defn|lambda)+)</string>
+			<string>(?:\()((?i:defn|defmacro|def|lambda)+)</string>
 			<key>name</key>
 			<string>meta.function.lisp</string>
 		</dict>
-		<!-- <dict>
-			<key>match</key>
-			<string>(?<=\s)(?i:&allow-other-keys|&aux|&body|&environment|&key|&optional|&rest|&whole)(?=\s+)</string>
-			<key>name</key>
-			<string>meta.function-parameters.lisp</string>
-		</dict> -->
-		<!-- <dict>
-			<key>captures</key>
-			<dict>
-				<key>1</key>
-				<dict>
-					<key>name</key>
-					<string>storage.type.function-type.lisp</string>
-				</dict>
-				<key>2</key>
-				<dict>
-					<key>name</key>
-					<string>entity.name.type.lisp</string>
-				</dict>
-			</dict>
-			<key>match</key>
-			<string>(?:\()((?i:deftype|defstruct|define-condition|defpackage|defclass))\s+((?:\w|[+\-<>/*&amp;=.?!$%:@\[\]^{}~#|])+)</string>
-		</dict> -->
 		<dict>
 			<key>captures</key>
 			<dict>

utils/local_install.sh

@@ -1,25 +1,34 @@
 #!/bin/sh
 
 echo "cleaning"
+rm -f CMakeCache.txt
 cmake -DCMAKE_BUILD_TYPE=Release .
 cmake --build ./ --target clean -j 4 --
 echo "building ml"
 cmake --build ./ --target all -j 4 --
 
 echo ""
+
+# echo "create dirs"
+# sudo mkdir -p /usr/local/var/mlisp/
+# sudo mkdir -p /usr/local/bin/
+# sudo chown -R vaclavt:admin /usr/local/bin/
+# sudo chown -R vaclavt:admin /usr/local/var/mlisp/
+
 echo "copying lsp files"
-mkdir -p /usr/local/var/mlisp/
 cp stdlib/*.lsp /usr/local/var/mlisp/
+
 echo "copying doc files"
 cp doc/*.md /usr/local/var/mlisp/
+
 echo "copying ml file"
 mv ./ml /usr/local/bin/ml
 
-
-
 echo ""
 echo "if syntax has changed you may use"
 echo "cp utils/Lisp.tmLanguage ~/.vscode/extensions/mattn.lisp-0.1.12/syntaxes/"
+echo "or"
+echo "cp utils/Lisp.tmLanguage ~/.local/share/code-server/extensions/mattn.lisp-0.1.11/syntaxes/Lisp.tmLanguage"
 echo ""
 
 ml -v

utils/resetCLion.sh

@@ -1,34 +0,0 @@
-#!/bin/bash
-#https://github.com/PythonicNinja/jetbrains-reset-trial-mac-osx/blob/master/runme.sh
-
-for product in CLion; do
-  echo "Closing $product"
-  ps aux | grep -i MacOs/$product | cut -d " " -f 5 | xargs kill -9
-
-  echo "Resetting trial period for $product"
-
-  echo "removing evaluation key..."
-  rm -rf ~/Library/Preferences/$product*/eval
-
-  # Above path not working on latest version. Fixed below
-  rm -rf ~/Library/Application\ Support/JetBrains/$product*/eval
-
-  echo "removing all evlsprt properties in options.xml..."
-  sed -i '' '/evlsprt/d' ~/Library/Preferences/$product*/options/other.xml
-
-  # Above path not working on latest version. Fixed below
-  sed -i '' '/evlsprt/d' ~/Library/Application\ Support/JetBrains/$product*/options/other.xml
-
-  echo
-done
-
-echo "removing additional plist files..."
-rm -f ~/Library/Preferences/com.apple.java.util.prefs.plist
-rm -f ~/Library/Preferences/com.jetbrains.*.plist
-rm -f ~/Library/Preferences/jetbrains.*.*.plist
-
-echo "restarting cfprefsd"
-killall cfprefsd
-
-echo
-echo "That's it, enjoy ;)"

wip.lsp

@@ -3,26 +3,6 @@
 (print (filter (lambda (eval e) (print e)) l))
 
 
-(thread-create
- 	(tcp-server 7777 (lambda (str) (list #f (+ "(print \"" (string-upcase str) "\")"))))
-)
-
-(thread-sleep 1)
-
-(thread-create
-	(def code (tcp-client "127.0.0.1" 7777 ("abcd" "xyz")))
-	(for c code
-		(print "executing code:" c)
- 		(eval (parse c))
-	)
-
-	(def code (tcp-client "127.0.0.1" 7777 "abcd"))
-	(print "executing code:" code)
- 	(eval (parse code))
-)
-(threads-join)
-(print "ok")
-
 
 ;; (usql "create table sf1 (symbol varchar(8), dimension varchar(3), calendar_date date, date_key date, report_period date, last_updated date, accoci float, assets float, assetsavg float, assetsc float, assetsnc float, assetturnover float, bvps float, capex float, cashneq float, cashnequsd float, cor float, consolinc float, currentratio float, de float, debt float, debtc float, debtnc float, debtusd float, deferredrev float, depamor float, deposits float, divyield float, dps float, ebit float, ebitda float, ebitdamargin float, ebitdausd float, ebitusd float, ebt float, eps float, epsdil float, epsusd float, equity float, equityavg float, equityusd float, ev float, evebit float, evebitda float, fcf float, fcfps float, fxusd float, gp float, grossmargin float, intangibles float, intexp float, invcap float, invcapavg float, inventory float, investments float, investmentsc float, investmentsnc float, liabilities float, liabilitiesc float, liabilitiesnc float, marketcap float, ncf float, ncfbus float, ncfcommon float, ncfdebt float, ncfdiv float, ncff float, ncfi float, ncfinv float, ncfo float, ncfx float, netinc float, netinccmn float, netinccmnusd float, netincdis float, netincnci float, netmargin float, opex float, opinc float, payables float, payoutratio float, pb float, pe float, pe1 float, ppnenet float, prefdivis float, price float, ps float, ps1 float, receivables float, retearn float, revenue float, revenueusd float, rnd float, roa float, roe float, roic float, ros float, sbcomp float, sgna float, sharefactor float, sharesbas float, shareswa float, shareswadil float, sps float, tangibles float, taxassets float, taxexp float, taxliabilities float, tbvps float, workingcapital float)")
 ;; (usql "set 'DATE_FORMAT' = '%Y-%m-%d'")