a bit od examples in doc

CMakeLists.txt

@@ -15,9 +15,9 @@ set(CPACK_PROJECT_VERSION ${PROJECT_VERSION})
 # set(CMAKE_CXX_FLAGS "-Wall -Wextra")
 set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG")
 
-include_directories(/usr/local/opt/openssl/include ${CMAKE_SOURCE_DIR}/clib ${CMAKE_SOURCE_DIR})
+include_directories(/opt/homebrew/opt/openssl@1.1/include ${CMAKE_SOURCE_DIR}/clib ${CMAKE_SOURCE_DIR})
 
-link_directories(/usr/local/lib /usr/local/opt/openssl/lib)
+link_directories(/opt/homebrew/opt/openssl@1.1/lib)
 
 
 project(ml)

Readme.md

@@ -65,11 +65,11 @@ utils/local_install.sh
 
 
 ### KNOWNN BUGS
-- server/client problem in std::string TcpNet::read_from_socket(int sockfd) - reading dows not know lenght to read
 - (read-url "https://api.nasdaq.com/api/calendar/dividends/") ; hangs in sslclient.cpp line 132
 
 ### TODO
 - order of arguments in functions like member and filter - filter swap lambda and list
+- add possibility to specify filename for input2 history file
 - better output of !e in repl (resp !e !ee)
 - unify -f and -run options
 - add debug support, at least call stack

clib/sslclient.cpp

@@ -62,7 +62,7 @@ std::pair<int, std::string> HttpClient::doRequest(const std::string &method, con
 
 	// and fetch the rest if not read completely
 	while (content_len > 0 && content_len > ssl_read_packet.length() - 4 - end_of_headers) {
-		auto read_bytes = sslRecvPacket();
+		/* auto read_bytes = */ sslRecvPacket();
 	}
 
 	// get body

clib/tcpnet.cpp

@@ -71,10 +71,7 @@ int TcpNet::server(int portno, const std::function<std::pair<bool, std::string>(
 			shutdown = response.first;
 			std::string response_str = response.second;
 
-			auto response_len = response_str.size();
+			write_to_socket(newsockfd, response_str);
-			auto n = write(newsockfd, response_str.c_str(), response_len);
-			if (n < 0 || response_len != n)
-				error("ERROR writing to socket");
 
 			requests_processed++;
 		}
@@ -134,11 +131,26 @@ std::string TcpNet::client(const std::string &address, int portno, const std::st
 	return response[0];
 }
 
+
+
+
+
+
 std::string TcpNet::read_from_socket(int sockfd) {
 	char buffer[TCPNET_BUFFER_SIZE];
 	std::string request;
 
+	// read length header
+	unsigned long long readdatalen = 0;
+	if (USE_LENGTH_HEADER) {
+		long n = read(sockfd, &readdatalen, sizeof(readdatalen));
+		if (n != 0 && n != sizeof(readdatalen))
+			error("ERROR reading length header failed");
+	}
+
+	// read data
 	long n;
+	long readed = 0;
 	do {
 		memset(buffer, 0, TCPNET_BUFFER_SIZE);
 		n = read(sockfd, buffer, TCPNET_BUFFER_SIZE - 1);
@@ -153,15 +165,26 @@ std::string TcpNet::read_from_socket(int sockfd) {
 
 		std::string part{buffer};
 		request.append(part);
-	} while (n == TCPNET_BUFFER_SIZE - 1);    // TODO what if data exactly of this size
+		readed += n;
+	} while ((USE_LENGTH_HEADER && readed < readdatalen) || n == TCPNET_BUFFER_SIZE - 1);    // TODO what if data exactly of this size
 
 	return request;
 }
 
 void TcpNet::write_to_socket(int sockfd, const std::string &str) {
 	const char *buffer = str.c_str();
-	int pos = 0;
 	long n;
+
+	// write length header
+	unsigned long long writedatalen = str.length(); 
+	if (USE_LENGTH_HEADER) {
+		n = write(sockfd, &writedatalen, (int) sizeof(writedatalen));
+		if (n < 0)
+			error("ERROR writing size number to socket");
+	}
+	
+	// write data
+	int pos = 0;
 	do {
 		n = write(sockfd, buffer + pos, (int) (str.length() - pos));
 		if (n < 0)

clib/tcpnet.h

@@ -17,6 +17,7 @@ public:
 	[[nodiscard]] std::vector<std::string> client(const std::string &address, int portno, const std::vector<std::string> &requests) const;
 
 private:
+	static constexpr bool USE_LENGTH_HEADER = true;
 	static std::string read_from_socket(int sockfd) ;
 	static void write_to_socket(int sockfd, const std::string &str) ;
 };

debug.lsp

@@ -1,18 +1,22 @@
+(dotimes i 4 (print i) )
 
-(def #sql_srv_address "stocksriddle.one")
+(def acc 0)
-(def #sql_srv_port 5388)
+(print (dotimes i 4 (do (set! acc (+ acc i)) (print acc))) )
 
-(defn exec-sql (sql)
+(defmacro until (cnd body)
-(do
+	(list 'while '(not (eval cnd)) 
-	(def r (tcp-client #sql_srv_address #sql_srv_port sql))
+		body
-	(print r)
-	(first (eval (parse r)))
     ))  
 
-(def sql "select distinct to_string(datetime, '%d.%m.%Y'),symbol,time,title from history_earnings_dates where datetime>=to_date('2022-03-02', '%Y-%m-%d') and datetime<=to_date('2022-03-19', '%Y-%m-%d')")
-(print "*" sql "*")
-(def row (exec-sql sql))
-(print row)
 
-(exit 1)
 
+(def stop nil)
+
+
+(until stop
+(do
+    (def line (input2 "=> "))
+	(if (= line "!q")
+		(set! stop #t)
+		(print line))
+))

doc/Doc.md

@@ -2,8 +2,8 @@
 |option|Description|
 |:-|-|
 |-h|Prints help|
-|-b|Skips loadin of std lib|
+|-b|Skips loading of std lib|
-|-c code|Runs given code|
+|-c code|Runs passed code|
 |-f source_file ..|Executes code in files|
 |-run source_file ..|Executes code in file, if first line of file is sheebang, it is skipped|
 |-i|Runs REPL|
@@ -23,7 +23,7 @@
 ## Syntax and Special Forms
 |Special Form|Argument Evaluations|Purpose|Section|
 |:-|-|-|-|
-|`(if cond a b)`|`if` only evaluates its `cond` argument. If `cond` is truthy (non-zero), then `a` is evaluated. Otherwise, `b` is evaluated.|This special form is the main method of control flow.|Language|
+|`(if cond a b)`|`if` only evaluates its `cond` argument. If `cond` is truthy, then `a` is evaluated. Otherwise, `b` is evaluated.|This special form is the main method of control flow.|Language|
 |`(cond (test1 action1 ..) (test2 action2 ..) ... (testn ..))`|The first clause whose test evaluates to non-nil is selected; all other clauses are ignored, and the consequents of the selected clause are evaluated in order. If none of the test conditions are evaluated to be true, then the cond statement returns nil.|This special form is method of control flow.|Language|
 |`(do a b c ...)`|`do` takes a list of s-expressions and evaluates them in the order they were given (in the current scope), and then returns the result of the last s-expression.|This special form allows lambda functions to have multi-step bodies.|Language|
 |`(scope a b c ...)`|`scope` takes a list of s-expressions and evaluates them in the order they were given _in a new scope_, and then returns the result of the last s-expression.|This special form allows the user to evaluate blocks of code in new scopes.|Language|
@@ -39,8 +39,15 @@
 |`(benchmark msg_string ...)`|`benchmark` takes a list of s-expressions and evaluates them in the order they were given (in the current scope), and then returns the result of the last s-expression and prints msg_string and timing info.||Language|
 |`(set! x)`|`set!` ...|....|Language|
 
+## Macros
+|Signature|Description|Example|Section|
+|:-|-|-|-|
+|`(unles cond body)`|`unless` only evaluates its `cond` argument. If `cond` is not truthy, then `body` is evaluated.|`>>> (unless #t (print "it is #f")) => nil`|Language|
+|`(dotimes v n body)`|Iterates over `v` from 0 below `n` and evaluated `body`. Returns the last value of v|`>>> (dotimes i 5 i) => 4`|Language|
+|`(until cond body)`|Evaluated `body` as long as `cond` is not truthy|`>>> (def i 0)(until (> i 3) (set! i (inc i))) => 4`|Language|
+
 ## Library
-|Signature|Description|Return values|Section|
+|Signature|Description|Example|Section|
 |:-|-|-|-|
 |`(= a b)`|Test whether two values are equal|`1` when equal otherwise `nil`|Language|
 |`(!= a b)`|Test whether two values are not equal|`1` when not equal otherwise `nil`|Language|
@@ -56,10 +63,10 @@
 |`(list ..)`|Create a list of values||List manipulation|
 |`(insert list index element)`|Insert an element into a list. Indexed from 0|new list with value inserted|List manipulation|
 |`(index list index)`|Return element at index in list. First element is at index 0|Element at index|List manipulation|
-|`(remove list index)`|Remove a value at an index from a list|List with element removed|List manipulation|
+|`(remove list index)`|Remove a value at an index from a list|`>>> (remove '(1 2 3 4) 1) => (1 3 4)`|List manipulation|
-|`(len list)`|Get the length of a list|list length|List manipulation|
+|`(len list)`|Get the length of a list|`>>> (len '(1 2 3 4)) => 4`|List manipulation|
-|`(push list element)`|Add an item to the end of a list|new list with element added|List manipulation|
+|`(push list element)`|Add an item to the end of a list|`>>> (push '(1 2 3 4) 5) => (1 2 3 4 5)`|List manipulation|
-|`(pop list)`|Returns last element of list|Last element|List manipulation|
+|`(pop list)`|Returns last element of list|`>>> (pop '(1 2 3 4)) => 4`|List manipulation|
 |`(head list)`|Returns first element of a list|`>>> (head '(1 2 3)) => 1`|List manipulation|
 |`(tail list)`|Return all elements of list except first one|`>>> (tail '(1 2 3)) => (2 3)`|List manipulation|
 |`(first list)`|Returns first element of a list|`>>> (first '(1 2 3)) => 1`|List manipulation|
@@ -76,28 +83,33 @@
 |`(find-val-in-list lst name)`|Returns tail od sublist which first element is name|`>>> (find-val-in-list '(("a" ("av" "avv")) ("b" "bv") (31 32 33) (41 42 43)) "b") => "bv" >>> `|List manipulation|
 |`(map ..)`|Returns list that is the result of executing lambda on its elements|`(map (lambda (x) (+ x 10)) '(1 2 3 4 5 6)) => (11 12 13 14 15 16)`|List manipulation|
 |`(filter lambda list)`|Returns list with elements for which passed lambda returns true|`(filter (lambda (x) (> x 2)) '(1 2 3 4 5)) => (3 4 5)`|List manipulation|
-|`(reduce lambda acumulator list)`|Reduces list|`>>> (reduce (lambda (x y) (+ (* x 10) y)) 0 '(1 2 3 4)) => 1234`|List manipulation|
+|`(reduce lambda acumulator list)`|Reduces list|`>>> (reduce (lambda (acc e) (+ (* acc 10) e)) 0 '(1 2 3 4)) => 1234`|List manipulation|
 |`(exit code)`|Exit the program with an integer code||System|
 |`(quit code)`|Same as (exit ..)||System|
 |`(print ..)`|Print one or several values separated by space and return the last one|`>>> (print "pi" "is" 3.14)\npi is 3.140000 => 3.140000`|IO|
 |`(random low high)`|Get a random number between two numbers inclusively|`>>> (random 1 6) => 5`|System|
 |`(include file)`|Read a file and execute its code||IO|
 |`(input [prompt])`|Get user input with an optional prompt||IO|
+|`(input2 [prompt])`|Get user input using libnoise with an optional prompt||IO|
 |`(read)`|Reads in the printed representation of a Lisp object from input-stream, builds a corresponding Lisp object, and returns the object||IO|
 |`(read-file filename)`|Get the contents of a file|`>>> (read-file "/tmp/a.txt") => "test"`|IO|
 |`(read-file-lines filename lambda)`|Reads file and for each line call lambda with passing the line as a parameter|`(read-file-lines "/tmp/f.txt" (lambda (ln) (print ln))`|IO|
 |`(write-file filename content-str)`|Write a string to a file|`>>> (write-file "/tmp/a.txt" "test") => #t`|IO|
 |`(read-url url [headers] [body] [method])`|Reads URL. Returns list (status-code content)||IO|
-|`(system-cmd command_str)`|Execute system command|`>>> (system-cmd "date") => (0 "Fri Feb 25 12:35:28 CET 2022\n")`|System|
+|`(system-cmd command_str)`|Execute system command|`>>> (system-cmd-fork "ml"  "-c" "(print 123) (sleep 1) (print \"aaa\")") => (0 "") >>> 123\n aaa`|System|
+|`(system-cmd-fork cmd [par ..])`|Execute system command as an independent process|`>>> (system-cmd "date") => (0 "Fri Feb 25 12:35:28 CET 2022\n")`|System|
 |`(ls-dir dir)`|List a dir|List of directory entries|`>>> (ls-dir "/tmp") => ("." ".." "vscode-ipc-cccbe1dd-8c71-4028-a863-df975ad5887b.sock" "vscode-ipc-1163bb52-d088-41dc-80a5-81b6a7c7fa36.sock"  "vscode-ipc-630f21df-26b5-43d4-8b2e-5175d53ce317.sock")`|IO|
 |`(is-file? filename)`|Returns true if passed filename is a file|`>>> (is-file? "/tmp") => nil`|IO|
 |`(is-dir? filename)`|Returns true if passed filename is a directory|`>>> (is-dir? "/tmp") => #t`|IO|
+|`(mk-dir dirname)`|Creates directory wirh dirname. Does not create missing parent directories. Returns true if creation successfull|`>>> (mk-dir "/tmp/testdir") => #t`|IO|
+|`(rm-dir dirname)`|Removes directory wirh dirname. Does not create missing parent directories. Returns true if removal successfull|`>>> (rm-dir "/tmp/testdir") => #t`|IO|
 |`(tcp-server port handler)`|Starts listening on port and when request comes calls passed lambda and writes its returned value back to client. Lambda must return either string or two element list where first element is boolean and second string.When first element is true it closes listening socker.|(`tcp-server 7777 (lambda (str) (list #t (string-upcase str))))`|IO|
 |`(tcp-client address port data)`|Opens connection to server on port, writes there data and returns response.|`(print (tcp-client "127.0.0.1" 7777 "abcd"))`|IO|
 |`(parse-csv string)`|Parse CSV string|`>>> (parse-csv "A,B\n1,1\n2,2") => ((1 1) (2 2))`|String manipulation|
 |`(parse-json json_string)`|Parse JSON string|`>>> (parse-json "{\"k1\":\"v1\", \"k2\":42, \"k3\":[\"a\",123,true,false,null]}") => (("k1" "v1") ("k2" 42) ("k3" ("a" 123 #t nil nil)))`|String manipulation|
 |`(make-csv ..)`|Returns list as csv string|`>>> (make-csv '(("A" "B")(1 1)(2 2))) => "A,B\n1,1\n2,2"`|String manipulation|
-|`(get-universal-time)`|Get current time as secs from epoch|`>>> (get-universal-time) => 1642152733`|Date and time|
+|`(get-universal-time)`|Get current time as number of seconds from epoch|`>>> (get-universal-time) => 1642152733`|Date and time|
+|`(get-universal-time-ms)`|Get current time as number of miliseconds from epoch|`>>> (get-universal-time-ms) => 1653575251097`|Date and time|
 |`(get-localtime-offset)`|Offset in seconds between local time and gmt time|`>>> (get-localtime-offset) => 3600`|Date and time|
 |`(date-to-str date format)`|Converts date to formated string. Format is strftime format (https://www.tutorialspoint.com/c_standard_library/c_function_strftime.htm)|`>>> (date-to-str (get-universal-time) "%Y-%m-%d %H:%M:%S") => "2021-03-13 19:53:01"`|Date and time|
 |`(str-to-date string format)`|Converst string to time of secs since epoch. |`>>> (str-to-date "2021-03-13 19:53:01" "%Y-%m-%d %H:%M:%S") => 1615665181`|Date and time|

doc/readme.txt

@@ -0,0 +1,22 @@
+to convert Doc.md to Doc.html use https://markdowntohtml.com/ with custom ccs style
+
+body {
+   color: black;
+}
+table {
+  border-collapse: collapse;
+}
+table, th, td {
+  border: 1px solid;
+}
+th {
+  text-align: left;
+}
+td {
+  height: 50px;
+  vertical-align: top;
+}
+code{
+    background: #0003;
+    color: #a31515;
+}

docker/Dockerfile

@@ -0,0 +1,31 @@
+# docker build -t ml:latest .
+# docker run --rm -it ml ml -v
+# docker run --rm -it --entrypoint sh ml
+
+
+FROM alpine:3.15.4 AS builder
+
+# Install all dependencies required for compiling ml
+RUN apk add --verbose build-base musl-dev openssl-dev make git cmake 
+
+RUN git clone http://gitea.stocksriddle.one/vaclavt/mlisp.git
+
+# Compile 
+RUN cd /mlisp \
+  && rm -f CMakeCache.txt \
+  && cmake -DCMAKE_BUILD_TYPE=Release . \
+  && cmake --build ./ --target clean -j 4 -- \
+  && cmake --build ./ --target all -j 4 --
+
+
+# Create image and copy compiled installation into it
+FROM alpine:3.15.4
+
+RUN apk add --no-cache openssl libstdc++
+
+CMD mkdir -p /usr/local/var/mlisp/
+COPY --from=builder /mlisp/stdlib/*.lsp /usr/local/var/mlisp/
+COPY --from=builder /mlisp/doc/*.md /usr/local/var/mlisp/
+
+CMD mkdir -p /usr/local/bin/
+COPY --from=builder /mlisp/ml /usr/local/bin/

ml.cpp

@@ -21,6 +21,7 @@
 #include <sstream>
 #include <cstdlib>
 #include <iostream>
+#include <fstream>
 #include <ctime>
 #include <chrono>
 #include <thread>
@@ -1134,6 +1135,24 @@ MlValue input(std::vector<MlValue> args, MlEnvironment &env) {
 	return MlValue::string(s);
 }
 
+// Get user input with an optional prompt using line noise
+MlValue input2(std::vector<MlValue> args, MlEnvironment &env) {
+	// TODO add setup etc
+	eval_args(args, env);
+
+	if (args.size() > 1)
+		throw MlError(MlValue("input2", input2), env, TOO_MANY_ARGS);
+
+	char *line = linenoise(args.empty() ? ">>> " : args[0].as_string().c_str());
+	if (line == nullptr) throw std::runtime_error("date_to_string, cannot initialise linenoise");
+
+	std::string input{line};
+	linenoise_add_to_history(input);
+
+
+	return MlValue::string(input);
+}
+
 // Get a random number between two numbers inclusively
 MlValue random(std::vector<MlValue> args, MlEnvironment &env) {
 	eval_args(args, env);
@@ -1279,6 +1298,16 @@ MlValue get_universal_time(std::vector<MlValue> args, MlEnvironment &env) {
 	return MlValue(now());
 }
 
+// Get current time as miliseconds from epoch
+MlValue get_universal_time_ms(std::vector<MlValue> args, MlEnvironment &env) {
+	eval_args(args, env);
+
+	if (!args.empty())
+		throw MlError(MlValue("get-universal-time-ms", get_universal_time_ms), env, TOO_MANY_ARGS);
+
+	return MlValue(now_ms());
+}
+
 // Get offsets in secs between local timezone and gmt
 MlValue get_localtime_offset(std::vector<MlValue> args, MlEnvironment &env) {
 	eval_args(args, env);
@@ -1331,6 +1360,22 @@ MlValue system_cmd(std::vector<MlValue> args, MlEnvironment &env) {
 	return exec_system_cmd(args[0].as_string());
 }
 
+// Execute system command as forked process so its independent from its parrent
+MlValue system_cmd_fork(std::vector<MlValue> args, MlEnvironment &env) {
+	eval_args(args, env);
+
+	// TODO add support for more params constructing options as one string
+	if (args.size() < 1)
+		throw MlError(MlValue("system-cmd-fork", system_cmd_fork), env, TOO_FEW_ARGS);
+
+	std::vector<std::string> exec_args;
+	exec_args.reserve(args.size());
+	for (auto const& a : args)
+			exec_args.emplace_back(a.as_string());
+
+	return exec_system_cmd_fork(exec_args);
+}
+
 
 // list directory
 MlValue ls_dir(std::vector<MlValue> args, MlEnvironment &env) {
@@ -1362,6 +1407,24 @@ MlValue is_dir(std::vector<MlValue> args, MlEnvironment &env) {
 	return MlValue(is_path_dir(args[0].as_string()));
 }
 
+MlValue mk_dir(std::vector<MlValue> args, MlEnvironment &env) {
+	eval_args(args, env);
+
+	if (args.size() != 1)
+		throw MlError(MlValue("mk-dir", mk_dir), env, args.size() > 1 ? TOO_MANY_ARGS : TOO_FEW_ARGS);
+
+	return MlValue((bool)mk_path_dir(args[0].as_string()));
+}
+
+MlValue rm_dir(std::vector<MlValue> args, MlEnvironment &env) {
+	eval_args(args, env);
+
+	if (args.size() != 1)
+		throw MlError(MlValue("rm-dir", rm_dir), env, args.size() > 1 ? TOO_MANY_ARGS : TOO_FEW_ARGS);
+
+	return MlValue((bool)rm_path_dir(args[0].as_string()));
+}
+
 // starts tcp listening server
 MlValue tcp_server(std::vector<MlValue> args, MlEnvironment &env) {
 	eval_args(args, env);
@@ -2133,9 +2196,6 @@ MlValue usql(std::vector<MlValue> args, MlEnvironment &env) {
 
 }	// namespace builtin
 
-void load_std_lib(MlEnvironment &env) {
-	run(STDLIB_LOADER, env);
-}
 
 // Does this environment, or its parent environment, have a variable?
 bool MlEnvironment::has(const std::string &name) const {
@@ -2217,15 +2277,19 @@ std::map<const std::string, Builtin> builtin_funcs
 	std::make_pair("random", builtin::random),
 	std::make_pair("include", builtin::include),
 	std::make_pair("input", builtin::input),
+	std::make_pair("input2", builtin::input2),
 	std::make_pair("read", builtin::read),
 	std::make_pair("read-file", builtin::read_file),
 	std::make_pair("read-file-lines", builtin::read_file_lines),
 	std::make_pair("write-file", builtin::write_file),
 	std::make_pair("read-url", builtin::read_url),
 	std::make_pair("system-cmd", builtin::system_cmd),
+	std::make_pair("system-cmd-fork", builtin::system_cmd_fork),
 	std::make_pair("ls-dir", builtin::ls_dir),
 	std::make_pair("is-file?", builtin::is_file),
 	std::make_pair("is-dir?", builtin::is_dir),
+	std::make_pair("mk-dir", builtin::mk_dir),
+	std::make_pair("rm-dir", builtin::rm_dir),
 	std::make_pair("tcp-server", builtin::tcp_server),
 	std::make_pair("tcp-client", builtin::tcp_client),
 	std::make_pair("get-env", builtin::get_env),
@@ -2237,6 +2301,7 @@ std::map<const std::string, Builtin> builtin_funcs
 
 	// Datetime operations
 	std::make_pair("get-universal-time", builtin::get_universal_time),
+	std::make_pair("get-universal-time-ms", builtin::get_universal_time_ms),
 	std::make_pair("get-localtime-offset", builtin::get_localtime_offset),
 	std::make_pair("date-to-str", builtin::date_to_str),
 	std::make_pair("str-to-date", builtin::str_to_date),
@@ -2323,15 +2388,14 @@ void repl(MlEnvironment &env) {
 	MlValue tmp;
 	std::vector<MlValue> parsed;
 
-	setup_linenoise(env, builtin_funcs);
+	setup_linenoise_repl(env, builtin_funcs);
 
 	while (true) {
 		char *line = linenoise(">>> ");
 		if (line == nullptr) break;
 
-		linenoise_line_read(line);
-
 		input = std::string(line);
+		linenoise_add_to_history(input);
 
 		if (input == "!quit" || input == "!q")
 			break;
@@ -2355,8 +2419,6 @@ void repl(MlEnvironment &env) {
 			}
 		}
 	}
-
-	close_linenoise();
 }
 
 
@@ -2375,6 +2437,12 @@ std::vector<std::string> getCmdOption(char *argv[], int argc, const std::string
 	return tokens;
 }
 
+
+void load_std_lib(MlEnvironment &env) {
+	run(STDLIB_LOADER, env);
+}
+
+
 int main(int argc, char *argv[]) {
 	MlEnvironment env;
 	std::vector<MlValue> args;
@@ -2387,11 +2455,13 @@ int main(int argc, char *argv[]) {
 	srand(time(NULL));
 	try {
 		// performance monitor on
-		if (cmdOptionExists(argv, argv + argc, "-p"))
+		if (cmdOptionExists(argv, argv + argc, "-p")) {
 			MlPerfMon::instance().turnOn();
+		}
 		// better stacktrace
-		if (cmdOptionExists(argv, argv + argc, "-d"))
+		if (cmdOptionExists(argv, argv + argc, "-d")) {
 			MlPerfMon::instance().debugOn();
+		}
 		// skip loading std lib
 		if (!cmdOptionExists(argv, argv + argc, "-b")) {
 			load_std_lib(env);
@@ -2406,6 +2476,9 @@ int main(int argc, char *argv[]) {
 			std::cout << VERSION << std::endl;
 			return 0;
 		}
+
+		setup_linenoise();
+		
 		// passed code
 		if (cmdOptionExists(argv, argv + argc, "-c")) {
 			std::vector<std::string> codes = getCmdOption(argv, argc, "-c");
@@ -2429,6 +2502,8 @@ int main(int argc, char *argv[]) {
 			repl(env);
 		}
 
+		close_linenoise();
+
 		MlPerfMon::instance().print_results();
 
 		return 0;

ml.h

@@ -7,9 +7,9 @@
 #include <exception>
 
 #ifdef NDEBUG
-const std::string VERSION = "ml 0.5 (" __DATE__ " " __TIME__ "), Release";
+const std::string VERSION = "ml 0.5.1 (" __DATE__ " " __TIME__ "), Release";
 #else
-const std::string VERSION = "ml 0.5 (" __DATE__ " " __TIME__ "), Debug";
+const std::string VERSION = "ml 0.5.1 (" __DATE__ " " __TIME__ "), Debug";
 #endif
 
 const std::string STDLIB_LOADER =

ml_date.cpp

@@ -2,6 +2,7 @@
 #include "ml_date.h"
 #include <cstring>
 #include <stdexcept>
+#include <chrono>
 
 long now() {	// get-universal-time
 	time_t t = std::time(nullptr);
@@ -10,6 +11,13 @@ long now() {	// get-universal-time
 	return now;
 }
 
+long now_ms() {	// get-universal-time-ms
+	using namespace std::chrono;
+	milliseconds ms = duration_cast< milliseconds >(system_clock::now().time_since_epoch());
+
+	return ms.count();
+}
+
 long get_gmt_localtime_offset() {
 	std::time_t current_time;
 	std::time(&current_time);

ml_date.h

@@ -6,6 +6,7 @@
 
 
 long now();
+long now_ms();
 
 long get_gmt_localtime_offset();
 

ml_io.cpp

@@ -1,10 +1,14 @@
 
 #include "ml_io.h"
 
+#include <cstring>
 #include <dirent.h>
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include <iostream>
+#include <fstream>
+
 
 std::string read_file_contents(const std::string &filename) {
 	std::ifstream f;
@@ -65,12 +69,22 @@ bool is_path_dir(const std::string &path) {
 	return (bool) S_ISDIR(buf.st_mode);
 }
 
-int mk_path_dir(const std::string &path) {
+bool mk_path_dir(const std::string &path) {
-	return ::mkdir(path.c_str(), 0755);
+	auto r = ::mkdir(path.c_str(), 0755);
+	if (r == -1) {
+		std::string err_msg = std::strerror(errno);
+		std::cerr << "mkdir failed: " << std::strerror(errno) << std::endl;
+	}
+	return r == 0;
 }
 
-int rm_path_dir(const std::string &path) {
+bool rm_path_dir(const std::string &path) {
-	return ::rmdir(path.c_str());
+	auto r = ::rmdir(path.c_str());
+	if (r == -1) {
+		std::string err_msg = std::strerror(errno);
+		std::cerr << "rmdir failed: " << std::strerror(errno) << std::endl;
+	}
+	return r == 0;
 }
 
 MlValue exec_system_cmd(const std::string &cmd) {
@@ -98,3 +112,32 @@ MlValue exec_system_cmd(const std::string &cmd) {
 
 	return std::vector<MlValue> { MlValue((long)cmd_retval), MlValue::string(cmd_output) };
 }
+
+
+MlValue exec_system_cmd_fork(const std::vector<std::string> &args) {
+	int pid = fork();
+	if (pid == 0) {
+	    // Child process, execute the command
+
+    	std::vector<char*> exec_args;
+		exec_args.reserve(args.size());
+
+	    for (auto const& a : args) 
+	        exec_args.emplace_back(const_cast<char*>(a.c_str()));
+
+    	exec_args.push_back(nullptr);	// exec must end with null
+
+    	execvp(args[0].c_str(), exec_args.data());
+
+		std::cerr << "execvp error: " << errno << ", " << strerror(errno) <<  std::endl;
+    	exit(1);	// indicate error, if here
+
+	} else if (pid > 0) {
+	    // Parent process, child process is now independent
+		return std::vector<MlValue> { MlValue((long)0), MlValue::string("") };
+
+	} else {
+	    // Fork error, still in parent process (there are no child process at this point)
+		return std::vector<MlValue> { MlValue((long)errno), MlValue::string(strerror(errno) ) };
+	}
+}

ml_io.h

@@ -4,8 +4,7 @@
 
 #include <string>
 #include <vector>
-#include <iostream>
+
-#include <fstream>
 
 std::string read_file_contents(const std::string &filename);
 
@@ -14,11 +13,10 @@ bool write_file_contents(const std::string &filename, const std::string &content
 MlValue list_dir(const std::string &path);
 
 bool is_path_file(const std::string &path);
-
 bool is_path_dir(const std::string &path);
 
-int mk_path_dir(const std::string &path);
+bool mk_path_dir(const std::string &path);
-
+bool rm_path_dir(const std::string &path);
-int rm_path_dir(const std::string &path);
 
 MlValue exec_system_cmd(const std::string &path);
+MlValue exec_system_cmd_fork(const std::vector<std::string> &args);

ml_profiler.cpp

@@ -79,7 +79,7 @@ void MlPerfMon::restore_callstack_position(size_t to_position) {
 }
 
 void MlPerfMon::clear_callstack() {
-    bool e = call_stack.empty();
+    auto r = call_stack.empty();
 }
 
 void MlPerfMon::print_results() const {

ml_util.cpp

@@ -1,13 +1,14 @@
 
 #include "ml_util.h"
+#include "string.h"
 
-#include <string>
 #include <vector>
 #include <algorithm>
 #include <iostream>
 // #include <filesystem>
 
 std::vector<std::string> commands {};
+std::string history_file;
 
 // kryplove z applu a jejich problemy s filesystemem
 /*
@@ -20,8 +21,7 @@ std::string get_history_file_dir() {
 }	
  */
 
-std::string get_history_file_dir() {
+std::string get_history_filepath(const std::string &file) {
-	std::string file{"/.ml_history.txt"};
 	const char *t = std::getenv("HOME");
 
 	if (t == nullptr) return "/tmp/" + file;
@@ -30,29 +30,33 @@ std::string get_history_file_dir() {
 
 MlEnvironment * repl_env = nullptr;
 
-void setup_linenoise(const MlEnvironment &env, const std::map<const std::string, Builtin> &builtins) {
+void setup_linenoise() {
-    repl_env = (MlEnvironment*) &env;
+	history_file = get_history_filepath("/.ml_history.txt");
-
+	linenoiseHistoryLoad(history_file.c_str()); 
-	std::string history_file = get_history_file_dir();
 
 	linenoiseHistorySetMaxLen(500);
+	linenoiseSetMultiLine(1);
+}
+
+void setup_linenoise_repl(const MlEnvironment &env, const std::map<const std::string, Builtin> &builtins) {
+	// basic linenoise setup must be already set
+
+    repl_env = (MlEnvironment*) &env;
+
 	linenoiseSetCompletionCallback(completion);
 	linenoiseSetHintsCallback(hints);
-	linenoiseSetMultiLine(1);
-	linenoiseHistoryLoad(history_file.c_str()); 
 
     commands.reserve(builtins.size());
     for(auto it = builtins.begin(); it != builtins.end(); ++it)
         commands.push_back(it->first);
 }
 
-void linenoise_line_read(char *line) {
+void linenoise_add_to_history(const std::string &line) {
-    linenoiseHistoryAdd(line);
+	if (!line.empty())
+    	linenoiseHistoryAdd(line.c_str());
 }
 
 void close_linenoise() {
-    std::string history_file = get_history_file_dir();
-
     linenoiseHistorySave(history_file.c_str());
 }
 

ml_util.h

@@ -1,11 +1,13 @@
 #pragma once
 
 #include "ml.h"
-
 #include "linenoise.h"
 
-void setup_linenoise(const MlEnvironment &env, const std::map<const std::string, Builtin> &builtins);
+#include <string>
-void linenoise_line_read(char *line);
+
+void setup_linenoise();
+void setup_linenoise_repl(const MlEnvironment &env, const std::map<const std::string, Builtin> &builtins);
+void linenoise_add_to_history(const std::string &line);
 void close_linenoise();
 
 

stdlib/doc.lsp

@@ -43,6 +43,12 @@
 	nil
 ))
 
+(defn doc::print-entry (entry)
+(do (def call (doc::strip-backticks (second entry)))
+	(def desc (doc::strip-backticks (third entry)))
+	(print (term-red (first entry)) "-" (term-green (doc::strip-backticks (second entry))) "-" (third entry))
+))
+
 (defn doc::man (what)
 (do	(def man (filter (lambda (x) (= (first x) what)) doc::doc_entries))
 	(if man
@@ -75,29 +81,54 @@
 	
 	(def sorted (quick-sort-by scores (lambda (a b) (< (first a) (first b)))))
 	(for e (take sorted 10)
-		(def entry (second e))
+		(doc::print-entry (second e)))
-		(def call (doc::strip-backticks (second entry)))
+
-		(def desc (doc::strip-backticks (third entry)))
-		(print (term-red (first entry)) "-" (term-green (doc::strip-backticks (second entry))) "-" (third entry))
-	)
 	(if (> (len sorted) 10) (print "..."))
 ))
 
+(defn doc::section (what)
+(do
+	(def entries '())
+	(for entry doc::doc_entries
+		; ("throw-exception" "`(throw-exception exp_desc)`" "Throws an exception with exp_desc describing what happened " "" "Exceptions"
+		; section matches
+		(if (= (string-downcase (fifth entry)) (string-downcase what))
+			(set! entries (push entries entry)))
+	)
+	
+	(for e (quick-sort-by entries (lambda (a b) (> (string-cmp (first a) (first b)) 0)))
+		(doc::print-entry e))
+))
+
+(defn doc::all ()
+	(for e (quick-sort-by doc::doc_entries (lambda (a b) (> (string-cmp (first a) (first b)) 0)))
+		(doc::print-entry e))
+)
+
 (defn doc::appropos (which)
 	(doc::look which))
 
 (defn doc::lookup (which)
 	(doc::look which))
 
-;(defn doc::section (which)
+(defn doc::doc ()
-;	(print (term-red "implement me!")))
+(do
+	(print "Usage:")
+	(print "\t(doc::doc)         - shows this help")
+	(print "\t(doc::man func)    - func must be a string, ie (doc::man \"for\")")
+	(print "\t(doc::look str)    - str must be a string, ie (doc::look \"length\")")
+	(print "\t(doc::lookup)      - alias for doc::look")
+	(print "\t(doc::appropos)    - alias for doc::look")
+	(print "\t(doc::all)         - show short info about all functions")
+	(print "\t(doc::section sec) - show help for section, sec is string one of:")
+	(print "\t\t\t\t\"List manipulation\" \"Language\" \"System\"")
+	(print "\t\t\t\t\"String manipulation\" \"Date and time\" \"IO\" \"Regex\"")
+	(print "\t\t\t\t\"Type casting\" \"Threading\" \"Exceptions\"")
+))
+
 
 (def doc::doc_entries '())	; must be here
 
 ; read doc into memory
 (doc::read-doc-file "/usr/local/var/mlisp/Doc.md")
 
-;;example
-; (doc::man "first")
-; (doc::look "string pad")
-; (doc::look "list flat")

stdlib/stdlib.lsp

@@ -78,6 +78,11 @@
 		body
     ))    
 
+(defmacro until (cnd body)
+	(list 'while '(not (eval cnd)) 
+		body
+    ))  
+
 
 ; return 1 when list contains item otherwise nil
 (defn member (lst itm) 

tests/cmd_fork.lsp

@@ -0,0 +1 @@
+(system-cmd-fork "ml"  "-c" "(print 123) (sleep 1) (print \"aaa\")")

tests/test.lsp

@@ -1,6 +1,6 @@
 (include "/usr/local/var/mlisp/ut.lsp")
 
-(def local_time_offset 3600)
+(def local_time_offset 7200)
 
 ;;  prepare some code to be used in tests
 (defn fact (n)
@@ -9,11 +9,11 @@
       (* n (fact (- n 1)))
     ))
 
-; for scitej 4000 stack must be 16MB, otherwise 1000 is ok
+; for adder 4000 stack must be 16MB, otherwise 1000 is ok
-(defn scitej (n)
+(defn adder (n)
    (if (<= n 1)
       1
-      (+ n (scitej (- n 1)))
+      (+ n (adder (- n 1)))
     ))
 
 (write-file "/tmp/f.txt" "line 1\nline 2\nline3")
@@ -29,6 +29,11 @@
 
 (thread-create (tcp-server 7778 (lambda (str) (list #t (+ "(print \"" (string-upcase str) "\")")))))
 
+
+(ut::define-test "result of (unless #t (print \"#f\"))" '(ut::assert-false (unless #t (print "#f"))))
+(ut::define-test "result of (dotimes i 5 i)" '(ut::assert-equal 4 (dotimes i 5 i)))
+(ut::define-test "result of (def i 0)(until (> i 3) (set! i (inc i)))" '(ut::assert-equal 4 (do (def i 0)(until (> i 3) (set! i (inc i))))))
+
 (ut::define-test "result of (and (> 2 1) (> 2 1))" '(ut::assert-true (and (> 2 1) (> 2 1))))
 (ut::define-test "result of (or  (> 2 1) (> 2 1))" '(ut::assert-true (or  (> 2 1) (> 2 1))))
 (ut::define-test "retult of (and (> 2 1) (> 1 2))" '(ut::assert-false (and (> 2 1) (> 1 2))))
@@ -47,6 +52,8 @@
 (ut::define-test "result of (member '(1 2 3) 3)" '(ut::assert-true (member '(1 2 3) 3)))
 (ut::define-test "result of (member '(1 2 3) 4)" '(ut::assert-false (member '(1 2 3) 4)))
 
+(ut::define-test "result of (reduce (lambda (acc e) (+ acc (string (+ \"<th>\" e \"</th>\")))) \"\" '(\"h1\" \"h2\" \"h3\"))" '(ut::assert-equal "<th>h1</th><th>h2</th><th>h3</th>" (reduce (lambda (acc e) (+ acc (string (+ "<th>" e "</th>")))) "" '("h1" "h2" "h3"))))
+
 (ut::define-test "result of (take '(1 2 3 4) 3)" '(ut::assert-equal '(1 2 3) (take '(1 2 3 4) 3)))
 
 (ut::define-test "result of (make-list 3)" '(ut::assert-equal '(nil nil nil) (make-list 3)))
@@ -55,7 +62,7 @@
 (ut::define-test "result of (range 1.5 5.5)" '(ut::assert-equal '(1.500000 2.500000 3.500000 4.500000) (range 1.5  5.5)))
 
 (ut::define-test "result of (fact 5)" '(ut::assert-equal 120 (fact 5)))
-(ut::define-test "result of (scitej 1000)" '(ut::assert-equal 500500 (scitej 1000)))
+(ut::define-test "result of (adder 1000)" '(ut::assert-equal 500500 (adder 1000)))
 
 (ut::define-test "result of (len json_list)" '(ut::assert-equal 3 (len json_list)))
 
@@ -121,6 +128,8 @@
 (ut::define-test "result of (is-dir? \"/tmp/file_whichnotex_ists\")" '(ut::assert-false (is-dir? "/tmp/file_whichnotex_ists")))
 (ut::define-test "result of (is-dir? \"/tmp\"" '(ut::assert-true (is-dir? "/tmp")))
 (ut::define-test "result of (is-file? \"/tmp\"" '(ut::assert-false (is-file? "/tmp")))
+(ut::define-test "result of (mk-dir \"/tmp/testdir\"" '(ut::assert-true (mk-dir "/tmp/testdir")))
+(ut::define-test "result of (rm-dir \"/tmp/testdir\"" '(ut::assert-true (rm-dir "/tmp/testdir")))
 
 (ut::define-test "result of (uniq '(1 2 2 2 3 4 5 4 4 1 2 2 2 3 4 5 4 4 61 2 2 2 3 4 5 4 4 66))" '(ut::assert-equal '(1 2 3 4 5 61 66) (uniq '(1 2 2 2 3 4 5 4 4 1 2 2 2 3 4 5 4 4 61 2 2 2 3 4 5 4 4 66))))
 (ut::define-test "result of (flatten '(1 2 (3 3 (4)) 5 6))" '(ut::assert-equal '(1 2 3 3 4 5 6) (flatten (1 2 (3 3 (4)) 5 6))))

tests/test_tcp.lsp

@@ -0,0 +1,19 @@
+(thread-create
+ 	(tcp-server 7777 (lambda (str) (list #f (+ "(print \"" (string-upcase str) "\")"))))
+)
+
+(thread-sleep 1)
+
+(thread-create
+	(def code (tcp-client "127.0.0.1" 7777 ("abcd" "xyz")))
+	(for c code
+		(print "executing code:" c)
+ 		(eval (parse c))
+	)
+
+	(def code (tcp-client "127.0.0.1" 7777 "abcd"))
+	(print "executing code:" code)
+ 	(eval (parse code))
+)
+(threads-join)
+(print "ok")

usql/lexer.cpp

@@ -162,7 +162,7 @@ TokenType Lexer::type(const std::string &token) {
 	if (token == "not") return TokenType::keyword_not;
 	if (token == "null") return TokenType::keyword_null;
 	if (token == "integer") return TokenType::keyword_integer;
-	if (token == "float") return TokenType::keyword_float;
+	if (token == "float" || token == "double") return TokenType::keyword_float;
 	if (token == "varchar") return TokenType::keyword_varchar;
 	if (token == "date") return TokenType::keyword_date;
 	if (token == "boolean") return TokenType::keyword_bool;

usql/parser.cpp

@@ -394,8 +394,7 @@ namespace usql {
 		std::unique_ptr<Node> left = parse_expression();
 		do {
 			left = parse_expression(std::move(left));
-
+		} while (m_lexer.tokenType() != TokenType::eof && m_lexer.tokenType() != TokenType::keyword_order && m_lexer.tokenType() != TokenType::keyword_offset && m_lexer.tokenType() != TokenType::keyword_limit && m_lexer.tokenType() != TokenType::semicolon);
-	} while (m_lexer.tokenType() != TokenType::eof && m_lexer.tokenType() != TokenType::keyword_order && m_lexer.tokenType() != TokenType::keyword_offset && m_lexer.tokenType() != TokenType::keyword_limit);
 	
 		return left;
     }

usql/parser.h

@@ -126,8 +126,10 @@ struct ColDefNode : Node {
 struct FunctionNode : Node {
 
 	enum class Type {
-		to_string,
+		to_char,
 		to_date,
+		to_int,
+		to_float,
 		date_add,
 		pp,
 		lower,
@@ -139,8 +141,10 @@ struct FunctionNode : Node {
 	};
 
 	static Type get_function(const std::string &str) {
-		if (str=="to_string")	return Type::to_string;
+		if (str=="to_char")		return Type::to_char;
 		if (str=="to_date")		return Type::to_date;
+		if (str=="to_int")		return Type::to_int;
+		if (str=="to_float")	return Type::to_float;
 		if (str=="date_add")	return Type::date_add;
 		if (str=="pp")			return Type::pp;
 		if (str=="lower")		return Type::lower;
@@ -154,8 +158,10 @@ struct FunctionNode : Node {
 	};
 
 	static std::string function_name(const Type type) {
-		if (type == Type::to_string)	return "to_string";
+		if (type == Type::to_char)		return "to_char";
 		if (type == Type::to_date)		return "to_date";
+		if (type == Type::to_int)		return "to_int";
+		if (type == Type::to_float)		return "to_float";
 		if (type == Type::date_add)		return "date_add";
 		if (type == Type::pp)			return "pp";
 		if (type == Type::lower)		return "lower";
@@ -169,9 +175,6 @@ struct FunctionNode : Node {
 	};
 
 
-	Type					function;
-	std::vector<std::unique_ptr<Node>>	params;
-
 	FunctionNode(std::string  func_name, std::vector<std::unique_ptr<Node>> pars) :
 	Node(NodeType::function), function(get_function(func_name)), params(std::move(pars)) {}
 
@@ -193,6 +196,9 @@ struct FunctionNode : Node {
 		}			
 		std::cout << ")" << std::endl;
 	}
+
+	Type								function;
+	std::vector<std::unique_ptr<Node>>	params;
 };
 
 struct TrueNode : Node {
@@ -575,7 +581,6 @@ struct CreateIndexNode : Node {
 };
 
 class Parser {
-private:
 public:
 	Parser();
 

usql/row.h

@@ -171,7 +171,7 @@ public:
 			case 5:
 				return (ColValue &) *std::get_if<ColBooleanValue>(&m_columns[i]);
 			default:
-				throw Exception("should not happen");
+				throw Exception("ColValue &operator[](int i), type index invalid :" + std::to_string(type_index));
 		}
 	}
 

usql/settings.cpp

@@ -12,7 +12,7 @@ std::vector<std::pair<std::string, std::string>> Settings::m_settings =
 	      std::make_pair("BOOL_TRUE_LITERAL", "Y"),
 	      std::make_pair("BOOL_FALSE_LITERAL", "N"),
           std::make_pair("DOUBLE_FORMAT", "%.2f"),
-	      std::make_pair("USE_INDEXSCAN", "N"),
+	      std::make_pair("USE_INDEXSCAN", "Y"),
 	      std::make_pair("MAX_PARALLELISM", "1") };		// values "AUTO" or number of workers; when number negative means std::thread::hardware_concurrency() - number 
 
 

usql/usql.cpp

@@ -6,9 +6,36 @@
 
 namespace usql {
 
+USql::USql() {
+	// create catalogue tables first
+
+	std::vector<std::string> k_debug_sql_commands {
+		"create table usql_tables(name varchar(32) not null, modified boolean not null)",
+		"create table usql_columns(table_name varchar(32) not null, column_name varchar(32) not null, column_type varchar(16) not null, column_length integer not null, nullable boolean not null, column_order integer not null)"
+	};
+
+	// create cataloque tables
+	for (const auto &command : k_debug_sql_commands) {
+		std::unique_ptr<Node> create_table_node = m_parser.parse(command);
+		const CreateTableNode &node = static_cast<CreateTableNode &>(*create_table_node); 
+
+		Table table{node.table_name, node.cols_defs};
+		m_tables.push_back(table);
+
+	}
+	// insert data into cataloque tables
+	for (const auto &command : k_debug_sql_commands) {
+		std::unique_ptr<Node> create_table_node = m_parser.parse(command);
+		const CreateTableNode &node = static_cast<CreateTableNode &>(*create_table_node); 
+
+		execute_create_table_sys_catalogue(node);
+	}
+}
+
 std::unique_ptr<Table> USql::execute(const std::string &command) {
 	try {
 		std::unique_ptr<Node> node = m_parser.parse(command);
+		// node->dump();
 		return execute(*node);
 
 	} catch (const std::exception &e) {
@@ -182,7 +209,9 @@ std::unique_ptr<ValueNode> USql::eval_function_value_node(Table *table, Row &row
 	if (fnc->function == FunctionNode::Type::lower) return lower_function(evaluatedPars);
 	if (fnc->function == FunctionNode::Type::upper) return upper_function(evaluatedPars);
 	if (fnc->function == FunctionNode::Type::to_date) return to_date_function(evaluatedPars);
-	if (fnc->function == FunctionNode::Type::to_string) return to_string_function(evaluatedPars);
+	if (fnc->function == FunctionNode::Type::to_char) return to_char_function(evaluatedPars);
+	if (fnc->function == FunctionNode::Type::to_int) return to_int_function(evaluatedPars);
+	if (fnc->function == FunctionNode::Type::to_float) return to_float_function(evaluatedPars);
 	if (fnc->function == FunctionNode::Type::date_add) return date_add_function(evaluatedPars);
 	if (fnc->function == FunctionNode::Type::pp) return pp_function(evaluatedPars);
 	if (fnc->function == FunctionNode::Type::count) return count_function(agg_func_value, evaluatedPars);

usql/usql.h

@@ -13,14 +13,14 @@ namespace usql {
 class USql {
 	
 public:
-    USql() = default;
+	USql();
-
     std::unique_ptr<Table> execute(const std::string &command);
 
 private:
     std::unique_ptr<Table> execute(Node &node);
 
     std::unique_ptr<Table> execute_create_table(const CreateTableNode &node);
+	bool execute_create_table_sys_catalogue(const CreateTableNode &node);
     std::unique_ptr<Table> execute_create_index(const CreateIndexNode &node);
     std::unique_ptr<Table> execute_create_table_as_table(const CreateTableAsSelectNode &node);
     std::unique_ptr<Table> execute_load(const LoadIntoTableNode &node);
@@ -59,9 +59,6 @@ private:
 	void check_index_not_exists(const std::string &index_name);
 
 private:
-    Parser m_parser;
-    std::list<Table> m_tables;
-
     static void execute_distinct(SelectFromTableNode &node, Table *result);
     static void execute_order_by(SelectFromTableNode &node, Table *result);
     static void execute_offset_limit(OffsetLimitNode &node, Table *result);
@@ -75,7 +72,9 @@ private:
     static std::unique_ptr<ValueNode> upper_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars);
     static std::unique_ptr<ValueNode> coalesce_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars);
     static std::unique_ptr<ValueNode> to_date_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars);
-    static std::unique_ptr<ValueNode> to_string_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars);
+    static std::unique_ptr<ValueNode> to_char_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars);
+    static std::unique_ptr<ValueNode> to_int_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars);
+    static std::unique_ptr<ValueNode> to_float_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars);
     static std::unique_ptr<ValueNode> date_add_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars);
     static std::unique_ptr<ValueNode> pp_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars);
 
@@ -95,6 +94,11 @@ private:
 	bool normalize_where(const Node *node) const;
 
     Table::rows_scanner get_iterator(Table *table, const Node *where) const;
+
+
+private:
+    Parser m_parser;
+    std::list<Table> m_tables;
 };
 
 

usql/usql_ddl.cpp

@@ -15,10 +15,35 @@ std::unique_ptr<Table> USql::execute_create_table(const CreateTableNode &node) {
 	Table table{node.table_name, node.cols_defs};
 	m_tables.push_back(table);
 
+	execute_create_table_sys_catalogue(node);
+
 	return create_stmt_result_table(0, "table created", 0);
 }
 
 
+bool USql::execute_create_table_sys_catalogue(const CreateTableNode &node) {
+	// usql_tables 
+	auto r = execute("insert into usql_tables(name, modified) values('" + node.table_name + "', false)");
+
+	// usql_columns 
+	for(const ColDefNode & col_def : node.cols_defs) {
+		std::string i {"insert into usql_columns(table_name, column_name, column_type, column_length, nullable, column_order) values("};
+					i += "'" + node.table_name + "', ";
+					i += "'" + col_def.name + "', ";
+					i += "'" + column_type_name(col_def.type) + "', ";
+					i += std::to_string(col_def.length) + ", ";
+					i += (col_def.null ?  "true, " : "false, ");
+					i += std::to_string(col_def.order);
+					i += ")";
+
+		auto r = execute(i);
+		// r->print();
+	}
+
+	return true;
+}
+
+
 std::unique_ptr<Table> USql::execute_create_index(const CreateIndexNode &node) {
 	Table *table_def = find_table(node.table_name);				// throws exception if not found
 	ColDefNode col_def = table_def->get_column_def(node.column_name);	// throws exception if not found
@@ -57,7 +82,6 @@ std::unique_ptr<Table> USql::execute_create_table_as_table(const CreateTableAsSe
 }
 
 
-
 std::unique_ptr<Table> USql::execute_drop(const DropTableNode &node) {
 	auto name_cmp = [node](const Table& t) { return t.m_name == node.table_name; };
 
@@ -70,11 +94,13 @@ std::unique_ptr<Table> USql::execute_drop(const DropTableNode &node) {
 	throw Exception("table not found (" + node.table_name + ")");
 }
 
+
 std::unique_ptr<Table> USql::execute_set(const SetNode &node) {
 	Settings::set_setting(node.name, node.value);
 	return create_stmt_result_table(0, "set succeeded", 1);
 }
 
+
 std::unique_ptr<Table> USql::execute_show(const ShowNode &node) {
 	std::string value = Settings::get_setting(node.name);
 	return create_stmt_result_table(0, "show succeeded: " + value, 1);

usql/usql_dml.cpp

@@ -35,7 +35,7 @@ std::pair<bool, std::vector<rowid_t>> USql::look_for_usable_index(const Node *wh
 					if (used_index != nullptr) {
 						std::vector<rowid_t> rowids = used_index->search((ValueNode *)ron->right.get());
 #ifndef NDEBUG
-						std::cout << "using index " << table->m_name << "(" << used_index->get_column_name() << "), " << rowids.size() << "/" << table->rows_count()  << std::endl;
+						std::cerr << "using index " << table->m_name << "(" << used_index->get_column_name() << "), " << rowids.size() << "/" << table->rows_count()  << std::endl;
 #endif
 						return std::make_pair(true, rowids);
 					}
@@ -155,8 +155,7 @@ void USql::setup_order_columns(std::vector<ColOrderNode> &node, Table *table) {
 void USql::execute_distinct(SelectFromTableNode &node, Table *result) {
 	if (!node.distinct) return;
 
-	auto compare_rows = [](const Row &a, const Row &b) { return a.compare(b) >= 0; };
+	std::sort(result->m_rows.begin(), result->m_rows.end(), [](const Row &a, const Row &b) { return a.compare(b) > 0; });
-	std::sort(result->m_rows.begin(), result->m_rows.end(), compare_rows);
 
 	result->m_rows.erase(std::unique(result->m_rows.begin(), result->m_rows.end()), result->m_rows.end());
 }
@@ -226,9 +225,15 @@ std::tuple<int, ColDefNode> USql::get_node_definition(Table *table, Node * node,
 	} else if (node->node_type == NodeType::function) {
 		auto func_node = static_cast<FunctionNode *>(node);
 
-		if (func_node->function == FunctionNode::Type::to_string) {
+		if (func_node->function == FunctionNode::Type::to_char) {
 			ColDefNode col_def = ColDefNode{col_name, ColumnType::varchar_type, col_order, 32, true};
 			return std::make_tuple(FUNCTION_CALL, col_def);
+		} else if (func_node->function == FunctionNode::Type::to_int) {
+			ColDefNode col_def = ColDefNode{col_name, ColumnType::integer_type, col_order, 1, true};
+			return std::make_tuple(FUNCTION_CALL, col_def);
+		} else if (func_node->function == FunctionNode::Type::to_float) {
+			ColDefNode col_def = ColDefNode{col_name, ColumnType::float_type, col_order, 1, true};
+			return std::make_tuple(FUNCTION_CALL, col_def);
 		} else if (func_node->function == FunctionNode::Type::to_date) {
 			ColDefNode col_def = ColDefNode{col_name, ColumnType::integer_type, col_order, 1, true};
 			return std::make_tuple(FUNCTION_CALL, col_def);
@@ -243,6 +248,7 @@ std::tuple<int, ColDefNode> USql::get_node_definition(Table *table, Node * node,
 			// TODO handle cases here
 			if (func_node->params.empty()) throw Exception("Coalesce without parameters");
 			if (func_node->params[0]->node_type != NodeType::database_value) throw Exception("Coalesce first parameter must be database column");
+
 			ColDefNode tbl_col_def = get_db_column_definition(table, func_node->params[0].get());
 			ColDefNode col_def = ColDefNode{col_name, tbl_col_def.type, col_order, tbl_col_def.length, true};
 			return std::make_tuple(FUNCTION_CALL, col_def);
@@ -250,11 +256,18 @@ std::tuple<int, ColDefNode> USql::get_node_definition(Table *table, Node * node,
 			auto col_type = ColumnType::float_type;
 			size_t col_len = 1;
 			auto &v = func_node->params[0];
+
 			if (v->node_type == NodeType::database_value) {
 				ColDefNode src_col_def = get_db_column_definition(table, v.get());
 				col_type = src_col_def.type;
 				col_len = src_col_def.length;
+
+			} else if (node->node_type == NodeType::function) {
+				std::tuple<int, ColDefNode> fun_type = get_node_definition(table, static_cast<Node*>(& *v), "", 0);
+				col_type = std::get<1>(fun_type).type;
+				col_len = std::get<1>(fun_type).length;
 			}
+
 			ColDefNode col_def = ColDefNode{col_name, col_type, col_order, col_len, true};
 			return std::make_tuple(FUNCTION_CALL, col_def);
 		} else if (func_node->function == FunctionNode::Type::count) {

usql/usql_function.cpp

@@ -7,12 +7,24 @@
 
 namespace usql {
 
-std::unique_ptr<ValueNode> USql::to_string_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars) {
+std::unique_ptr<ValueNode> USql::to_char_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars) {
+	if (evaluatedPars[0]->node_type == NodeType::int_value && evaluatedPars.size()==2) {		// TODO when propper date is introduced
 		long date = evaluatedPars[0]->getDateValue();
 		std::string format = evaluatedPars[1]->getStringValue();
 		std::string formatted_date = date_to_string(date, format);
 	
 		return std::make_unique<StringValueNode>(formatted_date);
+	} else {
+		return std::make_unique<StringValueNode>(evaluatedPars[0]->getStringValue());
+	}
+}
+
+std::unique_ptr<ValueNode> USql::to_int_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars) {
+	return std::make_unique<IntValueNode>(evaluatedPars[0]->getIntegerValue());
+}
+
+std::unique_ptr<ValueNode> USql::to_float_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars) {
+	return std::make_unique<DoubleValueNode>(evaluatedPars[0]->getDoubleValue());
 }
 
 std::unique_ptr<ValueNode> USql::to_date_function(const std::vector<std::unique_ptr<ValueNode>> &evaluatedPars) {

utils/Lisp.tmLanguage

@@ -55,29 +55,6 @@
 			<key>name</key>
 			<string>meta.function.lisp</string>
 		</dict>
-		<!-- <dict>
-			<key>match</key>
-			<string>(?<=\s)(?i:&allow-other-keys|&aux|&body|&environment|&key|&optional|&rest|&whole)(?=\s+)</string>
-			<key>name</key>
-			<string>meta.function-parameters.lisp</string>
-		</dict> -->
-		<!-- <dict>
-			<key>captures</key>
-			<dict>
-				<key>1</key>
-				<dict>
-					<key>name</key>
-					<string>storage.type.function-type.lisp</string>
-				</dict>
-				<key>2</key>
-				<dict>
-					<key>name</key>
-					<string>entity.name.type.lisp</string>
-				</dict>
-			</dict>
-			<key>match</key>
-			<string>(?:\()((?i:deftype|defstruct|define-condition|defpackage|defclass))\s+((?:\w|[+\-<>/*&amp;=.?!$%:@\[\]^{}~#|])+)</string>
-		</dict> -->
 		<dict>
 			<key>captures</key>
 			<dict>

utils/local_install.sh

@@ -8,19 +8,27 @@ echo "building ml"
 cmake --build ./ --target all -j 4 --
 
 echo ""
+
+# echo "create dirs"
+# sudo mkdir -p /usr/local/var/mlisp/
+# sudo mkdir -p /usr/local/bin/
+# sudo chown -R vaclavt:admin /usr/local/bin/
+# sudo chown -R vaclavt:admin /usr/local/var/mlisp/
+
 echo "copying lsp files"
-mkdir -p /usr/local/var/mlisp/
 cp stdlib/*.lsp /usr/local/var/mlisp/
+
 echo "copying doc files"
 cp doc/*.md /usr/local/var/mlisp/
+
 echo "copying ml file"
 mv ./ml /usr/local/bin/ml
 
 echo ""
 echo "if syntax has changed you may use"
 echo "cp utils/Lisp.tmLanguage ~/.vscode/extensions/mattn.lisp-0.1.12/syntaxes/"
-# echo "or"
+echo "or"
-# echo "cp utils/Lisp.tmLanguage ~/.local/share/code-server/extensions/mattn.lisp-0.1.11/syntaxes/Lisp.tmLanguage"
+echo "cp utils/Lisp.tmLanguage ~/.local/share/code-server/extensions/mattn.lisp-0.1.11/syntaxes/Lisp.tmLanguage"
 echo ""
 
 ml -v

utils/resetCLion.sh

@@ -1,34 +0,0 @@
-#!/bin/bash
-#https://github.com/PythonicNinja/jetbrains-reset-trial-mac-osx/blob/master/runme.sh
-
-for product in CLion; do
-  echo "Closing $product"
-  ps aux | grep -i MacOs/$product | cut -d " " -f 5 | xargs kill -9
-
-  echo "Resetting trial period for $product"
-
-  echo "removing evaluation key..."
-  rm -rf ~/Library/Preferences/$product*/eval
-
-  # Above path not working on latest version. Fixed below
-  rm -rf ~/Library/Application\ Support/JetBrains/$product*/eval
-
-  echo "removing all evlsprt properties in options.xml..."
-  sed -i '' '/evlsprt/d' ~/Library/Preferences/$product*/options/other.xml
-
-  # Above path not working on latest version. Fixed below
-  sed -i '' '/evlsprt/d' ~/Library/Application\ Support/JetBrains/$product*/options/other.xml
-
-  echo
-done
-
-echo "removing additional plist files..."
-rm -f ~/Library/Preferences/com.apple.java.util.prefs.plist
-rm -f ~/Library/Preferences/com.jetbrains.*.plist
-rm -f ~/Library/Preferences/jetbrains.*.*.plist
-
-echo "restarting cfprefsd"
-killall cfprefsd
-
-echo
-echo "That's it, enjoy ;)"

wip.lsp

@@ -3,26 +3,6 @@
 (print (filter (lambda (eval e) (print e)) l))
 
 
-(thread-create
- 	(tcp-server 7777 (lambda (str) (list #f (+ "(print \"" (string-upcase str) "\")"))))
-)
-
-(thread-sleep 1)
-
-(thread-create
-	(def code (tcp-client "127.0.0.1" 7777 ("abcd" "xyz")))
-	(for c code
-		(print "executing code:" c)
- 		(eval (parse c))
-	)
-
-	(def code (tcp-client "127.0.0.1" 7777 "abcd"))
-	(print "executing code:" code)
- 	(eval (parse code))
-)
-(threads-join)
-(print "ok")
-
 
 ;; (usql "create table sf1 (symbol varchar(8), dimension varchar(3), calendar_date date, date_key date, report_period date, last_updated date, accoci float, assets float, assetsavg float, assetsc float, assetsnc float, assetturnover float, bvps float, capex float, cashneq float, cashnequsd float, cor float, consolinc float, currentratio float, de float, debt float, debtc float, debtnc float, debtusd float, deferredrev float, depamor float, deposits float, divyield float, dps float, ebit float, ebitda float, ebitdamargin float, ebitdausd float, ebitusd float, ebt float, eps float, epsdil float, epsusd float, equity float, equityavg float, equityusd float, ev float, evebit float, evebitda float, fcf float, fcfps float, fxusd float, gp float, grossmargin float, intangibles float, intexp float, invcap float, invcapavg float, inventory float, investments float, investmentsc float, investmentsnc float, liabilities float, liabilitiesc float, liabilitiesnc float, marketcap float, ncf float, ncfbus float, ncfcommon float, ncfdebt float, ncfdiv float, ncff float, ncfi float, ncfinv float, ncfo float, ncfx float, netinc float, netinccmn float, netinccmnusd float, netincdis float, netincnci float, netmargin float, opex float, opinc float, payables float, payoutratio float, pb float, pe float, pe1 float, ppnenet float, prefdivis float, price float, ps float, ps1 float, receivables float, retearn float, revenue float, revenueusd float, rnd float, roa float, roe float, roic float, ros float, sbcomp float, sgna float, sharefactor float, sharesbas float, shareswa float, shareswadil float, sps float, tangibles float, taxassets float, taxexp float, taxliabilities float, tbvps float, workingcapital float)")
 ;; (usql "set 'DATE_FORMAT' = '%Y-%m-%d'")