tcp-server and tcp-client added

CMakeLists.txt

@@ -34,6 +34,7 @@ set(SOURCE
 	ml_usql.cpp
 	clib/csvparser.cpp
 	clib/sslclient.cpp
+	clib/tcpnet.cpp
 	clib/json11.cpp
 	clib/printf.cpp
 	clib/linenoise.c

Readme.md

@@ -65,6 +65,7 @@ utils/local_install.sh
 (read-url "https://api.nasdaq.com/api/calendar/dividends/") ; hangs in sslclient.cpp line 132
 
 ### TODO
+- tcp-client, add support for list of requests returning list of responses
 - add debug support, at least call stack
 - multiline editing (see kilocpp editor)
 - execute system command should capture stderr
@@ -72,6 +73,7 @@ utils/local_install.sh
 
 #### Code
 - MlEnvironment::get should be map instead of many if's
+- tcpnet should use RAII
 - add documentation
 - add more unit test, mainly for usql
 - add instrumentation (time, nr of evals, num of atoms, debug info, debug environment etc)

clib/tcpnet.cpp

@@ -0,0 +1,118 @@
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <netdb.h> 
+
+#include "tcpnet.h"
+
+
+void error(const char *msg) {
+	// perror(msg);
+	throw std::runtime_error(msg);
+}
+
+// TODO do it as RAII
+TcpNet::TcpNet() {}
+
+
+#define TCPNET_BUFFER_SIZE 16256
+
+
+int TcpNet::server(int portno, std::function<std::pair<bool, std::string>(std::string)> process_request) {
+	int sockfd, newsockfd;
+	socklen_t clilen;
+
+	char buffer[TCPNET_BUFFER_SIZE];
+ 	struct sockaddr_in serv_addr, cli_addr;
+ 
+ 	int n;
+	sockfd = socket(AF_INET, SOCK_STREAM, 0);
+	if (sockfd < 0) 
+		error("ERROR opening socket");
+
+ 	bzero((char *) &serv_addr, sizeof(serv_addr));
+	serv_addr.sin_family = AF_INET;
+	serv_addr.sin_addr.s_addr = INADDR_ANY;
+	serv_addr.sin_port = htons(portno);
+
+	if (bind(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) 
+		error("ERROR on binding");
+
+	listen(sockfd,5);
+	clilen = sizeof(cli_addr);
+
+	int requests_processed = 0;
+	bool shutdown = false;
+	while (!shutdown) {
+		newsockfd = accept(sockfd, (struct sockaddr *) &cli_addr, &clilen);
+		if (newsockfd < 0) 
+			error("ERROR on accept");
+	
+		bzero(buffer,TCPNET_BUFFER_SIZE);
+	
+		n = read(newsockfd,buffer,TCPNET_BUFFER_SIZE - 1);
+		if (n < 0) error("ERROR reading from socket");
+
+		std::string request{buffer};
+		std::pair<bool, std::string> response = process_request(request);
+		shutdown = response.first;
+		std::string response_str = response.second;
+
+		n = write(newsockfd, response_str.c_str(), response_str.size());
+		if (n < 0)
+			error("ERROR writing to socket");
+	
+		close(newsockfd);
+		requests_processed++;
+	}
+
+	close(sockfd);
+
+	return requests_processed; 
+}
+
+std::string TcpNet::client(const std::string address, int portno, const std::string &content) {
+	int sockfd, n;
+	struct sockaddr_in serv_addr;
+	struct hostent *server;
+
+    	char buffer[TCPNET_BUFFER_SIZE];
+    
+	sockfd = socket(AF_INET, SOCK_STREAM, 0);
+	if (sockfd < 0) 
+		error("ERROR opening socket");
+
+ 	server = gethostbyname(address.c_str());
+	if (server == NULL) {
+		fprintf(stderr,"ERROR, no such host\n");
+		exit(0);
+	}
+
+	bzero((char *) &serv_addr, sizeof(serv_addr));
+	serv_addr.sin_family = AF_INET;
+	bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length);
+	serv_addr.sin_port = htons(portno);
+	
+	if (connect(sockfd,(struct sockaddr *) &serv_addr,sizeof(serv_addr)) < 0) 
+ 		error("ERROR connecting");
+    
+
+	n = write(sockfd, content.c_str(), strlen(content.c_str()));
+	if (n < 0) 
+		error("ERROR writing to socket");
+		
+	bzero(buffer, TCPNET_BUFFER_SIZE);
+	n = read(sockfd,buffer, TCPNET_BUFFER_SIZE - 1);
+	if (n < 0)
+		error("ERROR reading from socket");
+ 
+	close(sockfd);
+
+	return std::string(buffer);
+}
+

clib/tcpnet.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include <string>
+
+
+class TcpNet {
+
+public:
+	TcpNet();
+
+	// starts listening on port
+	int server(int port, std::function<std::pair<bool, std::string>(std::string)> process_request);
+
+	// writes content to server on address:port and returns response
+	std::string client(const std::string address, int port, const std::string &content);
+
+	// TODO add support for vector of strings to be sent to server
+	// std::vector<std::string> client(const std::string address, int port, const std::vector<std::string> &content);
+
+};

debug.lsp

@@ -1,5 +1,27 @@
-(print (string-regex-list "<td class=\"xyz\">1</td><td>2</td>" "<td.*?>(.*?)</td>" "match" "ignore"))
+
-(print (string-regex-list "<td class=\"xyz\">1</td><td>2</td>" "<td.*?>(.*?)</td>" "token"))
+(thread-create
+ 	(tcp-server 7777 (lambda (str) (list #t (+ "(print \"" (string-upcase str) "\")"))))
+)
+
+(thread-sleep 1)
+
+(thread-create
+	(define code (tcp-client "127.0.0.1" 7777 "abcd"))
+	(print "executing code:" code)
+ 	(eval (parse code))
+)
+(threads-join)
+(print "ok")
+
+
+;; (usql "create table sf1 (symbol varchar(8), dimension varchar(3), calendar_date date, date_key date, report_period date, last_updated date, accoci float, assets float, assetsavg float, assetsc float, assetsnc float, assetturnover float, bvps float, capex float, cashneq float, cashnequsd float, cor float, consolinc float, currentratio float, de float, debt float, debtc float, debtnc float, debtusd float, deferredrev float, depamor float, deposits float, divyield float, dps float, ebit float, ebitda float, ebitdamargin float, ebitdausd float, ebitusd float, ebt float, eps float, epsdil float, epsusd float, equity float, equityavg float, equityusd float, ev float, evebit float, evebitda float, fcf float, fcfps float, fxusd float, gp float, grossmargin float, intangibles float, intexp float, invcap float, invcapavg float, inventory float, investments float, investmentsc float, investmentsnc float, liabilities float, liabilitiesc float, liabilitiesnc float, marketcap float, ncf float, ncfbus float, ncfcommon float, ncfdebt float, ncfdiv float, ncff float, ncfi float, ncfinv float, ncfo float, ncfx float, netinc float, netinccmn float, netinccmnusd float, netincdis float, netincnci float, netmargin float, opex float, opinc float, payables float, payoutratio float, pb float, pe float, pe1 float, ppnenet float, prefdivis float, price float, ps float, ps1 float, receivables float, retearn float, revenue float, revenueusd float, rnd float, roa float, roe float, roic float, ros float, sbcomp float, sgna float, sharefactor float, sharesbas float, shareswa float, shareswadil float, sps float, tangibles float, taxassets float, taxexp float, taxliabilities float, tbvps float, workingcapital float)")
+;; (usql "set 'DATE_FORMAT' = '%Y-%m-%d'")
+;; (usql "load into sf1 '/srv/SHARADAR_SF1.csv'")
+;; (print (usql "select dimension, to_string(calendar_date, '%d.%m.%Y'), pp(eps, \"%.2f\"), pp(shareswadil), pp(revenue), pp(netinc), pp(cashneq), pp(assets), pp(debt), pp(ncfdebt), pp(roe*100), pp(intangibles), calendar_date from sf1 where symbol = 'MU' and dimension = 'ARQ' order by dimension, calendar_date desc limit 5"))
+
+
+;; (print (string-regex-list "<td class=\"xyz\">1</td><td>2</td>" "<td.*?>(.*?)</td>" "match" "ignore"))
+;; (print (string-regex-list "<td class=\"xyz\">1</td><td>2</td>" "<td.*?>(.*?)</td>" "token"))
 
 
 ;; (include "/usr/local/var/mlisp/ut.lsp")

doc/Doc.md

@@ -84,6 +84,9 @@
 |`(ls-dir dir)`|List a dir|List of directory entries|
 |`(is-file? filename)`|Returns true if passed filename is a file||
 |`(is-dir? filename)`|Returns true if passed filename is a directory||
+|`(tcp-server port handler)`|Starts listening on port and when request comes calls passed lambda and writes its returned value back to client. Lambda must return either string or two element list where first element is boolean and second string.When first element is true it closes listening socker.|(`tcp-server 7777 (lambda (str) (list #t (string-upcase str))))`|
+|`(tcp-client address port data)`|Opens connection to server on port, writes there data and returns response.|`(print (tcp-client "127.0.0.1" 7777 "abcd"))`|
+|`(is-dir? filename)`|Returns true if passed filename is a directory||
 |`(parse-csv string)`|Parse CSV string||
 |`(parse-json json_string)`|Parse JSON string||
 |`(save-csv ..)`|||

ml.cpp

@@ -9,6 +9,7 @@
 
 #include "clib/csvparser.h"
 #include "clib/sslclient.h"
+#include "clib/tcpnet.h"
 #include "clib/json11.h"
 #include "clib/printf.h"
 
@@ -58,6 +59,11 @@
 #define LIST_TYPE "list"
 
 
+std::mutex register_mutex;
+std::vector<std::thread> threads_register;
+std::mutex interpreter_mutex;
+
+
 // Is this character a valid lisp symbol character
 bool is_symbol(char ch) {
 	return (isalpha(ch) || ispunct(ch)) && ch != '(' && ch != ')' && ch != '"' && ch != '\'';
@@ -1065,6 +1071,10 @@ MlValue exit(std::vector<MlValue> args, MlEnvironment &env) {
 	// Is not a special form, so we can evaluate our args.
 	eval_args(args, env);
 
+	std::lock_guard<std::mutex> lockGuard(register_mutex);
+	for(auto &t : threads_register)
+		t.detach();
+
 	std::exit(args.size() < 1 ? 0 : args[0].cast_to_int().as_int());
 	return MlValue();        // will not be called :-)
 }
@@ -1306,6 +1316,40 @@ MlValue is_dir(std::vector<MlValue> args, MlEnvironment &env) {
 	return MlValue(is_path_dir(args[0].as_string()));
 }
 
+MlValue tcp_server(std::vector<MlValue> args, MlEnvironment &env) {
+	eval_args(args, env);
+
+	if (args.size() != 2)
+		throw MlError(MlValue("tcp-server", tcp_server), env, args.size() > 2 ? TOO_MANY_ARGS : TOO_FEW_ARGS);
+
+	auto proccess_req = [&args, &env](const std::string &str) -> std::pair<bool, std::string> {
+		std::vector<MlValue> tmp {MlValue::string(str)};
+		MlValue result = args[1].apply(tmp, env);
+		// TODO more robust handling
+		if (result.is_list() && result.as_list().size() >= 2) {
+			std::vector<MlValue> list = result.as_list();
+			return std::make_pair(list[0].as_bool(), list[1].as_string());
+		} else {
+			return std::make_pair(false, result.as_string());
+		}
+	};
+
+	TcpNet server;
+	int r = server.server(args[0].as_int(), proccess_req);
+	return MlValue((long)r);
+}
+
+MlValue tcp_client(std::vector<MlValue> args, MlEnvironment &env) {
+	eval_args(args, env);
+
+	if (args.size() != 3)
+		throw MlError(MlValue("tcp-client", tcp_client), env, args.size() > 3 ? TOO_MANY_ARGS : TOO_FEW_ARGS);
+
+	TcpNet tcpclient;
+	std::string response = tcpclient.client(args[0].as_string(), args[1].as_int(), args[2].as_string());
+	return MlValue::string(response);
+}
+
 // Read a file and execute its code
 MlValue include(std::vector<MlValue> args, MlEnvironment &env) {
 	// Import is technically not a special form, it's more of a macro.
@@ -1895,11 +1939,6 @@ MlValue benchmark(std::vector<MlValue> args, MlEnvironment &env) {
 }
 
 
-std::mutex register_mutex;
-std::vector<std::thread> threads_register;
-std::vector<std::mutex> mutexes_register;
-std::mutex interpreter_mutex;
-
 MlValue thread_create(std::vector<MlValue> args, MlEnvironment &env) {
 	auto functor = [](std::vector<MlValue> args, MlEnvironment &env) -> void {
 	    try {
@@ -1907,6 +1946,7 @@ MlValue thread_create(std::vector<MlValue> args, MlEnvironment &env) {
 			    MlValue acc = args[i].eval(env);
 	    } catch (std::exception &e) {
 		    std::cerr << "thread exception: " << e.what() << std::endl;
+		    throw e;
 	    }
 	};
 
@@ -1953,7 +1993,9 @@ MlValue threads_join(std::vector<MlValue> args, MlEnvironment &env) {
 	if (args.size() != 0)
 		throw MlError(MlValue("threads-join", threads_join), env, TOO_MANY_ARGS);
 
-	std::lock_guard<std::mutex> lockGuard(register_mutex);
+	// here is a question about using lockGuard, when used it holds lockGuard locked until
+	// threads do not leave join
+	// std::lock_guard<std::mutex> lockGuard(register_mutex);
 	for (auto &th : threads_register)
 		if (th.joinable()) th.join();
 
@@ -2094,6 +2136,8 @@ MlValue MlEnvironment::get(const std::string &name) const {
 	if (name == "ls-dir") return MlValue("ls-dir", builtin::ls_dir);
 	if (name == "is-file?") return MlValue("is-file?", builtin::is_file);
 	if (name == "is-dir?") return MlValue("is-dir?", builtin::is_dir);
+	if (name == "tcp-server") return MlValue("tcp-client", builtin::tcp_server);
+	if (name == "tcp-client") return MlValue("tcp-client", builtin::tcp_client);
 
 	// parsing operations
 	if (name == "parse-csv") return MlValue("parse-csv", builtin::parse_csv);
@@ -2113,7 +2157,7 @@ MlValue MlEnvironment::get(const std::string &name) const {
 	if (name == "string-replace") return MlValue("string-replace", builtin::string_replace);
 	if (name == "string-replace-re") return MlValue("string-replace-re", builtin::string_replace_re);
 	if (name == "string-regex?") return MlValue("string-regex?", builtin::string_regex);
-	if (name == "string-regex-list") return MlValue("string-regex?", builtin::string_regex_list);
+	if (name == "string-regex-list") return MlValue("string-regex-list", builtin::string_regex_list);
 	if (name == "string-split") return MlValue("string-split", builtin::string_split);
 	if (name == "string-pad") return MlValue("string-pad", builtin::string_pad);
 	if (name == "string-rltrim") return MlValue("string-rltrim", builtin::string_rltrim);

tests/test.lsp

@@ -24,6 +24,8 @@
 (define json_list (parse-json "{\"k1\":\"v1\", \"k2\":42, \"k3\":[\"a\",123,true,false,null]}"))
 
 
+(thread-create (tcp-server 7777 (lambda (str) (list #t (+ "(print \"" (string-upcase str) "\")")))))
+
 (ut::define-test "result of (and (> 2 1) (> 2 1))" '(ut::assert-true (and (> 2 1) (> 2 1))))
 (ut::define-test "result of (or  (> 2 1) (> 2 1))" '(ut::assert-true (or  (> 2 1) (> 2 1))))
 (ut::define-test "retult of (and (> 2 1) (> 1 2))" '(ut::assert-false (and (> 2 1) (> 1 2))))
@@ -131,12 +133,16 @@
 
 (ut::define-test "(read-file-lines \"/tmp/f.txt\" counter)" '(ut::assert-equal 3 (read-file-lines "/tmp/f.txt" counter)))
 
-
 (ut::define-test "result of create table" '(ut::assert-equal ((0 "table created" 0)) (usql "create table a (i integer not null, s varchar(64), f float null, d date null,  b boolean)")))
 
+
+
+(ut::define-test "result tcp-client" '(ut::assert-equal "(print \"ABCD\")" (tcp-client "127.0.0.1" 7777 "abcd")))
+
 ;(ut::define-test "result of " '(ut::assert-true )
 (ut::run-tests)
 
+(threads-join)
 
 ;; (include "/usr/local/var/mlisp/terminal.lsp")
 

utils/Lisp.tmLanguage

@@ -136,7 +136,7 @@
 		</dict>
 		<dict>
 			<key>match</key>
-			<string>(?<=\()(?i:\*|\*\*|\*\*\*|\+|\+\+|\+\+\+|\-|/|//|///|/=|1\+|1\-|<|<=|=|>|>=|if|cond|do|for|while|scope|quote|defun|and|or|set!|eval|type|parse|list|insert|index|remove|len|push|pop|head|tail|first|last|range|map|filter|reduce|exit|quit|print|input|random|include|read-file|read-file-lines|write-file|read-url|system-cmd|ls-dir|is-file?|is-dir?|parse-csv|parse-json|get-universal-time|date-to-str|str-to-date|date-add|debug|sprintf|display|string-replace|string-regex?|string-split|string-pad|string-rltrim|string-case|string-len|string-substr|string-find|benchmark|thread-create|thread-under-lock|thread-sleep|threads-join|try|throw|usql|first|second|third|fourth|fifth|sixth|seventh|eight|nth|print|get-universal-time|not|is-pos?|is-neg?|neg|dec|inc|string-ltrim|string-rtrim|string-trim|string-rpad|string-lpad|string-upcase|string-downcase|string-join|itok|ktoi|sleep|get-env|member|make-list-of|make-list|empty-list?|uniq|flatten|quick-sort-by|quick-sort|quick-sort-reverse|start-of-day|end-of-day|start-of-month|start-of-next-month|end-of-next-month|end-of-month|start-of-prev-month|end-of-prev-month|start-of-year|end-of-year|make-csv)(?=\s+)</string>
+			<string>(?<=\()(?i:\*|\*\*|\*\*\*|\+|\+\+|\+\+\+|\-|/|//|///|/=|1\+|1\-|<|<=|=|>|>=|if|cond|do|for|while|scope|quote|defun|and|or|set!|eval|type|parse|list|insert|index|remove|len|push|pop|head|tail|first|last|range|map|filter|reduce|exit|quit|print|input|random|include|read-file|read-file-lines|write-file|read-url|system-cmd|ls-dir|is-file?|is-dir?|tcp-server|tcp-client|parse-csv|parse-json|get-universal-time|date-to-str|str-to-date|date-add|debug|sprintf|display|string-replace|string-replace-re|string-regex?|string-regex-list|string-split|string-pad|string-rltrim|string-case|string-len|string-substr|string-find|benchmark|thread-create|thread-under-lock|thread-sleep|threads-join|try|throw|usql|first|second|third|fourth|fifth|sixth|seventh|eight|nth|print|get-universal-time|not|is-pos?|is-neg?|neg|dec|inc|string-ltrim|string-rtrim|string-trim|string-rpad|string-lpad|string-upcase|string-downcase|string-join|itok|ktoi|sleep|get-env|member|make-list-of|make-list|empty-list?|uniq|flatten|quick-sort-by|quick-sort|quick-sort-reverse|start-of-day|end-of-day|start-of-month|start-of-next-month|end-of-next-month|end-of-month|start-of-prev-month|end-of-prev-month|start-of-year|end-of-year|make-csv)(?=\s+)</string>
 			<key>name</key>
 			<string>keyword.control.lisp</string>
 		</dict>

utils/local_install.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 gcc -std=c99 -c -O2 -o linenoise.o clib/linenoise.c
-c++ -c -O2  -I/usr/local/opt/openssl/include -Iclib -I./ --std=c++17  ml.cpp ml_io.cpp ml_date.cpp ml_string.cpp ml_util.cpp ml_profiler.cpp ml_usql.cpp clib/json11.cpp clib/csvparser.cpp clib/sslclient.cpp clib/printf.cpp usql/exception.cpp usql/lexer.cpp usql/parser.cpp usql/usql.cpp usql/table.cpp usql/table.h usql/row.cpp usql/csvreader.cpp usql/usql.cpp usql/usql_ddl.cpp usql/usql_dml.cpp usql/settings.cpp
+c++ -c -O2  -I/usr/local/opt/openssl/include -Iclib -I./ --std=c++17  ml.cpp ml_io.cpp ml_date.cpp ml_string.cpp ml_util.cpp ml_profiler.cpp ml_usql.cpp clib/json11.cpp clib/csvparser.cpp clib/sslclient.cpp clib/tcpnet.cpp clib/printf.cpp usql/exception.cpp usql/lexer.cpp usql/parser.cpp usql/usql.cpp usql/table.cpp usql/table.h usql/row.cpp usql/csvreader.cpp usql/usql.cpp usql/usql_ddl.cpp usql/usql_dml.cpp usql/settings.cpp
 c++ -o ml -O2 -L/usr/local/lib -L/usr/local/opt/openssl/lib -lm -lstdc++ -lcrypto -lssl *.o
 
 cp stdlib/*.lsp /usr/local/var/mlisp/
@@ -9,4 +9,9 @@ cp ml /usr/local/bin/ml
 
 rm *.o
 rm ./ml
-ml -v
+
+echo "if syntax has changed you may use"
+echo "cp utils/Lisp.tmLanguage ~/.vscode/extensions/mattn.lisp-0.1.11/syntaxes/"
+echo ""
+
+ml -v

utils/remote_install.sh

@@ -24,7 +24,7 @@ fi
 
 echo "Building ml"
 ssh -p 5333 root@46.28.109.184 "cd /tmp/mlisp; gcc -std=c99 -c -O2 -o linenoise.o clib/linenoise.c"
-ssh -p 5333 root@46.28.109.184 "cd /tmp/mlisp; c++ -c -O2  -I/usr/local/opt/openssl/include -Iclib -I./ --std=c++17  ml.cpp ml_io.cpp ml_date.cpp ml_string.cpp ml_util.cpp ml_profiler.cpp ml_usql.cpp  clib/json11.cpp clib/csvparser.cpp clib/sslclient.cpp clib/printf.cpp usql/exception.cpp usql/lexer.cpp usql/parser.cpp usql/usql.cpp usql/table.cpp usql/table.h usql/row.cpp usql/csvreader.cpp usql/usql.cpp usql/usql_ddl.cpp usql/usql_dml.cpp usql/settings.cpp"
+ssh -p 5333 root@46.28.109.184 "cd /tmp/mlisp; c++ -c -O2  -I/usr/local/opt/openssl/include -Iclib -I./ --std=c++17  ml.cpp ml_io.cpp ml_date.cpp ml_string.cpp ml_util.cpp ml_profiler.cpp ml_usql.cpp  clib/json11.cpp clib/csvparser.cpp clib/sslclient.cpp clib/tcpnet.cpp clib/printf.cpp usql/exception.cpp usql/lexer.cpp usql/parser.cpp usql/usql.cpp usql/table.cpp usql/table.h usql/row.cpp usql/csvreader.cpp usql/usql.cpp usql/usql_ddl.cpp usql/usql_dml.cpp usql/settings.cpp"
 ssh -p 5333 root@46.28.109.184 "cd /tmp/mlisp; c++ -o ml -O2 -L/usr/local/lib -L/usr/local/opt/openssl/lib -lm -lstdc++ -lcrypto -lssl *.o"