From 159845bb9b892e753306674fa8dfb9196995d20f Mon Sep 17 00:00:00 2001 From: VaclavT Date: Sun, 31 Oct 2021 09:53:48 +0100 Subject: [PATCH] tcp-server and tcp-client added --- CMakeLists.txt | 1 + Readme.md | 2 + clib/tcpnet.cpp | 118 ++++++++++++++++++++++++++++++++++++++++ clib/tcpnet.h | 20 +++++++ debug.lsp | 26 ++++++++- doc/Doc.md | 3 + ml.cpp | 58 +++++++++++++++++--- tests/test.lsp | 8 ++- utils/Lisp.tmLanguage | 2 +- utils/local_install.sh | 9 ++- utils/remote_install.sh | 2 +- 11 files changed, 235 insertions(+), 14 deletions(-) create mode 100644 clib/tcpnet.cpp create mode 100644 clib/tcpnet.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 38e5306..bd82555 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,6 +34,7 @@ set(SOURCE ml_usql.cpp clib/csvparser.cpp clib/sslclient.cpp + clib/tcpnet.cpp clib/json11.cpp clib/printf.cpp clib/linenoise.c diff --git a/Readme.md b/Readme.md index d3d6672..2f938d0 100644 --- a/Readme.md +++ b/Readme.md @@ -65,6 +65,7 @@ utils/local_install.sh (read-url "https://api.nasdaq.com/api/calendar/dividends/") ; hangs in sslclient.cpp line 132 ### TODO +- tcp-client, add support for list of requests returning list of responses - add debug support, at least call stack - multiline editing (see kilocpp editor) - execute system command should capture stderr @@ -72,6 +73,7 @@ utils/local_install.sh #### Code - MlEnvironment::get should be map instead of many if's +- tcpnet should use RAII - add documentation - add more unit test, mainly for usql - add instrumentation (time, nr of evals, num of atoms, debug info, debug environment etc) diff --git a/clib/tcpnet.cpp b/clib/tcpnet.cpp new file mode 100644 index 0000000..2a16187 --- /dev/null +++ b/clib/tcpnet.cpp @@ -0,0 +1,118 @@ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "tcpnet.h" + + +void error(const char *msg) { + // perror(msg); + throw std::runtime_error(msg); +} + +// TODO do it as RAII +TcpNet::TcpNet() {} + + +#define TCPNET_BUFFER_SIZE 16256 + + +int TcpNet::server(int portno, std::function(std::string)> process_request) { + int sockfd, newsockfd; + socklen_t clilen; + + char buffer[TCPNET_BUFFER_SIZE]; + struct sockaddr_in serv_addr, cli_addr; + + int n; + sockfd = socket(AF_INET, SOCK_STREAM, 0); + if (sockfd < 0) + error("ERROR opening socket"); + + bzero((char *) &serv_addr, sizeof(serv_addr)); + serv_addr.sin_family = AF_INET; + serv_addr.sin_addr.s_addr = INADDR_ANY; + serv_addr.sin_port = htons(portno); + + if (bind(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) + error("ERROR on binding"); + + listen(sockfd,5); + clilen = sizeof(cli_addr); + + int requests_processed = 0; + bool shutdown = false; + while (!shutdown) { + newsockfd = accept(sockfd, (struct sockaddr *) &cli_addr, &clilen); + if (newsockfd < 0) + error("ERROR on accept"); + + bzero(buffer,TCPNET_BUFFER_SIZE); + + n = read(newsockfd,buffer,TCPNET_BUFFER_SIZE - 1); + if (n < 0) error("ERROR reading from socket"); + + std::string request{buffer}; + std::pair response = process_request(request); + shutdown = response.first; + std::string response_str = response.second; + + n = write(newsockfd, response_str.c_str(), response_str.size()); + if (n < 0) + error("ERROR writing to socket"); + + close(newsockfd); + requests_processed++; + } + + close(sockfd); + + return requests_processed; +} + +std::string TcpNet::client(const std::string address, int portno, const std::string &content) { + int sockfd, n; + struct sockaddr_in serv_addr; + struct hostent *server; + + char buffer[TCPNET_BUFFER_SIZE]; + + sockfd = socket(AF_INET, SOCK_STREAM, 0); + if (sockfd < 0) + error("ERROR opening socket"); + + server = gethostbyname(address.c_str()); + if (server == NULL) { + fprintf(stderr,"ERROR, no such host\n"); + exit(0); + } + + bzero((char *) &serv_addr, sizeof(serv_addr)); + serv_addr.sin_family = AF_INET; + bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); + serv_addr.sin_port = htons(portno); + + if (connect(sockfd,(struct sockaddr *) &serv_addr,sizeof(serv_addr)) < 0) + error("ERROR connecting"); + + + n = write(sockfd, content.c_str(), strlen(content.c_str())); + if (n < 0) + error("ERROR writing to socket"); + + bzero(buffer, TCPNET_BUFFER_SIZE); + n = read(sockfd,buffer, TCPNET_BUFFER_SIZE - 1); + if (n < 0) + error("ERROR reading from socket"); + + close(sockfd); + + return std::string(buffer); +} + diff --git a/clib/tcpnet.h b/clib/tcpnet.h new file mode 100644 index 0000000..1023d3c --- /dev/null +++ b/clib/tcpnet.h @@ -0,0 +1,20 @@ +#pragma once + +#include + + +class TcpNet { + +public: + TcpNet(); + + // starts listening on port + int server(int port, std::function(std::string)> process_request); + + // writes content to server on address:port and returns response + std::string client(const std::string address, int port, const std::string &content); + + // TODO add support for vector of strings to be sent to server + // std::vector client(const std::string address, int port, const std::vector &content); + +}; diff --git a/debug.lsp b/debug.lsp index b1b934a..8211e97 100644 --- a/debug.lsp +++ b/debug.lsp @@ -1,5 +1,27 @@ -(print (string-regex-list "12" "(.*?)" "match" "ignore")) -(print (string-regex-list "12" "(.*?)" "token")) + +(thread-create + (tcp-server 7777 (lambda (str) (list #t (+ "(print \"" (string-upcase str) "\")")))) +) + +(thread-sleep 1) + +(thread-create + (define code (tcp-client "127.0.0.1" 7777 "abcd")) + (print "executing code:" code) + (eval (parse code)) +) +(threads-join) +(print "ok") + + +;; (usql "create table sf1 (symbol varchar(8), dimension varchar(3), calendar_date date, date_key date, report_period date, last_updated date, accoci float, assets float, assetsavg float, assetsc float, assetsnc float, assetturnover float, bvps float, capex float, cashneq float, cashnequsd float, cor float, consolinc float, currentratio float, de float, debt float, debtc float, debtnc float, debtusd float, deferredrev float, depamor float, deposits float, divyield float, dps float, ebit float, ebitda float, ebitdamargin float, ebitdausd float, ebitusd float, ebt float, eps float, epsdil float, epsusd float, equity float, equityavg float, equityusd float, ev float, evebit float, evebitda float, fcf float, fcfps float, fxusd float, gp float, grossmargin float, intangibles float, intexp float, invcap float, invcapavg float, inventory float, investments float, investmentsc float, investmentsnc float, liabilities float, liabilitiesc float, liabilitiesnc float, marketcap float, ncf float, ncfbus float, ncfcommon float, ncfdebt float, ncfdiv float, ncff float, ncfi float, ncfinv float, ncfo float, ncfx float, netinc float, netinccmn float, netinccmnusd float, netincdis float, netincnci float, netmargin float, opex float, opinc float, payables float, payoutratio float, pb float, pe float, pe1 float, ppnenet float, prefdivis float, price float, ps float, ps1 float, receivables float, retearn float, revenue float, revenueusd float, rnd float, roa float, roe float, roic float, ros float, sbcomp float, sgna float, sharefactor float, sharesbas float, shareswa float, shareswadil float, sps float, tangibles float, taxassets float, taxexp float, taxliabilities float, tbvps float, workingcapital float)") +;; (usql "set 'DATE_FORMAT' = '%Y-%m-%d'") +;; (usql "load into sf1 '/srv/SHARADAR_SF1.csv'") +;; (print (usql "select dimension, to_string(calendar_date, '%d.%m.%Y'), pp(eps, \"%.2f\"), pp(shareswadil), pp(revenue), pp(netinc), pp(cashneq), pp(assets), pp(debt), pp(ncfdebt), pp(roe*100), pp(intangibles), calendar_date from sf1 where symbol = 'MU' and dimension = 'ARQ' order by dimension, calendar_date desc limit 5")) + + +;; (print (string-regex-list "12" "(.*?)" "match" "ignore")) +;; (print (string-regex-list "12" "(.*?)" "token")) ;; (include "/usr/local/var/mlisp/ut.lsp") diff --git a/doc/Doc.md b/doc/Doc.md index f1e2749..9e36c6a 100644 --- a/doc/Doc.md +++ b/doc/Doc.md @@ -84,6 +84,9 @@ |`(ls-dir dir)`|List a dir|List of directory entries| |`(is-file? filename)`|Returns true if passed filename is a file|| |`(is-dir? filename)`|Returns true if passed filename is a directory|| +|`(tcp-server port handler)`|Starts listening on port and when request comes calls passed lambda and writes its returned value back to client. Lambda must return either string or two element list where first element is boolean and second string.When first element is true it closes listening socker.|(`tcp-server 7777 (lambda (str) (list #t (string-upcase str))))`| +|`(tcp-client address port data)`|Opens connection to server on port, writes there data and returns response.|`(print (tcp-client "127.0.0.1" 7777 "abcd"))`| +|`(is-dir? filename)`|Returns true if passed filename is a directory|| |`(parse-csv string)`|Parse CSV string|| |`(parse-json json_string)`|Parse JSON string|| |`(save-csv ..)`||| diff --git a/ml.cpp b/ml.cpp index 9b38143..6501545 100644 --- a/ml.cpp +++ b/ml.cpp @@ -9,6 +9,7 @@ #include "clib/csvparser.h" #include "clib/sslclient.h" +#include "clib/tcpnet.h" #include "clib/json11.h" #include "clib/printf.h" @@ -58,6 +59,11 @@ #define LIST_TYPE "list" +std::mutex register_mutex; +std::vector threads_register; +std::mutex interpreter_mutex; + + // Is this character a valid lisp symbol character bool is_symbol(char ch) { return (isalpha(ch) || ispunct(ch)) && ch != '(' && ch != ')' && ch != '"' && ch != '\''; @@ -1065,6 +1071,10 @@ MlValue exit(std::vector args, MlEnvironment &env) { // Is not a special form, so we can evaluate our args. eval_args(args, env); + std::lock_guard lockGuard(register_mutex); + for(auto &t : threads_register) + t.detach(); + std::exit(args.size() < 1 ? 0 : args[0].cast_to_int().as_int()); return MlValue(); // will not be called :-) } @@ -1306,6 +1316,40 @@ MlValue is_dir(std::vector args, MlEnvironment &env) { return MlValue(is_path_dir(args[0].as_string())); } +MlValue tcp_server(std::vector args, MlEnvironment &env) { + eval_args(args, env); + + if (args.size() != 2) + throw MlError(MlValue("tcp-server", tcp_server), env, args.size() > 2 ? TOO_MANY_ARGS : TOO_FEW_ARGS); + + auto proccess_req = [&args, &env](const std::string &str) -> std::pair { + std::vector tmp {MlValue::string(str)}; + MlValue result = args[1].apply(tmp, env); + // TODO more robust handling + if (result.is_list() && result.as_list().size() >= 2) { + std::vector list = result.as_list(); + return std::make_pair(list[0].as_bool(), list[1].as_string()); + } else { + return std::make_pair(false, result.as_string()); + } + }; + + TcpNet server; + int r = server.server(args[0].as_int(), proccess_req); + return MlValue((long)r); +} + +MlValue tcp_client(std::vector args, MlEnvironment &env) { + eval_args(args, env); + + if (args.size() != 3) + throw MlError(MlValue("tcp-client", tcp_client), env, args.size() > 3 ? TOO_MANY_ARGS : TOO_FEW_ARGS); + + TcpNet tcpclient; + std::string response = tcpclient.client(args[0].as_string(), args[1].as_int(), args[2].as_string()); + return MlValue::string(response); +} + // Read a file and execute its code MlValue include(std::vector args, MlEnvironment &env) { // Import is technically not a special form, it's more of a macro. @@ -1895,11 +1939,6 @@ MlValue benchmark(std::vector args, MlEnvironment &env) { } -std::mutex register_mutex; -std::vector threads_register; -std::vector mutexes_register; -std::mutex interpreter_mutex; - MlValue thread_create(std::vector args, MlEnvironment &env) { auto functor = [](std::vector args, MlEnvironment &env) -> void { try { @@ -1907,6 +1946,7 @@ MlValue thread_create(std::vector args, MlEnvironment &env) { MlValue acc = args[i].eval(env); } catch (std::exception &e) { std::cerr << "thread exception: " << e.what() << std::endl; + throw e; } }; @@ -1953,7 +1993,9 @@ MlValue threads_join(std::vector args, MlEnvironment &env) { if (args.size() != 0) throw MlError(MlValue("threads-join", threads_join), env, TOO_MANY_ARGS); - std::lock_guard lockGuard(register_mutex); + // here is a question about using lockGuard, when used it holds lockGuard locked until + // threads do not leave join + // std::lock_guard lockGuard(register_mutex); for (auto &th : threads_register) if (th.joinable()) th.join(); @@ -2094,6 +2136,8 @@ MlValue MlEnvironment::get(const std::string &name) const { if (name == "ls-dir") return MlValue("ls-dir", builtin::ls_dir); if (name == "is-file?") return MlValue("is-file?", builtin::is_file); if (name == "is-dir?") return MlValue("is-dir?", builtin::is_dir); + if (name == "tcp-server") return MlValue("tcp-client", builtin::tcp_server); + if (name == "tcp-client") return MlValue("tcp-client", builtin::tcp_client); // parsing operations if (name == "parse-csv") return MlValue("parse-csv", builtin::parse_csv); @@ -2113,7 +2157,7 @@ MlValue MlEnvironment::get(const std::string &name) const { if (name == "string-replace") return MlValue("string-replace", builtin::string_replace); if (name == "string-replace-re") return MlValue("string-replace-re", builtin::string_replace_re); if (name == "string-regex?") return MlValue("string-regex?", builtin::string_regex); - if (name == "string-regex-list") return MlValue("string-regex?", builtin::string_regex_list); + if (name == "string-regex-list") return MlValue("string-regex-list", builtin::string_regex_list); if (name == "string-split") return MlValue("string-split", builtin::string_split); if (name == "string-pad") return MlValue("string-pad", builtin::string_pad); if (name == "string-rltrim") return MlValue("string-rltrim", builtin::string_rltrim); diff --git a/tests/test.lsp b/tests/test.lsp index 2b8518b..b95294b 100644 --- a/tests/test.lsp +++ b/tests/test.lsp @@ -24,6 +24,8 @@ (define json_list (parse-json "{\"k1\":\"v1\", \"k2\":42, \"k3\":[\"a\",123,true,false,null]}")) +(thread-create (tcp-server 7777 (lambda (str) (list #t (+ "(print \"" (string-upcase str) "\")"))))) + (ut::define-test "result of (and (> 2 1) (> 2 1))" '(ut::assert-true (and (> 2 1) (> 2 1)))) (ut::define-test "result of (or (> 2 1) (> 2 1))" '(ut::assert-true (or (> 2 1) (> 2 1)))) (ut::define-test "retult of (and (> 2 1) (> 1 2))" '(ut::assert-false (and (> 2 1) (> 1 2)))) @@ -131,12 +133,16 @@ (ut::define-test "(read-file-lines \"/tmp/f.txt\" counter)" '(ut::assert-equal 3 (read-file-lines "/tmp/f.txt" counter))) - (ut::define-test "result of create table" '(ut::assert-equal ((0 "table created" 0)) (usql "create table a (i integer not null, s varchar(64), f float null, d date null, b boolean)"))) + + +(ut::define-test "result tcp-client" '(ut::assert-equal "(print \"ABCD\")" (tcp-client "127.0.0.1" 7777 "abcd"))) + ;(ut::define-test "result of " '(ut::assert-true ) (ut::run-tests) +(threads-join) ;; (include "/usr/local/var/mlisp/terminal.lsp") diff --git a/utils/Lisp.tmLanguage b/utils/Lisp.tmLanguage index d399193..955119e 100644 --- a/utils/Lisp.tmLanguage +++ b/utils/Lisp.tmLanguage @@ -136,7 +136,7 @@ match - (?<=\()(?i:\*|\*\*|\*\*\*|\+|\+\+|\+\+\+|\-|/|//|///|/=|1\+|1\-|<|<=|=|>|>=|if|cond|do|for|while|scope|quote|defun|and|or|set!|eval|type|parse|list|insert|index|remove|len|push|pop|head|tail|first|last|range|map|filter|reduce|exit|quit|print|input|random|include|read-file|read-file-lines|write-file|read-url|system-cmd|ls-dir|is-file?|is-dir?|parse-csv|parse-json|get-universal-time|date-to-str|str-to-date|date-add|debug|sprintf|display|string-replace|string-regex?|string-split|string-pad|string-rltrim|string-case|string-len|string-substr|string-find|benchmark|thread-create|thread-under-lock|thread-sleep|threads-join|try|throw|usql|first|second|third|fourth|fifth|sixth|seventh|eight|nth|print|get-universal-time|not|is-pos?|is-neg?|neg|dec|inc|string-ltrim|string-rtrim|string-trim|string-rpad|string-lpad|string-upcase|string-downcase|string-join|itok|ktoi|sleep|get-env|member|make-list-of|make-list|empty-list?|uniq|flatten|quick-sort-by|quick-sort|quick-sort-reverse|start-of-day|end-of-day|start-of-month|start-of-next-month|end-of-next-month|end-of-month|start-of-prev-month|end-of-prev-month|start-of-year|end-of-year|make-csv)(?=\s+) + (?<=\()(?i:\*|\*\*|\*\*\*|\+|\+\+|\+\+\+|\-|/|//|///|/=|1\+|1\-|<|<=|=|>|>=|if|cond|do|for|while|scope|quote|defun|and|or|set!|eval|type|parse|list|insert|index|remove|len|push|pop|head|tail|first|last|range|map|filter|reduce|exit|quit|print|input|random|include|read-file|read-file-lines|write-file|read-url|system-cmd|ls-dir|is-file?|is-dir?|tcp-server|tcp-client|parse-csv|parse-json|get-universal-time|date-to-str|str-to-date|date-add|debug|sprintf|display|string-replace|string-replace-re|string-regex?|string-regex-list|string-split|string-pad|string-rltrim|string-case|string-len|string-substr|string-find|benchmark|thread-create|thread-under-lock|thread-sleep|threads-join|try|throw|usql|first|second|third|fourth|fifth|sixth|seventh|eight|nth|print|get-universal-time|not|is-pos?|is-neg?|neg|dec|inc|string-ltrim|string-rtrim|string-trim|string-rpad|string-lpad|string-upcase|string-downcase|string-join|itok|ktoi|sleep|get-env|member|make-list-of|make-list|empty-list?|uniq|flatten|quick-sort-by|quick-sort|quick-sort-reverse|start-of-day|end-of-day|start-of-month|start-of-next-month|end-of-next-month|end-of-month|start-of-prev-month|end-of-prev-month|start-of-year|end-of-year|make-csv)(?=\s+) name keyword.control.lisp diff --git a/utils/local_install.sh b/utils/local_install.sh index 1aa9bf7..dbf6d95 100755 --- a/utils/local_install.sh +++ b/utils/local_install.sh @@ -1,7 +1,7 @@ #!/bin/sh gcc -std=c99 -c -O2 -o linenoise.o clib/linenoise.c -c++ -c -O2 -I/usr/local/opt/openssl/include -Iclib -I./ --std=c++17 ml.cpp ml_io.cpp ml_date.cpp ml_string.cpp ml_util.cpp ml_profiler.cpp ml_usql.cpp clib/json11.cpp clib/csvparser.cpp clib/sslclient.cpp clib/printf.cpp usql/exception.cpp usql/lexer.cpp usql/parser.cpp usql/usql.cpp usql/table.cpp usql/table.h usql/row.cpp usql/csvreader.cpp usql/usql.cpp usql/usql_ddl.cpp usql/usql_dml.cpp usql/settings.cpp +c++ -c -O2 -I/usr/local/opt/openssl/include -Iclib -I./ --std=c++17 ml.cpp ml_io.cpp ml_date.cpp ml_string.cpp ml_util.cpp ml_profiler.cpp ml_usql.cpp clib/json11.cpp clib/csvparser.cpp clib/sslclient.cpp clib/tcpnet.cpp clib/printf.cpp usql/exception.cpp usql/lexer.cpp usql/parser.cpp usql/usql.cpp usql/table.cpp usql/table.h usql/row.cpp usql/csvreader.cpp usql/usql.cpp usql/usql_ddl.cpp usql/usql_dml.cpp usql/settings.cpp c++ -o ml -O2 -L/usr/local/lib -L/usr/local/opt/openssl/lib -lm -lstdc++ -lcrypto -lssl *.o cp stdlib/*.lsp /usr/local/var/mlisp/ @@ -9,4 +9,9 @@ cp ml /usr/local/bin/ml rm *.o rm ./ml -ml -v + +echo "if syntax has changed you may use" +echo "cp utils/Lisp.tmLanguage ~/.vscode/extensions/mattn.lisp-0.1.11/syntaxes/" +echo "" + +ml -v \ No newline at end of file diff --git a/utils/remote_install.sh b/utils/remote_install.sh index 2886658..6163cd4 100755 --- a/utils/remote_install.sh +++ b/utils/remote_install.sh @@ -24,7 +24,7 @@ fi echo "Building ml" ssh -p 5333 root@46.28.109.184 "cd /tmp/mlisp; gcc -std=c99 -c -O2 -o linenoise.o clib/linenoise.c" -ssh -p 5333 root@46.28.109.184 "cd /tmp/mlisp; c++ -c -O2 -I/usr/local/opt/openssl/include -Iclib -I./ --std=c++17 ml.cpp ml_io.cpp ml_date.cpp ml_string.cpp ml_util.cpp ml_profiler.cpp ml_usql.cpp clib/json11.cpp clib/csvparser.cpp clib/sslclient.cpp clib/printf.cpp usql/exception.cpp usql/lexer.cpp usql/parser.cpp usql/usql.cpp usql/table.cpp usql/table.h usql/row.cpp usql/csvreader.cpp usql/usql.cpp usql/usql_ddl.cpp usql/usql_dml.cpp usql/settings.cpp" +ssh -p 5333 root@46.28.109.184 "cd /tmp/mlisp; c++ -c -O2 -I/usr/local/opt/openssl/include -Iclib -I./ --std=c++17 ml.cpp ml_io.cpp ml_date.cpp ml_string.cpp ml_util.cpp ml_profiler.cpp ml_usql.cpp clib/json11.cpp clib/csvparser.cpp clib/sslclient.cpp clib/tcpnet.cpp clib/printf.cpp usql/exception.cpp usql/lexer.cpp usql/parser.cpp usql/usql.cpp usql/table.cpp usql/table.h usql/row.cpp usql/csvreader.cpp usql/usql.cpp usql/usql_ddl.cpp usql/usql_dml.cpp usql/settings.cpp" ssh -p 5333 root@46.28.109.184 "cd /tmp/mlisp; c++ -o ml -O2 -L/usr/local/lib -L/usr/local/opt/openssl/lib -lm -lstdc++ -lcrypto -lssl *.o"