changes here, changes there

This commit is contained in:
VaclavT 2021-02-16 19:53:39 +01:00
parent c1532e78b1
commit 4b93f39375
9 changed files with 332 additions and 121 deletions

View File

@ -20,10 +20,11 @@ set(PROJECT_NAME ml)
set(SOURCE
ml.cpp
ml_io.cpp
ml_io.cpp
ml_date.cpp
clib/csvparser.cpp
clib/sslclient.cpp
clib/json11.cpp)
clib/json11.cpp ml_date.h ml_date.cpp)
add_executable(${PROJECT_NAME} ${SOURCE})

View File

@ -1,38 +1,37 @@
### TODO
- move system_cmd into ml_io
- date support
- decode-universal-time
- add save-csv
- logical operators
- documentation
- add url of source/inspiration to clib/*.cpp
- load std lib when starting
- add more command line args
- add stdtest
- support for including lisp libs
- add stdtest - to test every functionality
- rename ivaluize
- add benchmark
- add some debug support??
- add instrumentation (time, nr of evals, debug info, debug environment etc)
- in cating functions (as_string..) better exception description
- add fuctions asecond,third, fourth...
- add better print (syntaxhighligting)
- add options -h -v
#### Functionality
- in casting functions (as_string..) better exception description
- add better print (syntax high-lighting)
- format (printf)
- add comman line options -h -v
- readline
- execute system command
- printf
- support for including lisp lib
- date support
- execute system command should capture stderr
- add sleep
- file functions
- name it here
- string funtions
- string functions
- name it here
- regexp functions
- add hash datatype
- add support for nil
- conversion functions like parse-integer
- list function nth
#### Performance
- push_back - repeately without reserving size
- push_back - repeatedly without reserving size

View File

@ -10,14 +10,14 @@
; (print (fact 5))
; (define json_list (read-json "{\"k1\":\"v1\", \"k2\":42, \"k3\":[\"a\",123,true,false,null]}"))
; (define json_list (parse-json "{\"k1\":\"v1\", \"k2\":42, \"k3\":[\"a\",123,true,false,null]}"))
; (print json_list)
; (for x json_list
; (print x))
; nefunguje - api.nasdaq.com
;(define nasdaq_headers (quote (("Accept" "application/json, text/plain, */*") ("Connection" "keep-alive") ("Origin" "https://www.nasdaq.com/") ("User-Agent" "Mozilla/5.0 (Windows NT 10.0)"))))
;(define json_list2 (read-json (read-url "https://api.nasdaq.com/api/calendar/dividends/?date=2021-02-01" nasdaq_headers)))
;(define json_list2 (parse-json (read-url "https://api.nasdaq.com/api/calendar/dividends/?date=2021-02-01" nasdaq_headers)))
; (for x json_list2
; (print x))
@ -36,40 +36,45 @@
;; (print fdx_list)
(define curl "curl -H \"Accept: application/json, text/plain, */*\" -H \"Origin: https://www.nasdaq.com/\" -H \"User-Agent: Mozilla/5.0 (Windows NT 10.0)\" https://api.nasdaq.com/api/calendar/dividends/?date=2021-02-01 2>/dev/null")
(define curl_out (system-cmd curl))
(define json_list (read-json (last curl_out)))
;(print json_list)
(define data (first json_list))
;(print data)
;(print (index data 1)) ; druhy prvek
(define calendar (index data 1))
;(print calendar)
(define calendar_data (index (index calendar 0) 1))
;(print calendar_data)
(define header (index (index calendar_data 0) 1))
;(print header)
(define rows (index (index calendar_data 1) 1))
;(print rows)
(for e rows
; (("announcement_Date" "01/07/2021") ("companyName" "Itau Unibanco Banco Holding SA") ("dividend_Ex_Date" "02/01/2021") ("dividend_Rate" 0.003) ("indicated_Annual_Dividend" 0.033) ("payment_Date" "03/11/2021") ("record_Date" "02/02/2021") ("symbol" "ITUB"))
(define symbol (second (first (filter (lambda (x) (= (first x) "symbol")) e))))
(define divrate (second (first (filter (lambda (x) (= (first x) "dividend_Rate")) e))))
(define adate (second (first (filter (lambda (x) (= (first x) "announcement_Date")) e))))
(define name (second (first (filter (lambda (x) (= (first x) "companyName")) e))))
(define edate (second (first (filter (lambda (x) (= (first x) "dividend_Ex_Date")) e))))
(define pdate (second (first (filter (lambda (x) (= (first x) "payment_Date")) e))))
(define rdate (second (first (filter (lambda (x) (= (first x) "record_Date")) e))))
;; (define curl "curl -H \"Accept: application/json, text/plain, */*\" -H \"Origin: https://www.nasdaq.com/\" -H \"User-Agent: Mozilla/5.0 (Windows NT 10.0)\" https://api.nasdaq.com/api/calendar/dividends/?date=2021-02-01 2>/dev/null")
;; (define curl_out (system-cmd curl))
;; (define json_list (parse-json (last curl_out)))
;; ;(print json_list)
;; (define data (first json_list))
;; ;(print data)
;; ;(print (index data 1)) ; druhy prvek
;; (define calendar (index data 1))
;; ;(print calendar)
;; (define calendar_data (index (index calendar 0) 1))
;; ;(print calendar_data)
;; (define header (index (index calendar_data 0) 1))
;; ;(print header)
;; (define rows (index (index calendar_data 1) 1))
;; ;(print rows)
;; (for e rows
;; ; (("announcement_Date" "01/07/2021") ("companyName" "Itau Unibanco Banco Holding SA") ("dividend_Ex_Date" "02/01/2021") ("dividend_Rate" 0.003) ("indicated_Annual_Dividend" 0.033) ("payment_Date" "03/11/2021") ("record_Date" "02/02/2021") ("symbol" "ITUB"))
;; (define symbol (second (first (filter (lambda (x) (= (first x) "symbol")) e))))
;; (define divrate (second (first (filter (lambda (x) (= (first x) "dividend_Rate")) e))))
;; (define adate (second (first (filter (lambda (x) (= (first x) "announcement_Date")) e))))
;; (define name (second (first (filter (lambda (x) (= (first x) "companyName")) e))))
;; (define edate (second (first (filter (lambda (x) (= (first x) "dividend_Ex_Date")) e))))
;; (define pdate (second (first (filter (lambda (x) (= (first x) "payment_Date")) e))))
;; (define rdate (second (first (filter (lambda (x) (= (first x) "record_Date")) e))))
;(print symbol edate pdate divrate)
'(symbol edate pdate divrate)
)
;; ;(print symbol edate pdate divrate)
;; '(symbol edate pdate divrate)
;; )
(print (ls-dir "/tmp"))
(print (is-file? "/tmp/file"))
(print (is-file? "/tmp/hovinko"))
(print (is-dir? "/tmp/powerlog"))
;(print (ls-dir "/tmp"))
;(write-file "/tmp/file" "write-file test\n")
;(print (is-file? "/tmp/file"))
;(print (is-file? "/tmp/hovinko"))
;(print (is-dir? "/tmp/powerlog"))
(print (get-universal-time))
(print (date-to-str (get-universal-time) "%d.%m.%Y"))
(print (str-to-date "01.01.1970" "%d.%m.%Y"))
(print (date-add (str-to-date "01.01.1970" "%d.%m.%Y") 1 "day"))
(print "Debug ends")

137
ml.cpp
View File

@ -1,6 +1,7 @@
#include "ml.h"
#include "ml_io.h"
#include "ml_date.h"
#include "csvparser.h"
#include "sslclient.h"
@ -30,7 +31,7 @@
#define BAD_CAST "cannot cast"
#define ATOM_NOT_DEFINED "atom not defined"
#define EVAL_EMPTY_LIST "evaluated empty list"
#define INTERNAL_ERROR "interal virtual machine error"
#define INTERNAL_ERROR "internal virtual machine error"
#define INDEX_OUT_OF_RANGE "index out of range"
#define MALFORMED_PROGRAM "malformed program"
@ -1057,16 +1058,9 @@ namespace builtin {
eval_args(args, env);
if (args.size() != 2)
throw MlError(MlValue("write-file", write_file), env,
args.size() > 1 ? TOO_MANY_ARGS : TOO_FEW_ARGS);
throw MlError(MlValue("write-file", write_file), env, args.size() > 2 ? TOO_MANY_ARGS : TOO_FEW_ARGS);
std::ofstream f;
// The first argument is the file name
f.open(args[0].as_string().c_str());
// The second argument is the contents of the file to write
MlValue result = MlValue((f << args[1].as_string()) ? 1 : 0);
f.close();
return result;
return MlValue(write_file_contents(args[0].as_string(), args[1].as_string()));
}
// Read URL to (code content)
@ -1097,12 +1091,13 @@ namespace builtin {
}
// Parse JSON string
MlValue read_json(std::vector<MlValue> args, MlEnvironment &env) {
MlValue parse_json(std::vector<MlValue> args, MlEnvironment &env) {
eval_args(args, env);
// TODO add support for more params specifying options
if (args.size() != 1)
throw MlError(MlValue("read-json", read_json), env, args.size() > 1 ? TOO_MANY_ARGS : TOO_FEW_ARGS);
throw MlError(MlValue("parse-json", parse_json), env,
args.size() > 1 ? TOO_MANY_ARGS : TOO_FEW_ARGS);
std::string str = args[0].as_string();
std::string err;
@ -1116,48 +1111,59 @@ namespace builtin {
return json.ivalualize();
}
// get current time as secs from epoch
MlValue get_universal_time(std::vector<MlValue> args, MlEnvironment &env) {
eval_args(args, env);
if (args.size() != 0)
throw MlError(MlValue("get-universal-time", get_universal_time), env, TOO_MANY_ARGS);
return MlValue(now());
}
MlValue date_to_str(std::vector<MlValue> args, MlEnvironment &env) {
eval_args(args, env);
if (args.size() != 2)
throw MlError(MlValue("date_to_str", date_to_str), env,
args.size() > 2 ? TOO_MANY_ARGS : TOO_FEW_ARGS);
return MlValue::string(date_to_string(args[0].as_int(), args[1].as_string()));
}
MlValue str_to_date(std::vector<MlValue> args, MlEnvironment &env) {
eval_args(args, env);
if (args.size() != 2)
throw MlError(MlValue("str-to-date", str_to_date), env,
args.size() > 2 ? TOO_MANY_ARGS : TOO_FEW_ARGS);
return MlValue(string_to_date(args[0].as_string(), args[1].as_string()));
}
MlValue date_add(std::vector<MlValue> args, MlEnvironment &env) {
eval_args(args, env);
if (args.size() != 3)
throw MlError(MlValue("date-add", date_add), env, args.size() > 3 ? TOO_MANY_ARGS : TOO_FEW_ARGS);
return MlValue(add_to_date(args[0].as_int(), args[1].as_int(), args[2].as_string()));
}
// Execute system command
MlValue system_cmd(std::vector<MlValue> args, MlEnvironment &env) {
eval_args(args, env);
// TODO add support for more params constructing options as one string
// TODO add support for stderr
if (args.size() != 1)
throw MlError(MlValue("system-cmd", system_cmd), env,
args.size() > 1 ? TOO_MANY_ARGS : TOO_FEW_ARGS);
std::string cmd = args[0].as_string();
std::string cmd_output;
int stat;
// TODO better parameter here
// TODO handle reading of stderr
// https://stackoverflow.com/questions/478898/how-do-i-execute-a-command-and-get-the-output-of-the-command-within-c-using-po
// https://jineshkj.wordpress.com/2006/12/22/how-to-capture-stdin-stdout-and-stderr-of-child-program/
char buffer[128];
FILE *pipe = popen(cmd.c_str(), "r");
if (!pipe)
throw std::runtime_error("popen() failed!");
try {
while (!std::feof(pipe)) {
if (std::fgets(buffer, 128, pipe) != nullptr)
cmd_output += buffer;
}
} catch (...) {
stat = pclose(pipe);
throw;
}
stat = pclose(pipe);
int cmd_retval = WEXITSTATUS(stat);
// TODO add helper function for this
std::vector<MlValue> lst;
lst.push_back(MlValue(cmd_retval));
lst.push_back(MlValue::string(cmd_output));
return lst;
return exec_system_cmd(args[0].as_string());
}
// list directory
MlValue ls_dir(std::vector<MlValue> args, MlEnvironment &env) {
eval_args(args, env);
@ -1172,7 +1178,7 @@ namespace builtin {
MlValue is_file(std::vector<MlValue> args, MlEnvironment &env) {
eval_args(args, env);
// TODO add suppor for more params and list params
// TODO add support for more params and list params
if (args.size() != 1)
throw MlError(MlValue("is-file?", is_file), env, args.size() > 1 ? TOO_MANY_ARGS : TOO_FEW_ARGS);
@ -1183,7 +1189,7 @@ namespace builtin {
MlValue is_dir(std::vector<MlValue> args, MlEnvironment &env) {
eval_args(args, env);
// TODO add suppor for more params and list params
// TODO add support for more params and list params
if (args.size() != 1)
throw MlError(MlValue("is-dir?", is_dir), env, args.size() > 1 ? TOO_MANY_ARGS : TOO_FEW_ARGS);
@ -1583,10 +1589,7 @@ void repl(MlEnvironment &env) {
std::cout << "File to export to: ";
std::getline(std::cin, input);
std::ofstream f;
f.open(input.c_str(), std::ofstream::out);
f << code;
f.close();
write_file_contents(input, code);
} else if (input != "") {
try {
tmp = run(input, env);
@ -1601,6 +1604,16 @@ void repl(MlEnvironment &env) {
}
}
// load lisp std library
void load_std_lib(MlEnvironment &env) {
// TODO iterate through dir with files
std::string filename = "stdlib/stdlib.lisp";
MlEnvironment e;
MlValue result = run(read_file_contents(filename), e);
env.combine(e);
}
// Does this environment, or its parent environment, have a variable?
bool MlEnvironment::has(std::string name) const {
// Find the value in the map
@ -1653,9 +1666,7 @@ MlValue MlEnvironment::get(const std::string &name) const {
if (name == "insert") return MlValue("insert", builtin::insert);
if (name == "index") return MlValue("index", builtin::index);
if (name == "remove") return MlValue("remove", builtin::remove);
if (name == "len") return MlValue("len", builtin::len);
if (name == "push") return MlValue("push", builtin::push);
if (name == "pop") return MlValue("pop", builtin::pop);
if (name == "head") return MlValue("head", builtin::head);
@ -1676,15 +1687,23 @@ MlValue MlEnvironment::get(const std::string &name) const {
if (name == "input") return MlValue("input", builtin::input);
if (name == "random") return MlValue("random", builtin::random);
if (name == "include") return MlValue("include", builtin::include);
if (name == "parse-csv") return MlValue("parse-csv", builtin::parse_csv);
if (name == "read-file") return MlValue("read-file", builtin::read_file);
if (name == "write-file") return MlValue("write-file", builtin::write_file);
if (name == "read-url") return MlValue("read-url", builtin::read_url);
if (name == "read-json") return MlValue("read-json", builtin::read_json);
if (name == "system-cmd") return MlValue("system-cmd", builtin::system_cmd);
if (name == "ls-dir") return MlValue("ls-dir", builtin::ls_dir);
if (name == "is-file?") return MlValue("is-file?", builtin::is_file);
if (name == "is-dir?") return MlValue("is-der?", builtin::is_dir);
if (name == "is-dir?") return MlValue("is-dir?", builtin::is_dir);
// parsing operations
if (name == "parse-csv") return MlValue("parse-csv", builtin::parse_csv);
if (name == "parse-json") return MlValue("parse-json", builtin::parse_json);
// Datetime operations
if (name == "get-universal-time") return MlValue("get-universal-time", builtin::get_universal_time);
if (name == "date-to-str") return MlValue("date-to-str", builtin::date_to_str);
if (name == "str-to-date") return MlValue("str-to-date", builtin::str_to_date);
if (name == "date-add") return MlValue("date-add", builtin::date_add);
// String operations
if (name == "debug") return MlValue("debug", builtin::debug);
@ -1709,16 +1728,6 @@ MlValue MlEnvironment::get(const std::string &name) const {
throw MlError(MlValue::atom(name), *this, ATOM_NOT_DEFINED);
}
// load lisp std library
void load_std_lib(MlEnvironment &env) {
// TODO iterate through dir with files
std::string filename = "stdlib/stdlib.lisp";
MlEnvironment e;
MlValue result = run(read_file_contents(filename), e);
env.combine(e);
}
int main(int argc, const char **argv) {
MlEnvironment env;
std::vector<MlValue> args;

58
ml_date.cpp Normal file
View File

@ -0,0 +1,58 @@
#include "ml_date.h"
int now() {
// get-universal-time
time_t t = std::time(0);
long int now = static_cast<long int>(t);
return (int) now;
}
std::string date_to_string(const int datetime, const std::string format) {
// std::locale::global(std::locale("en-US.UTF8"));
time_t timestamp = datetime;
char mbstr[128];
if (std::strftime(mbstr, sizeof(mbstr), format.c_str(), std::localtime(&timestamp))) {
std::string result = {mbstr};
return result;
}
// TODO exception here
}
int string_to_date(const std::string &datestr, const std::string &format) {
// format for example "%d.%m.%Y";
std::istringstream in{datestr.c_str()};
date::sys_seconds tp;
in >> date::parse(format, tp);
return (int) tp.time_since_epoch().count();
}
int add_to_date(const int datetime, const int quantity, const std::string &part) {
// part is one of 'year', 'month', 'day', 'hour', 'minute', 'second', or 'millisecond'
// very basic implementation, just for now - no timezones DST etc
time_t base = datetime;
struct tm *tm = localtime(&base);
if (part == "year") {
tm->tm_year += quantity;
} else if (part == "month") {
tm->tm_mon += quantity;
} else if (part == "day") {
tm->tm_mday += quantity;
} else if (part == "hour") {
tm->tm_hour += quantity;
} else if (part == "minute") {
tm->tm_min += quantity;
} else if (part == "second") {
tm->tm_sec += quantity;
} else {
// TODO exception here
}
return (int) mktime(tm);
}

19
ml_date.h Normal file
View File

@ -0,0 +1,19 @@
#pragma once
#include "ml.h"
#include "clib/date.h"
#include <string>
#include <vector>
int now();
// TODO temporary solution before format will be implemented
std::string date_to_string(const int datetime, const std::string format);
int string_to_date(const std::string &datestr, const std::string &format);
int add_to_date(const int datetime, const int quantity, const std::string &part);

View File

@ -23,6 +23,14 @@ std::string read_file_contents(const std::string &filename) {
return contents;
}
int write_file_contents(const std::string &filename, const std::string &content) {
std::ofstream f;
f.open(filename.c_str());
int result = (f << content) ? 1 : 0;
f.close();
return result;
}
MlValue list_dir(const std::string &path) {
std::vector<MlValue> entries;
@ -50,13 +58,42 @@ bool is_path_dir(const std::string &path) {
return (bool) S_ISDIR(buf.st_mode);
}
int mk_dir(const std::string &path) {
int mk_path_dir(const std::string &path) {
int r = ::mkdir(path.c_str(), 0755);
return r;
}
int rm_dir(const std::string &path) {
int rm_path_dir(const std::string &path) {
int r = ::rmdir(path.c_str());
return r;
}
MlValue exec_system_cmd(const std::string &cmd) {
// TODO handle reading of stderr
// https://stackoverflow.com/questions/478898/how-do-i-execute-a-command-and-get-the-output-of-the-command-within-c-using-po
// https://jineshkj.wordpress.com/2006/12/22/how-to-capture-stdin-stdout-and-stderr-of-child-program/
std::string cmd_output;
int stat;
char buffer[128];
FILE *pipe = popen(cmd.c_str(), "r");
if (!pipe)
throw std::runtime_error("popen() failed!");
try {
while (!feof(pipe)) {
if (fgets(buffer, 128, pipe) != nullptr)
cmd_output += buffer;
}
} catch (...) {
stat = pclose(pipe);
throw;
}
stat = pclose(pipe);
int cmd_retval = WEXITSTATUS(stat);
// TODO add helper function for this
std::vector<MlValue> lst;
lst.push_back(MlValue(cmd_retval));
lst.push_back(MlValue::string(cmd_output));
return lst;
}

View File

@ -9,13 +9,16 @@
std::string read_file_contents(const std::string &filename);
int write_file_contents(const std::string &filename, const std::string &content);
MlValue list_dir(const std::string &path);
bool is_path_file(const std::string &path);
bool is_path_dir(const std::string &path);
int mk_dir(const std::string &path);
int mk_path_dir(const std::string &path);
int rm_dir(const std::string &path);
int rm_path_dir(const std::string &path);
MlValue exec_system_cmd(const std::string &path);

80
tests/test.lisp Normal file
View File

@ -0,0 +1,80 @@
(print "Debug starts")
; define a function `fact` that takes an argument `n`
; (defun fact (n)
; (if (<= n 1)
; 1
; (* n (fact (- n 1)))
; ))
; (print (fact 5))
; (define json_list (parse-json "{\"k1\":\"v1\", \"k2\":42, \"k3\":[\"a\",123,true,false,null]}"))
; (print json_list)
; (for x json_list
; (print x))
; nefunguje - api.nasdaq.com
;(define nasdaq_headers (quote (("Accept" "application/json, text/plain, */*") ("Connection" "keep-alive") ("Origin" "https://www.nasdaq.com/") ("User-Agent" "Mozilla/5.0 (Windows NT 10.0)"))))
;(define json_list2 (parse-json (read-url "https://api.nasdaq.com/api/calendar/dividends/?date=2021-02-01" nasdaq_headers)))
; (for x json_list2
; (print x))
;; (include "tmp/example.lisp")
;; (print "sorted: " (qs '(10 9 8 7 6 5 4 3 2 1)))
; (define csv (read-file "tmp/data.csv"))
; (define csv_list (parse-csv csv))
; (for x csv_list
; (print x))
;; (define web_page (read-url "https://query1.finance.yahoo.com/v7/finance/download/FDX?period1=1581272585&period2=1612894985&interval=1d&events=history&includeAdjustedClose=true"))
;; (print web_page)
;; (define fdx_list (parse-csv (index web_page 1)))
;; (print fdx_list)
;; (define curl "curl -H \"Accept: application/json, text/plain, */*\" -H \"Origin: https://www.nasdaq.com/\" -H \"User-Agent: Mozilla/5.0 (Windows NT 10.0)\" https://api.nasdaq.com/api/calendar/dividends/?date=2021-02-01 2>/dev/null")
;; (define curl_out (system-cmd curl))
;; (define json_list (parse-json (last curl_out)))
;; ;(print json_list)
;; (define data (first json_list))
;; ;(print data)
;; ;(print (index data 1)) ; druhy prvek
;; (define calendar (index data 1))
;; ;(print calendar)
;; (define calendar_data (index (index calendar 0) 1))
;; ;(print calendar_data)
;; (define header (index (index calendar_data 0) 1))
;; ;(print header)
;; (define rows (index (index calendar_data 1) 1))
;; ;(print rows)
;; (for e rows
;; ; (("announcement_Date" "01/07/2021") ("companyName" "Itau Unibanco Banco Holding SA") ("dividend_Ex_Date" "02/01/2021") ("dividend_Rate" 0.003) ("indicated_Annual_Dividend" 0.033) ("payment_Date" "03/11/2021") ("record_Date" "02/02/2021") ("symbol" "ITUB"))
;; (define symbol (second (first (filter (lambda (x) (= (first x) "symbol")) e))))
;; (define divrate (second (first (filter (lambda (x) (= (first x) "dividend_Rate")) e))))
;; (define adate (second (first (filter (lambda (x) (= (first x) "announcement_Date")) e))))
;; (define name (second (first (filter (lambda (x) (= (first x) "companyName")) e))))
;; (define edate (second (first (filter (lambda (x) (= (first x) "dividend_Ex_Date")) e))))
;; (define pdate (second (first (filter (lambda (x) (= (first x) "payment_Date")) e))))
;; (define rdate (second (first (filter (lambda (x) (= (first x) "record_Date")) e))))
;; ;(print symbol edate pdate divrate)
;; '(symbol edate pdate divrate)
;; )
;(print (ls-dir "/tmp"))
;(write-file "/tmp/file" "write-file test\n")
;(print (is-file? "/tmp/file"))
;(print (is-file? "/tmp/hovinko"))
;(print (is-dir? "/tmp/powerlog"))
(print (get-universal-time))
(print (date-to-str (get-universal-time) "%d.%m.%Y"))
(print (str-to-date "01.01.1970" "%d.%m.%Y"))
(print (date-add (str-to-date "01.01.1970" "%d.%m.%Y") 1 "day"))
(print "Debug ends")