vaclavt
/
mlisp
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fixes & enhandcements 

(benchmark code..) implemented
repl completion very first version
(string xx) added
some builtin renames
a bit of comments
a bit of cocumentation
(sleep interval) added
This commit is contained in:
VaclavT 2021-03-14 16:15:04 +01:00
parent 8fad428a4b
commit c4e4522492
12 changed files with 276 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CMakeLists.txt
View File

@ -13,6 +13,9 @@ set(CPACK_PROJECT_VERSION ${PROJECT_VERSION})
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-stack_size -Wl,0x1000000")
# otool -lV build/ml | grep stack
# set(CMAKE_CXX_FLAGS "-Wall -Wextra")
# set(CMAKE_CXX_FLAGS_RELEASE "-O3")
include_directories(/usr/local/opt/openssl/include ${CMAKE_SOURCE_DIR}/clib ${CMAKE_SOURCE_DIR} )
link_directories(/usr/local/lib /usr/local/opt/openssl/lib)
@ -26,6 +29,7 @@ set(SOURCE
ml_io.cpp
ml_date.cpp
ml_string.cpp
ml_util.cpp
clib/csvparser.cpp
clib/sslclient.cpp
clib/json11.cpp

23
Readme.md
View File

@ -1,27 +1,20 @@
### BUGS
- (read-file "nonexisting/file.csv") shows only "could not open file"
- (read-file "nonexisting/file.csv") shows only "could not open file" - should print filename
### TODO
- support for (), nil, t, in lsp code replace 0 by nil in logicals
- casting to string (like ti int and to float)
- some performance functionality (at least counting how many times symbol was evaluated)
- documentation
- add url of source/inspiration to clib/*.cpp
- add stdtest - to test every functionality
- rename ivaluize
- add benchmark
- add instrumentation (time, nr of evals, num of atoms, debug info, debug environment etc)
- in casting functions (as_string..)
- better exception description
- add debug support function call could keep call stack
- add better print (coloring output)
- add command line options -h for help, -v for version, -b for no stdlib.lsp on startup
- add readline like functionality (libnoise)
- multiline editting (kilo editor)
- execute system command should capture stderr
- add builtin or lib function sleep
- add built in for and, or
- add debug support
- add some mem stats to benchmark
- support for (), nil, t, in lsp code replace 0 by nil in logicals
- file functions
- name it here
- string functions
@ -35,17 +28,17 @@
- env functions
- get env
- set env
- add hash datatype
- conversion functions like parse-integer
- in test.lisp some explaining prints
- format (sprintf)
- setq
- mapcar (funcall, apply)
- syntax highlighting do VS Code
- add hash datatype
#### Performance
- push_back - repeatedly without reserving size
- range - with for(int i...) and reserving result size can be 3times faster on (range 1 10000)
- mini_sprintf - unnecesary copying between vector and list
- (do ..) repeatedly assign to acc, maybe somewhere else
#### Install
```
@ -55,7 +48,7 @@ cp stdlib/*.lsp /usr/local/var/mlisp/
#### Compile
```
gcc -o ml -I/usr/local/opt/openssl/include -Iclib -L/usr/local/lib -L/usr/local/opt/openssl/lib -lm -lstdc++ -lcrypto -lssl -Wl,-stack_size -Wl,0x1000000 --std=c++17 ml.cpp ml_io.cpp ml_date.cpp ml_string.cpp clib/json11.cpp clib/csvparser.cpp clib/sslclient.cpp clib/printf.cpp clib/linenoise.c
gcc -o ml -I/usr/local/opt/openssl/include -Iclib -L/usr/local/lib -L/usr/local/opt/openssl/lib -lm -lstdc++ -lcrypto -lssl -Wl,-stack_size -Wl,0x1000000 --std=c++17 ml.cpp ml_io.cpp ml_date.cpp ml_string.cpp ml_util.cpp clib/json11.cpp clib/csvparser.cpp clib/sslclient.cpp clib/printf.cpp clib/linenoise.c
```
or
cmake

2
clib/linenoise.h
View File

@ -43,6 +43,8 @@
extern "C" {
#endif
#include "stddef.h"
typedef struct linenoiseCompletions {
size_t len;
char **cvec;

7
debug.lsp
View File

@ -21,3 +21,10 @@
(term-blue (sprintf "%.2f" (list 1.11))) " "
(term-yellow (sprintf "%.2f" (list 1.11))) " "
))
(benchmark "benchmark makelist 1000 : " (make-list 1000))
(benchmark "benchmark range 1000 : " (range 1 1000))
(sleep 1.5)

25
doc/Doc.md
View File

@ -64,19 +64,26 @@
|`(parse-json json_string)`|Parse JSON string||
|`(save-csv ..)`|||
|`(get-universal-time)`|Get current time as secs from epoch||
|`(date-to-str ..)`|||
|`(str-to-date ..)`|||
|`(date-add ..)`|||
|`(date-to-str date format)`|Converts date to formated string. Format is strftime format (https://www.tutorialspoint.com/c_standard_library/c_function_strftime.htm)|`>>> (date-to-str (get-universal-time) "%Y-%m-%d %H:%M:%S")
=> "2021-03-13 19:53:01"`|
|`(str-to-date string format)`|Converst string to time of secs since epoch. |`>>> (str-to-date "2021-03-13 19:53:01" "%Y-%m-%d %H:%M:%S")
=> 1615665181`|
|`(date-add ..)`|Add number of units to date. A unit is one of 'year', 'month', 'day', 'hour', 'minute' or 'second'|`>>> (date-to-str (date-add (str-to-date "2021-03-13 19:53:01" "%Y-%m-%d %H:%M:%S") 10 "day") "%Y-%m-%d %H:%M:%S")
=> "2021-03-23 20:53:01"`|
|`(debug ..)`|||
|`(display ..)`|||
|`(replace ..)`|||
|`(regex-search? ..)`|||
|`(string-replace source substr replacement)`|Replace a substring with a replacement string in a source string|`>>> (string-replace "abcdefg" "de" "DE") => "abcDEfg"`|
|`(string-regex? where regex)`| Returns true if where contains regex|`>>> (string-regex? "aba123cdefg" "[0-9]+") => 1`|
|`(string-pad str len char rpad_lpad)`|||
|`(int ..)`|||
|`(float ..)`|||
|`(int value)`|Cast an item to an int|`>>> (int 3.41) => 3`|
|`(float value)`|Cast item to a float|`>>> (int 3.41) => 3.14`|
|`(string value)`|Cast int or float item to a string|`>>> (string 3.14) => "3.14"`|
|`(eval ..)`|||
|`(type ..)`|||
|`(parse ..)`|||
|`(make-list size)`|||
|`(make-list-of size value)`|||
|`(make-list-of size value)`|Makes list with size elements of values||
|`(make-list size)`|Makes list of nil values with size length||
|`(uniq list)`|||
|`(flatten list)`|||
|`(benchmark msg_string code..)`|Benchmarks a block of expressions|Returns value od benchmarked code|
|`(xx ..)`|||

183
ml.cpp
View File

@ -3,13 +3,13 @@
#include "ml_io.h"
#include "ml_date.h"
#include "ml_string.h"
#include "ml_util.h"
#include "clib/csvparser.h"
#include "clib/sslclient.h"
#include "clib/json11.h"
#include "clib/printf.h"
#include "linenoise.h"
#include <cmath>
#include <map>
@ -19,7 +19,7 @@
#include <cstdlib>
#include <iostream>
#include <ctime>
#include <chrono>
#define TOO_FEW_ARGS "too few arguments to function"
#define TOO_MANY_ARGS "too many arguments to function"
@ -257,6 +257,20 @@ MlValue MlValue::cast_to_float() const {
}
}
// Cast this to a string
MlValue MlValue::cast_to_string() const {
switch (type) {
case INT:
return MlValue::string(to_string(stack_data.i));
case FLOAT:
return MlValue::string(to_string(stack_data.f));
case STRING:
return *this;
default:
throw MlError(*this, MlEnvironment(), BAD_CAST);
}
}
bool MlValue::operator==(MlValue other) const {
// If either of these values are floats, promote the
@ -1110,6 +1124,7 @@ namespace builtin {
return MlValue(now());
}
// Converts date to formated string.
MlValue date_to_str(std::vector<MlValue> args, MlEnvironment &env) {
eval_args(args, env);
@ -1119,6 +1134,7 @@ namespace builtin {
return MlValue::string(date_to_string(args[0].as_int(), args[1].as_string()));
}
// Converst string to time of secs since epoch
MlValue str_to_date(std::vector<MlValue> args, MlEnvironment &env) {
eval_args(args, env);
@ -1129,6 +1145,7 @@ namespace builtin {
return MlValue(string_to_date(args[0].as_string(), args[1].as_string()));
}
// Add number of units to date. A unit is one of 'year', 'month', 'day', 'hour', 'minute' or 'second'
MlValue date_add(std::vector<MlValue> args, MlEnvironment &env) {
eval_args(args, env);
@ -1353,6 +1370,16 @@ namespace builtin {
return args[0].cast_to_int();
}
// Cast an item to a string
MlValue cast_to_string(std::vector<MlValue> args, MlEnvironment &env) {
eval_args(args, env);
if (args.size() != 1)
throw MlError(MlValue(STRING_TYPE, cast_to_string), env, args.size() > 1 ? TOO_MANY_ARGS : TOO_FEW_ARGS);
return args[0].cast_to_string();
}
// Index a list
MlValue index(std::vector<MlValue> args, MlEnvironment &env) {
eval_args(args, env);
@ -1474,22 +1501,24 @@ namespace builtin {
return MlValue(parsed);
}
MlValue replace(std::vector<MlValue> args, MlEnvironment &env) {
// Replace a substring with a replacement string in a source string
MlValue string_replace(std::vector<MlValue> args, MlEnvironment &env) {
eval_args(args, env);
if (args.size() != 3)
throw MlError(MlValue("replace", replace), env, args.size() > 3 ? TOO_MANY_ARGS : TOO_FEW_ARGS);
throw MlError(MlValue("string-replace", string_replace), env, args.size() > 3 ? TOO_MANY_ARGS : TOO_FEW_ARGS);
std::string src = args[0].as_string();
replace_substring(src, args[1].as_string(), args[2].as_string());
return MlValue::string(src);
}
MlValue regex_search(std::vector<MlValue> args, MlEnvironment &env) {
// Returns true if where contains regex
MlValue string_regex(std::vector<MlValue> args, MlEnvironment &env) {
eval_args(args, env);
if (args.size() != 2) // if (args.size() < 2 || args.size() > 3)
throw MlError(MlValue("regex_search", regex_search), env, args.size() > 2 ? TOO_MANY_ARGS : TOO_FEW_ARGS);
throw MlError(MlValue("string-regex?", string_regex), env, args.size() > 2 ? TOO_MANY_ARGS : TOO_FEW_ARGS);
return MlValue(regexp_search(args[0].as_string(), args[1].as_string()));
}
@ -1608,53 +1637,39 @@ namespace builtin {
}
return MlValue(result);
}
// Benchmarks a block of expressions in the current environment (SPECIAL FORM)
MlValue benchmark(std::vector<MlValue> args, MlEnvironment &env) {
// TODO add some memory stats
using namespace std::chrono;
high_resolution_clock::time_point t1 = high_resolution_clock::now();
MlValue acc;
for (size_t i = 1; i < args.size(); i++)
acc = args[i].eval(env);
high_resolution_clock::time_point t2 = high_resolution_clock::now();
duration<double, std::milli> time_span = t2 - t1;
std::cout << args[0].as_string() << " " << time_span.count() << " ms" << std::endl;
return acc;
}
}
// void completion(const char *buf, linenoiseCompletions *lc) {
// if (buf[0] == 'h') {
// linenoiseAddCompletion(lc,"hello");
// linenoiseAddCompletion(lc,"hello there");
// }
// }
// char *hints(const char *buf, int *color, int *bold) {
// if (!strcasecmp(buf,"hello")) {
// *color = 35;
// *bold = 0;
// return " World";
// }
// return NULL;
// }
void repl(MlEnvironment &env) {
std::string code;
std::string input;
MlValue tmp;
std::vector<MlValue> parsed;
char *line;
// TODO not portable and in function
std::string history_file;
std::string file{"/.ml_history.txt"};
const char *t = std::getenv("HOME");
if (t == nullptr)
history_file = "/tmp/" + file;
else
history_file = std::string{t} + "/" + file;
// linenoiseHistorySetMaxLen(500);
// linenoiseSetCompletionCallback(completion);
// linenoiseSetHintsCallback(hints);
linenoiseSetMultiLine(1);
linenoiseHistoryLoad(history_file.c_str());
setup_linenoise(env);
while (true) {
// std::cout << ">>> ";
// std::getline(std::cin, input);
char *line = linenoise(">>> ");
if (line == nullptr) break;
line = linenoise(">>> ");
linenoiseHistoryAdd(line);
linenoise_line_read(line);
input = std::string(line);
@ -1680,23 +1695,11 @@ void repl(MlEnvironment &env) {
}
}
linenoiseHistorySave(history_file.c_str());
close_linenoise();
}
void load_std_lib(MlEnvironment &env) {
std::string loader =
R"( (define ___lib_path '("/usr/local/var/mlisp"))
(for d ___lib_path
(if (is-dir? d)
(for f (ls-dir d)
(if (regex-search? f "^.*\.l(i)?sp$")
(include (+ d "/" f))
'())
)
'()))
)";
run(loader, env);
run(STDLIB_LOADER, env);
}
// Does this environment, or its parent environment, have a variable?
@ -1730,6 +1733,7 @@ MlValue MlEnvironment::get(const std::string &name) const {
if (name == "defun") return MlValue("defun", builtin::defun);
if (name == "define") return MlValue("define", builtin::define);
if (name == "lambda") return MlValue("lambda", builtin::lambda);
if (name == "benchmark") return MlValue("benchmark", builtin::benchmark);
// Comparison operations
if (name == "=") return MlValue("=", builtin::eq);
@ -1794,13 +1798,14 @@ MlValue MlEnvironment::get(const std::string &name) const {
if (name == "debug") return MlValue("debug", builtin::debug);
if (name == "sprintf") return MlValue("sprintf", builtin::sprintf);
if (name == "display") return MlValue("display", builtin::display);
if (name == "replace") return MlValue("replace", builtin::replace);
if (name == "regex-search?") return MlValue("regex-search?", builtin::regex_search);
if (name == "string-replace") return MlValue("string-replace", builtin::string_replace);
if (name == "string-regex?") return MlValue("string-regex?", builtin::string_regex);
if (name == "string-pad") return MlValue("string-pad", builtin::string_pad);
// Casting operations
if (name == "int") return MlValue("int", builtin::cast_to_int);
if (name == "float") return MlValue("float", builtin::cast_to_float);
if (name == "string") return MlValue("string", builtin::cast_to_string);
// Constants
if (name == "endl") return MlValue::string("\n");
@ -1816,9 +1821,41 @@ MlValue MlEnvironment::get(const std::string &name) const {
throw MlError(MlValue::atom(name), *this, ATOM_NOT_DEFINED);
}
// Get vector of executables in this scope
std::vector<std::string> MlEnvironment::get_lambdas_list() const {
std::vector<std::string> lambdas {128};
for (auto it = defs.begin(); it != defs.end(); it++) {
if (it->second.get_type_name()==FUNCTION_TYPE) {
lambdas.push_back(it->first);
}
}
std::vector<std::string> commands {
"eval", "type", "parse", "do", "if", "for", "while", "scope", "quote", "defun",
"define", "lambda", "benchmark", "=", "!=", ">", "<", ">=", "<=", "+", "-", "*", "/", "%",
"list", "insert", "index", "remove", "len", "push", "pop", "head", "tail", "first", "last",
"range", "map", "filter", "reduce", "exit", "quit", "print", "input", "random", "include",
"read-file", "write-file", "read-url", "system-cmd", "ls-dir", "is-file?", "is-dir?",
"parse-csv", "parse-json", "get-universal-time", "date-to-str", "str-to-date", "date-add", "debug",
"sprintf", "display", "string-replace", "string-regex?", "string-pad", "int", "float", "string" };
lambdas.insert(end(lambdas), begin(commands), end(commands));
return lambdas;
}
bool cmdOptionExists(char **begin, char **end, const std::string &option) { return std::find(begin, end, option) != end; }
std::vector<std::string> getCmdOption(char *argv[], int argc, const std::string &option) {
std::vector<std::string> tokens;
for (int i = 1; i < argc; ++i) {
if (option == argv[i] && i + 1 < argc) {
i++;
tokens.push_back(std::string(argv[i]));
}
}
return tokens;
}
int main(int argc, char *argv[]) {
MlEnvironment env;
std::vector<MlValue> args;
@ -1828,29 +1865,35 @@ int main(int argc, char *argv[]) {
srand(time(NULL));
try {
// skip loading std lib
if (!cmdOptionExists(argv, argv + argc, "-b")) {
load_std_lib(env);
// for xcode profiling
// run(read_file_contents("/Users/vaclavt/Development/mlisp/tests/test.lsp"), env);
}
// help
if (cmdOptionExists(argv, argv + argc, "-h")) {
std::cout << "Usage:\n\t-h print this help\n\t-f source_file - executes code in file\n\t-c code - runs passed code\n\t-i runs repl\n\t-v prints version string\n\n";
std::cout << "Usage:\n\t-h print this help\n\t-f source_file - executes code in file\n\t-c code - runs passed code\n\t-i runs repl\n\t-b skip stdlib loading\n\t-v prints version string\n\n";
return 0;
}
// version
if (cmdOptionExists(argv, argv + argc, "-v")) {
std::cout << VERSION << std::endl;
return 0;
}
if (argc == 1 || (argc == 2 && std::string(argv[1]) == "-i"))
// passed code
if (cmdOptionExists(argv, argv + argc, "-c")) {
std::vector<std::string> codes = getCmdOption(argv, argc, "-c");
for (size_t i = 0; i < codes.size(); i++)
run(codes[i], env);
// run files
} else if (cmdOptionExists(argv, argv + argc, "-f")) {
std::vector<std::string> files = getCmdOption(argv, argc, "-f");
for (size_t i = 0; i < files.size(); i++)
run(read_file_contents(files[i]), env);
// repl
} else {
repl(env);
else if (argc == 3 && std::string(argv[1]) == "-c")
run(argv[2], env);
else if (argc == 3 && std::string(argv[1]) == "-f")
run(read_file_contents(argv[2]), env);
else std::cerr << "invalid arguments" << std::endl;
}
} catch (MlError &e) {
std::cerr << e.description() << std::endl;
} catch (std::runtime_error &e) {

18
ml.h
View File

@ -9,6 +9,18 @@
const std::string VERSION = "mi 0.1 (" __DATE__ " " __TIME__ ")";
const std::string STDLIB_LOADER =
R"( (define ___lib_path '("/usr/local/var/mlisp"))
(for d ___lib_path
(if (is-dir? d)
(for f (ls-dir d)
(if (string-regex? f "^.*\.l(i)?sp$")
(include (+ d "/" f))
'())
)
'()))
)";
// Forward declaration for MlEnvironment class definition
class MlValue;
@ -32,6 +44,9 @@ public:
// Set the value associated with this name in this scope
void set(const std::string &name, MlValue value);
// Get vector of executables in this scope
std::vector<std::string> get_lambdas_list() const;
void combine(MlEnvironment const &other);
void set_parent_scope(MlEnvironment *parent) {
@ -153,6 +168,9 @@ public:
// Cast this to a floating point value
MlValue cast_to_float() const;
// Cast this to a string
MlValue cast_to_string() const;
bool operator==(MlValue other) const;

2
ml_date.cpp
View File

@ -33,7 +33,7 @@ int string_to_date(const std::string &datestr, const std::string &format) {
}
int add_to_date(const int datetime, const int quantity, const std::string &part) {
// part is one of 'year', 'month', 'day', 'hour', 'minute', 'second', or 'millisecond'
// part is one of 'year', 'month', 'day', 'hour', 'minute' or 'second'
// very basic implementation, just for now - no timezones DST etc
time_t base = datetime;

78
ml_util.cpp Normal file
View File

@ -0,0 +1,78 @@
#include "ml_util.h"
#include <string>
#include <vector>
std::string get_history_file_dir() {
// TODO not portable and in function
std::string file{"/.ml_history.txt"};
const char *t = std::getenv("HOME");
if (t == nullptr) return "/tmp/" + file;
else return std::string{t} + "/" + file;
}
// TODO fujtajbl
MlEnvironment * repl_env = nullptr;
void setup_linenoise(const MlEnvironment &env) {
repl_env = (MlEnvironment*) &env;
std::string history_file = get_history_file_dir();
linenoiseHistorySetMaxLen(500);
linenoiseSetCompletionCallback(completion);
linenoiseSetHintsCallback(hints);
linenoiseSetMultiLine(1);
linenoiseHistoryLoad(history_file.c_str());
}
void linenoise_line_read(char *line) {
linenoiseHistoryAdd(line);
}
void close_linenoise() {
std::string history_file = get_history_file_dir();
linenoiseHistorySave(history_file.c_str());
}
size_t last_token_index( std::string str ) {
// remove trailing white space
while( !str.empty() && std::isspace( str.back() ) ) str.pop_back() ;
// locate the last white space
return str.find_last_of( "() \t\n" ) ;
}
void completion(const char *buf, linenoiseCompletions *lc) {
if (buf != nullptr) {
std::string str{buf};
const auto pos = last_token_index(str);
if (pos == std::string::npos)
return; // cannot find what to complete
std::string token = str.substr(pos+1);
std::string begining = str.substr(0, pos+1);
std::vector<std::string> lambdas = repl_env->get_lambdas_list();
for (std::vector<std::string>::iterator t = lambdas.begin(); t != lambdas.end(); ++t) {
if(t->find(token) == 0) {
std::string completion_string = begining + *t;
linenoiseAddCompletion(lc, completion_string.c_str());
}
}
}
}
char *hints(const char *buf, int *color, int *bold) {
// if (!strcasecmp(buf,"hello")) {
// *color = 35;
// *bold = 0;
// return " World";
// }
return nullptr;
}

13
ml_util.h Normal file
View File

@ -0,0 +1,13 @@
#pragma once
#include "ml.h"
#include "linenoise.h"
void setup_linenoise(const MlEnvironment &env);
void linenoise_line_read(char *line);
void close_linenoise();
void completion(const char *buf, linenoiseCompletions *lc);
char *hints(const char *buf, int *color, int *bold);

5
stdlib/stdlib.lsp
View File

@ -1,4 +1,3 @@
; not a bool
(defun not (x) (if x 0 1))
@ -9,7 +8,6 @@
(defun or (a b) (if a 1 (if b 1 0)))
; negate a number
(defun neg (n) (- 0 n))
@ -35,6 +33,9 @@
(string-pad str length pad_char "lpad"))
; pause for interval
(defun sleep (time)
(system-cmd (+ "sleep " (string time))))
; return second element of list

4
tests/test.lsp
View File

@ -116,7 +116,7 @@
(print (str-to-date "01.01.1970" "%d.%m.%Y"))
(print (date-add (str-to-date "01.01.1970" "%d.%m.%Y") 1 "day"))
(print (regex-search? "test.lsp" "^.*\.l(i)?sp$"))
(print (string-regex? "test.lsp" "^.*\.l(i)?sp$"))
(if (> 2 1)
@ -131,7 +131,7 @@
(define csv_list '())
(for f (ls-dir "tests/divi")
(if (regex-search? f "^divi.*\.csv$")
(if (string-regex? f "^divi.*\.csv$")
(do
(define filename (+ "tests/divi/" f))
; (print filename)