a bit od examples in doc

CMakeLists.txt

@@ -15,9 +15,9 @@ set(CPACK_PROJECT_VERSION ${PROJECT_VERSION})
 # set(CMAKE_CXX_FLAGS "-Wall -Wextra")
 set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG")
 
-include_directories(/usr/local/opt/openssl/include ${CMAKE_SOURCE_DIR}/clib ${CMAKE_SOURCE_DIR})
+include_directories(/opt/homebrew/opt/openssl@1.1/include ${CMAKE_SOURCE_DIR}/clib ${CMAKE_SOURCE_DIR})
 
-link_directories(/usr/local/lib /usr/local/opt/openssl/lib)
+link_directories(/opt/homebrew/opt/openssl@1.1/lib)
 
 
 project(ml)

Readme.md

@@ -69,6 +69,7 @@ utils/local_install.sh
 
 ### TODO
 - order of arguments in functions like member and filter - filter swap lambda and list
+- add possibility to specify filename for input2 history file
 - better output of !e in repl (resp !e !ee)
 - unify -f and -run options
 - add debug support, at least call stack

doc/Doc.md

@@ -40,14 +40,14 @@
 |`(set! x)`|`set!` ...|....|Language|
 
 ## Macros
-|Signature|Description|Return values|Section|
+|Signature|Description|Example|Section|
 |:-|-|-|-|
 |`(unles cond body)`|`unless` only evaluates its `cond` argument. If `cond` is not truthy, then `body` is evaluated.|`>>> (unless #t (print "it is #f")) => nil`|Language|
 |`(dotimes v n body)`|Iterates over `v` from 0 below `n` and evaluated `body`. Returns the last value of v|`>>> (dotimes i 5 i) => 4`|Language|
 |`(until cond body)`|Evaluated `body` as long as `cond` is not truthy|`>>> (def i 0)(until (> i 3) (set! i (inc i))) => 4`|Language|
 
 ## Library
-|Signature|Description|Return values|Section|
+|Signature|Description|Example|Section|
 |:-|-|-|-|
 |`(= a b)`|Test whether two values are equal|`1` when equal otherwise `nil`|Language|
 |`(!= a b)`|Test whether two values are not equal|`1` when not equal otherwise `nil`|Language|
@@ -63,10 +63,10 @@
 |`(list ..)`|Create a list of values||List manipulation|
 |`(insert list index element)`|Insert an element into a list. Indexed from 0|new list with value inserted|List manipulation|
 |`(index list index)`|Return element at index in list. First element is at index 0|Element at index|List manipulation|
-|`(remove list index)`|Remove a value at an index from a list|List with element removed|List manipulation|
-|`(len list)`|Get the length of a list|list length|List manipulation|
-|`(push list element)`|Add an item to the end of a list|new list with element added|List manipulation|
-|`(pop list)`|Returns last element of list|Last element|List manipulation|
+|`(remove list index)`|Remove a value at an index from a list|`>>> (remove '(1 2 3 4) 1) => (1 3 4)`|List manipulation|
+|`(len list)`|Get the length of a list|`>>> (len '(1 2 3 4)) => 4`|List manipulation|
+|`(push list element)`|Add an item to the end of a list|`>>> (push '(1 2 3 4) 5) => (1 2 3 4 5)`|List manipulation|
+|`(pop list)`|Returns last element of list|`>>> (pop '(1 2 3 4)) => 4`|List manipulation|
 |`(head list)`|Returns first element of a list|`>>> (head '(1 2 3)) => 1`|List manipulation|
 |`(tail list)`|Return all elements of list except first one|`>>> (tail '(1 2 3)) => (2 3)`|List manipulation|
 |`(first list)`|Returns first element of a list|`>>> (first '(1 2 3)) => 1`|List manipulation|

doc/readme.txt

@@ -0,0 +1,22 @@
+to convert Doc.md to Doc.html use https://markdowntohtml.com/ with custom ccs style
+
+body {
+   color: black;
+}
+table {
+  border-collapse: collapse;
+}
+table, th, td {
+  border: 1px solid;
+}
+th {
+  text-align: left;
+}
+td {
+  height: 50px;
+  vertical-align: top;
+}
+code{
+    background: #0003;
+    color: #a31515;
+}

ml.cpp

@@ -2196,9 +2196,6 @@ MlValue usql(std::vector<MlValue> args, MlEnvironment &env) {
 
 }	// namespace builtin
 
-void load_std_lib(MlEnvironment &env) {
-	run(STDLIB_LOADER, env);
-}
 
 // Does this environment, or its parent environment, have a variable?
 bool MlEnvironment::has(const std::string &name) const {
@@ -2440,6 +2437,12 @@ std::vector<std::string> getCmdOption(char *argv[], int argc, const std::string
 	return tokens;
 }
 
+
+void load_std_lib(MlEnvironment &env) {
+	run(STDLIB_LOADER, env);
+}
+
+
 int main(int argc, char *argv[]) {
 	MlEnvironment env;
 	std::vector<MlValue> args;

ml_profiler.cpp

@@ -79,7 +79,7 @@ void MlPerfMon::restore_callstack_position(size_t to_position) {
 }
 
 void MlPerfMon::clear_callstack() {
-    call_stack.empty();
+    auto r = call_stack.empty();
 }
 
 void MlPerfMon::print_results() const {

stdlib/doc.lsp

@@ -43,6 +43,12 @@
 	nil
 ))
 
+(defn doc::print-entry (entry)
+(do (def call (doc::strip-backticks (second entry)))
+	(def desc (doc::strip-backticks (third entry)))
+	(print (term-red (first entry)) "-" (term-green (doc::strip-backticks (second entry))) "-" (third entry))
+))
+
 (defn doc::man (what)
 (do	(def man (filter (lambda (x) (= (first x) what)) doc::doc_entries))
 	(if man
@@ -75,29 +81,54 @@
 	
 	(def sorted (quick-sort-by scores (lambda (a b) (< (first a) (first b)))))
 	(for e (take sorted 10)
-		(def entry (second e))
-		(def call (doc::strip-backticks (second entry)))
-		(def desc (doc::strip-backticks (third entry)))
-		(print (term-red (first entry)) "-" (term-green (doc::strip-backticks (second entry))) "-" (third entry))
-	)
+		(doc::print-entry (second e)))
+
 	(if (> (len sorted) 10) (print "..."))
 ))
 
+(defn doc::section (what)
+(do
+	(def entries '())
+	(for entry doc::doc_entries
+		; ("throw-exception" "`(throw-exception exp_desc)`" "Throws an exception with exp_desc describing what happened " "" "Exceptions"
+		; section matches
+		(if (= (string-downcase (fifth entry)) (string-downcase what))
+			(set! entries (push entries entry)))
+	)
+	
+	(for e (quick-sort-by entries (lambda (a b) (> (string-cmp (first a) (first b)) 0)))
+		(doc::print-entry e))
+))
+
+(defn doc::all ()
+	(for e (quick-sort-by doc::doc_entries (lambda (a b) (> (string-cmp (first a) (first b)) 0)))
+		(doc::print-entry e))
+)
+
 (defn doc::appropos (which)
 	(doc::look which))
 
 (defn doc::lookup (which)
 	(doc::look which))
 
-;(defn doc::section (which)
-;	(print (term-red "implement me!")))
+(defn doc::doc ()
+(do
+	(print "Usage:")
+	(print "\t(doc::doc)         - shows this help")
+	(print "\t(doc::man func)    - func must be a string, ie (doc::man \"for\")")
+	(print "\t(doc::look str)    - str must be a string, ie (doc::look \"length\")")
+	(print "\t(doc::lookup)      - alias for doc::look")
+	(print "\t(doc::appropos)    - alias for doc::look")
+	(print "\t(doc::all)         - show short info about all functions")
+	(print "\t(doc::section sec) - show help for section, sec is string one of:")
+	(print "\t\t\t\t\"List manipulation\" \"Language\" \"System\"")
+	(print "\t\t\t\t\"String manipulation\" \"Date and time\" \"IO\" \"Regex\"")
+	(print "\t\t\t\t\"Type casting\" \"Threading\" \"Exceptions\"")
+))
+
 
 (def doc::doc_entries '())	; must be here
 
 ; read doc into memory
 (doc::read-doc-file "/usr/local/var/mlisp/Doc.md")
 
-;;example
-; (doc::man "first")
-; (doc::look "string pad")
-; (doc::look "list flat")

usql/lexer.cpp

@@ -162,7 +162,7 @@ TokenType Lexer::type(const std::string &token) {
 	if (token == "not") return TokenType::keyword_not;
 	if (token == "null") return TokenType::keyword_null;
 	if (token == "integer") return TokenType::keyword_integer;
-	if (token == "float") return TokenType::keyword_float;
+	if (token == "float" || token == "double") return TokenType::keyword_float;
 	if (token == "varchar") return TokenType::keyword_varchar;
 	if (token == "date") return TokenType::keyword_date;
 	if (token == "boolean") return TokenType::keyword_bool;

usql/parser.cpp

@@ -394,8 +394,7 @@ namespace usql {
 		std::unique_ptr<Node> left = parse_expression();
 		do {
 			left = parse_expression(std::move(left));
-
-	} while (m_lexer.tokenType() != TokenType::eof && m_lexer.tokenType() != TokenType::keyword_order && m_lexer.tokenType() != TokenType::keyword_offset && m_lexer.tokenType() != TokenType::keyword_limit);
+		} while (m_lexer.tokenType() != TokenType::eof && m_lexer.tokenType() != TokenType::keyword_order && m_lexer.tokenType() != TokenType::keyword_offset && m_lexer.tokenType() != TokenType::keyword_limit && m_lexer.tokenType() != TokenType::semicolon);
 	
 		return left;
     }

usql/parser.h

@@ -175,9 +175,6 @@ struct FunctionNode : Node {
 	};
 
 
-	Type								function;
-	std::vector<std::unique_ptr<Node>>	params;
-
 	FunctionNode(std::string  func_name, std::vector<std::unique_ptr<Node>> pars) :
 	Node(NodeType::function), function(get_function(func_name)), params(std::move(pars)) {}
 
@@ -199,6 +196,9 @@ struct FunctionNode : Node {
 		}			
 		std::cout << ")" << std::endl;
 	}
+
+	Type								function;
+	std::vector<std::unique_ptr<Node>>	params;
 };
 
 struct TrueNode : Node {
@@ -581,7 +581,6 @@ struct CreateIndexNode : Node {
 };
 
 class Parser {
-private:
 public:
 	Parser();
 

usql/row.h

@@ -171,7 +171,7 @@ public:
 			case 5:
 				return (ColValue &) *std::get_if<ColBooleanValue>(&m_columns[i]);
 			default:
-				throw Exception("should not happen");
+				throw Exception("ColValue &operator[](int i), type index invalid :" + std::to_string(type_index));
 		}
 	}
 

usql/settings.cpp

@@ -12,7 +12,7 @@ std::vector<std::pair<std::string, std::string>> Settings::m_settings =
 	      std::make_pair("BOOL_TRUE_LITERAL", "Y"),
 	      std::make_pair("BOOL_FALSE_LITERAL", "N"),
           std::make_pair("DOUBLE_FORMAT", "%.2f"),
-	      std::make_pair("USE_INDEXSCAN", "N"),
+	      std::make_pair("USE_INDEXSCAN", "Y"),
 	      std::make_pair("MAX_PARALLELISM", "1") };		// values "AUTO" or number of workers; when number negative means std::thread::hardware_concurrency() - number 
 
 

usql/usql.cpp

@@ -6,9 +6,36 @@
 
 namespace usql {
 
+USql::USql() {
+	// create catalogue tables first
+
+	std::vector<std::string> k_debug_sql_commands {
+		"create table usql_tables(name varchar(32) not null, modified boolean not null)",
+		"create table usql_columns(table_name varchar(32) not null, column_name varchar(32) not null, column_type varchar(16) not null, column_length integer not null, nullable boolean not null, column_order integer not null)"
+	};
+
+	// create cataloque tables
+	for (const auto &command : k_debug_sql_commands) {
+		std::unique_ptr<Node> create_table_node = m_parser.parse(command);
+		const CreateTableNode &node = static_cast<CreateTableNode &>(*create_table_node); 
+
+		Table table{node.table_name, node.cols_defs};
+		m_tables.push_back(table);
+
+	}
+	// insert data into cataloque tables
+	for (const auto &command : k_debug_sql_commands) {
+		std::unique_ptr<Node> create_table_node = m_parser.parse(command);
+		const CreateTableNode &node = static_cast<CreateTableNode &>(*create_table_node); 
+
+		execute_create_table_sys_catalogue(node);
+	}
+}
+
 std::unique_ptr<Table> USql::execute(const std::string &command) {
 	try {
 		std::unique_ptr<Node> node = m_parser.parse(command);
+		// node->dump();
 		return execute(*node);
 
 	} catch (const std::exception &e) {

usql/usql.h

@@ -13,14 +13,14 @@ namespace usql {
 class USql {
 	
 public:
-    USql() = default;
-
+	USql();
     std::unique_ptr<Table> execute(const std::string &command);
 
 private:
     std::unique_ptr<Table> execute(Node &node);
 
     std::unique_ptr<Table> execute_create_table(const CreateTableNode &node);
+	bool execute_create_table_sys_catalogue(const CreateTableNode &node);
     std::unique_ptr<Table> execute_create_index(const CreateIndexNode &node);
     std::unique_ptr<Table> execute_create_table_as_table(const CreateTableAsSelectNode &node);
     std::unique_ptr<Table> execute_load(const LoadIntoTableNode &node);
@@ -59,9 +59,6 @@ private:
 	void check_index_not_exists(const std::string &index_name);
 
 private:
-    Parser m_parser;
-    std::list<Table> m_tables;
-
     static void execute_distinct(SelectFromTableNode &node, Table *result);
     static void execute_order_by(SelectFromTableNode &node, Table *result);
     static void execute_offset_limit(OffsetLimitNode &node, Table *result);
@@ -97,6 +94,11 @@ private:
 	bool normalize_where(const Node *node) const;
 
     Table::rows_scanner get_iterator(Table *table, const Node *where) const;
+
+
+private:
+    Parser m_parser;
+    std::list<Table> m_tables;
 };
 
 

usql/usql_ddl.cpp

@@ -15,10 +15,35 @@ std::unique_ptr<Table> USql::execute_create_table(const CreateTableNode &node) {
 	Table table{node.table_name, node.cols_defs};
 	m_tables.push_back(table);
 
+	execute_create_table_sys_catalogue(node);
+
 	return create_stmt_result_table(0, "table created", 0);
 }
 
 
+bool USql::execute_create_table_sys_catalogue(const CreateTableNode &node) {
+	// usql_tables 
+	auto r = execute("insert into usql_tables(name, modified) values('" + node.table_name + "', false)");
+
+	// usql_columns 
+	for(const ColDefNode & col_def : node.cols_defs) {
+		std::string i {"insert into usql_columns(table_name, column_name, column_type, column_length, nullable, column_order) values("};
+					i += "'" + node.table_name + "', ";
+					i += "'" + col_def.name + "', ";
+					i += "'" + column_type_name(col_def.type) + "', ";
+					i += std::to_string(col_def.length) + ", ";
+					i += (col_def.null ?  "true, " : "false, ");
+					i += std::to_string(col_def.order);
+					i += ")";
+
+		auto r = execute(i);
+		// r->print();
+	}
+
+	return true;
+}
+
+
 std::unique_ptr<Table> USql::execute_create_index(const CreateIndexNode &node) {
 	Table *table_def = find_table(node.table_name);				// throws exception if not found
 	ColDefNode col_def = table_def->get_column_def(node.column_name);	// throws exception if not found
@@ -57,7 +82,6 @@ std::unique_ptr<Table> USql::execute_create_table_as_table(const CreateTableAsSe
 }
 
 
-
 std::unique_ptr<Table> USql::execute_drop(const DropTableNode &node) {
 	auto name_cmp = [node](const Table& t) { return t.m_name == node.table_name; };
 
@@ -70,11 +94,13 @@ std::unique_ptr<Table> USql::execute_drop(const DropTableNode &node) {
 	throw Exception("table not found (" + node.table_name + ")");
 }
 
+
 std::unique_ptr<Table> USql::execute_set(const SetNode &node) {
 	Settings::set_setting(node.name, node.value);
 	return create_stmt_result_table(0, "set succeeded", 1);
 }
 
+
 std::unique_ptr<Table> USql::execute_show(const ShowNode &node) {
 	std::string value = Settings::get_setting(node.name);
 	return create_stmt_result_table(0, "show succeeded: " + value, 1);

usql/usql_dml.cpp

@@ -35,7 +35,7 @@ std::pair<bool, std::vector<rowid_t>> USql::look_for_usable_index(const Node *wh
 					if (used_index != nullptr) {
 						std::vector<rowid_t> rowids = used_index->search((ValueNode *)ron->right.get());
 #ifndef NDEBUG
-						std::cout << "using index " << table->m_name << "(" << used_index->get_column_name() << "), " << rowids.size() << "/" << table->rows_count()  << std::endl;
+						std::cerr << "using index " << table->m_name << "(" << used_index->get_column_name() << "), " << rowids.size() << "/" << table->rows_count()  << std::endl;
 #endif
 						return std::make_pair(true, rowids);
 					}
@@ -155,8 +155,7 @@ void USql::setup_order_columns(std::vector<ColOrderNode> &node, Table *table) {
 void USql::execute_distinct(SelectFromTableNode &node, Table *result) {
 	if (!node.distinct) return;
 
-	auto compare_rows = [](const Row &a, const Row &b) { return a.compare(b) >= 0; };
-	std::sort(result->m_rows.begin(), result->m_rows.end(), compare_rows);
+	std::sort(result->m_rows.begin(), result->m_rows.end(), [](const Row &a, const Row &b) { return a.compare(b) > 0; });
 
 	result->m_rows.erase(std::unique(result->m_rows.begin(), result->m_rows.end()), result->m_rows.end());
 }

utils/Lisp.tmLanguage

@@ -55,29 +55,6 @@
 			<key>name</key>
 			<string>meta.function.lisp</string>
 		</dict>
-		<!-- <dict>
-			<key>match</key>
-			<string>(?<=\s)(?i:&allow-other-keys|&aux|&body|&environment|&key|&optional|&rest|&whole)(?=\s+)</string>
-			<key>name</key>
-			<string>meta.function-parameters.lisp</string>
-		</dict> -->
-		<!-- <dict>
-			<key>captures</key>
-			<dict>
-				<key>1</key>
-				<dict>
-					<key>name</key>
-					<string>storage.type.function-type.lisp</string>
-				</dict>
-				<key>2</key>
-				<dict>
-					<key>name</key>
-					<string>entity.name.type.lisp</string>
-				</dict>
-			</dict>
-			<key>match</key>
-			<string>(?:\()((?i:deftype|defstruct|define-condition|defpackage|defclass))\s+((?:\w|[+\-<>/*&amp;=.?!$%:@\[\]^{}~#|])+)</string>
-		</dict> -->
 		<dict>
 			<key>captures</key>
 			<dict>

utils/local_install.sh

@@ -8,19 +8,27 @@ echo "building ml"
 cmake --build ./ --target all -j 4 --
 
 echo ""
+
+# echo "create dirs"
+# sudo mkdir -p /usr/local/var/mlisp/
+# sudo mkdir -p /usr/local/bin/
+# sudo chown -R vaclavt:admin /usr/local/bin/
+# sudo chown -R vaclavt:admin /usr/local/var/mlisp/
+
 echo "copying lsp files"
-mkdir -p /usr/local/var/mlisp/
 cp stdlib/*.lsp /usr/local/var/mlisp/
+
 echo "copying doc files"
 cp doc/*.md /usr/local/var/mlisp/
+
 echo "copying ml file"
 mv ./ml /usr/local/bin/ml
 
 echo ""
 echo "if syntax has changed you may use"
 echo "cp utils/Lisp.tmLanguage ~/.vscode/extensions/mattn.lisp-0.1.12/syntaxes/"
-# echo "or"
-# echo "cp utils/Lisp.tmLanguage ~/.local/share/code-server/extensions/mattn.lisp-0.1.11/syntaxes/Lisp.tmLanguage"
+echo "or"
+echo "cp utils/Lisp.tmLanguage ~/.local/share/code-server/extensions/mattn.lisp-0.1.11/syntaxes/Lisp.tmLanguage"
 echo ""
 
 ml -v