vaclavt/mlisp
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

REMOVED USER_STD macros, code formating, csv changes, few file functions

Browse Source
This commit is contained in:
VaclavT
2021-02-15 23:20:20 +01:00
parent 125c73ac33
commit c1532e78b1
15 changed files with 10550 additions and 2101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
clib/csvparser.cpp
View File

@@ -12,11 +12,13 @@ CsvParser::CsvParser(bool skip_hdr, char field_sep, char quote_ch, char line_sep
header_skiped = false;
}
void CsvParser::parseCSV(const std::string &csvSource, std::vector< std::vector<std::string> > &lines) {
MlValue CsvParser::parseCSV(const std::string &csvSource) {
bool inQuote(false);
bool newLine(false);
std::string field;
lines.clear();
// PERF optimize it for memory usage and performance
std::vector<std::vector<std::string>> parsed_data; // TODO some default size here
std::vector<std::string> line;
std::string::const_iterator aChar = csvSource.begin();
@@ -38,7 +40,7 @@ void CsvParser::parseCSV(const std::string &csvSource, std::vector< std::vector<
} else {
if (newLine == false) {
line.push_back(field);
addLine(line, lines);
addLine(line, parsed_data);
field.clear();
line.clear();
newLine = true;
@@ -55,37 +57,39 @@ void CsvParser::parseCSV(const std::string &csvSource, std::vector< std::vector<
if (field.size())
line.push_back(field);
addLine(line, lines);
addLine(line, parsed_data);
return ivalualize(parsed_data);
}
MlValue CsvParser::ivalualize(std::vector< std::vector<std::string> > &parsed_data) const {
int rows = parsed_data.size();
int cols = rows > 0 ? parsed_data[0].size() : 0;
MlValue CsvParser::ivalualize(std::vector<std::vector<std::string> > &parsed_data) const {
int rows = parsed_data.size();
int cols = rows > 0 ? parsed_data[0].size() : 0;
std::vector<MlValue> result;
std::vector<MlValue> result;
if (rows > 0 && cols > 0) {
for (int r = 0; r < rows; r++) {
std::vector<MlValue> row;
for (int c = 0; c < cols; c++) {
std::string value = parsed_data[r][c];
if (is_string_int(value)) {
row.push_back(MlValue(stoi(value)));
}
if (is_string_float(value)) {
row.push_back(MlValue(std::stod(value)));
} else {
row.push_back(MlValue::string(value));
}
}
result.push_back(row);
}
}
if (rows > 0 && cols > 0) {
for (int r = 0; r < rows; r++) {
std::vector<MlValue> row;
for (int c = 0; c < cols; c++) {
std::string value = parsed_data[r][c];
if (is_string_int(value)) {
row.push_back(MlValue(stoi(value)));
}
if (is_string_float(value)) {
row.push_back(MlValue(std::stod(value)));
} else {
row.push_back(MlValue::string(value));
}
}
result.push_back(row);
}
}
return result;
return result;
}
void CsvParser::addLine(const std::vector<std::string> &line, std::vector< std::vector<std::string> > &lines) {
void CsvParser::addLine(const std::vector<std::string> &line, std::vector<std::vector<std::string> > &lines) {
if (skip_header && !header_skiped) {
header_skiped = true;
} else {
@@ -101,10 +105,10 @@ std::regex double_regex("[0-9]+\\.[0-9]+");
// Is string representing int value
bool CsvParser::is_string_int(const std::string &str) const {
return std::regex_match(str, int_regex);
return std::regex_match(str, int_regex);
}
// Is string representing float value
bool CsvParser::is_string_float(const std::string &str) const {
return std::regex_match(str, double_regex);
return std::regex_match(str, double_regex);
}

23
clib/csvparser.h
View File

@@ -8,23 +8,26 @@
class CsvParser {
private:
char field_separator;
char line_separator;
char line_separator2;
char quote_character;
char field_separator;
char line_separator;
char line_separator2;
char quote_character;
bool skip_header;
bool header_skiped;
bool skip_header;
bool header_skiped;
public:
CsvParser(bool skip_hdr = false, char field_sep = ',', char quote_ch = '"', char line_sep = '\r', char line_sep2 = '\n');
CsvParser(bool skip_hdr = false, char field_sep = ',', char quote_ch = '"', char line_sep = '\r',
char line_sep2 = '\n');
void parseCSV(const std::string &csvSource, std::vector< std::vector<std::string> > &lines);
MlValue parseCSV(const std::string &csvSource);
MlValue ivalualize(std::vector< std::vector<std::string> > &parsed_data) const;
private:
void addLine(const std::vector<std::string> &line, std::vector< std::vector<std::string> > &lines);
void addLine(const std::vector<std::string> &line, std::vector<std::vector<std::string> > &lines);
MlValue ivalualize(std::vector<std::vector<std::string> > &parsed_data) const;
bool is_string_int(const std::string &str) const;
bool is_string_float(const std::string &str) const;
};

8319
clib/date.h Normal file
View File

File diff suppressed because it is too large Load Diff

1221
clib/json11.cpp
View File

File diff suppressed because it is too large Load Diff

293
clib/json11.h
View File

@@ -11,165 +11,200 @@
namespace json11 {
enum JsonParse {
STANDARD, COMMENTS
STANDARD, COMMENTS
};
class JsonValue;
class Json final {
public:
// Types
enum Type {
NUL, NUMBER, BOOL, STRING, ARRAY, OBJECT
};
// Types
enum Type {
NUL, NUMBER, BOOL, STRING, ARRAY, OBJECT
};
// Array and object typedefs
typedef std::vector<Json> array;
typedef std::map<std::string, Json> object;
// Array and object typedefs
typedef std::vector<Json> array;
typedef std::map<std::string, Json> object;
// Constructors for the various types of JSON value.
Json() noexcept; // NUL
Json(std::nullptr_t) noexcept; // NUL
Json(double value); // NUMBER
Json(int value); // NUMBER
Json(bool value); // BOOL
Json(const std::string &value); // STRING
Json(std::string &&value); // STRING
Json(const char * value); // STRING
Json(const array &values); // ARRAY
Json(array &&values); // ARRAY
Json(const object &values); // OBJECT
Json(object &&values); // OBJECT
// Constructors for the various types of JSON value.
Json() noexcept; // NUL
Json(std::nullptr_t) noexcept; // NUL
Json(double value); // NUMBER
Json(int value); // NUMBER
Json(bool value); // BOOL
Json(const std::string &value); // STRING
Json(std::string &&value); // STRING
Json(const char *value); // STRING
Json(const array &values); // ARRAY
Json(array &&values); // ARRAY
Json(const object &values); // OBJECT
Json(object &&values); // OBJECT
// Implicit constructor: anything with a to_json() function.
template <class T, class = decltype(&T::to_json)>
Json(const T & t) : Json(t.to_json()) {}
// Implicit constructor: anything with a to_json() function.
template<class T, class = decltype(&T::to_json)>
Json(const T &t) : Json(t.to_json()) {}
// Implicit constructor: map-like objects (std::map, std::unordered_map, etc)
template <class M, typename std::enable_if<
std::is_constructible<std::string, decltype(std::declval<M>().begin()->first)>::value
&& std::is_constructible<Json, decltype(std::declval<M>().begin()->second)>::value,
int>::type = 0>
Json(const M & m) : Json(object(m.begin(), m.end())) {}
// Implicit constructor: map-like objects (std::map, std::unordered_map, etc)
template<class M, typename std::enable_if<
std::is_constructible<std::string, decltype(std::declval<M>().begin()->first)>::value
&& std::is_constructible<Json, decltype(std::declval<M>().begin()->second)>::value,
int>::type = 0>
Json(const M &m) : Json(object(m.begin(), m.end())) {}
// Implicit constructor: vector-like objects (std::list, std::vector, std::set, etc)
template <class V, typename std::enable_if<
std::is_constructible<Json, decltype(*std::declval<V>().begin())>::value,
int>::type = 0>
Json(const V & v) : Json(array(v.begin(), v.end())) {}
// Implicit constructor: vector-like objects (std::list, std::vector, std::set, etc)
template<class V, typename std::enable_if<
std::is_constructible<Json, decltype(*std::declval<V>().begin())>::value,
int>::type = 0>
Json(const V &v) : Json(array(v.begin(), v.end())) {}
// This prevents Json(some_pointer) from accidentally producing a bool. Use
// Json(bool(some_pointer)) if that behavior is desired.
Json(void *) = delete;
// This prevents Json(some_pointer) from accidentally producing a bool. Use
// Json(bool(some_pointer)) if that behavior is desired.
Json(void *) = delete;
// Accessors
Type type() const;
// Accessors
Type type() const;
bool is_null() const { return type() == NUL; }
bool is_number() const { return type() == NUMBER; }
bool is_bool() const { return type() == BOOL; }
bool is_string() const { return type() == STRING; }
bool is_array() const { return type() == ARRAY; }
bool is_object() const { return type() == OBJECT; }
bool is_null() const { return type() == NUL; }
// Return the enclosed value if this is a number, 0 otherwise. Note that json11 does not
// distinguish between integer and non-integer numbers - number_value() and int_value()
// can both be applied to a NUMBER-typed object.
double number_value() const;
int int_value() const;
bool is_number() const { return type() == NUMBER; }
// Return the enclosed value if this is a boolean, false otherwise.
bool bool_value() const;
// Return the enclosed string if this is a string, "" otherwise.
const std::string &string_value() const;
// Return the enclosed std::vector if this is an array, or an empty vector otherwise.
const array &array_items() const;
// Return the enclosed std::map if this is an object, or an empty map otherwise.
const object &object_items() const;
bool is_bool() const { return type() == BOOL; }
// Return a reference to arr[i] if this is an array, Json() otherwise.
const Json & operator[](size_t i) const;
// Return a reference to obj[key] if this is an object, Json() otherwise.
const Json & operator[](const std::string &key) const;
bool is_string() const { return type() == STRING; }
// Serialize.
void dump(std::string &out) const;
std::string dump() const {
std::string out;
dump(out);
return out;
}
bool is_array() const { return type() == ARRAY; }
MlValue ivalualize() const;
bool is_object() const { return type() == OBJECT; }
// Parse. If parse fails, return Json() and assign an error message to err.
static Json parse(const std::string & in,
std::string & err,
JsonParse strategy = JsonParse::STANDARD);
static Json parse(const char * in,
std::string & err,
JsonParse strategy = JsonParse::STANDARD) {
if (in) {
return parse(std::string(in), err, strategy);
} else {
err = "null input";
return nullptr;
}
}
// Parse multiple objects, concatenated or separated by whitespace
static std::vector<Json> parse_multi(
const std::string & in,
std::string::size_type & parser_stop_pos,
std::string & err,
JsonParse strategy = JsonParse::STANDARD);
// Return the enclosed value if this is a number, 0 otherwise. Note that json11 does not
// distinguish between integer and non-integer numbers - number_value() and int_value()
// can both be applied to a NUMBER-typed object.
double number_value() const;
static inline std::vector<Json> parse_multi(
const std::string & in,
std::string & err,
JsonParse strategy = JsonParse::STANDARD) {
std::string::size_type parser_stop_pos;
return parse_multi(in, parser_stop_pos, err, strategy);
}
int int_value() const;
bool operator== (const Json &rhs) const;
bool operator< (const Json &rhs) const;
bool operator!= (const Json &rhs) const { return !(*this == rhs); }
bool operator<= (const Json &rhs) const { return !(rhs < *this); }
bool operator> (const Json &rhs) const { return (rhs < *this); }
bool operator>= (const Json &rhs) const { return !(*this < rhs); }
// Return the enclosed value if this is a boolean, false otherwise.
bool bool_value() const;
/* has_shape(types, err)
*
* Return true if this is a JSON object and, for each item in types, has a field of
* the given type. If not, return false and set err to a descriptive message.
*/
typedef std::initializer_list<std::pair<std::string, Type>> shape;
bool has_shape(const shape & types, std::string & err) const;
// Return the enclosed string if this is a string, "" otherwise.
const std::string &string_value() const;
// Return the enclosed std::vector if this is an array, or an empty vector otherwise.
const array &array_items() const;
// Return the enclosed std::map if this is an object, or an empty map otherwise.
const object &object_items() const;
// Return a reference to arr[i] if this is an array, Json() otherwise.
const Json &operator[](size_t i) const;
// Return a reference to obj[key] if this is an object, Json() otherwise.
const Json &operator[](const std::string &key) const;
// Serialize.
void dump(std::string &out) const;
std::string dump() const {
std::string out;
dump(out);
return out;
}
MlValue ivalualize() const;
// Parse. If parse fails, return Json() and assign an error message to err.
static Json parse(const std::string &in,
std::string &err,
JsonParse strategy = JsonParse::STANDARD);
static Json parse(const char *in,
std::string &err,
JsonParse strategy = JsonParse::STANDARD) {
if (in) {
return parse(std::string(in), err, strategy);
} else {
err = "null input";
return nullptr;
}
}
// Parse multiple objects, concatenated or separated by whitespace
static std::vector<Json> parse_multi(
const std::string &in,
std::string::size_type &parser_stop_pos,
std::string &err,
JsonParse strategy = JsonParse::STANDARD);
static inline std::vector<Json> parse_multi(
const std::string &in,
std::string &err,
JsonParse strategy = JsonParse::STANDARD) {
std::string::size_type parser_stop_pos;
return parse_multi(in, parser_stop_pos, err, strategy);
}
bool operator==(const Json &rhs) const;
bool operator<(const Json &rhs) const;
bool operator!=(const Json &rhs) const { return !(*this == rhs); }
bool operator<=(const Json &rhs) const { return !(rhs < *this); }
bool operator>(const Json &rhs) const { return (rhs < *this); }
bool operator>=(const Json &rhs) const { return !(*this < rhs); }
/* has_shape(types, err)
*
* Return true if this is a JSON object and, for each item in types, has a field of
* the given type. If not, return false and set err to a descriptive message.
*/
typedef std::initializer_list<std::pair<std::string, Type>> shape;
bool has_shape(const shape &types, std::string &err) const;
private:
std::shared_ptr<JsonValue> m_ptr;
std::shared_ptr<JsonValue> m_ptr;
};
// Internal class hierarchy - JsonValue objects are not exposed to users of this API.
class JsonValue {
protected:
friend class Json;
friend class JsonInt;
friend class JsonDouble;
virtual Json::Type type() const = 0;
virtual bool equals(const JsonValue * other) const = 0;
virtual bool less(const JsonValue * other) const = 0;
virtual void dump(std::string &out) const = 0;
virtual MlValue ivalualize() const = 0;
virtual double number_value() const;
virtual int int_value() const;
virtual bool bool_value() const;
virtual const std::string &string_value() const;
virtual const Json::array &array_items() const;
virtual const Json &operator[](size_t i) const;
virtual const Json::object &object_items() const;
virtual const Json &operator[](const std::string &key) const;
virtual ~JsonValue() {}
friend class Json;
friend class JsonInt;
friend class JsonDouble;
virtual Json::Type type() const = 0;
virtual bool equals(const JsonValue *other) const = 0;
virtual bool less(const JsonValue *other) const = 0;
virtual void dump(std::string &out) const = 0;
virtual MlValue ivalualize() const = 0;
virtual double number_value() const;
virtual int int_value() const;
virtual bool bool_value() const;
virtual const std::string &string_value() const;
virtual const Json::array &array_items() const;
virtual const Json &operator[](size_t i) const;
virtual const Json::object &object_items() const;
virtual const Json &operator[](const std::string &key) const;
virtual ~JsonValue() {}
};
} // namespace json11

95
clib/sslclient.cpp
View File

@@ -17,59 +17,60 @@
#include <string>
HttpClient::HttpClient(){};
HttpClient::HttpClient() {};
std::pair<int, std::string> HttpClient::doGetRequest(const std::string &url, const std::unordered_map<std::string, std::string> &headers) {
std::pair<int, std::string>
HttpClient::doGetRequest(const std::string &url, const std::unordered_map<std::string, std::string> &headers) {
// https://stackoverflow.com/questions/25896916/parse-http-headers-in-c
std::regex rgx{R"(^(?:((?:https?|s?ftp):)//)([^:/\s]+)(?::(\d*))?(?:/([^\s?#]+)?([?][^?#]*)?(#.*)?)?)"};
std::smatch matches;
if (std::regex_search(url, matches, rgx)) {
for (size_t i = 0; i < matches.size(); ++i) {
switch (i) {
case 0:
full_url = matches[i].str();
break;
case 1:
proto = matches[i].str();
break;
case 2:
server = matches[i].str();
break;
case 3:
port = matches[i].str();
break;
case 4:
uri = matches[i].str();
break;
case 5:
params = matches[i].str();
break;
case 6:
href = matches[i].str();
break;
case 0:
full_url = matches[i].str();
break;
case 1:
proto = matches[i].str();
break;
case 2:
server = matches[i].str();
break;
case 3:
port = matches[i].str();
break;
case 4:
uri = matches[i].str();
break;
case 5:
params = matches[i].str();
break;
case 6:
href = matches[i].str();
break;
}
// std::cout << i << ": '" << matches[i].str() << "'\n";
}
} else {
std::cerr << "Match not found" << std::endl; // TODO better message
std::cerr << "Match not found" << std::endl; // TODO better message
}
std::string headers_string = "";
for (auto it = headers.begin(); it != headers.end(); ++it) {
headers_string.append("\r\n" + it->first + ": " + it->second);
// std::cerr << "KEY: `" << it->first << "`, VALUE: `" << it->second << '`' << std::endl;
}
}
std::string request = "GET " + full_url + " HTTP/1.0\r\nHost: " + server + headers_string + "\r\n\r\n";
std::string request = "GET " + full_url + " HTTP/1.0\r\nHost: " + server + headers_string + "\r\n\r\n";
// TODO memory leaks ???
int bytes_read = sslRequest(server, request);
if (bytes_read <= 0) {
std::cerr << "no data read" << std::endl;
return std::make_pair(403, "");
std::cerr << "no data read" << std::endl;
return std::make_pair(403, "");
}
std::string::size_type position = ssl_read_packet.find("\r\n\r\n");
@@ -88,8 +89,8 @@ std::pair<int, std::string> HttpClient::doGetRequest(const std::string &url, con
std::smatch status_matches;
if (std::regex_search(status_str, status_matches, status_rgx)) {
if (status_matches.size() > 1) {
auto sta = status_matches[1].str(); // string "200"
// std::cout << "status: " << sta << std::endl;
auto sta = status_matches[1].str(); // string "200"
// std::cout << "status: " << sta << std::endl;
}
}
@@ -116,7 +117,7 @@ std::string HttpClient::inetAddress(std::string hostname) {
std::cerr << hostname << " is unavailable" << std::endl;
exit(1);
}
in_addr *address = (in_addr *)record->h_addr;
in_addr *address = (in_addr *) record->h_addr;
std::string ip_address = inet_ntoa(*address);
return ip_address;
@@ -131,8 +132,8 @@ int HttpClient::sslRecvPacket() {
do {
len = SSL_read(ssl, buf, len);
if (len >= 0) {
buf[len] = 0;
ssl_read_packet.append((const char *)buf);
buf[len] = 0;
ssl_read_packet.append((const char *) buf);
}
} while (len > 0);
@@ -154,15 +155,15 @@ int HttpClient::sslSendPacket(std::string buf) {
if (len < 0) {
int err = SSL_get_error(ssl, len);
switch (err) {
case SSL_ERROR_WANT_WRITE:
return 0;
case SSL_ERROR_WANT_READ:
return 0;
case SSL_ERROR_ZERO_RETURN:
case SSL_ERROR_SYSCALL:
case SSL_ERROR_SSL:
default:
return -1;
case SSL_ERROR_WANT_WRITE:
return 0;
case SSL_ERROR_WANT_READ:
return 0;
case SSL_ERROR_ZERO_RETURN:
case SSL_ERROR_SYSCALL:
case SSL_ERROR_SSL:
default:
return -1;
}
}
@@ -188,7 +189,7 @@ int HttpClient::sslRequest(const std::string &server_name, const std::string &re
socklen_t socklen = sizeof(sa);
// connect to server
if (connect(s, (struct sockaddr *)&sa, socklen)) {
if (connect(s, (struct sockaddr *) &sa, socklen)) {
printf("MlError connecting to server.\n");
return -1;
}
@@ -206,8 +207,8 @@ int HttpClient::sslRequest(const std::string &server_name, const std::string &re
}
sock = SSL_get_fd(ssl);
SSL_set_fd(ssl, s);
SSL_ctrl(ssl, SSL_CTRL_SET_TLSEXT_HOSTNAME, TLSEXT_NAMETYPE_host_name, (void*)server.c_str());
SSL_ctrl(ssl, SSL_CTRL_SET_TLSEXT_HOSTNAME, TLSEXT_NAMETYPE_host_name, (void *) server.c_str());
int err = SSL_connect(ssl);
if (err <= 0) {
@@ -221,7 +222,7 @@ int HttpClient::sslRequest(const std::string &server_name, const std::string &re
// printf ("SSL connection using %s\n", SSL_get_cipher (ssl));
// send request
//std::err << request << std::endl;
//std::err << request << std::endl;
sslSendPacket(request);
// read response and return its length

30
clib/sslclient.h
View File

@@ -7,26 +7,30 @@
class HttpClient {
// TODO at this moment only https is implemented
// TODO at this moment only https is implemented
private:
SSL *ssl;
int sock;
SSL *ssl;
int sock;
std::string full_url, proto, server, port, uri, params, href;
std::basic_string<char> ssl_read_packet;
std::unordered_map<std::string, std::string> headers_map;
std::string full_url, proto, server, port, uri, params, href;
std::basic_string<char> ssl_read_packet;
std::unordered_map<std::string, std::string> headers_map;
public:
HttpClient();
HttpClient();
std::pair<int, std::string> doGetRequest(const std::string &url, const std::unordered_map<std::string, std::string> &headers);
std::pair<int, std::string>
doGetRequest(const std::string &url, const std::unordered_map<std::string, std::string> &headers);
private:
std::string inetAddress(std::string hostname);
std::string inetAddress(std::string hostname);
int sslRecvPacket();
int sslSendPacket(std::string buf);
int sslRequest(const std::string &server_name, const std::string &request);
void log_ssl();
int sslRecvPacket();
int sslSendPacket(std::string buf);
int sslRequest(const std::string &server_name, const std::string &request);
void log_ssl();
};