vaclavt/usql
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

more strict parsing

Browse Source
This commit is contained in:
VaclavT
2021-08-31 18:53:49 +02:00
parent be89b55b17
commit 4e54c6d134
3 changed files with 40 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
lexer.cpp
View File

@@ -63,12 +63,18 @@ namespace usql {
Token Lexer::currentToken() { return m_tokens[m_index]; }
Token Lexer::consumeCurrentToken() {
Token Lexer::consumeToken() {
int i = m_index;
nextToken();
return m_tokens[i];
}
Token Lexer::consumeToken(TokenType type) {
int i = m_index;
skipToken(type);
return m_tokens[i];
}
void Lexer::nextToken() {
if (m_index < m_tokens.size()) {
m_index++;
@@ -79,8 +85,7 @@ namespace usql {
if (tokenType() == type) {
nextToken();
} else {
throw Exception("ERROR unexpected token " + consumeCurrentToken().token_string + ", instead of " +
typeToString(type));
throw Exception("ERROR unexpected token " + consumeToken().token_string + ", instead of " + typeToString(type));
}
}
@@ -215,8 +220,8 @@ namespace usql {
(token.at(token.length() - 1) == '\n' || token.at(token.length() - 1) == '\r'))
return TokenType::comment;
// if (token.length() >= 2 && token.at(0) == '"' && token.at(token.length() - 1) == '"')
// return TokenType::string_literal;
if (token.length() >= 2 && token.at(0) == '"' && token.at(token.length() - 1) == '"')
return TokenType::string_literal;
if (token.length() >= 2 && token.at(0) == '\'' && token.at(token.length() - 1) == '\'')
return TokenType::string_literal;
@@ -233,9 +238,6 @@ namespace usql {
if (std::regex_match(token, k_identifier_regex))
return TokenType::identifier;
if (m_index + 1 >= m_tokens.size())
return TokenType::eof;
return TokenType::undef;
}