Lexer: Added ::isContiguous for word-like matching

This commit is contained in:
Paul Beckingham 2015-06-22 21:34:57 -04:00
parent e66ad50e7e
commit d9bcbdee0a
2 changed files with 30 additions and 7 deletions

View file

@ -746,9 +746,9 @@ bool Lexer::isPair (std::string& token, Lexer::Type& type)
{ {
_cursor++; _cursor++;
if (isString (ignoredToken, ignoredType, '\'') || if (isString (ignoredToken, ignoredType, '\'') ||
isString (ignoredToken, ignoredType, '"') || isString (ignoredToken, ignoredType, '"') ||
isWord (ignoredToken, ignoredType)) isContiguous (ignoredToken, ignoredType))
{ {
token = _text.substr (marker, _cursor - marker); token = _text.substr (marker, _cursor - marker);
type = Lexer::Type::pair; type = Lexer::Type::pair;
@ -763,10 +763,10 @@ bool Lexer::isPair (std::string& token, Lexer::Type& type)
{ {
_cursor++; _cursor++;
if (isString (ignoredToken, ignoredType, '\'') || if (isString (ignoredToken, ignoredType, '\'') ||
isString (ignoredToken, ignoredType, '"') || isString (ignoredToken, ignoredType, '"') ||
isWord (ignoredToken, ignoredType) || isContiguous (ignoredToken, ignoredType) ||
_eos == _cursor || _eos == _cursor ||
_text[_cursor] == ' ') _text[_cursor] == ' ')
{ {
token = _text.substr (marker, _cursor - marker); token = _text.substr (marker, _cursor - marker);
@ -1161,6 +1161,28 @@ bool Lexer::isWord (std::string& token, Lexer::Type& type)
return false; return false;
} }
////////////////////////////////////////////////////////////////////////////////
// Lexer::Type::word
// [^\s]+
bool Lexer::isContiguous (std::string& token, Lexer::Type& type)
{
std::size_t marker = _cursor;
while (_text[marker] &&
! isWhitespace (_text[marker]))
utf8_next_char (_text, marker);
if (marker > _cursor)
{
token = _text.substr (_cursor, marker - _cursor);
type = Lexer::Type::word;
_cursor = marker;
return true;
}
return false;
}
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
// Static // Static
std::string Lexer::typeToString (Lexer::Type type) std::string Lexer::typeToString (Lexer::Type type)

View file

@ -102,6 +102,7 @@ public:
bool isDOM (std::string&, Lexer::Type&); bool isDOM (std::string&, Lexer::Type&);
bool isIdentifier (std::string&, Lexer::Type&); bool isIdentifier (std::string&, Lexer::Type&);
bool isWord (std::string&, Lexer::Type&); bool isWord (std::string&, Lexer::Type&);
bool isContiguous (std::string&, Lexer::Type&);
private: private:
std::string _text; std::string _text;