Lexer: Added ::isContiguous for word-like matching

src/Lexer.cpp

@@ -748,7 +748,7 @@ bool Lexer::isPair (std::string& token, Lexer::Type& type)
 
         if (isString     (ignoredToken, ignoredType, '\'') ||
             isString     (ignoredToken, ignoredType, '"')  ||
-            isWord   (ignoredToken, ignoredType))
+            isContiguous (ignoredToken, ignoredType))
         {
           token = _text.substr (marker, _cursor - marker);
           type = Lexer::Type::pair;
@@ -765,7 +765,7 @@ bool Lexer::isPair (std::string& token, Lexer::Type& type)
 
       if (isString     (ignoredToken, ignoredType, '\'') ||
           isString     (ignoredToken, ignoredType, '"')  ||
-          isWord   (ignoredToken, ignoredType)       ||
+          isContiguous (ignoredToken, ignoredType)       ||
           _eos == _cursor                                ||
           _text[_cursor] == ' ')
       {
@@ -1161,6 +1161,28 @@ bool Lexer::isWord (std::string& token, Lexer::Type& type)
   return false;
 }
 
+////////////////////////////////////////////////////////////////////////////////
+// Lexer::Type::word
+//   [^\s]+
+bool Lexer::isContiguous (std::string& token, Lexer::Type& type)
+{
+  std::size_t marker = _cursor;
+
+  while (_text[marker] &&
+         ! isWhitespace (_text[marker]))
+    utf8_next_char (_text, marker);
+
+  if (marker > _cursor)
+  {
+    token = _text.substr (_cursor, marker - _cursor);
+    type = Lexer::Type::word;
+    _cursor = marker;
+    return true;
+  }
+
+  return false;
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 // Static
 std::string Lexer::typeToString (Lexer::Type type)

src/Lexer.h

@@ -102,6 +102,7 @@ public:
   bool isDOM          (std::string&, Lexer::Type&);
   bool isIdentifier   (std::string&, Lexer::Type&);
   bool isWord         (std::string&, Lexer::Type&);
+  bool isContiguous   (std::string&, Lexer::Type&);
 
 private:
   std::string _text;