Rules: Added ::tokenizeLine

src/Rules.cpp

@@ -343,3 +343,17 @@ unsigned int Rules::getIndentation (const std::string& line)
 }
 
 ////////////////////////////////////////////////////////////////////////////////
+// Tokenize the line. This loses whitespace information and token types.
+std::vector <std::string> Rules::tokenizeLine (const std::string& line)
+{
+  std::vector <std::string> tokens;
+  std::string token;
+  Lexer::Type type;
+  Lexer lexer (line);
+  while (lexer.token (token, type))
+    tokens.push_back (token);
+
+  return tokens;
+}
+
+////////////////////////////////////////////////////////////////////////////////

src/Rules.h

@@ -62,6 +62,7 @@ private:
   void parseRuleTag        (const std::vector <std::string>&);
 
   unsigned int getIndentation (const std::string&);
+  std::vector <std::string> tokenizeLine (const std::string&);
 
 private:
   std::string                         _original_file {};