Lexer: Added static ::tokenize method to reduce a string to tokens and types

src/Lexer.cpp

@@ -64,6 +64,20 @@ bool Lexer::token (std::string& token, Lexer::Type& type)
   return false;
 }
 
+////////////////////////////////////////////////////////////////////////////////
+std::vector <std::tuple <std::string, Lexer::Type>> Lexer::tokenize (const std::string& input)
+{
+  std::vector <std::tuple <std::string, Lexer::Type>> tokens;
+
+  std::string token;
+  Lexer::Type type;
+  Lexer lexer (input);
+  while (lexer.token (token, type))
+    tokens.push_back (std::tuple <std::string, Lexer::Type> (token, type));
+
+  return tokens;
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 // No L10N - these are for internal purposes.
 const std::string Lexer::typeName (const Lexer::Type& type)

src/Lexer.h

@@ -48,6 +48,7 @@ public:
   static std::string typeToString (Lexer::Type);
 
   // Static helpers.
+  static std::vector <std::tuple <std::string, Lexer::Type>> tokenize (const std::string&);
   static const std::string typeName          (const Lexer::Type&);
   static bool isSingleCharOperator           (int);
   static bool isDoubleCharOperator           (int, int, int);