diff --git a/src/Lexer.cpp b/src/Lexer.cpp index a33b48f8..b6ffca58 100644 --- a/src/Lexer.cpp +++ b/src/Lexer.cpp @@ -64,6 +64,20 @@ bool Lexer::token (std::string& token, Lexer::Type& type) return false; } +//////////////////////////////////////////////////////////////////////////////// +std::vector > Lexer::tokenize (const std::string& input) +{ + std::vector > tokens; + + std::string token; + Lexer::Type type; + Lexer lexer (input); + while (lexer.token (token, type)) + tokens.push_back (std::tuple (token, type)); + + return tokens; +} + //////////////////////////////////////////////////////////////////////////////// // No L10N - these are for internal purposes. const std::string Lexer::typeName (const Lexer::Type& type) diff --git a/src/Lexer.h b/src/Lexer.h index 2261eb0a..0a05c904 100644 --- a/src/Lexer.h +++ b/src/Lexer.h @@ -48,6 +48,7 @@ public: static std::string typeToString (Lexer::Type); // Static helpers. + static std::vector > tokenize (const std::string&); static const std::string typeName (const Lexer::Type&); static bool isSingleCharOperator (int); static bool isDoubleCharOperator (int, int, int);