From 248add6cead6d145ca5a276525bac2f34e06ddf7 Mon Sep 17 00:00:00 2001
From: Paul Beckingham <paul@beckingham.net>
Date: Sat, 19 Mar 2016 15:45:50 -0400
Subject: [PATCH] Lexer: Added static ::tokenize method to reduce a string to
 tokens and types

---
 src/Lexer.cpp | 14 ++++++++++++++
 src/Lexer.h   |  1 +
 2 files changed, 15 insertions(+)

diff --git a/src/Lexer.cpp b/src/Lexer.cpp
index a33b48f8..b6ffca58 100644
--- a/src/Lexer.cpp
+++ b/src/Lexer.cpp
@@ -64,6 +64,20 @@ bool Lexer::token (std::string& token, Lexer::Type& type)
   return false;
 }
 
+////////////////////////////////////////////////////////////////////////////////
+std::vector <std::tuple <std::string, Lexer::Type>> Lexer::tokenize (const std::string& input)
+{
+  std::vector <std::tuple <std::string, Lexer::Type>> tokens;
+
+  std::string token;
+  Lexer::Type type;
+  Lexer lexer (input);
+  while (lexer.token (token, type))
+    tokens.push_back (std::tuple <std::string, Lexer::Type> (token, type));
+
+  return tokens;
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 // No L10N - these are for internal purposes.
 const std::string Lexer::typeName (const Lexer::Type& type)
diff --git a/src/Lexer.h b/src/Lexer.h
index 2261eb0a..0a05c904 100644
--- a/src/Lexer.h
+++ b/src/Lexer.h
@@ -48,6 +48,7 @@ public:
   static std::string typeToString (Lexer::Type);
 
   // Static helpers.
+  static std::vector <std::tuple <std::string, Lexer::Type>> tokenize (const std::string&);
   static const std::string typeName          (const Lexer::Type&);
   static bool isSingleCharOperator           (int);
   static bool isDoubleCharOperator           (int, int, int);