From 7d4e166277e4beff9d036dd7655e55d919678989 Mon Sep 17 00:00:00 2001
From: Paul Beckingham <paul@beckingham.net>
Date: Sat, 14 Jun 2014 13:46:10 -0400
Subject: [PATCH] Lexer
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Implemented an overload of ::token_split that preserveѕ types.
---
 src/Lexer.cpp | 13 +++++++++++++
 src/Lexer.h   |  1 +
 2 files changed, 14 insertions(+)

diff --git a/src/Lexer.cpp b/src/Lexer.cpp
index 1711fb547..022a5ea30 100644
--- a/src/Lexer.cpp
+++ b/src/Lexer.cpp
@@ -662,6 +662,19 @@ void Lexer::token_split (std::vector <std::string>& words, const std::string& in
     words.push_back (word);
 }
 
+////////////////////////////////////////////////////////////////////////////////
+// Split 'input' into 'tokens', preserving type.
+void Lexer::token_split (std::vector <std::pair <std::string, Lexer::Type> >& lexemes, const std::string& input)
+{
+  lexemes.clear ();
+
+  std::string word;
+  Lexer::Type type;
+  Lexer lex (input);
+  while (lex.token (word, type))
+    lexemes.push_back (std::pair <std::string, Lexer::Type>(word, type));
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 bool Lexer::is_punct (int c) const
 {
diff --git a/src/Lexer.h b/src/Lexer.h
index 0d6dbf2f6..1d21161db 100644
--- a/src/Lexer.h
+++ b/src/Lexer.h
@@ -67,6 +67,7 @@ public:
   static bool is_ws (int);
   static void word_split (std::vector <std::string>&, const std::string&);
   static void token_split (std::vector <std::string>&, const std::string&);
+  static void token_split (std::vector <std::pair <std::string, Lexer::Type> >&, const std::string&);
 
 private:
   bool is_punct (int) const;