From 562fd8ce3c8a7ca1ff57c747ec25efe79bfd89ca Mon Sep 17 00:00:00 2001
From: Paul Beckingham <paul@beckingham.net>
Date: Sat, 10 Sep 2011 13:24:54 -0400
Subject: [PATCH] Helpers

- Implemented isTokenEnd, as a special case of isWordEnd, but considers
  consecutive punctuation to be a set of individual tokens.
---
 src/text.cpp | 21 ++++++++++++++++++++-
 src/text.h   |  1 +
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/src/text.cpp b/src/text.cpp
index f56dc7986..ab6aa948a 100644
--- a/src/text.cpp
+++ b/src/text.cpp
@@ -601,7 +601,7 @@ bool isWordStart (const std::string& input, std::string::size_type pos)
 // Result for pos: ....y......y
 bool isWordEnd (const std::string& input, std::string::size_type pos)
 {
-  // Short circuit: no input means no word start.
+  // Short circuit: no input means no word end.
   if (input.length () == 0)
     return false;
 
@@ -618,6 +618,25 @@ bool isWordEnd (const std::string& input, std::string::size_type pos)
   return false;
 }
 
+////////////////////////////////////////////////////////////////////////////////
+//          Input: hello, world
+// Result for pos: ....y......y
+//
+//          Input: (one) two
+// Result for pos: y..yy...y
+bool isTokenEnd (const std::string& input, std::string::size_type pos)
+{
+  // Delegate.
+  if (isWordEnd (input, pos))
+    return true;
+
+  // Punctuation divides tokens.
+  if (pos < input.length () && isPunctuation (input[pos]))
+    return true;
+
+  return false;
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 // Override of ispunct, that considers #, $ and @ not to be punctuation.
 //
diff --git a/src/text.h b/src/text.h
index a73734374..e2d8c170b 100644
--- a/src/text.h
+++ b/src/text.h
@@ -61,6 +61,7 @@ bool noSpaces (const std::string&);
 bool noVerticalSpace (const std::string&);
 bool isWordStart (const std::string&, std::string::size_type);
 bool isWordEnd (const std::string&, std::string::size_type);
+bool isTokenEnd (const std::string&, std::string::size_type);
 bool isPunctuation (char);
 bool compare (const std::string&, const std::string&, bool sensitive = true);
 bool closeEnough (const std::string&, const std::string&);