From a97f8194236ff80d9815abdc4e2d48a08df49bd7 Mon Sep 17 00:00:00 2001
From: Paul Beckingham <paul@beckingham.net>
Date: Sun, 20 Dec 2015 12:50:41 -0500
Subject: [PATCH] Tests: Added Lexer tests

---
 test/.gitignore     |  1 +
 test/CMakeLists.txt |  2 +-
 test/lexer.t.cpp    | 83 +++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 85 insertions(+), 1 deletion(-)
 create mode 100644 test/lexer.t.cpp

diff --git a/test/.gitignore b/test/.gitignore
index 726ed1b1..b017134e 100644
--- a/test/.gitignore
+++ b/test/.gitignore
@@ -1,5 +1,6 @@
 all.log
 color.t
+lexer.t
 rx.t
 text.t
 utf8.t
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 779b1236..ef0c94c8 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -9,7 +9,7 @@ include_directories (${CMAKE_SOURCE_DIR}
                      ${CMAKE_SOURCE_DIR}/test
                      ${TIMEW_INCLUDE_DIRS})
 
-set (test_SRCS color.t rx.t text.t utf8.t)
+set (test_SRCS color.t lexer.t rx.t text.t utf8.t)
 
 add_custom_target (test ./run_all --verbose
                         DEPENDS ${test_SRCS} timew_executable
diff --git a/test/lexer.t.cpp b/test/lexer.t.cpp
new file mode 100644
index 00000000..0fcc6e85
--- /dev/null
+++ b/test/lexer.t.cpp
@@ -0,0 +1,83 @@
+////////////////////////////////////////////////////////////////////////////////
+//
+// Copyright 2013 - 2015, Göteborg Bit Factory.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included
+// in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//
+// http://www.opensource.org/licenses/mit-license.php
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#include <cmake.h>
+#include <Lexer.h>
+#include <iostream>
+#include <vector>
+#include <string.h>
+#include <test.h>
+
+////////////////////////////////////////////////////////////////////////////////
+int main (int, char**)
+{
+  UnitTest t (29);
+
+  std::vector <std::pair <std::string, Lexer::Type>> tokens;
+  std::string token;
+  Lexer::Type type;
+
+  // White space detection.
+  t.notok (Lexer::isWhitespace (0x0041), "U+0041 (A) ! isWhitespace");
+  t.ok (Lexer::isWhitespace (0x0020), "U+0020 isWhitespace");
+  t.ok (Lexer::isWhitespace (0x0009), "U+0009 isWhitespace");
+  t.ok (Lexer::isWhitespace (0x000A), "U+000A isWhitespace");
+  t.ok (Lexer::isWhitespace (0x000B), "U+000B isWhitespace");
+  t.ok (Lexer::isWhitespace (0x000C), "U+000C isWhitespace");
+  t.ok (Lexer::isWhitespace (0x000D), "U+000D isWhitespace");
+  t.ok (Lexer::isWhitespace (0x0085), "U+0085 isWhitespace");
+  t.ok (Lexer::isWhitespace (0x00A0), "U+00A0 isWhitespace");
+  t.ok (Lexer::isWhitespace (0x1680), "U+1680 isWhitespace"); // 10
+  t.ok (Lexer::isWhitespace (0x180E), "U+180E isWhitespace");
+  t.ok (Lexer::isWhitespace (0x2000), "U+2000 isWhitespace");
+  t.ok (Lexer::isWhitespace (0x2001), "U+2001 isWhitespace");
+  t.ok (Lexer::isWhitespace (0x2002), "U+2002 isWhitespace");
+  t.ok (Lexer::isWhitespace (0x2003), "U+2003 isWhitespace");
+  t.ok (Lexer::isWhitespace (0x2004), "U+2004 isWhitespace");
+  t.ok (Lexer::isWhitespace (0x2005), "U+2005 isWhitespace");
+  t.ok (Lexer::isWhitespace (0x2006), "U+2006 isWhitespace");
+  t.ok (Lexer::isWhitespace (0x2007), "U+2007 isWhitespace");
+  t.ok (Lexer::isWhitespace (0x2008), "U+2008 isWhitespace"); // 20
+  t.ok (Lexer::isWhitespace (0x2009), "U+2009 isWhitespace");
+  t.ok (Lexer::isWhitespace (0x200A), "U+200A isWhitespace");
+  t.ok (Lexer::isWhitespace (0x2028), "U+2028 isWhitespace");
+  t.ok (Lexer::isWhitespace (0x2029), "U+2029 isWhitespace");
+  t.ok (Lexer::isWhitespace (0x202F), "U+202F isWhitespace");
+  t.ok (Lexer::isWhitespace (0x205F), "U+205F isWhitespace");
+  t.ok (Lexer::isWhitespace (0x3000), "U+3000 isWhitespace");
+
+  // Should result in no tokens.
+  Lexer l0 ("");
+  t.notok (l0.token (token, type), "'' --> no tokens");
+
+  // Should result in no tokens.
+  Lexer l1 ("       \t ");
+  t.notok (l1.token (token, type), "'       \\t ' --> no tokens");
+
+  return 0;
+}
+
+////////////////////////////////////////////////////////////////////////////////