TI-5: Unicode tags not working.

- Thanks to Sergey Trofimov.

AUTHORS

@@ -11,6 +11,7 @@ The following submitted code, packages or analysis, and deserve special thanks:
   Dirk Deimeke
   Wim Schuermann
   Jelle van der Waa
+  Sergey Trofimov
 
 Thanks to the following, who submitted detailed bug reports and excellent
 suggestions:

ChangeLog

@@ -7,6 +7,8 @@
           (thanks to Jelle van der Waa).
 - TI-2    Two line tags causes false entry in database
           (thanks to Sebastian Uharek).
+- TI-5    Unicode tags not working.
+          (thanks to Sergey Trofimov).
 - Added 'continue' command.
 - Added 'diagnostics' command.
 - Added 'export' command.

src/CLI.cpp

@@ -30,6 +30,7 @@
 #include <Pig.h>
 #include <shared.h>
 #include <format.h>
+#include <utf8.h>
 #include <sstream>
 #include <algorithm>
 
@@ -145,10 +146,16 @@ void CLI::entity (const std::string& category, const std::string& name)
 void CLI::add (const std::string& argument)
 {
   // Sanitize the input: Convert control charts to spaces. Then trim.
-  auto clean = argument;
-  for (auto& c : clean)
-    if (c <= 32)
-      c = ' ';
+  std::string clean;
+  std::string::size_type i = 0;
+  int character;
+  while ((character = utf8_next_char (argument.c_str (), i)))
+  {
+    if (character <= 32)
+       clean += ' ';
+    else
+       clean += utf8_character (character);
+  }
 
   A2 arg (Lexer::trim (clean), Lexer::Type::word);
   arg.tag ("ORIGINAL");