GothenburgBitFactory/timewarrior
1
0
Fork 0
mirror of https://github.com/GothenburgBitFactory/timewarrior.git synced 2025-07-07 20:06:39 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Lexer: Migrated from ::iѕDigit to unicodeLatinDigit

Browse source
This commit is contained in:
Paul Beckingham 2016-01-14 22:04:44 -05:00
parent 9f12807526
commit d2629584d5
2 changed files with 12 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

33
src/Lexer.cpp
View file

@ -84,15 +84,6 @@ const std::string Lexer::typeName (const Lexer::Type& type)
return "unknown"; return "unknown";
} }
////////////////////////////////////////////////////////////////////////////////
// Digits 0-9.
//
// TODO This list should be derived from the Unicode database.
bool Lexer::isDigit (int c)
{
return c >= 0x30 && c <= 0x39;
}
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
// Digits 0-9 a-f A-F. // Digits 0-9 a-f A-F.
bool Lexer::isHexDigit (int c) bool Lexer::isHexDigit (int c)
@ -112,19 +103,19 @@ bool Lexer::isNumber (std::string& token, Lexer::Type& type)
{ {
std::size_t marker = _cursor; std::size_t marker = _cursor;
if (isDigit (_text[marker])) if (unicodeLatinDigit (_text[marker]))
{ {
++marker; ++marker;
while (isDigit (_text[marker])) while (unicodeLatinDigit (_text[marker]))
utf8_next_char (_text, marker); utf8_next_char (_text, marker);
if (_text[marker] == '.') if (_text[marker] == '.')
{ {
++marker; ++marker;
if (isDigit (_text[marker])) if (unicodeLatinDigit (_text[marker]))
{ {
++marker; ++marker;
while (isDigit (_text[marker])) while (unicodeLatinDigit (_text[marker]))
utf8_next_char (_text, marker); utf8_next_char (_text, marker);
} }
} }
@ -138,19 +129,19 @@ bool Lexer::isNumber (std::string& token, Lexer::Type& type)
_text[marker] == '-') _text[marker] == '-')
++marker; ++marker;
if (isDigit (_text[marker])) if (unicodeLatinDigit (_text[marker]))
{ {
++marker; ++marker;
while (isDigit (_text[marker])) while (unicodeLatinDigit (_text[marker]))
utf8_next_char (_text, marker); utf8_next_char (_text, marker);
if (_text[marker] == '.') if (_text[marker] == '.')
{ {
++marker; ++marker;
if (isDigit (_text[marker])) if (unicodeLatinDigit (_text[marker]))
{ {
++marker; ++marker;
while (isDigit (_text[marker])) while (unicodeLatinDigit (_text[marker]))
utf8_next_char (_text, marker); utf8_next_char (_text, marker);
} }
} }
@ -180,10 +171,10 @@ bool Lexer::isInteger (std::string& token, Lexer::Type& type)
{ {
std::size_t marker = _cursor; std::size_t marker = _cursor;
if (isDigit (_text[marker])) if (unicodeLatinDigit (_text[marker]))
{ {
++marker; ++marker;
while (isDigit (_text[marker])) while (unicodeLatinDigit (_text[marker]))
utf8_next_char (_text, marker); utf8_next_char (_text, marker);
token = _text.substr (_cursor, marker - _cursor); token = _text.substr (_cursor, marker - _cursor);
@ -242,7 +233,7 @@ bool Lexer::isBoundary (int left, int right)
// XOR // XOR
if (unicodeLatinAlpha (left) != unicodeLatinAlpha (right)) return true; if (unicodeLatinAlpha (left) != unicodeLatinAlpha (right)) return true;
if (isDigit (left) != isDigit (right)) return true; if (unicodeLatinDigit (left) != unicodeLatinDigit (right)) return true;
if (unicodeWhitespace (left) != unicodeWhitespace (right)) return true; if (unicodeWhitespace (left) != unicodeWhitespace (right)) return true;
// OR // OR
@ -277,7 +268,7 @@ bool Lexer::isPunctuation (int c)
c != '#' && c != '#' &&
c != '$' && c != '$' &&
c != '_' && c != '_' &&
! isDigit (c) && ! unicodeLatinDigit (c) &&
! unicodeLatinAlpha (c); ! unicodeLatinAlpha (c);
} }

1
src/Lexer.h
View file

@ -49,7 +49,6 @@ public:
// Static helpers. // Static helpers.
static const std::string typeName (const Lexer::Type&); static const std::string typeName (const Lexer::Type&);
static bool isDigit (int);
static bool isHexDigit (int); static bool isHexDigit (int);
static bool isSingleCharOperator (int); static bool isSingleCharOperator (int);
static bool isDoubleCharOperator (int, int, int); static bool isDoubleCharOperator (int, int, int);