From 85d991704b41f47882aed39d0a9af8aaed46da62 Mon Sep 17 00:00:00 2001 From: Shaun Ruffell Date: Sat, 9 May 2020 14:50:43 -0500 Subject: [PATCH 001/492] Speed up deserialization of Intervals When the Lexer breaks a line into tokens, it also wants to return the type of the token. This information isn't used by the IntervalFactory and it slows down the operation since dates end up being parsed at least twice, once by the Lexer to determine that the string is a date, then again in the IntervalFactory to actually construct the Date. Before are the before and after results when exporting a database with 100 lines. The number of instructions executed went from roughly 31,552,467 to 12,952,372 on debug builds. Release builds saw a change from around 14K to 7K instructions. Before: $ rm -fr ~/.timewarrior; src/timew :yes >/dev/null; for x in {100..1}; do src/timew start ${x}sec ago proj_${x} >/dev/null; done; $ sudo chrt -f 99 valgrind --tool=callgrind --callgrind-out-file=callgrind.out src/timew export >/dev/null ==20888== Callgrind, a call-graph generating cache profiler ==20888== Copyright (C) 2002-2017, and GNU GPL'd, by Josef Weidendorfer et al. ==20888== Using Valgrind-3.13.0 and LibVEX; rerun with -h for copyright info ==20888== Command: src/timew export ==20888== ==20888== For interactive control, run 'callgrind_control -h'. ==20888== ==20888== Events : Ir ==20888== Collected : 31552467 ==20888== ==20888== I refs: 31,552,467 After: $ sudo chrt -f 99 valgrind --tool=callgrind --callgrind-out-file=callgrind.out src/timew export >/dev/null ==24088== Callgrind, a call-graph generating cache profiler ==24088== Copyright (C) 2002-2017, and GNU GPL'd, by Josef Weidendorfer et al. ==24088== Using Valgrind-3.13.0 and LibVEX; rerun with -h for copyright info ==24088== Command: src/timew export ==24088== ==24088== For interactive control, run 'callgrind_control -h'. ==24088== ==24088== Events : Ir ==24088== Collected : 12952372 ==24088== ==24088== I refs: 12,952,372 Signed-off-by: Shaun Ruffell --- src/IntervalFactory.cpp | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/src/IntervalFactory.cpp b/src/IntervalFactory.cpp index f69f3960..1fce8fb8 100644 --- a/src/IntervalFactory.cpp +++ b/src/IntervalFactory.cpp @@ -29,21 +29,40 @@ #include #include -//////////////////////////////////////////////////////////////////////////////// -// Syntax: -// 'inc' [ [ '-' ]] [ '#' [ ... ]] -Interval IntervalFactory::fromSerialization (const std::string& line) +static std::vector tokenizeSerialization (const std::string& line) { - Lexer lexer (line); std::vector tokens; + + Lexer lexer (line); std::string token; Lexer::Type type; + + // When parsing the serialization, we only need the lexer to look for strings + // and words since we're not using the provided type information + lexer.noDate (); + lexer.noDuration (); + lexer.noUUID (); + lexer.noHexNumber (); + lexer.noURL (); + lexer.noPath (); + lexer.noPattern (); + lexer.noOperator (); while (lexer.token (token, type)) { tokens.push_back (Lexer::dequote (token)); } + return tokens; +} + +//////////////////////////////////////////////////////////////////////////////// +// Syntax: +// 'inc' [ [ '-' ]] [ '#' [ ... ]] +Interval IntervalFactory::fromSerialization (const std::string& line) +{ + std::vector tokens = tokenizeSerialization (line); + // Minimal requirement 'inc'. if (!tokens.empty () && tokens[0] == "inc") { From 42ede4104c94500728f99b080159f2d02e37cabe Mon Sep 17 00:00:00 2001 From: Shaun Ruffell Date: Sat, 9 May 2020 14:45:59 -0500 Subject: [PATCH 002/492] Remove naked pointer in initializeTagDatabase The json::object pointer was allocated in the parse function but never freed. The way timwarrior is used currently, this leak does not cause any problems, but... Signed-off-by: Shaun Ruffell --- src/Database.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Database.cpp b/src/Database.cpp index f0a95d68..8d8defcc 100644 --- a/src/Database.cpp +++ b/src/Database.cpp @@ -468,7 +468,7 @@ void Database::initializeTagDatabase () { try { - json::object *json = dynamic_cast (json::parse (content)); + std::unique_ptr json (dynamic_cast (json::parse (content))); if (content.empty () || (json == nullptr)) { From 29cc9e8a0a98540ee00dbde14cf9b83463c82526 Mon Sep 17 00:00:00 2001 From: Shaun Ruffell Date: Sun, 10 May 2020 06:18:33 -0500 Subject: [PATCH 003/492] Fix another place where json::parse() result was leaked Signed-off-by: Shaun Ruffell --- src/IntervalFactory.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IntervalFactory.cpp b/src/IntervalFactory.cpp index 1fce8fb8..0a2d2c36 100644 --- a/src/IntervalFactory.cpp +++ b/src/IntervalFactory.cpp @@ -127,7 +127,7 @@ Interval IntervalFactory::fromJson (const std::string& jsonString) if (!jsonString.empty ()) { - auto* json = (json::object*) json::parse (jsonString); + std::unique_ptr json (dynamic_cast (json::parse (jsonString))); json::array* tags = (json::array*) json->_data["tags"]; From 422e49bacb7481038009134a2a49764b6b00f317 Mon Sep 17 00:00:00 2001 From: Thomas Lauf Date: Fri, 8 May 2020 13:47:39 +0200 Subject: [PATCH 004/492] Extract function `getDomReferences` and move it to CLI Signed-off-by: Thomas Lauf --- src/CLI.cpp | 16 ++++++++++++++++ src/CLI.h | 1 + src/commands/CmdGet.cpp | 19 ++++++++----------- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/src/CLI.cpp b/src/CLI.cpp index c0f1149a..cf31c523 100644 --- a/src/CLI.cpp +++ b/src/CLI.cpp @@ -606,3 +606,19 @@ Duration CLI::getDuration () const } //////////////////////////////////////////////////////////////////////////////// + +std::vector CLI::getDomReferences () const +{ + std::vector references; + + for (auto &arg : _args) + { + if (arg.hasTag ("TAG") && + arg.hasTag ("FILTER")) + { + references.emplace_back (arg.attribute ("raw")); + } + } + + return references; +} diff --git a/src/CLI.h b/src/CLI.h index f831d882..a5e393ee 100644 --- a/src/CLI.h +++ b/src/CLI.h @@ -70,6 +70,7 @@ public: std::vector getTags () const; std::string getAnnotation() const; Duration getDuration() const; + std::vector getDomReferences () const; std::string dump (const std::string& title = "CLI Parser") const; private: diff --git a/src/commands/CmdGet.cpp b/src/commands/CmdGet.cpp index 501aee8e..05218b8c 100644 --- a/src/commands/CmdGet.cpp +++ b/src/commands/CmdGet.cpp @@ -38,18 +38,15 @@ int CmdGet ( Database& database) { std::vector results; - for (auto& arg : cli._args) - { - if (arg.hasTag ("TAG") && - arg.hasTag ("FILTER")) - { - std::string reference = arg.attribute ("raw"); - std::string value; - if (! domGet (database, rules, reference, value)) - throw format ("DOM reference '{1}' is not valid.", reference); + std::vector references = cli.getDomReferences (); - results.push_back (value); - } + for (auto& reference : references) + { + std::string value; + if (! domGet (database, rules, reference, value)) + throw format ("DOM reference '{1}' is not valid.", reference); + + results.push_back (value); } std::cout << join (" ", results) << '\n'; From 5c234c95f9e23426392c9583ecde8cea8511a230 Mon Sep 17 00:00:00 2001 From: Thomas Lauf Date: Fri, 8 May 2020 14:50:32 +0200 Subject: [PATCH 005/492] Move test setup to the test methods Signed-off-by: Thomas Lauf --- test/dom.t | 50 +++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/test/dom.t b/test/dom.t index 080e4b3c..878fd8ef 100755 --- a/test/dom.t +++ b/test/dom.t @@ -154,83 +154,119 @@ class TestDOM(TestCase): class TestDOMTracked(TestCase): - @classmethod - def setUpClass(cls): - """Executed before each test in the class""" - cls.t = Timew() - cls.t("track :yesterday one two") #2 - cls.t("start") #1 - def setUp(self): """Executed before each test in the class""" + self.t = Timew() def test_dom_tracked_count_some(self): """Test dom.tracked.count with an active interval""" + self.t("track :yesterday one two") + self.t("start") + code, out, err = self.t("get dom.tracked.count") self.assertEqual('2\n', out) def test_dom_tracked_N_tag_count_zero(self): """Test dom.tracked.N.tag.count with zero tags""" + self.t("track :yesterday one two") + self.t("start") + code, out, err = self.t("get dom.tracked.1.tag.count") self.assertEqual('0\n', out) def test_dom_tracked_N_tag_count_two(self): """Test dom.tracked.N.tag.count with two tags""" + self.t("track :yesterday one two") + self.t("start") + code, out, err = self.t("get dom.tracked.2.tag.count") self.assertEqual('2\n', out) def test_dom_tracked_N_tag_N_none(self): """Test dom.tracked.N.tag.N with no data""" + self.t("track :yesterday one two") + self.t("start") + code, out, err = self.t.runError("get dom.tracked.1.tag.1") self.assertIn("DOM reference 'dom.tracked.1.tag.1' is not valid.", err) def test_dom_tracked_N_tag_N_two(self): """Test dom.tracked.N.tag.N with two tags""" + self.t("track :yesterday one two") + self.t("start") + code, out, err = self.t("get dom.tracked.2.tag.2") self.assertEqual('two\n', out) def test_dom_tracked_N_start_inactive(self): """Test dom.tracked.N.start with no active track""" + self.t("track :yesterday one two") + self.t("start") + code, out, err = self.t.runError("get dom.tracked.3.start") self.assertIn("DOM reference 'dom.tracked.3.start' is not valid.", err) def test_dom_tracked_N_start_active(self): """Test dom.tracked.N.start with active track""" + self.t("track :yesterday one two") + self.t("start") + code, out, err = self.t("get dom.tracked.1.start") self.assertRegex(out, r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}') def test_dom_tracked_N_end_invalid(self): """Test dom.tracked.N.end with no active track""" + self.t("track :yesterday one two") + self.t("start") + code, out, err = self.t.runError("get dom.tracked.3.end") self.assertIn("DOM reference 'dom.tracked.3.end' is not valid.", err) def test_dom_tracked_N_end_inactive(self): """Test dom.tracked.N.end with active track""" + self.t("track :yesterday one two") + self.t("start") + code, out, err = self.t("get dom.tracked.2.end") self.assertRegex(out, r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}') def test_dom_tracked_N_end_active(self): """Test dom.tracked.N.end with active track""" + self.t("track :yesterday one two") + self.t("start") + code, out, err = self.t("get dom.tracked.1.end") self.assertEqual('\n', out) def test_dom_tracked_N_duration_inactive(self): """Test dom.tracked.N.duration of closed track""" + self.t("track :yesterday one two") + self.t("start") + code, out, err = self.t("get dom.tracked.2.duration") self.assertRegex(out, r'P1D') def test_dom_tracked_N_duration_active(self): """Test dom.tracked.N.duration with open track""" + self.t("track :yesterday one two") + self.t("start") + code, out, err = self.t("get dom.tracked.1.duration") self.assertRegex(out, r'PT\d+S') def test_dom_tracked_N_json_inactive(self): """Test dom.tracked.N.json of closed track""" + self.t("track :yesterday one two") + self.t("start") + code, out, err = self.t("get dom.tracked.2.json") self.assertRegex(out, r'{"id":2,"start":"\d{8}T\d{6}Z","end":"\d{8}T\d{6}Z","tags":\["one","two"\]}') def test_dom_tracked_N_json_active(self): """Test dom.tracked.N.json of open track""" + self.t("track :yesterday one two") + self.t("start") + code, out, err = self.t("get dom.tracked.1.json") self.assertRegex(out, r'{"id":1,"start":"\d{8}T\d{6}Z"}') From 6e5236d4723643b707be237c3574bfaeb87e9727 Mon Sep 17 00:00:00 2001 From: Thomas Lauf Date: Fri, 8 May 2020 15:14:27 +0200 Subject: [PATCH 006/492] Add DOM query `dom.tracked.tags` Signed-off-by: Thomas Lauf --- src/dom.cpp | 23 +++++++++++++++++++++++ test/dom.t | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/src/dom.cpp b/src/dom.cpp index 20e5103c..24a5ce9f 100644 --- a/src/dom.cpp +++ b/src/dom.cpp @@ -108,6 +108,29 @@ bool domGet ( auto tracked = getTracked (database, rules, filter); int count = static_cast (tracked.size ()); + // dom.tracked.tags + if (pig.skipLiteral ("tags")) + { + std::set tags; + for (const auto& interval : tracked) + { + for (const auto &tag : interval.tags ()) + { + tags.insert (tag); + } + } + + std::stringstream s; + + for (const auto& tag : tags) + { + s << format ( "{1} ", tag ); + } + + value = s.str(); + return true; + } + // dom.tracked.count if (pig.skipLiteral ("count")) { diff --git a/test/dom.t b/test/dom.t index 878fd8ef..f31466d5 100755 --- a/test/dom.t +++ b/test/dom.t @@ -27,9 +27,10 @@ ############################################################################### import os +import sys import unittest -import sys +from datetime import datetime, timedelta # Ensure python finds the local simpletap module sys.path.append(os.path.dirname(os.path.abspath(__file__))) @@ -166,6 +167,36 @@ class TestDOMTracked(TestCase): code, out, err = self.t("get dom.tracked.count") self.assertEqual('2\n', out) + def test_dom_tracked_tags_with_emtpy_database(self): + """Test dom.tracked.tags with empty database""" + code, out, err = self.t("get dom.tracked.tags") + self.assertEqual("\n", out) + + def test_dom_tracked_tags_with_no_tags(self): + """Test dom.tracked.tags with no tags""" + now_utc = datetime.now().utcnow() + four_hours_before_utc = now_utc - timedelta(hours=4) + five_hours_before_utc = now_utc - timedelta(hours=5) + + self.t("track {:%Y-%m-%dT%H:%M:%S}Z - {:%Y-%m-%dT%H:%M:%S}Z".format(five_hours_before_utc, four_hours_before_utc)) + + code, out, err = self.t("get dom.tracked.tags") + self.assertEqual("\n", out) + + def test_dom_tracked_tags_with_tags(self): + """Test dom.tracked.tags with tags""" + now_utc = datetime.now().utcnow() + two_hours_before_utc = now_utc - timedelta(hours=2) + three_hours_before_utc = now_utc - timedelta(hours=3) + four_hours_before_utc = now_utc - timedelta(hours=4) + five_hours_before_utc = now_utc - timedelta(hours=5) + + self.t("track {:%Y-%m-%dT%H:%M:%S}Z - {:%Y-%m-%dT%H:%M:%S}Z foo".format(five_hours_before_utc, four_hours_before_utc)) + self.t("track {:%Y-%m-%dT%H:%M:%S}Z - {:%Y-%m-%dT%H:%M:%S}Z bar".format(three_hours_before_utc, two_hours_before_utc)) + + code, out, err = self.t("get dom.tracked.tags") + self.assertEqual("bar foo \n", out) + def test_dom_tracked_N_tag_count_zero(self): """Test dom.tracked.N.tag.count with zero tags""" self.t("track :yesterday one two") From 547bda5ef87672e88ff6ff5d0b5c6c95153bd6bf Mon Sep 17 00:00:00 2001 From: Thomas Lauf Date: Fri, 8 May 2020 16:27:12 +0200 Subject: [PATCH 007/492] Extract filter so it can be injected Signed-off-by: Thomas Lauf --- src/commands/CmdGet.cpp | 3 ++- src/dom.cpp | 2 +- src/timew.h | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/commands/CmdGet.cpp b/src/commands/CmdGet.cpp index 05218b8c..90c7aed3 100644 --- a/src/commands/CmdGet.cpp +++ b/src/commands/CmdGet.cpp @@ -40,10 +40,11 @@ int CmdGet ( std::vector results; std::vector references = cli.getDomReferences (); + Interval filter; for (auto& reference : references) { std::string value; - if (! domGet (database, rules, reference, value)) + if (! domGet (database, filter, rules, reference, value)) throw format ("DOM reference '{1}' is not valid.", reference); results.push_back (value); diff --git a/src/dom.cpp b/src/dom.cpp index 24a5ce9f..c1cdb026 100644 --- a/src/dom.cpp +++ b/src/dom.cpp @@ -35,6 +35,7 @@ //////////////////////////////////////////////////////////////////////////////// bool domGet ( Database& database, + Interval& filter, const Rules& rules, const std::string& reference, std::string& value) @@ -104,7 +105,6 @@ bool domGet ( else if (pig.skipLiteral ("tracked.")) { - Interval filter; auto tracked = getTracked (database, rules, filter); int count = static_cast (tracked.size ()); diff --git a/src/timew.h b/src/timew.h index c2715f20..6c61c0e4 100644 --- a/src/timew.h +++ b/src/timew.h @@ -97,6 +97,6 @@ std::string joinQuotedIfNeeded(const std::string& glue, const std::set & array); // dom.cpp -bool domGet (Database&, const Rules&, const std::string&, std::string&); +bool domGet (Database&, Interval&, const Rules&, const std::string&, std::string&); #endif From 9ae3ace109fe18b6459a60a9d7b4c8ff7c1a68a8 Mon Sep 17 00:00:00 2001 From: Thomas Lauf Date: Fri, 8 May 2020 17:08:02 +0200 Subject: [PATCH 008/492] Make getFilter(cli) a method of CLI --- src/CLI.cpp | 188 +++++++++++++++++++++++++++++++++++ src/CLI.h | 2 + src/commands/CmdChart.cpp | 6 +- src/commands/CmdContinue.cpp | 2 +- src/commands/CmdExport.cpp | 2 +- src/commands/CmdGaps.cpp | 2 +- src/commands/CmdModify.cpp | 2 +- src/commands/CmdReport.cpp | 2 +- src/commands/CmdStart.cpp | 2 +- src/commands/CmdStop.cpp | 2 +- src/commands/CmdSummary.cpp | 2 +- src/commands/CmdTags.cpp | 2 +- src/commands/CmdTrack.cpp | 2 +- src/data.cpp | 186 ---------------------------------- src/timew.h | 1 - src/validate.cpp | 2 +- 16 files changed, 204 insertions(+), 201 deletions(-) diff --git a/src/CLI.cpp b/src/CLI.cpp index cf31c523..bc9d9315 100644 --- a/src/CLI.cpp +++ b/src/CLI.cpp @@ -35,6 +35,7 @@ #include #include #include +#include //////////////////////////////////////////////////////////////////////////////// A2::A2 (const std::string& raw, Lexer::Type lextype) @@ -622,3 +623,190 @@ std::vector CLI::getDomReferences () const return references; } + +//////////////////////////////////////////////////////////////////////////////// +// A filter is just another interval, containing start, end and tags. +// +// Supported interval forms: +// ["from"] ["to"|"-" ] +// ["from"] "for" +// ["before"|"after" ] +// "ago" +// +Interval CLI::getFilter () const +{ + // One instance, so we can directly compare. + Datetime now; + + Interval filter; + std::string start; + std::string end; + std::string duration; + std::vector args; + + for (auto& arg : _args) + { + if (arg.hasTag ("BINARY") || + arg.hasTag ("CMD") || + arg.hasTag ("EXT")) + continue; + + if (arg.hasTag ("FILTER")) + { + auto canonical = arg.attribute ("canonical"); + auto raw = arg.attribute ("raw"); + + if (arg.hasTag ("HINT")) + { + Range range; + if (expandIntervalHint (canonical, range)) + { + start = range.start.toISO (); + end = range.end.toISO (); + + args.push_back (""); + args.push_back ("-"); + args.push_back (""); + } + + // Hints that are not expandable to a date range are ignored. + } + else if (arg._lextype == Lexer::Type::date) + { + if (start.empty ()) + start = raw; + else if (end.empty ()) + end = raw; + + args.push_back (""); + } + else if (arg._lextype == Lexer::Type::duration) + { + if (duration.empty ()) + duration = raw; + + args.push_back (""); + } + else if (arg.hasTag ("KEYWORD")) + { + // Note: that KEYWORDS are not entities (why not?) and there is a list + // in CLI.cpp of them that must be maintained and synced with this + // function. + args.push_back (raw); + } + else if (arg.hasTag ("ID")) + { + // Not part of a filter. + } + else + { + filter.tag (raw); + } + } + } + + // + if (args.size () == 1 && + args[0] == "") + { + filter.setRange ({Datetime (start), 0}); + } + + // from + else if (args.size () == 2 && + args[0] == "from" && + args[1] == "") + { + filter.setRange ({Datetime (start), 0}); + } + + // to/- + else if (args.size () == 3 && + args[0] == "" && + (args[1] == "to" || args[1] == "-") && + args[2] == "") + { + filter.setRange ({Datetime (start), Datetime (end)}); + } + + // from to/- + else if (args.size () == 4 && + args[0] == "from" && + args[1] == "" && + (args[2] == "to" || args[2] == "-") && + args[3] == "") + { + filter.setRange ({Datetime (start), Datetime (end)}); + } + + // for + else if (args.size () == 3 && + args[0] == "" && + args[1] == "for" && + args[2] == "") + { + filter.setRange ({Datetime (start), Datetime (start) + Duration (duration).toTime_t ()}); + } + + // from for + else if (args.size () == 4 && + args[0] == "from" && + args[1] == "" && + args[2] == "for" && + args[3] == "") + { + filter.setRange ({Datetime (start), Datetime (start) + Duration (duration).toTime_t ()}); + } + + // before + else if (args.size () == 3 && + args[0] == "" && + args[1] == "before" && + args[2] == "") + { + filter.setRange ({Datetime (start) - Duration (duration).toTime_t (), Datetime (start)}); + } + + // after + else if (args.size () == 3 && + args[0] == "" && + args[1] == "after" && + args[2] == "") + { + filter.setRange ({Datetime (start), Datetime (start) + Duration (duration).toTime_t ()}); + } + + // ago + else if (args.size () == 2 && + args[0] == "" && + args[1] == "ago") + { + filter.setRange ({now - Duration (duration).toTime_t (), 0}); + } + + // for + else if (args.size () == 2 && + args[0] == "for" && + args[1] == "") + { + filter.setRange ({now - Duration (duration).toTime_t (), now}); + } + + // + else if (args.size () == 1 && + args[0] == "") + { + filter.setRange ({now - Duration (duration).toTime_t (), now}); + } + + // Unrecognized date range construct. + else if (! args.empty ()) + { + throw std::string ("Unrecognized date range: '") + join (" ", args) + "'."; + } + + if (filter.end != 0 && filter.start > filter.end) + throw std::string ("The end of a date range must be after the start."); + + return filter; +} diff --git a/src/CLI.h b/src/CLI.h index a5e393ee..cde10c5b 100644 --- a/src/CLI.h +++ b/src/CLI.h @@ -33,6 +33,7 @@ #include #include #include +#include "Interval.h" // Represents a single argument. class A2 @@ -71,6 +72,7 @@ public: std::string getAnnotation() const; Duration getDuration() const; std::vector getDomReferences () const; + Interval getFilter () const; std::string dump (const std::string& title = "CLI Parser") const; private: diff --git a/src/commands/CmdChart.cpp b/src/commands/CmdChart.cpp index 41508918..9e350ebd 100644 --- a/src/commands/CmdChart.cpp +++ b/src/commands/CmdChart.cpp @@ -46,7 +46,7 @@ int CmdChartDay ( Database& database) { // Create a filter, and if empty, choose the current day. - auto filter = getFilter (cli); + auto filter = cli.getFilter (); if (! filter.is_started ()) { @@ -63,7 +63,7 @@ int CmdChartWeek ( Database& database) { // Create a filter, and if empty, choose the current week. - auto filter = getFilter (cli); + auto filter = cli.getFilter (); if (! filter.is_started ()) { @@ -80,7 +80,7 @@ int CmdChartMonth ( Database& database) { // Create a filter, and if empty, choose the current month. - auto filter = getFilter (cli); + auto filter = cli.getFilter (); if (! filter.is_started ()) { diff --git a/src/commands/CmdContinue.cpp b/src/commands/CmdContinue.cpp index 10307ce7..1c204e6f 100644 --- a/src/commands/CmdContinue.cpp +++ b/src/commands/CmdContinue.cpp @@ -81,7 +81,7 @@ int CmdContinue ( to_copy = latest; } - auto filter = getFilter (cli); + auto filter = cli.getFilter (); Datetime start_time; Datetime end_time; diff --git a/src/commands/CmdExport.cpp b/src/commands/CmdExport.cpp index e971f25a..d836abc1 100644 --- a/src/commands/CmdExport.cpp +++ b/src/commands/CmdExport.cpp @@ -34,7 +34,7 @@ int CmdExport ( Rules& rules, Database& database) { - auto filter = getFilter (cli); + auto filter = cli.getFilter (); std::cout << jsonFromIntervals (getTracked (database, rules, filter)); return 0; } diff --git a/src/commands/CmdGaps.cpp b/src/commands/CmdGaps.cpp index 507eeca0..75e84d0a 100644 --- a/src/commands/CmdGaps.cpp +++ b/src/commands/CmdGaps.cpp @@ -40,7 +40,7 @@ int CmdGaps ( auto verbose = rules.getBoolean ("verbose"); // If filter is empty, choose 'today'. - auto filter = getFilter (cli); + auto filter = cli.getFilter (); if (! filter.is_started ()) { if (rules.has ("reports.gaps.range")) diff --git a/src/commands/CmdModify.cpp b/src/commands/CmdModify.cpp index ddd8798b..035876aa 100644 --- a/src/commands/CmdModify.cpp +++ b/src/commands/CmdModify.cpp @@ -39,7 +39,7 @@ int CmdModify ( { bool verbose = rules.getBoolean ("verbose"); - auto filter = getFilter (cli); + auto filter = cli.getFilter (); std::set ids = cli.getIds (); std::vector words = cli.getWords (); enum { MODIFY_START, MODIFY_END } op = MODIFY_START; diff --git a/src/commands/CmdReport.cpp b/src/commands/CmdReport.cpp index cf7e029e..4ff79292 100644 --- a/src/commands/CmdReport.cpp +++ b/src/commands/CmdReport.cpp @@ -90,7 +90,7 @@ int CmdReport ( throw std::string ("Specify which report to run."); // Compose Header info. - auto filter = getFilter (cli); + auto filter = cli.getFilter (); auto tracked = getTracked (database, rules, filter); rules.set ("temp.report.start", filter.start.toEpoch () > 0 ? filter.start.toISO () : ""); diff --git a/src/commands/CmdStart.cpp b/src/commands/CmdStart.cpp index 6e6e447c..00b42baf 100644 --- a/src/commands/CmdStart.cpp +++ b/src/commands/CmdStart.cpp @@ -37,7 +37,7 @@ int CmdStart ( { auto verbose = rules.getBoolean ("verbose"); - auto filter = getFilter (cli); + auto filter = cli.getFilter (); auto now = Datetime (); diff --git a/src/commands/CmdStop.cpp b/src/commands/CmdStop.cpp index 66418de5..02166641 100644 --- a/src/commands/CmdStop.cpp +++ b/src/commands/CmdStop.cpp @@ -50,7 +50,7 @@ int CmdStop ( auto verbose = rules.getBoolean ("verbose"); // Load the most recent interval. - auto filter = getFilter (cli); + auto filter = cli.getFilter (); auto latest = getLatestInterval (database); std::set ids = cli.getIds (); diff --git a/src/commands/CmdSummary.cpp b/src/commands/CmdSummary.cpp index 6b729d42..89a1e805 100644 --- a/src/commands/CmdSummary.cpp +++ b/src/commands/CmdSummary.cpp @@ -45,7 +45,7 @@ int CmdSummary ( auto verbose = rules.getBoolean ("verbose"); // Create a filter, and if empty, choose 'today'. - auto filter = getFilter (cli); + auto filter = cli.getFilter (); if (! filter.is_started ()) filter.setRange (Datetime ("today"), Datetime ("tomorrow")); diff --git a/src/commands/CmdTags.cpp b/src/commands/CmdTags.cpp index 514c15a1..c024a40f 100644 --- a/src/commands/CmdTags.cpp +++ b/src/commands/CmdTags.cpp @@ -41,7 +41,7 @@ int CmdTags ( auto verbose = rules.getBoolean ("verbose"); // Create a filter, with no default range. - auto filter = getFilter (cli); + auto filter = cli.getFilter (); // Generate a unique, ordered list of tags. std::set tags; diff --git a/src/commands/CmdTrack.cpp b/src/commands/CmdTrack.cpp index f5069a1b..5e9d6e8a 100644 --- a/src/commands/CmdTrack.cpp +++ b/src/commands/CmdTrack.cpp @@ -37,7 +37,7 @@ int CmdTrack ( { auto boolean = rules.getBoolean ("verbose"); - auto filter = getFilter (cli); + auto filter = cli.getFilter (); // If this is not a proper closed interval, then the user is trying to make // the 'track' command behave like 'start', so delegate to CmdStart. diff --git a/src/data.cpp b/src/data.cpp index ab9c9f6a..12e34322 100644 --- a/src/data.cpp +++ b/src/data.cpp @@ -37,192 +37,6 @@ #include #include -//////////////////////////////////////////////////////////////////////////////// -// A filter is just another interval, containing start, end and tags. -// -// Supported interval forms: -// ["from"] ["to"|"-" ] -// ["from"] "for" -// ["before"|"after" ] -// "ago" -// -Interval getFilter (const CLI& cli) -{ - // One instance, so we can directly compare. - Datetime now; - - Interval filter; - std::string start; - std::string end; - std::string duration; - std::vector args; - for (auto& arg : cli._args) - { - if (arg.hasTag ("BINARY") || - arg.hasTag ("CMD") || - arg.hasTag ("EXT")) - continue; - - if (arg.hasTag ("FILTER")) - { - auto canonical = arg.attribute ("canonical"); - auto raw = arg.attribute ("raw"); - - if (arg.hasTag ("HINT")) - { - Range range; - if (expandIntervalHint (canonical, range)) - { - start = range.start.toISO (); - end = range.end.toISO (); - - args.push_back (""); - args.push_back ("-"); - args.push_back (""); - } - - // Hints that are not expandable to a date range are ignored. - } - else if (arg._lextype == Lexer::Type::date) - { - if (start.empty ()) - start = raw; - else if (end.empty ()) - end = raw; - - args.push_back (""); - } - else if (arg._lextype == Lexer::Type::duration) - { - if (duration.empty ()) - duration = raw; - - args.push_back (""); - } - else if (arg.hasTag ("KEYWORD")) - { - // Note: that KEYWORDS are not entities (why not?) and there is a list - // in CLI.cpp of them that must be maintained and synced with this - // function. - args.push_back (raw); - } - else if (arg.hasTag ("ID")) - { - // Not part of a filter. - } - else - { - filter.tag (raw); - } - } - } - - // - if (args.size () == 1 && - args[0] == "") - { - filter.setRange ({Datetime (start), 0}); - } - - // from - else if (args.size () == 2 && - args[0] == "from" && - args[1] == "") - { - filter.setRange ({Datetime (start), 0}); - } - - // to/- - else if (args.size () == 3 && - args[0] == "" && - (args[1] == "to" || args[1] == "-") && - args[2] == "") - { - filter.setRange ({Datetime (start), Datetime (end)}); - } - - // from to/- - else if (args.size () == 4 && - args[0] == "from" && - args[1] == "" && - (args[2] == "to" || args[2] == "-") && - args[3] == "") - { - filter.setRange ({Datetime (start), Datetime (end)}); - } - - // for - else if (args.size () == 3 && - args[0] == "" && - args[1] == "for" && - args[2] == "") - { - filter.setRange ({Datetime (start), Datetime (start) + Duration (duration).toTime_t ()}); - } - - // from for - else if (args.size () == 4 && - args[0] == "from" && - args[1] == "" && - args[2] == "for" && - args[3] == "") - { - filter.setRange ({Datetime (start), Datetime (start) + Duration (duration).toTime_t ()}); - } - - // before - else if (args.size () == 3 && - args[0] == "" && - args[1] == "before" && - args[2] == "") - { - filter.setRange ({Datetime (start) - Duration (duration).toTime_t (), Datetime (start)}); - } - - // after - else if (args.size () == 3 && - args[0] == "" && - args[1] == "after" && - args[2] == "") - { - filter.setRange ({Datetime (start), Datetime (start) + Duration (duration).toTime_t ()}); - } - - // ago - else if (args.size () == 2 && - args[0] == "" && - args[1] == "ago") - { - filter.setRange ({now - Duration (duration).toTime_t (), 0}); - } - - // for - else if (args.size () == 2 && - args[0] == "for" && - args[1] == "") - { - filter.setRange ({now - Duration (duration).toTime_t (), now}); - } - - // - else if (args.size () == 1 && - args[0] == "") - { - filter.setRange ({now - Duration (duration).toTime_t (), now}); - } - - // Unrecognized date range construct. - else if (! args.empty ()) - { - throw std::string ("Unrecognized date range: '") + join (" ", args) + "'."; - } - - if (filter.end != 0 && filter.start > filter.end) - throw std::string ("The end of a date range must be after the start."); - - return filter; -} - //////////////////////////////////////////////////////////////////////////////// // Read rules and extract all holiday definitions. Create a Range for each // one that spans from midnight to midnight. diff --git a/src/timew.h b/src/timew.h index 6c61c0e4..cf35bf27 100644 --- a/src/timew.h +++ b/src/timew.h @@ -37,7 +37,6 @@ #include // data.cpp -Interval getFilter (const CLI&); std::vector getHolidays (const Rules&); std::vector getAllExclusions (const Rules&, const Range&); std::vector getIntervalsByIds (Database&, const Rules&, const std::set &); diff --git a/src/validate.cpp b/src/validate.cpp index a68e577a..0c9cd6c4 100644 --- a/src/validate.cpp +++ b/src/validate.cpp @@ -173,7 +173,7 @@ void validate ( Interval& interval) { // Create a filter, and if empty, choose 'today'. - auto filter = getFilter (cli); + auto filter = cli.getFilter (); if (! filter.is_started ()) filter.setRange (Datetime ("today"), Datetime ("tomorrow")); From 13f1de2c495a7610c3474cc19f502e05fa983e5b Mon Sep 17 00:00:00 2001 From: Thomas Lauf Date: Fri, 8 May 2020 17:05:04 +0200 Subject: [PATCH 009/492] Identify DOM references by their prefix ('dom.<...>') and mark them as such when analyzing the command line Signed-off-by: Thomas Lauf --- src/CLI.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/CLI.cpp b/src/CLI.cpp index bc9d9315..89730923 100644 --- a/src/CLI.cpp +++ b/src/CLI.cpp @@ -522,6 +522,10 @@ void CLI::identifyFilter () a.tag ("KEYWORD"); } + else if (raw.rfind("dom.",0) == 0) + { + a.tag ("DOM"); + } else { a.tag ("FILTER"); @@ -614,8 +618,7 @@ std::vector CLI::getDomReferences () const for (auto &arg : _args) { - if (arg.hasTag ("TAG") && - arg.hasTag ("FILTER")) + if (arg.hasTag ("DOM")) { references.emplace_back (arg.attribute ("raw")); } From a508dd21eac0e78498003dc36465147acab50a5b Mon Sep 17 00:00:00 2001 From: Thomas Lauf Date: Fri, 8 May 2020 17:08:49 +0200 Subject: [PATCH 010/492] Get and use filter from CLI when processing DOM references - Closes #188 Signed-off-by: Thomas Lauf --- src/commands/CmdGet.cpp | 2 +- test/dom.t | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/commands/CmdGet.cpp b/src/commands/CmdGet.cpp index 90c7aed3..6912593c 100644 --- a/src/commands/CmdGet.cpp +++ b/src/commands/CmdGet.cpp @@ -39,8 +39,8 @@ int CmdGet ( { std::vector results; std::vector references = cli.getDomReferences (); + Interval filter = cli.getFilter (); - Interval filter; for (auto& reference : references) { std::string value; diff --git a/test/dom.t b/test/dom.t index f31466d5..a09697ef 100755 --- a/test/dom.t +++ b/test/dom.t @@ -197,6 +197,38 @@ class TestDOMTracked(TestCase): code, out, err = self.t("get dom.tracked.tags") self.assertEqual("bar foo \n", out) + def test_dom_tracked_tags_filtered_by_time(self): + """Test dom.tracked.tags with tags filtered by time""" + now_utc = datetime.now().utcnow() + one_hour_before_utc = now_utc - timedelta(hours=1) + two_hours_before_utc = now_utc - timedelta(hours=2) + three_hours_before_utc = now_utc - timedelta(hours=3) + four_hours_before_utc = now_utc - timedelta(hours=4) + five_hours_before_utc = now_utc - timedelta(hours=5) + + self.t("track {:%Y-%m-%dT%H:%M:%S}Z - {:%Y-%m-%dT%H:%M:%S}Z foo".format(five_hours_before_utc, four_hours_before_utc)) + self.t("track {:%Y-%m-%dT%H:%M:%S}Z - {:%Y-%m-%dT%H:%M:%S}Z bar".format(three_hours_before_utc, two_hours_before_utc)) + self.t("track {:%Y-%m-%dT%H:%M:%S}Z - {:%Y-%m-%dT%H:%M:%S}Z baz".format(two_hours_before_utc, one_hour_before_utc)) + + code, out, err = self.t("get dom.tracked.tags {:%Y-%m-%dT%H:%M:%S}Z - {:%Y-%m-%dT%H:%M:%S}Z".format(five_hours_before_utc, two_hours_before_utc)) + self.assertEqual("bar foo \n", out) + + def test_dom_tracked_tags_filtered_by_tag(self): + """Test dom.tracked.tags with tags filtered by tag""" + now_utc = datetime.now().utcnow() + one_hour_before_utc = now_utc - timedelta(hours=1) + two_hours_before_utc = now_utc - timedelta(hours=2) + three_hours_before_utc = now_utc - timedelta(hours=3) + four_hours_before_utc = now_utc - timedelta(hours=4) + five_hours_before_utc = now_utc - timedelta(hours=5) + + self.t("track {:%Y-%m-%dT%H:%M:%S}Z - {:%Y-%m-%dT%H:%M:%S}Z foo".format(five_hours_before_utc, four_hours_before_utc)) + self.t("track {:%Y-%m-%dT%H:%M:%S}Z - {:%Y-%m-%dT%H:%M:%S}Z bar".format(three_hours_before_utc, two_hours_before_utc)) + self.t("track {:%Y-%m-%dT%H:%M:%S}Z - {:%Y-%m-%dT%H:%M:%S}Z foo baz".format(two_hours_before_utc, one_hour_before_utc)) + + code, out, err = self.t("get dom.tracked.tags foo") + self.assertEqual("baz foo \n", out) + def test_dom_tracked_N_tag_count_zero(self): """Test dom.tracked.N.tag.count with zero tags""" self.t("track :yesterday one two") From 2169e8e9282fd724cda2153f5ca243584676ebc9 Mon Sep 17 00:00:00 2001 From: Thomas Lauf Date: Fri, 8 May 2020 17:31:39 +0200 Subject: [PATCH 011/492] Add/update descriptive comments Signed-off-by: Thomas Lauf --- src/dom.cpp | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/dom.cpp b/src/dom.cpp index c1cdb026..3960000e 100644 --- a/src/dom.cpp +++ b/src/dom.cpp @@ -43,6 +43,7 @@ bool domGet ( Pig pig (reference); if (pig.skipLiteral ("dom.")) { + // dom.active if (pig.skipLiteral ("active")) { auto latest = getLatestInterval (database); @@ -103,6 +104,7 @@ bool domGet ( } } + // dom.tracked.<...> else if (pig.skipLiteral ("tracked.")) { auto tracked = getTracked (database, rules, filter); @@ -139,25 +141,26 @@ bool domGet ( } int n; + // dom.tracked..<...> if (pig.getDigits (n) && n <= count && pig.skipLiteral (".")) { - // dom.tracked.N.tag.count + // dom.tracked..tag.count if (pig.skipLiteral ("tag.count")) { value = format ("{1}", tracked[count - n].tags ().size ()); return true; } - // dom.tracked.N.start + // dom.tracked..start if (pig.skipLiteral ("start")) { value = tracked[count - n].start.toISOLocalExtended (); return true; } - // dom.tracked.N.end + // dom.tracked..end if (pig.skipLiteral ("end")) { if (tracked[count -n].is_open ()) @@ -167,14 +170,14 @@ bool domGet ( return true; } - // dom.tracked.N.duration + // dom.tracked..duration if (pig.skipLiteral ("duration")) { value = Duration (tracked[count - n].total ()).formatISO (); return true; } - // dom.tracked.N.json + // dom.tracked..json if (pig.skipLiteral ("json")) { value = tracked[count - n].json (); @@ -182,6 +185,7 @@ bool domGet ( } int n; + // dom.tracked..tag. if (pig.skipLiteral ("tag.") && pig.getDigits (n)) { @@ -198,6 +202,7 @@ bool domGet ( } } + // dom.tag.<...> else if (pig.skipLiteral ("tag.")) { // get unique, ordered list of tags. @@ -210,8 +215,8 @@ bool domGet ( return true; } - // dom.tag. int n; + // dom.tag. if (pig.getDigits (n)) { if (n <= static_cast (tags.size ())) From c71eeb6dfd080ba09547a281a8bddd6bac0392d1 Mon Sep 17 00:00:00 2001 From: Thomas Lauf Date: Fri, 8 May 2020 17:32:28 +0200 Subject: [PATCH 012/492] Add DOM-query for ids - Closes #126 Signed-off-by: Thomas Lauf --- src/dom.cpp | 12 ++++++++++++ test/dom.t | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/src/dom.cpp b/src/dom.cpp index 3960000e..9e494550 100644 --- a/src/dom.cpp +++ b/src/dom.cpp @@ -133,6 +133,18 @@ bool domGet ( return true; } + // dom.tracked.ids + if (pig.skipLiteral ("ids")) + { + std::stringstream s; + for (auto& interval : tracked) + { + s << format ( "@{1} ", interval.id ); + } + value = s.str(); + return true; + } + // dom.tracked.count if (pig.skipLiteral ("count")) { diff --git a/test/dom.t b/test/dom.t index a09697ef..dfe947ba 100755 --- a/test/dom.t +++ b/test/dom.t @@ -229,6 +229,54 @@ class TestDOMTracked(TestCase): code, out, err = self.t("get dom.tracked.tags foo") self.assertEqual("baz foo \n", out) + def test_dom_tracked_ids_with_emtpy_database(self): + """Test dom.tracked.ids with empty database""" + code, out, err = self.t("get dom.tracked.tags") + self.assertEqual("\n", out) + + def test_dom_tracked_ids(self): + """Test dom.tracked.ids""" + now_utc = datetime.now().utcnow() + four_hours_before_utc = now_utc - timedelta(hours=4) + five_hours_before_utc = now_utc - timedelta(hours=5) + + self.t("track {:%Y-%m-%dT%H:%M:%S}Z - {:%Y-%m-%dT%H:%M:%S}Z".format(five_hours_before_utc, four_hours_before_utc)) + + code, out, err = self.t("get dom.tracked.ids") + self.assertEqual("@1 \n", out) + + def test_dom_tracked_ids_filtered_by_time(self): + """Test dom.tracked.ids filtered by time""" + now_utc = datetime.now().utcnow() + one_hour_before_utc = now_utc - timedelta(hours=1) + two_hours_before_utc = now_utc - timedelta(hours=2) + three_hours_before_utc = now_utc - timedelta(hours=3) + four_hours_before_utc = now_utc - timedelta(hours=4) + five_hours_before_utc = now_utc - timedelta(hours=5) + + self.t("track {:%Y-%m-%dT%H:%M:%S}Z - {:%Y-%m-%dT%H:%M:%S}Z foo".format(five_hours_before_utc, four_hours_before_utc)) + self.t("track {:%Y-%m-%dT%H:%M:%S}Z - {:%Y-%m-%dT%H:%M:%S}Z bar".format(three_hours_before_utc, two_hours_before_utc)) + self.t("track {:%Y-%m-%dT%H:%M:%S}Z - {:%Y-%m-%dT%H:%M:%S}Z baz".format(two_hours_before_utc, one_hour_before_utc)) + + code, out, err = self.t("get dom.tracked.ids {:%Y-%m-%dT%H:%M:%S}Z - {:%Y-%m-%dT%H:%M:%S}Z".format(five_hours_before_utc, two_hours_before_utc)) + self.assertEqual("@3 @2 \n", out) + + def test_dom_tracked_ids_filtered_by_tag(self): + """Test dom.tracked.ids filtered by tag""" + now_utc = datetime.now().utcnow() + one_hour_before_utc = now_utc - timedelta(hours=1) + two_hours_before_utc = now_utc - timedelta(hours=2) + three_hours_before_utc = now_utc - timedelta(hours=3) + four_hours_before_utc = now_utc - timedelta(hours=4) + five_hours_before_utc = now_utc - timedelta(hours=5) + + self.t("track {:%Y-%m-%dT%H:%M:%S}Z - {:%Y-%m-%dT%H:%M:%S}Z foo".format(five_hours_before_utc, four_hours_before_utc)) + self.t("track {:%Y-%m-%dT%H:%M:%S}Z - {:%Y-%m-%dT%H:%M:%S}Z bar".format(three_hours_before_utc, two_hours_before_utc)) + self.t("track {:%Y-%m-%dT%H:%M:%S}Z - {:%Y-%m-%dT%H:%M:%S}Z foo baz".format(two_hours_before_utc, one_hour_before_utc)) + + code, out, err = self.t("get dom.tracked.ids foo") + self.assertEqual("@3 @1 \n", out) + def test_dom_tracked_N_tag_count_zero(self): """Test dom.tracked.N.tag.count with zero tags""" self.t("track :yesterday one two") From 7e11fde9921e779eac3db58d8deea751498fa52a Mon Sep 17 00:00:00 2001 From: Thomas Lauf Date: Fri, 22 May 2020 08:57:33 +0200 Subject: [PATCH 013/492] Remove template.t from test folder - This file was development only, now any other test file can serve as template for new tests - Closes #303 Signed-off-by: Thomas Lauf --- test/template.t | 120 ------------------------------------------------ 1 file changed, 120 deletions(-) delete mode 100644 test/template.t diff --git a/test/template.t b/test/template.t deleted file mode 100644 index a96588e3..00000000 --- a/test/template.t +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env python3 - -############################################################################### -# -# Copyright 2016 - 2019, Thomas Lauf, Paul Beckingham, Federico Hernandez. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# https://www.opensource.org/licenses/mit-license.php -# -############################################################################### - -import os -import sys -import unittest - -from datetime import datetime - -# Ensure python finds the local simpletap module -sys.path.append(os.path.dirname(os.path.abspath(__file__))) - -from basetest import Timew, TestCase - - -# Test methods available: -# self.assertEqual(a, b) -# self.assertNotEqual(a, b) -# self.assertTrue(x) -# self.assertFalse(x) -# self.assertIs(a, b) -# self.assertIsNot(substring, text) -# self.assertIsNone(x) -# self.assertIsNotNone(x) -# self.assertIn(substring, text) -# self.assertNotIn(substring, text -# self.assertRaises(e) -# self.assertRegex(text, pattern) -# self.assertNotRegexpMatches(text, pattern) -# self.tap("") - - -class TestBugNumber(TestCase): - @classmethod - def setUpClass(cls): - """Executed once before any test in the class""" - # Used to initialize objects that can be shared across tests - # Also useful if none of the tests of the current TestCase performs - # data alterations. See tw-285.t for an example - - def setUp(self): - """Executed before each test in the class""" - # Used to initialize objects that should be re-initialized or - # re-created for each individual test - self.t = Timew() - - def test_foo(self): - """Test foo""" - code, out, err = self.t("foo") - self.tap(out) - self.tap(err) - - def test_faketime(self): - """Running tests using libfaketime - - WARNING: - faketime version 0.9.6 and later correctly propagates non-zero - exit codes. Please don't combine faketime tests and - self.t.runError(). - - https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=750721 - """ - self.t.faketime("-2y") - - command = ("insert test here") - self.t(command) - - # Remove FAKETIME settings - self.t.faketime() - - code, out, err = self.t("insert test here") - expected = "2.0y" - self.assertIn(expected, out) - - @unittest.skipIf(1 != 0, "This machine has sane logic") - def test_skipped(self): - """Test all logic of the world""" - - @unittest.expectedFailure - def test_expected_failure(self): - """Test something that fails and we know or expect that""" - self.assertEqual(1, 0) - - def tearDown(self): - """Executed after each test in the class""" - - @classmethod - def tearDownClass(cls): - """Executed once after all tests in the class""" - - -if __name__ == "__main__": - from simpletap import TAPTestRunner - - unittest.main(testRunner=TAPTestRunner()) From 6117ce1a847a1daf936541d62a7d25aba0059a5a Mon Sep 17 00:00:00 2001 From: Thomas Lauf Date: Wed, 3 Jun 2020 17:38:41 +0200 Subject: [PATCH 014/492] Introduce default range parameter when requesting filter from CLI Signed-off-by: Thomas Lauf --- src/CLI.cpp | 9 +++++++-- src/CLI.h | 2 +- src/commands/CmdChart.cpp | 30 ++++++++++++------------------ src/commands/CmdSummary.cpp | 4 +--- 4 files changed, 21 insertions(+), 24 deletions(-) diff --git a/src/CLI.cpp b/src/CLI.cpp index 89730923..30ea1f66 100644 --- a/src/CLI.cpp +++ b/src/CLI.cpp @@ -636,7 +636,7 @@ std::vector CLI::getDomReferences () const // ["before"|"after" ] // "ago" // -Interval CLI::getFilter () const +Interval CLI::getFilter (const Range& default_range) const { // One instance, so we can directly compare. Datetime now; @@ -708,8 +708,13 @@ Interval CLI::getFilter () const } } + if (args.empty ()) + { + filter.setRange(default_range); + } + // - if (args.size () == 1 && + else if (args.size () == 1 && args[0] == "") { filter.setRange ({Datetime (start), 0}); diff --git a/src/CLI.h b/src/CLI.h index cde10c5b..b1d4b676 100644 --- a/src/CLI.h +++ b/src/CLI.h @@ -72,7 +72,7 @@ public: std::string getAnnotation() const; Duration getDuration() const; std::vector getDomReferences () const; - Interval getFilter () const; + Interval getFilter (const Range& = {}) const; std::string dump (const std::string& title = "CLI Parser") const; private: diff --git a/src/commands/CmdChart.cpp b/src/commands/CmdChart.cpp index 9e350ebd..e52d0986 100644 --- a/src/commands/CmdChart.cpp +++ b/src/commands/CmdChart.cpp @@ -45,13 +45,11 @@ int CmdChartDay ( Rules& rules, Database& database) { - // Create a filter, and if empty, choose the current day. - auto filter = cli.getFilter (); + Range default_range = {}; + expandIntervalHint (rules.get ("reports.day.range", ":day"), default_range); - if (! filter.is_started ()) - { - expandIntervalHint (rules.get ("reports.day.range", ":day"), filter); - } + // Create a filter, and if empty, choose the current day. + auto filter = cli.getFilter (default_range); return renderChart (cli, "day", filter, rules, database); } @@ -62,13 +60,11 @@ int CmdChartWeek ( Rules& rules, Database& database) { - // Create a filter, and if empty, choose the current week. - auto filter = cli.getFilter (); + Range default_range = {}; + expandIntervalHint (rules.get ("reports.week.range", ":week"), default_range); - if (! filter.is_started ()) - { - expandIntervalHint (rules.get ("reports.week.range", ":week"), filter); - } + // Create a filter, and if empty, choose the current week. + auto filter = cli.getFilter (default_range); return renderChart (cli, "week", filter, rules, database); } @@ -79,13 +75,11 @@ int CmdChartMonth ( Rules& rules, Database& database) { - // Create a filter, and if empty, choose the current month. - auto filter = cli.getFilter (); + Range default_range = {}; + expandIntervalHint (rules.get ("reports.month.range", ":month"), default_range); - if (! filter.is_started ()) - { - expandIntervalHint (rules.get ("reports.month.range", ":month"), filter); - } + // Create a filter, and if empty, choose the current month. + auto filter = cli.getFilter (default_range); return renderChart (cli, "month", filter, rules, database); } diff --git a/src/commands/CmdSummary.cpp b/src/commands/CmdSummary.cpp index 89a1e805..15536acf 100644 --- a/src/commands/CmdSummary.cpp +++ b/src/commands/CmdSummary.cpp @@ -45,9 +45,7 @@ int CmdSummary ( auto verbose = rules.getBoolean ("verbose"); // Create a filter, and if empty, choose 'today'. - auto filter = cli.getFilter (); - if (! filter.is_started ()) - filter.setRange (Datetime ("today"), Datetime ("tomorrow")); + auto filter = cli.getFilter (Range { Datetime ("today"), Datetime ("tomorrow") }); if (! filter.is_ended()) filter.end = filter.start + Duration("1d").toTime_t(); From c9eec956e4866c8af47f73b27c2f3975ffd01a35 Mon Sep 17 00:00:00 2001 From: Thomas Lauf Date: Wed, 3 Jun 2020 17:59:11 +0200 Subject: [PATCH 015/492] Add hint `:all` - Closes #206 Signed-off-by: Thomas Lauf --- src/CLI.cpp | 23 ++++++++++++++++++----- src/helper.cpp | 8 ++++++++ src/init.cpp | 1 + 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/src/CLI.cpp b/src/CLI.cpp index 30ea1f66..4ead2d8f 100644 --- a/src/CLI.cpp +++ b/src/CLI.cpp @@ -664,12 +664,19 @@ Interval CLI::getFilter (const Range& default_range) const Range range; if (expandIntervalHint (canonical, range)) { - start = range.start.toISO (); - end = range.end.toISO (); + if (range.is_empty ()) + { + args.push_back (""); + } + else + { + start = range.start.toISO (); + end = range.end.toISO (); - args.push_back (""); - args.push_back ("-"); - args.push_back (""); + args.push_back (""); + args.push_back ("-"); + args.push_back (""); + } } // Hints that are not expandable to a date range are ignored. @@ -807,6 +814,12 @@ Interval CLI::getFilter (const Range& default_range) const filter.setRange ({now - Duration (duration).toTime_t (), now}); } + // :all + else if (args.size () == 1 && args[0] == "") + { + filter.setRange (0, 0); + } + // Unrecognized date range construct. else if (! args.empty ()) { diff --git a/src/helper.cpp b/src/helper.cpp index fc901318..10ff71f9 100644 --- a/src/helper.cpp +++ b/src/helper.cpp @@ -176,6 +176,14 @@ bool expandIntervalHint ( return true; } + if (hint == ":all") + { + range.start = 0; + range.end = 0; + + return true; + } + // Some require math. if (hint == ":lastweek") { diff --git a/src/init.cpp b/src/init.cpp index 90c5384b..881136da 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -88,6 +88,7 @@ void initializeEntities (CLI& cli) cli.entity ("extension", "week"); // Hint entities. + cli.entity ("hint", ":all"); cli.entity ("hint", ":adjust"); cli.entity ("hint", ":blank"); cli.entity ("hint", ":color"); From 83ec55cdc6d5acff27a9d83557606db2cbd0cf25 Mon Sep 17 00:00:00 2001 From: Thomas Lauf Date: Wed, 3 Jun 2020 18:40:29 +0200 Subject: [PATCH 016/492] Remove unused filter Signed-off-by: Thomas Lauf --- src/validate.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/validate.cpp b/src/validate.cpp index 0c9cd6c4..ee754ae1 100644 --- a/src/validate.cpp +++ b/src/validate.cpp @@ -172,14 +172,11 @@ void validate ( Database& database, Interval& interval) { - // Create a filter, and if empty, choose 'today'. - auto filter = cli.getFilter (); - if (! filter.is_started ()) - filter.setRange (Datetime ("today"), Datetime ("tomorrow")); - // All validation performed here. if (findHint (cli, ":fill")) + { autoFill (rules, database, interval); + } autoAdjust (findHint (cli, ":adjust"), rules, database, interval); } From 59d0f1263f1ea9eb585773833ebbc410d958931f Mon Sep 17 00:00:00 2001 From: Shaun Ruffell Date: Sat, 9 May 2020 14:42:18 -0500 Subject: [PATCH 017/492] Mask signals while updating database In order to keep the database consistent, we really want all the AtomicFiles to be finalized as a group. This will prevent an inopportune signal from interrupting this process. Signed-off-by: Shaun Ruffell --- src/AtomicFile.cpp | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/AtomicFile.cpp b/src/AtomicFile.cpp index 8e12ecef..e38cc496 100644 --- a/src/AtomicFile.cpp +++ b/src/AtomicFile.cpp @@ -24,11 +24,13 @@ // //////////////////////////////////////////////////////////////////////////////// +#include #include #include #include #include #include +#include #include #include @@ -231,6 +233,7 @@ void AtomicFile::impl::finalize () { if (is_temp_active && impl::allow_atomics) { + debug (format ("Moving '{1}' -> '{2}'", temp_file._data, real_file._data)); if (std::rename (temp_file._data.c_str (), real_file._data.c_str ())) { throw format("Failed copying '{1}' to '{2}'. Database corruption possible.", @@ -415,13 +418,23 @@ void AtomicFile::finalize_all () file->close (); } - atomic_files_t new_atomic_files; - // Step 2: Rename the temp files to the *real* files + sigset_t new_mask; + sigset_t old_mask; + sigfillset (&new_mask); + + // Step 2: Rename the temp files to the *real* file + sigprocmask (SIG_SETMASK, &new_mask, &old_mask); for (auto& file : impl::atomic_files) { file->finalize (); + } + sigprocmask (SIG_SETMASK, &old_mask, nullptr); + // Step 3: Cleanup any references + atomic_files_t new_atomic_files; + for (auto& file : impl::atomic_files) + { // Delete entry if we are holding the last reference if (file.use_count () > 1) { From 90081afb9783151c27006113c283434cb9980771 Mon Sep 17 00:00:00 2001 From: George Buckingham Date: Thu, 11 Jun 2020 12:22:51 +0200 Subject: [PATCH 018/492] Update docs links from taskwarrior.org to timewarrior.net Signed-off-by: George Buckingham --- CONTRIBUTING.md | 2 +- doc/man1/timew.1.in | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1dfe0fb7..460d8e60 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -96,7 +96,7 @@ Extensions are standalone programs or scripts that consume Timewarrior data outp Extensions are the easiest way to contribute as you only need to adhere to the extension API, but are otherwise free in your choice of programming language and style. -Consult the [documentation](https://taskwarrior.org/docs/timewarrior/api.html) on how you can use the extension API. +Consult the [documentation](https://timewarrior.net/docs/api.html) on how you can use the extension API. ### Tests, Bug-fixes and Features diff --git a/doc/man1/timew.1.in b/doc/man1/timew.1.in index 57db9c73..710b4116 100644 --- a/doc/man1/timew.1.in +++ b/doc/man1/timew.1.in @@ -16,7 +16,7 @@ It allows you to easily track your time and generate summary reports. This is a reference, not a tutorial. If you are looking for a tutorial, check the online documentation here: .br - https://taskwarrior.org/docs/timewarrior + https://timewarrior.net/docs/ .br When run without arguments or options, the default command is run, which indicates whether there is any active tracking, and if so, shows a summary, then exits with a code 0. If there is no active time tracking, exit code is 1. @@ -193,7 +193,7 @@ Display week chart For examples please see the online documentation starting at: . .RS - + .RE . Note that the online documentation can be more detailed and more current than this man page. From 2906f368304b701096dca92c20318514a9b4c4a4 Mon Sep 17 00:00:00 2001 From: Thomas Lauf Date: Fri, 5 Jun 2020 10:26:25 +0200 Subject: [PATCH 019/492] Add test for command `summary` with `:all` hint Signed-off-by: Thomas Lauf --- test/summary.t | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/test/summary.t b/test/summary.t index fa21017b..e77680e9 100755 --- a/test/summary.t +++ b/test/summary.t @@ -27,11 +27,10 @@ ############################################################################### import os +import sys import unittest from datetime import datetime, timedelta -import sys - # Ensure python finds the local simpletap module sys.path.append(os.path.dirname(os.path.abspath(__file__))) @@ -155,6 +154,29 @@ W10 2017-03-09 Thu @4 Tag1 8:43:08 9:38:15 0:55:07 1:09:03 """, out) + def test_with_all_hint(self): + """Summary should work with :all hint""" + now = datetime.now() + yesterday = now - timedelta(days=1) + tomorrow = now + timedelta(days=1) + + self.t("track {0:%Y-%m-%d}T10:00:00 - {0:%Y-%m-%d}T11:00:00 FOO".format(yesterday)) + self.t("track {0:%Y-%m-%d}T10:00:00 - {0:%Y-%m-%d}T11:00:00 BAR".format(now)) + self.t("track {0:%Y-%m-%d}T10:00:00 - {0:%Y-%m-%d}T11:00:00 BAZ".format(tomorrow)) + + code, out, err = self.t("summary :ids :all") + + self.assertIn(""" +Wk Date Day ID Tags Start End Time Total +--- ---------- --- -- ---- -------- -------- ------- ------- +W{3} {0:%Y-%m-%d} {0:%a} @3 FOO 10:00:00 11:00:00 1:00:00 1:00:00 +W{4} {1:%Y-%m-%d} {1:%a} @2 BAR 10:00:00 11:00:00 1:00:00 1:00:00 +W{5} {2:%Y-%m-%d} {2:%a} @1 BAZ 10:00:00 11:00:00 1:00:00 1:00:00 + + 3:00:00 +""".format(yesterday, now, tomorrow, + yesterday.isocalendar()[1], now.isocalendar()[1], tomorrow.isocalendar()[1]), out) + def test_with_day_gap(self): """Summary should skip days with no data""" self.t("track 2017-03-09T10:00:00 - 2017-03-09T11:00:00") From 63f230013a73843e9fbfd0e40b2116ebead49eb0 Mon Sep 17 00:00:00 2001 From: Thomas Lauf Date: Fri, 5 Jun 2020 13:35:19 +0200 Subject: [PATCH 020/492] Detect whether a date is meant as range - Add DatetimeParser::parse_range: If a date contains no time, it is assumed to be a fixed range, else an open range starting at given datetime - Add tests for summary with named dates 'yesterday' and 'today' - Remove closing of filter in CmdSummary.cpp - Closes #333 Signed-off-by: Thomas Lauf --- src/CLI.cpp | 6 +- src/CMakeLists.txt | 1 + src/DatetimeParser.cpp | 2924 +++++++++++++++++++++++++++++++++++ src/DatetimeParser.h | 148 ++ src/commands/CmdSummary.cpp | 9 +- test/CMakeLists.txt | 2 +- test/DatetimeParser.t.cpp | 634 ++++++++ test/summary.t | 40 + 8 files changed, 3757 insertions(+), 7 deletions(-) create mode 100644 src/DatetimeParser.cpp create mode 100644 src/DatetimeParser.h create mode 100644 test/DatetimeParser.t.cpp diff --git a/src/CLI.cpp b/src/CLI.cpp index 4ead2d8f..020c8538 100644 --- a/src/CLI.cpp +++ b/src/CLI.cpp @@ -36,6 +36,7 @@ #include #include #include +#include "DatetimeParser.h" //////////////////////////////////////////////////////////////////////////////// A2::A2 (const std::string& raw, Lexer::Type lextype) @@ -724,7 +725,9 @@ Interval CLI::getFilter (const Range& default_range) const else if (args.size () == 1 && args[0] == "") { - filter.setRange ({Datetime (start), 0}); + DatetimeParser dtp; + Range range = dtp.parse_range(start); + filter.setRange (range); } // from @@ -734,7 +737,6 @@ Interval CLI::getFilter (const Range& default_range) const { filter.setRange ({Datetime (start), 0}); } - // to/- else if (args.size () == 3 && args[0] == "" && diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 268b7e5f..f0df1cfb 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -11,6 +11,7 @@ set (timew_SRCS AtomicFile.cpp AtomicFile.h ChartConfig.h Database.cpp Database.h Datafile.cpp Datafile.h + DatetimeParser.cpp DatetimeParser.h Exclusion.cpp Exclusion.h Extensions.cpp Extensions.h Interval.cpp Interval.h diff --git a/src/DatetimeParser.cpp b/src/DatetimeParser.cpp new file mode 100644 index 00000000..c715f652 --- /dev/null +++ b/src/DatetimeParser.cpp @@ -0,0 +1,2924 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// Copyright 2020, Thomas Lauf, Paul Beckingham, Federico Hernandez +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// http://www.opensource.org/licenses/mit-license.php +// +//////////////////////////////////////////////////////////////////////////////// + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static std::vector dayNames { + "sunday", + "monday", + "tuesday", + "wednesday", + "thursday", + "friday", + "saturday"}; + +static std::vector monthNames { + "january", + "february", + "march", + "april", + "may", + "june", + "july", + "august", + "september", + "october", + "november", + "december"}; + +//////////////////////////////////////////////////////////////////////////////// +Range DatetimeParser::parse_range (const std::string& input) +{ + clear (); + std::string::size_type start = 0; + auto i = start; + Pig pig (input); + if (i) + pig.skipN (static_cast (i)); + + auto checkpoint = pig.cursor (); + + // Parse epoch first, as it's the most common scenario. + if (parse_epoch (pig)) + { + // ::validate and ::resolve are not needed in this case. + start = pig.cursor (); + return Range {}; + } + + // Allow parse_date_time and parse_date_time_ext regardless of + // DatetimeParser::isoEnabled setting, because these formats are relied upon by + // the 'import' command, JSON parser and hook system. + if (parse_date_time_ext (pig) || // Strictest first. + parse_date_time (pig)) + { + // Check the values and determine time_t. + if (validate ()) + { + start = pig.cursor (); + resolve (); + return Range { Datetime {_date}, 0 }; + } + } + + // Allow parse_date_time and parse_date_time_ext regardless of + // DatetimeParser::isoEnabled setting, because these formats are relied upon by + // the 'import' command, JSON parser and hook system. + if (Datetime::isoEnabled && + ( parse_date_ext (pig) || + (Datetime::standaloneDateEnabled && parse_date (pig)) + ) + ) + { + // Check the values and determine time_t. + if (validate ()) + { + start = pig.cursor (); + resolve (); + + if (_day != 0) + { + auto start_date = Datetime (_date); + auto end_date = start_date + Duration ("1d").toTime_t (); + return Range{start_date, end_date }; + } + else if (_month != 0) + { + auto start_date = Datetime (_date); + auto end_date = Datetime(start_date.year(), start_date.month()+1, 1); + return Range { start_date, end_date }; + } + else if (_year != 0) + { + auto start_date = Datetime (_date); + auto end_date = Datetime(start_date.year()+1, 1, 1); + return Range { start_date, end_date }; + } + return Range {}; + } + } + + // Allow parse_date_time and parse_date_time_ext regardless of + // DatetimeParser::isoEnabled setting, because these formats are relied upon by + // the 'import' command, JSON parser and hook system. + if (Datetime::isoEnabled && + ( parse_time_utc_ext (pig) || + parse_time_utc (pig) || + parse_time_off_ext (pig) || + parse_time_off (pig) || + parse_time_ext (pig) || + (Datetime::standaloneTimeEnabled && parse_time (pig)) // Time last, as it is the most permissive. + ) + ) + { + // Check the values and determine time_t. + if (validate ()) + { + start = pig.cursor (); + resolve (); + return Range { Datetime (_date), 0 }; + } + } + + pig.restoreTo (checkpoint); + + if (parse_informal_time (pig)) + { + return Range { Datetime {_date}, 0 }; + } + + if (parse_named_day (pig)) + { + // ::validate and ::resolve are not needed in this case. + start = pig.cursor (); + return Range { Datetime (_date), Datetime (_date) + Duration ("1d").toTime_t () }; + } + + if (parse_named_month (pig)) + { + // ::validate and ::resolve are not needed in this case. + start = pig.cursor (); + auto begin = Datetime (_date); + auto month = (begin.month() + 1) % 13 + (begin.month() == 12); + auto year = (begin.year() + (begin.month() == 12)); + auto end = Datetime (year, month, 1); + return Range { begin, end }; + } + + if (parse_named (pig)) + { + // ::validate and ::resolve are not needed in this case. + start = pig.cursor (); + return Range { Datetime (_date), 0 }; + } + + throw format ("'{1}' is not a valid range.", input); +} + +//////////////////////////////////////////////////////////////////////////////// +void DatetimeParser::clear () +{ + _year = 0; + _month = 0; + _week = 0; + _weekday = 0; + _julian = 0; + _day = 0; + _seconds = 0; + _offset = 0; + _utc = false; + _date = 0; +} + +//////////////////////////////////////////////////////////////////////////////// +// Note how these are all single words. +// +// Examples and descriptions, assuming now == 2017-03-05T12:34:56. +// +// Example Notes +// ------------------- ------------------ +// yesterday 2017-03-04T00:00:00 Unaffected +// today 2017-03-05T00:00:00 Unaffected +// tomorrow 2017-03-06T00:00:00 Unaffected +// 12th 2017-03-12T00:00:00 +// monday 2017-03-06T00:00:00 +// easter 2017-04-16T00:00:00 +// eastermonday 2017-04-16T00:00:00 +// ascension 2017-05-25T00:00:00 +// pentecost 2017-06-04T00:00:00 +// goodfriday 2017-04-14T00:00:00 +// midsommar 2017-06-24T00:00:00 midnight, 1st Saturday after 20th June +// midsommarafton 2017-06-23T00:00:00 midnight, 1st Friday after 19th June +// juhannus 2017-06-23T00:00:00 midnight, 1st Friday after 19th June +bool DatetimeParser::parse_named_day (Pig& pig) +{ + auto checkpoint = pig.cursor (); + + if (initializeYesterday (pig) || + initializeToday (pig) || + initializeTomorrow (pig) || + initializeOrdinal (pig) || + initializeDayName (pig) || + initializeEaster (pig) || + initializeMidsommar (pig) || + initializeMidsommarafton (pig)) + { + return true; + } + + pig.restoreTo (checkpoint); + return false; +} + +//////////////////////////////////////////////////////////////////////////////// +bool DatetimeParser::parse_named_month (Pig& pig) +{ + auto checkpoint = pig.cursor (); + + if (initializeMonthName (pig)) + { + return true; + } + + pig.restoreTo (checkpoint); + return false; +} + +//////////////////////////////////////////////////////////////////////////////// +bool DatetimeParser::parse_informal_time (Pig& pig) +{ + auto checkpoint = pig.cursor (); + + if (initializeInformalTime (pig)) + { + return true; + } + + pig.restoreTo (checkpoint); + return false; +} + +//////////////////////////////////////////////////////////////////////////////// +// Note how these are all single words. +// +// Examples and descriptions, assuming now == 2017-03-05T12:34:56. +// +// Example Notes +// ------------------- ------------------ +// now 2017-03-05T12:34:56 Unaffected +// yesterday 2017-03-04T00:00:00 Unaffected +// today 2017-03-05T00:00:00 Unaffected +// tomorrow 2017-03-06T00:00:00 Unaffected +// 12th 2017-03-12T00:00:00 +// monday 2017-03-06T00:00:00 +// april 2017-04-01T00:00:00 +// later 2038-01-18T00:00:00 Unaffected +// someday 2038-01-18T00:00:00 Unaffected +// sopd 2017-03-04T00:00:00 Unaffected +// sod 2017-03-05T00:00:00 Unaffected +// sond 2017-03-06T00:00:00 Unaffected +// eopd 2017-03-05T00:00:00 Unaffected +// eod 2017-03-06T00:00:00 Unaffected +// eond 2017-03-07T00:00:00 Unaffected +// sopw 2017-02-26T00:00:00 Unaffected +// sow 2017-03-05T00:00:00 Unaffected +// sonw 2017-03-12T00:00:00 Unaffected +// eopw 2017-03-05T00:00:00 Unaffected +// eow 2017-03-12T00:00:00 Unaffected +// eonw 2017-03-19T00:00:00 Unaffected +// sopww 2017-02-27T00:00:00 Unaffected +// soww 2017-03-06T00:00:00 +// sonww 2017-03-06T00:00:00 Unaffected +// eopww 2017-03-03T00:00:00 Unaffected +// eoww 2017-03-10T00:00:00 +// eonww 2017-03-17T00:00:00 Unaffected +// sopm 2017-02-01T00:00:00 Unaffected +// som 2017-03-01T00:00:00 Unaffected +// sonm 2017-04-01T00:00:00 Unaffected +// eopm 2017-03-01T00:00:00 Unaffected +// eom 2017-04-01T00:00:00 Unaffected +// eonm 2017-05-01T00:00:00 Unaffected +// sopq 2017-10-01T00:00:00 Unaffected +// soq 2017-01-01T00:00:00 Unaffected +// sonq 2017-04-01T00:00:00 Unaffected +// eopq 2017-01-01T00:00:00 Unaffected +// eoq 2017-04-01T00:00:00 Unaffected +// eonq 2017-07-01T00:00:00 Unaffected +// sopy 2016-01-01T00:00:00 Unaffected +// soy 2017-01-01T00:00:00 Unaffected +// sony 2018-01-01T00:00:00 Unaffected +// eopy 2017-01-01T00:00:00 Unaffected +// eoy 2018-01-01T00:00:00 Unaffected +// eony 2019-01-01T00:00:00 Unaffected +// easter 2017-04-16T00:00:00 +// eastermonday 2017-04-16T00:00:00 +// ascension 2017-05-25T00:00:00 +// pentecost 2017-06-04T00:00:00 +// goodfriday 2017-04-14T00:00:00 +// midsommar 2017-06-24T00:00:00 midnight, 1st Saturday after 20th June +// midsommarafton 2017-06-23T00:00:00 midnight, 1st Friday after 19th June +// juhannus 2017-06-23T00:00:00 midnight, 1st Friday after 19th June +// +bool DatetimeParser::parse_named (Pig& pig) +{ + auto checkpoint = pig.cursor (); + + // Experimental handling of date phrases, such as "first monday in march". + // Note that this requires that phrases are delimited by EOS or WS. + std::string token; + std::vector tokens; + while (pig.getUntilWS (token)) + { + tokens.push_back (token); + if (! pig.skipWS ()) + break; + } + +/* + // This group contains "1st monday ..." which must be processed before + // initializeOrdinal below. + if (initializeNthDayInMonth (tokens)) + { + return true; + } +*/ + + // Restoration necessary because of the tokenization. + pig.restoreTo (checkpoint); + + if (initializeNow (pig) || + initializeLater (pig) || + initializeSopd (pig) || + initializeSod (pig) || + initializeSond (pig) || + initializeEopd (pig) || + initializeEod (pig) || + initializeEond (pig) || + initializeSopw (pig) || + initializeSow (pig) || + initializeSonw (pig) || + initializeEopw (pig) || + initializeEow (pig) || + initializeEonw (pig) || + initializeSopww (pig) || // Must appear after sopw + initializeSonww (pig) || // Must appear after sonw + initializeSoww (pig) || // Must appear after sow + initializeEopww (pig) || // Must appear after eopw + initializeEonww (pig) || // Must appear after eonw + initializeEoww (pig) || // Must appear after eow + initializeSopm (pig) || + initializeSom (pig) || + initializeSonm (pig) || + initializeEopm (pig) || + initializeEom (pig) || + initializeEonm (pig) || + initializeSopq (pig) || + initializeSoq (pig) || + initializeSonq (pig) || + initializeEopq (pig) || + initializeEoq (pig) || + initializeEonq (pig) || + initializeSopy (pig) || + initializeSoy (pig) || + initializeSony (pig) || + initializeEopy (pig) || + initializeEoy (pig) || + initializeEony (pig) || + initializeInformalTime (pig)) + { + return true; + } + + pig.restoreTo (checkpoint); + return false; +} + +//////////////////////////////////////////////////////////////////////////////// +// Valid epoch values are unsigned integers after 1980-01-01T00:00:00Z. This +// restriction means that '12' will not be identified as an epoch date. +bool DatetimeParser::parse_epoch (Pig& pig) +{ + auto checkpoint = pig.cursor (); + + int epoch {}; + if (pig.getDigits (epoch) && + ! unicodeLatinAlpha (pig.peek ()) && + epoch >= 315532800) + { + _date = static_cast (epoch); + return true; + } + + pig.restoreTo (checkpoint); + return false; +} + +//////////////////////////////////////////////////////////////////////////////// +// date_ext 'T' time_utc_ext 'Z' +// date_ext 'T' time_off_ext +// date_ext 'T' time_ext +bool DatetimeParser::parse_date_time_ext (Pig& pig) +{ + auto checkpoint = pig.cursor (); + + if (parse_date_ext (pig) && + pig.skip ('T') && + (parse_time_utc_ext (pig) || + parse_time_off_ext (pig) || + parse_time_ext (pig))) + { + return true; + } + + pig.restoreTo (checkpoint); + return false; +} + +//////////////////////////////////////////////////////////////////////////////// +// YYYY-MM-DD +// YYYY-MM +// YYYY-DDD +// YYYY-Www-D +// YYYY-Www +bool DatetimeParser::parse_date_ext (Pig& pig) +{ + auto checkpoint = pig.cursor (); + + int year {}; + if (parse_year (pig, year) && + pig.skip ('-')) + { + auto checkpointYear = pig.cursor (); + + int month {}; + int day {}; + int julian {}; + + if (pig.skip ('W') && + parse_week (pig, _week)) + { + if (pig.skip ('-') && + pig.getDigit (_weekday)) + { + // What is happening here - must be something to do? + } + + if (! unicodeLatinDigit (pig.peek ())) + { + _year = year; + return true; + } + } + + pig.restoreTo (checkpointYear); + + if (parse_month (pig, month) && + pig.skip ('-') && + parse_day (pig, day) && + ! unicodeLatinDigit (pig.peek ())) + { + _year = year; + _month = month; + _day = day; + return true; + } + + pig.restoreTo (checkpointYear); + + if (parse_julian (pig, julian) && + ! unicodeLatinDigit (pig.peek ())) + { + _year = year; + _julian = julian; + return true; + } + + pig.restoreTo (checkpointYear); + + if (parse_month (pig, month) && + pig.peek () != '-' && + ! unicodeLatinDigit (pig.peek ())) + { + _year = year; + _month = month; + _day = 1; + return true; + } + } + + pig.restoreTo (checkpoint); + return false; +} + +//////////////////////////////////////////////////////////////////////////////// +// ±hh[:mm] +bool DatetimeParser::parse_off_ext (Pig& pig) +{ + auto checkpoint = pig.cursor (); + + int sign = pig.peek (); + if (sign == '+' || sign == '-') + { + pig.skipN (1); + + int hour {0}; + int minute {0}; + + if (parse_off_hour (pig, hour)) + { + if (pig.skip (':')) + { + if (! parse_off_minute (pig, minute)) + { + pig.restoreTo (checkpoint); + return false; + } + } + + _offset = (hour * 3600) + (minute * 60); + if (sign == '-') + _offset = - _offset; + + if (! unicodeLatinDigit (pig.peek ())) + return true; + } + } + + pig.restoreTo (checkpoint); + return false; +} + +//////////////////////////////////////////////////////////////////////////////// +// hh:mm[:ss] +bool DatetimeParser::parse_time_ext (Pig& pig, bool terminated) +{ + auto checkpoint = pig.cursor (); + + int hour {}; + int minute {}; + if (parse_hour (pig, hour) && + pig.skip (':') && + parse_minute (pig, minute)) + { + if (pig.skip (':')) + { + int second {}; + if (parse_second (pig, second) && + ! unicodeLatinDigit (pig.peek ()) && + (! terminated || (pig.peek () != '-' && pig.peek () != '+'))) + { + _seconds = (hour * 3600) + (minute * 60) + second; + return true; + } + + pig.restoreTo (checkpoint); + return false; + } + + auto following = pig.peek (); + if (! unicodeLatinDigit (following) && + (! terminated || (following != '+' && following != '-')) && + following != 'A' && + following != 'a' && + following != 'P' && + following != 'p') + { + _seconds = (hour * 3600) + (minute * 60); + return true; + } + } + + pig.restoreTo (checkpoint); + return false; +} + +//////////////////////////////////////////////////////////////////////////////// +// time-ext 'Z' +bool DatetimeParser::parse_time_utc_ext (Pig& pig) +{ + auto checkpoint = pig.cursor (); + + if (parse_time_ext (pig, false) && + pig.skip ('Z')) + { + if (! unicodeLatinDigit (pig.peek ())) + { + _utc = true; + return true; + } + } + + pig.restoreTo (checkpoint); + return false; +} + +//////////////////////////////////////////////////////////////////////////////// +// time-ext off-ext +bool DatetimeParser::parse_time_off_ext (Pig& pig) +{ + auto checkpoint = pig.cursor (); + + if (parse_time_ext (pig, false) && + parse_off_ext (pig)) + { + return true; + } + + pig.restoreTo (checkpoint); + return false; +} + +//////////////////////////////////////////////////////////////////////////////// +// YYYYMMDDTHHMMSSZ +// YYYYMMDDTHHMMSS +bool DatetimeParser::parse_date_time (Pig& pig) +{ + auto checkpoint = pig.cursor (); + + if (parse_date (pig) && + pig.skip ('T') && + (parse_time_utc (pig) || + parse_time_off (pig) || + parse_time (pig))) + { + return true; + } + + pig.restoreTo (checkpoint); + return false; +} + +//////////////////////////////////////////////////////////////////////////////// +// YYYYWww +// YYYYDDD +// YYYYMMDD +// YYYYMM +bool DatetimeParser::parse_date (Pig& pig) +{ + auto checkpoint = pig.cursor (); + + int year {}; + int month {}; + int julian {}; + int week {}; + int weekday {}; + int day {}; + if (parse_year (pig, year)) + { + auto checkpointYear = pig.cursor (); + + if (pig.skip ('W') && + parse_week (pig, week)) + { + if (pig.getDigit (weekday)) + _weekday = weekday; + + if (! unicodeLatinDigit (pig.peek ())) + { + _year = year; + _week = week; + return true; + } + } + + pig.restoreTo (checkpointYear); + + if (parse_julian (pig, julian) && + ! unicodeLatinDigit (pig.peek ())) + { + _year = year; + _julian = julian; + return true; + } + + pig.restoreTo (checkpointYear); + + if (parse_month (pig, month)) + { + if (parse_day (pig, day)) + { + if (! unicodeLatinDigit (pig.peek ())) + { + _year = year; + _month = month; + _day = day; + return true; + } + } + else + { + if (! unicodeLatinDigit (pig.peek ())) + { + _year = year; + _month = month; + _day = 1; + return true; + } + } + } + } + + pig.restoreTo (checkpoint); + return false; +} + +//////////////////////////////////////////////////////////////////////////////// +//