Support importing Taskwarrior v2.x data files (#3724)

This should ease the pain of upgrading from v2.x to v3.x.

doc/man/task.1.in

@@ -414,6 +414,11 @@ few example scripts, such as:
   import-yaml.pl
 .fi
 
+.TP
+.B task import-v2
+Imports tasks from the Taskwarrior v2.x format. This is used when upgrading from
+version 2.x to version 3.x.
+
 .TP
 .B task log <mods>
 Adds a new task that is already completed, to the task list. It is affected by

src/CMakeLists.txt

@@ -14,6 +14,7 @@ add_library (task STATIC CLI2.cpp CLI2.h
                   Hooks.cpp Hooks.h
                   Lexer.cpp Lexer.h
                   Operation.cpp Operation.h
+                  TF2.cpp TF2.h
                   TDB2.cpp TDB2.h
                   Task.cpp Task.h
                   Variant.cpp Variant.h

src/TF2.cpp

@@ -0,0 +1,184 @@
+////////////////////////////////////////////////////////////////////////////////
+//
+// Copyright 2006 - 2021, Tomas Babej, Paul Beckingham, Federico Hernandez.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included
+// in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//
+// https://www.opensource.org/licenses/mit-license.php
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#include <Color.h>
+#include <Context.h>
+#include <Datetime.h>
+#include <TF2.h>
+#include <Table.h>
+#include <cmake.h>
+#include <format.h>
+#include <main.h>
+#include <shared.h>
+#include <signal.h>
+#include <stdlib.h>
+#include <util.h>
+
+#include <algorithm>
+#include <iostream>
+#include <list>
+#include <set>
+#include <sstream>
+
+#define STRING_TDB2_REVERTED "Modified task reverted."
+
+////////////////////////////////////////////////////////////////////////////////
+TF2::TF2() : _loaded_tasks(false), _loaded_lines(false) {}
+
+////////////////////////////////////////////////////////////////////////////////
+TF2::~TF2() {}
+
+////////////////////////////////////////////////////////////////////////////////
+void TF2::target(const std::string& f) { _file = File(f); }
+
+////////////////////////////////////////////////////////////////////////////////
+const std::vector<std::map<std::string, std::string>>& TF2::get_tasks() {
+  if (!_loaded_tasks) load_tasks();
+
+  return _tasks;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Attempt an FF4 parse.
+//
+// Note that FF1, FF2, FF3, and JSON are no longer supported.
+//
+// start --> [ --> Att --> ] --> end
+//              ^       |
+//              +-------+
+//
+std::map<std::string, std::string> TF2::load_task(const std::string& input) {
+  std::map<std::string, std::string> data;
+
+  // File format version 4, from 2009-5-16 - now, v1.7.1+
+  // This is the parse format tried first, because it is most used.
+  data.clear();
+
+  if (input[0] == '[') {
+    // Not using Pig to parse here (which would be idiomatic), because we
+    // don't need to differentiate betwen utf-8 and normal characters.
+    // Pig's scanning the string can be expensive.
+    auto ending_bracket = input.find_last_of(']');
+    if (ending_bracket != std::string::npos) {
+      std::string line = input.substr(1, ending_bracket);
+
+      if (line.length() == 0) throw std::string("Empty record in input.");
+
+      Pig attLine(line);
+      std::string name;
+      std::string value;
+      while (!attLine.eos()) {
+        if (attLine.getUntilAscii(':', name) && attLine.skip(':') &&
+            attLine.getQuoted('"', value)) {
+#ifdef PRODUCT_TASKWARRIOR
+          legacyAttributeMap(name);
+#endif
+
+          data[name] = decode(json::decode(value));
+        }
+
+        attLine.skip(' ');
+      }
+
+      std::string remainder;
+      attLine.getRemainder(remainder);
+      if (remainder.length()) throw std::string("Unrecognized characters at end of line.");
+    }
+  } else {
+    throw std::string("Record not recognized as format 4.");
+  }
+
+  // for compatibility, include all tags in `tags` as `tag_..` attributes
+  if (data.find("tags") != data.end()) {
+    for (auto& tag : split(data["tags"], ',')) {
+      data[Task::tag2Attr(tag)] = "x";
+    }
+  }
+
+  // same for `depends` / `dep_..`
+  if (data.find("depends") != data.end()) {
+    for (auto& dep : split(data["depends"], ',')) {
+      data[Task::dep2Attr(dep)] = "x";
+    }
+  }
+
+  return data;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Decode values after parse.
+//   [  <- &open;
+//   ]  <- &close;
+const std::string TF2::decode(const std::string& value) const {
+  if (value.find('&') == std::string::npos) return value;
+
+  auto modified = str_replace(value, "&open;", "[");
+  return str_replace(modified, "&close;", "]");
+}
+
+////////////////////////////////////////////////////////////////////////////////
+void TF2::load_tasks() {
+  Timer timer;
+
+  if (!_loaded_lines) {
+    load_lines();
+  }
+
+  // Reduce unnecessary allocations/copies.
+  // Calling it on _tasks is the right thing to do even when from_gc is set.
+  _tasks.reserve(_lines.size());
+
+  int line_number = 0;  // Used for error message in catch block.
+  try {
+    for (auto& line : _lines) {
+      ++line_number;
+      auto task = load_task(line);
+      _tasks.push_back(task);
+    }
+
+    _loaded_tasks = true;
+  }
+
+  catch (const std::string& e) {
+    throw e + format(" in {1} at line {2}", _file._data, line_number);
+  }
+
+  Context::getContext().time_load_us += timer.total_us();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+void TF2::load_lines() {
+  if (_file.open()) {
+    if (Context::getContext().config.getBoolean("locking")) _file.lock();
+
+    _file.read(_lines);
+    _file.close();
+    _loaded_lines = true;
+  }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// vim: ts=2 et sw=2

src/TF2.h

@@ -0,0 +1,66 @@
+////////////////////////////////////////////////////////////////////////////////
+//
+// Copyright 2006 - 2024, Tomas Babej, Paul Beckingham, Federico Hernandez.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included
+// in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//
+// https://www.opensource.org/licenses/mit-license.php
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef INCLUDED_TF2
+#define INCLUDED_TF2
+
+#include <FS.h>
+#include <Task.h>
+#include <stdio.h>
+
+#include <map>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+// TF2 Class represents a single 2.x-style file in the task database.
+//
+// This is only used for importing tasks from 2.x. It only reads format 4, based
+// on a stripped-down version of the TF2 class from v2.6.2.
+class TF2 {
+ public:
+  TF2();
+  ~TF2();
+
+  void target(const std::string&);
+
+  const std::vector<std::map<std::string, std::string>>& get_tasks();
+
+  std::map<std::string, std::string> load_task(const std::string&);
+  void load_tasks();
+  void load_lines();
+  const std::string decode(const std::string& value) const;
+
+  bool _loaded_tasks;
+  bool _loaded_lines;
+  std::vector<std::map<std::string, std::string>> _tasks;
+  std::vector<std::string> _lines;
+  File _file;
+};
+
+#endif
+////////////////////////////////////////////////////////////////////////////////

src/commands/CMakeLists.txt

@@ -35,6 +35,7 @@ set (commands_SRCS Command.cpp        Command.h
                    CmdHistory.cpp     CmdHistory.h
                    CmdIDs.cpp         CmdIDs.h
                    CmdImport.cpp      CmdImport.h
+                   CmdImportV2.cpp    CmdImportV2.h
                    CmdInfo.cpp        CmdInfo.h
                    CmdLog.cpp         CmdLog.h
                    CmdLogo.cpp        CmdLogo.h

src/commands/CmdCustom.cpp

@@ -241,7 +241,8 @@ int CmdCustom::execute(std::string& output) {
     Color warning = Color(Context::getContext().config.get("color.warning"));
     std::cerr << warning.colorize(format("Found existing '*.data' files in {1}", location)) << "\n";
     std::cerr << "  Taskwarrior's storage format changed in 3.0, requiring a manual migration.\n";
-    std::cerr << "  See https://taskwarrior.org/docs/upgrade-3/\n";
+    std::cerr << "  See https://taskwarrior.org/docs/upgrade-3/. Run `task import-v2` to import\n";
+    std::cerr << "  the tasks into the Taskwarrior-3.x format\n";
   }
 
   feedback_backlog();

src/commands/CmdImportV2.cpp

@@ -0,0 +1,135 @@
+////////////////////////////////////////////////////////////////////////////////
+//
+// Copyright 2006 - 2021, Tomas Babej, Paul Beckingham, Federico Hernandez.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included
+// in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//
+// https://www.opensource.org/licenses/mit-license.php
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#include <cmake.h>
+// cmake.h include header must come first
+
+#include <CmdImportV2.h>
+#include <CmdModify.h>
+#include <Context.h>
+#include <TF2.h>
+#include <format.h>
+#include <shared.h>
+#include <util.h>
+
+#include <iostream>
+#include <unordered_map>
+
+////////////////////////////////////////////////////////////////////////////////
+CmdImportV2::CmdImportV2() {
+  _keyword = "import-v2";
+  _usage = "task          import-v2";
+  _description = "Imports Taskwarrior v2.x files";
+  _read_only = false;
+  _displays_id = false;
+  _needs_gc = false;
+  _uses_context = false;
+  _accepts_filter = false;
+  _accepts_modifications = false;
+  _accepts_miscellaneous = true;
+  _category = Command::Category::migration;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+int CmdImportV2::execute(std::string&) {
+  std::vector<std::map<std::string, std::string>> task_data;
+
+  std::string location = (Context::getContext().data_dir);
+  File pending_file = File(location + "/pending.data");
+  if (pending_file.exists()) {
+    TF2 pending_tf;
+    pending_tf.target(pending_file);
+    auto& pending_tasks = pending_tf.get_tasks();
+    task_data.insert(task_data.end(), pending_tasks.begin(), pending_tasks.end());
+  }
+  File completed_file = File(location + "/completed.data");
+  if (completed_file.exists()) {
+    TF2 completed_tf;
+    completed_tf.target(completed_file);
+    auto& completed_tasks = completed_tf.get_tasks();
+    task_data.insert(task_data.end(), completed_tasks.begin(), completed_tasks.end());
+  }
+
+  auto count = import(task_data);
+
+  Context::getContext().footnote(
+      format("Imported {1} tasks from `*.data` files. You may now delete these files.", count));
+  return 0;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+int CmdImportV2::import(const std::vector<std::map<std::string, std::string>>& task_data) {
+  auto count = 0;
+  const std::string uuid_key = "uuid";
+  const std::string id_key = "id";
+  const std::string descr_key = "description";
+  auto& replica = Context::getContext().tdb2.replica();
+  rust::Vec<tc::Operation> ops;
+  tc::add_undo_point(ops);
+
+  for (auto& task : task_data) {
+    auto uuid_iter = task.find(uuid_key);
+    if (uuid_iter == task.end()) {
+      std::cout << " err  - Task with no UUID\n";
+      continue;
+    }
+    auto uuid_str = uuid_iter->second;
+    auto uuid = tc::uuid_from_string(uuid_str);
+
+    bool added_task = false;
+    auto maybe_task_data = replica->get_task_data(uuid);
+    auto task_data = maybe_task_data.is_some() ? maybe_task_data.take() : [&]() {
+      added_task = true;
+      return tc::create_task(uuid, ops);
+    }();
+
+    for (auto& attr : task) {
+      if (attr.first == uuid_key || attr.first == id_key) {
+        continue;
+      }
+      task_data->update(attr.first, attr.second, ops);
+    }
+    count++;
+
+    if (added_task) {
+      std::cout << " add ";
+    } else {
+      std::cout << " mod ";
+    }
+    std::cout << uuid_str << ' ';
+    if (auto descr_iter = task.find(descr_key); descr_iter != task.end()) {
+      std::cout << descr_iter->second;
+    } else {
+      std::cout << "(no description)";
+    }
+    std::cout << "\n";
+  }
+
+  replica->commit_operations(std::move(ops));
+  return count;
+}
+
+////////////////////////////////////////////////////////////////////////////////

src/commands/CmdImportV2.h

@@ -0,0 +1,46 @@
+////////////////////////////////////////////////////////////////////////////////
+//
+// Copyright 2006 - 2021, Tomas Babej, Paul Beckingham, Federico Hernandez.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included
+// in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//
+// https://www.opensource.org/licenses/mit-license.php
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef INCLUDED_CMDIMPORTV2
+#define INCLUDED_CMDIMPORTV2
+
+#include <Command.h>
+#include <JSON.h>
+
+#include <string>
+#include <unordered_map>
+
+class CmdImportV2 : public Command {
+ public:
+  CmdImportV2();
+  int execute(std::string &);
+
+ private:
+  int import(const std::vector<std::map<std::string, std::string>> &task_data);
+};
+
+#endif
+////////////////////////////////////////////////////////////////////////////////

src/commands/Command.cpp

@@ -66,6 +66,7 @@
 #include <CmdHistory.h>
 #include <CmdIDs.h>
 #include <CmdImport.h>
+#include <CmdImportV2.h>
 #include <CmdInfo.h>
 #include <CmdLog.h>
 #include <CmdLogo.h>
@@ -188,6 +189,8 @@ void Command::factory(std::map<std::string, Command*>& all) {
   all[c->keyword()] = c;
   c = new CmdImport();
   all[c->keyword()] = c;
+  c = new CmdImportV2();
+  all[c->keyword()] = c;
   c = new CmdInfo();
   all[c->keyword()] = c;
   c = new CmdLog();

test/CMakeLists.txt

@@ -136,6 +136,7 @@ set (pythonTests
   hyphenate.test.py
   ids.test.py
   import.test.py
+  import-v2.test.py
   info.test.py
   limit.test.py
   list.all.projects.test.py

test/import-v2.test.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+###############################################################################
+#
+# Copyright 2006 - 2021, Tomas Babej, Paul Beckingham, Federico Hernandez.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+# https://www.opensource.org/licenses/mit-license.php
+#
+###############################################################################
+
+import sys
+import os
+import unittest
+import json
+
+# Ensure python finds the local simpletap module
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
+
+from basetest import Task, TestCase
+from basetest.utils import mkstemp
+
+
+class TestImport(TestCase):
+    def setUp(self):
+        self.t = Task()
+        self.t.config("dateformat", "m/d/Y")
+
+        # Multiple tasks.
+        self.pending = """\
+[description:"bing" due:"1734480000" entry:"1734397061" modified:"1734397061" status:"pending" uuid:"ad7f7585-bff3-4b57-a116-abfc9f71ee4a"]
+[description:"baz" entry:"1734397063" modified:"1734397063" status:"pending" uuid:"591ccfee-dd8d-44e9-908a-40618257cf54"]\
+"""
+        self.completed = """\
+[description:"foo" end:"1734397073" entry:"1734397054" modified:"1734397074" status:"deleted" uuid:"6849568f-55d7-4152-8db0-00356e39f0bb"]
+[description:"bar" end:"1734397065" entry:"1734397056" modified:"1734397065" status:"completed" uuid:"51921813-7abb-412d-8ada-7c1417d01209"]\
+"""
+
+    def test_import_v2(self):
+        with open(os.path.join(self.t.datadir, "pending.data"), "w") as f:
+            f.write(self.pending)
+        with open(os.path.join(self.t.datadir, "completed.data"), "w") as f:
+            f.write(self.completed)
+        code, out, err = self.t("import-v2")
+        self.assertIn("Imported 4 tasks", err)
+
+        code, out, err = self.t("list")
+        self.assertIn("bing", out)
+        self.assertIn("baz", out)
+        self.assertNotIn("foo", out)
+        self.assertNotIn("bar", out)
+
+        code, out, err = self.t("completed")
+        self.assertNotIn("bing", out)
+        self.assertNotIn("baz", out)
+        self.assertNotIn("foo", out)  # deleted, not in the completed report
+        self.assertIn("bar", out)
+
+
+if __name__ == "__main__":
+    from simpletap import TAPTestRunner
+
+    unittest.main(testRunner=TAPTestRunner())
+
+# vim: ai sts=4 et sw=4 ft=python