Reorganize taskchampion crate for docs and tests

The public API of the taskchampion crate now contains the expected parts
and no more, and has some better documentation.

This moves the crate's external `tests/` into internal tests, as the
TaskDB is no longer exposed as part of the crate API.

cli/src/main.rs

@@ -1,6 +1,6 @@
 use clap::{App, Arg, SubCommand};
 use std::path::Path;
-use taskchampion::{taskstorage, Replica, Status, DB};
+use taskchampion::{taskstorage, Replica, Status};
 use uuid::Uuid;
 
 fn main() {
@@ -20,12 +20,9 @@ fn main() {
         .subcommand(SubCommand::with_name("gc").about("run garbage collection"))
         .get_matches();
 
-    let mut replica = Replica::new(
+    let mut replica = Replica::new(Box::new(
-        DB::new(Box::new(
         taskstorage::KVStorage::new(Path::new("/tmp/tasks")).unwrap(),
-        ))
+    ));
-        .into(),
-    );
 
     match matches.subcommand() {
         ("add", Some(matches)) => {

taskchampion/src/lib.rs

@@ -1,14 +1,38 @@
+/*!
+
+This crate implements the core of TaskChampion, the [replica](crate::Replica).
+
+A TaskChampion replica is a local copy of a user's task data.  As the name suggests, several
+replicas of the same data can exist (such as on a user's laptop and on their phone) and can
+synchronize with one another.
+
+# Task Storage
+
+The [`taskstorage`](crate::taskstorage) module supports pluggable storage for a replica's data.
+An implementation is provided, but users of this crate can provide their own implementation as well.
+
+# Server
+
+Replica synchronization takes place against a server.
+The [`server`](crate::server) module defines the interface a server must meet.
+
+# See Also
+
+See the [TaskChampion Book](https://github.com/djmitche/taskchampion/blob/main/docs/src/SUMMARY.md)
+for more information about the design and usage of the tool.
+
+ */
 mod errors;
-mod operation;
 mod replica;
 pub mod server;
 mod task;
 mod taskdb;
 pub mod taskstorage;
 
-pub use operation::Operation;
 pub use replica::Replica;
 pub use task::Priority;
 pub use task::Status;
-pub use task::Task;
+pub use task::{Task, TaskMut};
-pub use taskdb::DB;
+
+#[cfg(test)]
+pub(crate) mod testing;

taskchampion/src/replica.rs

@@ -1,21 +1,29 @@
 use crate::errors::Error;
-use crate::operation::Operation;
+use crate::server::Server;
 use crate::task::{Status, Task};
 use crate::taskdb::DB;
-use crate::taskstorage::TaskMap;
+use crate::taskstorage::{Operation, TaskMap, TaskStorage};
 use chrono::Utc;
 use failure::Fallible;
 use std::collections::HashMap;
 use uuid::Uuid;
 
-/// A replica represents an instance of a user's task data.
+/// A replica represents an instance of a user's task data, providing an easy interface
+/// for querying and modifying that data.
 pub struct Replica {
-    taskdb: Box<DB>,
+    taskdb: DB,
 }
 
 impl Replica {
-    pub fn new(taskdb: Box<DB>) -> Replica {
+    pub fn new(storage: Box<dyn TaskStorage>) -> Replica {
-        return Replica { taskdb };
+        return Replica {
+            taskdb: DB::new(storage),
+        };
+    }
+
+    #[cfg(test)]
+    pub fn new_inmemory() -> Replica {
+        Replica::new(Box::new(crate::taskstorage::InMemoryStorage::new()))
     }
 
     /// Update an existing task.  If the value is Some, the property is added or updated.  If the
@@ -45,7 +53,7 @@ impl Replica {
     }
 
     /// Get all tasks represented as a map keyed by UUID
-    pub fn all_tasks<'a>(&'a mut self) -> Fallible<HashMap<Uuid, Task>> {
+    pub fn all_tasks(&mut self) -> Fallible<HashMap<Uuid, Task>> {
         let mut res = HashMap::new();
         for (uuid, tm) in self.taskdb.all_tasks()?.drain(..) {
             res.insert(uuid.clone(), Task::new(uuid.clone(), tm));
@@ -54,7 +62,7 @@ impl Replica {
     }
 
     /// Get the UUIDs of all tasks
-    pub fn all_task_uuids<'a>(&'a mut self) -> Fallible<Vec<Uuid>> {
+    pub fn all_task_uuids(&mut self) -> Fallible<Vec<Uuid>> {
         self.taskdb.all_task_uuids()
     }
 
@@ -124,6 +132,11 @@ impl Replica {
         self.taskdb.apply(Operation::Delete { uuid })
     }
 
+    /// Synchronize this replica against the given server.
+    pub fn sync(&mut self, username: &str, server: &mut dyn Server) -> Fallible<()> {
+        self.taskdb.sync(username, server)
+    }
+
     /// Perform "garbage collection" on this replica.  In particular, this renumbers the working
     /// set to contain only pending tasks.
     pub fn gc(&mut self) -> Fallible<()> {
@@ -138,12 +151,11 @@ impl Replica {
 mod tests {
     use super::*;
     use crate::task::Status;
-    use crate::taskdb::DB;
     use uuid::Uuid;
 
     #[test]
     fn new_task() {
-        let mut rep = Replica::new(DB::new_inmemory().into());
+        let mut rep = Replica::new_inmemory();
         let uuid = Uuid::new_v4();
 
         let t = rep
@@ -156,7 +168,7 @@ mod tests {
 
     #[test]
     fn modify_task() {
-        let mut rep = Replica::new(DB::new_inmemory().into());
+        let mut rep = Replica::new_inmemory();
         let uuid = Uuid::new_v4();
 
         let t = rep
@@ -183,7 +195,7 @@ mod tests {
 
     #[test]
     fn delete_task() {
-        let mut rep = Replica::new(DB::new_inmemory().into());
+        let mut rep = Replica::new_inmemory();
         let uuid = Uuid::new_v4();
 
         rep.new_task(uuid.clone(), Status::Pending, "a task".into())
@@ -195,7 +207,7 @@ mod tests {
 
     #[test]
     fn get_and_modify() {
-        let mut rep = Replica::new(DB::new_inmemory().into());
+        let mut rep = Replica::new_inmemory();
         let uuid = Uuid::new_v4();
 
         rep.new_task(uuid.clone(), Status::Pending, "another task".into())
@@ -215,7 +227,7 @@ mod tests {
 
     #[test]
     fn new_pending_adds_to_working_set() {
-        let mut rep = Replica::new(DB::new_inmemory().into());
+        let mut rep = Replica::new_inmemory();
         let uuid = Uuid::new_v4();
 
         rep.new_task(uuid.clone(), Status::Pending, "to-be-pending".into())
@@ -233,7 +245,7 @@ mod tests {
 
     #[test]
     fn get_does_not_exist() {
-        let mut rep = Replica::new(DB::new_inmemory().into());
+        let mut rep = Replica::new_inmemory();
         let uuid = Uuid::new_v4();
         assert_eq!(rep.get_task(&uuid).unwrap(), None);
     }

taskchampion/src/task.rs

@@ -1,34 +1,37 @@
 use crate::replica::Replica;
 use crate::taskstorage::TaskMap;
 use chrono::prelude::*;
-use failure::{format_err, Fallible};
+use failure::Fallible;
-use std::convert::TryFrom;
 use uuid::Uuid;
 
 pub type Timestamp = DateTime<Utc>;
 
+/// The priority of a task
 #[derive(Debug, PartialEq)]
 pub enum Priority {
+    /// Low
     L,
+    /// Medium
     M,
+    /// High
     H,
 }
 
-impl TryFrom<&str> for Priority {
+#[allow(dead_code)]
-    type Error = failure::Error;
+impl Priority {
-
+    /// Get a Priority from the 1-character value in a TaskMap,
-    fn try_from(s: &str) -> Result<Self, Self::Error> {
+    /// defaulting to M
+    pub(crate) fn from_taskmap(s: &str) -> Priority {
         match s {
-            "L" => Ok(Priority::L),
+            "L" => Priority::L,
-            "M" => Ok(Priority::M),
+            "M" => Priority::M,
-            "H" => Ok(Priority::H),
+            "H" => Priority::H,
-            _ => Err(format_err!("invalid status {}", s)),
+            _ => Priority::M,
         }
     }
-}
 
-impl AsRef<str> for Priority {
+    /// Get the 1-character value for this priority to use in the TaskMap.
-    fn as_ref(&self) -> &str {
+    pub(crate) fn to_taskmap(&self) -> &str {
         match self {
             Priority::L => "L",
             Priority::M => "M",
@@ -36,6 +39,8 @@ impl AsRef<str> for Priority {
         }
     }
 }
+
+/// The status of a task.  The default status in "Pending".
 #[derive(Debug, PartialEq)]
 pub enum Status {
     Pending,
@@ -86,8 +91,8 @@ pub struct Task {
     taskmap: TaskMap,
 }
 
-/// A mutable task, with setter methods.  Calling a setter will update the Replica, as well as the
+/// A mutable task, with setter methods.  Most methods are simple setters and not further
-/// included Task.
+/// described.  Calling a setter will update the Replica, as well as the included Task.
 pub struct TaskMut<'r> {
     task: Task,
     replica: &'r mut Replica,
@@ -145,7 +150,8 @@ impl Task {
 }
 
 impl<'r> TaskMut<'r> {
-    /// Get the immutable task
+    /// Get the immutable version of this object.  Note that TaskMut [`std::ops::Deref`]s to
+    /// [`crate::task::Task`], so all of that struct's getter methods can be used on TaskMut.
     pub fn into_immut(self) -> Task {
         self.task
     }
@@ -160,12 +166,10 @@ impl<'r> TaskMut<'r> {
         self.set_string("status", Some(String::from(status.to_taskmap())))
     }
 
-    /// Set the task's description
     pub fn set_description(&mut self, description: String) -> Fallible<()> {
         self.set_string("description", Some(description))
     }
 
-    /// Set the task's description
     pub fn set_modified(&mut self, modified: DateTime<Utc>) -> Fallible<()> {
         self.set_timestamp("modified", Some(modified))
     }
@@ -213,3 +217,28 @@ impl<'r> std::ops::Deref for TaskMut<'r> {
         &self.task
     }
 }
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_priority() {
+        assert_eq!(Priority::L.to_taskmap(), "L");
+        assert_eq!(Priority::M.to_taskmap(), "M");
+        assert_eq!(Priority::H.to_taskmap(), "H");
+        assert_eq!(Priority::from_taskmap("L"), Priority::L);
+        assert_eq!(Priority::from_taskmap("M"), Priority::M);
+        assert_eq!(Priority::from_taskmap("H"), Priority::H);
+    }
+
+    #[test]
+    fn test_status() {
+        assert_eq!(Status::Pending.to_taskmap(), "P");
+        assert_eq!(Status::Completed.to_taskmap(), "C");
+        assert_eq!(Status::Deleted.to_taskmap(), "D");
+        assert_eq!(Status::from_taskmap("P"), Status::Pending);
+        assert_eq!(Status::from_taskmap("C"), Status::Completed);
+        assert_eq!(Status::from_taskmap("D"), Status::Deleted);
+    }
+}

taskchampion/src/taskdb.rs

@@ -1,7 +1,6 @@
 use crate::errors::Error;
-use crate::operation::Operation;
 use crate::server::{Server, VersionAdd};
-use crate::taskstorage::{TaskMap, TaskStorage, TaskStorageTxn};
+use crate::taskstorage::{Operation, TaskMap, TaskStorage, TaskStorageTxn};
 use failure::Fallible;
 use serde::{Deserialize, Serialize};
 use std::collections::HashSet;
@@ -268,7 +267,8 @@ impl DB {
 
     // functions for supporting tests
 
-    pub fn sorted_tasks(&mut self) -> Vec<(Uuid, Vec<(String, String)>)> {
+    #[cfg(test)]
+    pub(crate) fn sorted_tasks(&mut self) -> Vec<(Uuid, Vec<(String, String)>)> {
         let mut res: Vec<(Uuid, Vec<(String, String)>)> = self
             .all_tasks()
             .unwrap()
@@ -286,7 +286,8 @@ impl DB {
         res
     }
 
-    pub fn operations(&mut self) -> Vec<Operation> {
+    #[cfg(test)]
+    pub(crate) fn operations(&mut self) -> Vec<Operation> {
         let mut txn = self.storage.txn().unwrap();
         txn.operations()
             .unwrap()
@@ -299,7 +300,10 @@ impl DB {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::taskstorage::InMemoryStorage;
+    use crate::testing::testserver::TestServer;
     use chrono::Utc;
+    use proptest::prelude::*;
     use std::collections::HashMap;
     use uuid::Uuid;
 
@@ -508,4 +512,182 @@ mod tests {
 
         Ok(())
     }
+
+    fn newdb() -> DB {
+        DB::new(Box::new(InMemoryStorage::new()))
+    }
+
+    #[test]
+    fn test_sync() {
+        let mut server = TestServer::new();
+
+        let mut db1 = newdb();
+        db1.sync("me", &mut server).unwrap();
+
+        let mut db2 = newdb();
+        db2.sync("me", &mut server).unwrap();
+
+        // make some changes in parallel to db1 and db2..
+        let uuid1 = Uuid::new_v4();
+        db1.apply(Operation::Create { uuid: uuid1 }).unwrap();
+        db1.apply(Operation::Update {
+            uuid: uuid1,
+            property: "title".into(),
+            value: Some("my first task".into()),
+            timestamp: Utc::now(),
+        })
+        .unwrap();
+
+        let uuid2 = Uuid::new_v4();
+        db2.apply(Operation::Create { uuid: uuid2 }).unwrap();
+        db2.apply(Operation::Update {
+            uuid: uuid2,
+            property: "title".into(),
+            value: Some("my second task".into()),
+            timestamp: Utc::now(),
+        })
+        .unwrap();
+
+        // and synchronize those around
+        db1.sync("me", &mut server).unwrap();
+        db2.sync("me", &mut server).unwrap();
+        db1.sync("me", &mut server).unwrap();
+        assert_eq!(db1.sorted_tasks(), db2.sorted_tasks());
+
+        // now make updates to the same task on both sides
+        db1.apply(Operation::Update {
+            uuid: uuid2,
+            property: "priority".into(),
+            value: Some("H".into()),
+            timestamp: Utc::now(),
+        })
+        .unwrap();
+        db2.apply(Operation::Update {
+            uuid: uuid2,
+            property: "project".into(),
+            value: Some("personal".into()),
+            timestamp: Utc::now(),
+        })
+        .unwrap();
+
+        // and synchronize those around
+        db1.sync("me", &mut server).unwrap();
+        db2.sync("me", &mut server).unwrap();
+        db1.sync("me", &mut server).unwrap();
+        assert_eq!(db1.sorted_tasks(), db2.sorted_tasks());
+    }
+
+    #[test]
+    fn test_sync_create_delete() {
+        let mut server = TestServer::new();
+
+        let mut db1 = newdb();
+        db1.sync("me", &mut server).unwrap();
+
+        let mut db2 = newdb();
+        db2.sync("me", &mut server).unwrap();
+
+        // create and update a task..
+        let uuid = Uuid::new_v4();
+        db1.apply(Operation::Create { uuid }).unwrap();
+        db1.apply(Operation::Update {
+            uuid: uuid,
+            property: "title".into(),
+            value: Some("my first task".into()),
+            timestamp: Utc::now(),
+        })
+        .unwrap();
+
+        // and synchronize those around
+        db1.sync("me", &mut server).unwrap();
+        db2.sync("me", &mut server).unwrap();
+        db1.sync("me", &mut server).unwrap();
+        assert_eq!(db1.sorted_tasks(), db2.sorted_tasks());
+
+        // delete and re-create the task on db1
+        db1.apply(Operation::Delete { uuid }).unwrap();
+        db1.apply(Operation::Create { uuid }).unwrap();
+        db1.apply(Operation::Update {
+            uuid: uuid,
+            property: "title".into(),
+            value: Some("my second task".into()),
+            timestamp: Utc::now(),
+        })
+        .unwrap();
+
+        // and on db2, update a property of the task
+        db2.apply(Operation::Update {
+            uuid: uuid,
+            property: "project".into(),
+            value: Some("personal".into()),
+            timestamp: Utc::now(),
+        })
+        .unwrap();
+
+        db1.sync("me", &mut server).unwrap();
+        db2.sync("me", &mut server).unwrap();
+        db1.sync("me", &mut server).unwrap();
+        assert_eq!(db1.sorted_tasks(), db2.sorted_tasks());
+    }
+
+    #[derive(Debug)]
+    enum Action {
+        Op(Operation),
+        Sync,
+    }
+
+    fn action_sequence_strategy() -> impl Strategy<Value = Vec<(Action, u8)>> {
+        // Create, Update, Delete, or Sync on client 1, 2, .., followed by a round of syncs
+        "([CUDS][123])*S1S2S3S1S2".prop_map(|seq| {
+            let uuid = Uuid::parse_str("83a2f9ef-f455-4195-b92e-a54c161eebfc").unwrap();
+            seq.as_bytes()
+                .chunks(2)
+                .map(|action_on| {
+                    let action = match action_on[0] {
+                        b'C' => Action::Op(Operation::Create { uuid }),
+                        b'U' => Action::Op(Operation::Update {
+                            uuid,
+                            property: "title".into(),
+                            value: Some("foo".into()),
+                            timestamp: Utc::now(),
+                        }),
+                        b'D' => Action::Op(Operation::Delete { uuid }),
+                        b'S' => Action::Sync,
+                        _ => unreachable!(),
+                    };
+                    let acton = action_on[1] - b'1';
+                    (action, acton)
+                })
+                .collect::<Vec<(Action, u8)>>()
+        })
+    }
+
+    proptest! {
+        #[test]
+        // check that various sequences of operations on mulitple db's do not get the db's into an
+        // incompatible state.  The main concern here is that there might be a sequence of create
+        // and delete operations that results in a task existing in one DB but not existing in
+        // another.  So, the generated sequences focus on a single task UUID.
+        fn transform_sequences_of_operations(action_sequence in action_sequence_strategy()) {
+            let mut server = TestServer::new();
+            let mut dbs = [newdb(), newdb(), newdb()];
+
+            for (action, db) in action_sequence {
+                println!("{:?} on db {}", action, db);
+
+                let db = &mut dbs[db as usize];
+                match action {
+                    Action::Op(op) => {
+                        if let Err(e) = db.apply(op) {
+                            println!("  {:?} (ignored)", e);
+                        }
+                    },
+                    Action::Sync => db.sync("me", &mut server).unwrap(),
+                }
+            }
+
+            assert_eq!(dbs[0].sorted_tasks(), dbs[0].sorted_tasks());
+            assert_eq!(dbs[1].sorted_tasks(), dbs[2].sorted_tasks());
+        }
+    }
 }

taskchampion/src/taskstorage/inmemory.rs

@@ -1,5 +1,4 @@
-use crate::operation::Operation;
+use crate::taskstorage::{Operation, TaskMap, TaskStorage, TaskStorageTxn};
-use crate::taskstorage::{TaskMap, TaskStorage, TaskStorageTxn};
 use failure::{format_err, Fallible};
 use std::collections::hash_map::Entry;
 use std::collections::HashMap;
@@ -139,6 +138,8 @@ impl<'t> TaskStorageTxn for Txn<'t> {
     }
 }
 
+/// InMemoryStorage is a simple in-memory task storage implementation.  It is not useful for
+/// production data, but is useful for testing purposes.
 #[derive(PartialEq, Debug, Clone)]
 pub struct InMemoryStorage {
     data: Data,

taskchampion/src/taskstorage/kv.rs

@@ -1,5 +1,4 @@
-use crate::operation::Operation;
+use crate::taskstorage::{Operation, TaskMap, TaskStorage, TaskStorageTxn};
-use crate::taskstorage::{TaskMap, TaskStorage, TaskStorageTxn};
 use failure::{format_err, Fallible};
 use kv::msgpack::Msgpack;
 use kv::{Bucket, Config, Error, Integer, Serde, Store, ValueBuf};

taskchampion/src/taskstorage/mod.rs

@@ -1,14 +1,16 @@
-use crate::Operation;
 use failure::Fallible;
 use std::collections::HashMap;
 use uuid::Uuid;
 
 mod inmemory;
 mod kv;
+mod operation;
 
 pub use self::kv::KVStorage;
 pub use inmemory::InMemoryStorage;
 
+pub use operation::Operation;
+
 /// An in-memory representation of a task as a simple hashmap
 pub type TaskMap = HashMap<String, String>;
 
@@ -22,10 +24,18 @@ fn taskmap_with(mut properties: Vec<(String, String)>) -> TaskMap {
 }
 
 /// A TaskStorage transaction, in which storage operations are performed.
-/// Serializable consistency is maintained, and implementations do not optimize
+///
-/// for concurrent access so some may simply apply a mutex to limit access to
+/// # Concurrency
-/// one transaction at a time.  Transactions are aborted if they are dropped.
+///
-/// It's safe to drop transactions that did not modify any data.
+/// Serializable consistency must be maintained.  Concurrent access is unusual
+/// and some implementations may simply apply a mutex to limit access to
+/// one transaction at a time.
+///
+/// # Commiting and Aborting
+///
+/// A transaction is not visible to other readers until it is committed with
+/// [`crate::taskstorage::TaskStorageTxn::commit`].  Transactions are aborted if they are dropped.
+/// It is safe and performant to drop transactions that did not modify any data without committing.
 pub trait TaskStorageTxn {
     /// Get an (immutable) task, if it is in the storage
     fn get_task(&mut self, uuid: &Uuid) -> Fallible<Option<TaskMap>>;
@@ -83,9 +93,8 @@ pub trait TaskStorageTxn {
     fn commit(&mut self) -> Fallible<()>;
 }
 
-/// A trait for objects able to act as backing storage for a DB.  This API is optimized to be
+/// A trait for objects able to act as task storage.  Most of the interesting behavior is in the
-/// easy to implement, with all of the semantic meaning of the data located in the DB
+/// [`crate::taskstorage::TaskStorageTxn`] trait.
-/// implementation, which is the sole consumer of this trait.
 pub trait TaskStorage {
     /// Begin a transaction
     fn txn<'a>(&'a mut self) -> Fallible<Box<dyn TaskStorageTxn + 'a>>;

taskchampion/src/taskstorage/operation.rs

@@ -129,7 +129,9 @@ impl Operation {
 mod test {
     use super::*;
     use crate::taskdb::DB;
+    use crate::taskstorage::InMemoryStorage;
     use chrono::{Duration, Utc};
+    use proptest::prelude::*;
 
     // note that `tests/operation_transform_invariant.rs` tests the transform function quite
     // thoroughly, so this testing is light.
@@ -273,4 +275,81 @@ mod test {
             None,
         );
     }
+
+    fn uuid_strategy() -> impl Strategy<Value = Uuid> {
+        prop_oneof![
+            Just(Uuid::parse_str("83a2f9ef-f455-4195-b92e-a54c161eebfc").unwrap()),
+            Just(Uuid::parse_str("56e0be07-c61f-494c-a54c-bdcfdd52d2a7").unwrap()),
+            Just(Uuid::parse_str("4b7ed904-f7b0-4293-8a10-ad452422c7b3").unwrap()),
+            Just(Uuid::parse_str("9bdd0546-07c8-4e1f-a9bc-9d6299f4773b").unwrap()),
+        ]
+    }
+
+    fn operation_strategy() -> impl Strategy<Value = Operation> {
+        prop_oneof![
+            uuid_strategy().prop_map(|uuid| Operation::Create { uuid }),
+            uuid_strategy().prop_map(|uuid| Operation::Delete { uuid }),
+            (uuid_strategy(), "(title|project|status)").prop_map(|(uuid, property)| {
+                Operation::Update {
+                    uuid,
+                    property,
+                    value: Some("true".into()),
+                    timestamp: Utc::now(),
+                }
+            }),
+        ]
+    }
+
+    proptest! {
+        #![proptest_config(ProptestConfig {
+          cases: 1024, .. ProptestConfig::default()
+        })]
+        #[test]
+        // check that the two operation sequences have the same effect, enforcing the invariant of
+        // the transform function.
+        fn transform_invariant_holds(o1 in operation_strategy(), o2 in operation_strategy()) {
+            let (o1p, o2p) = Operation::transform(o1.clone(), o2.clone());
+
+            let mut db1 = DB::new(Box::new(InMemoryStorage::new()));
+            let mut db2 = DB::new(Box::new(InMemoryStorage::new()));
+
+            // Ensure that any expected tasks already exist
+            if let Operation::Update{ ref uuid, .. } = o1 {
+                let _ = db1.apply(Operation::Create{uuid: uuid.clone()});
+                let _ = db2.apply(Operation::Create{uuid: uuid.clone()});
+            }
+
+            if let Operation::Update{ ref uuid, .. } = o2 {
+                let _ = db1.apply(Operation::Create{uuid: uuid.clone()});
+                let _ = db2.apply(Operation::Create{uuid: uuid.clone()});
+            }
+
+            if let Operation::Delete{ ref uuid } = o1 {
+                let _ = db1.apply(Operation::Create{uuid: uuid.clone()});
+                let _ = db2.apply(Operation::Create{uuid: uuid.clone()});
+            }
+
+            if let Operation::Delete{ ref uuid } = o2 {
+                let _ = db1.apply(Operation::Create{uuid: uuid.clone()});
+                let _ = db2.apply(Operation::Create{uuid: uuid.clone()});
+            }
+
+            // if applying the initial operations fail, that indicates the operation was invalid
+            // in the base state, so consider the case successful.
+            if let Err(_) = db1.apply(o1) {
+                return Ok(());
+            }
+            if let Err(_) = db2.apply(o2) {
+                return Ok(());
+            }
+
+            if let Some(o) = o2p {
+                db1.apply(o).map_err(|e| TestCaseError::Fail(format!("Applying to db1: {}", e).into()))?;
+            }
+            if let Some(o) = o1p {
+                db2.apply(o).map_err(|e| TestCaseError::Fail(format!("Applying to db2: {}", e).into()))?;
+            }
+            assert_eq!(db1.sorted_tasks(), db2.sorted_tasks());
+        }
+    }
 }

taskchampion/src/testing/mod.rs

@@ -0,0 +1 @@
+pub mod testserver;

taskchampion/src/testing/testserver.rs

@@ -1,7 +1,7 @@
+use crate::server::{Blob, Server, VersionAdd};
 use std::collections::HashMap;
-use taskchampion::server::{Blob, Server, VersionAdd};
 
-pub struct TestServer {
+pub(crate) struct TestServer {
     users: HashMap<String, User>,
 }
 

taskchampion/tests/data/tdb2-test.data

@@ -1,2 +0,0 @@
-[description:"https:\/\/phabricator.services.example.com\/D7364 &open;taskgraph&close; Download debian packages" end:"1541705209" entry:"1538520624" modified:"1541705209" phabricatorid:"D7364" priority:"M" project:"moz" status:"completed" tags:"phabricator,respond" uuid:"ca33f6d6-1688-4503-90be-3b3526a32b5a" wait:"1570118809"]
-[annotation_1541461824:"https:\/\/github.com\/taskcluster\/taskcluster-worker-manager\/pull\/3" description:"https:\/\/github.com\/taskcluster\/taskcluster-worker-manager\/pull\/3 More changes" end:"1541702602" entry:"1541451283" githubbody:"some notes:\n\n1.  This is a huge PR, so I'm not expecting a quick turn around at all.  If you have questions, let me know and I can hope on Vidyo.\n1. Data persistence is written in a semi-janky way.  My intention is to use the time while this is under review to make progress on postgres stuff so that the next review cycle will be using new Postgres things. Which means....  There's a lot of bugs in the concurrency because there's no synchronisation at all in this janky-ish model.\n1. The API is the minimum api required to get provisioning-ish things working\n1. I intend to write a system for automatically testing provider and bidding strategy implementations, so that you can do instantiate a provider\/strategy, stub\/spy it as needed then run a test suite against it and have it do its thing.  The idea is that each provider will need to mock their underlying api system in their own way, but the set of tests we run for Provider API conformance would be pretty standardized.  This should make writing tests for new providers a lot easier.\n1. The provider\/strategy loading system is intentionally simple.  The idea is that these aren't general purpose plugins, but rather special ones.  The idea is that the config files would essentially declare instances and then provide constructor arguments to initialize them all... This would make enabling\/disabling providers\/strategies fairly trivial\n1. I decided to drop fake implementations of providers and strategies for testing the provisioning logic and instead opt for Sinon stubs, which I think give us a better testing story\n1. I still intend to have fake providers and bidding strategies for doing API testing.\n\nLet me know, and again, I don't expect or need a quick turn around on this PR.\n" githubcreatedon:"1541451283" githubnamespace:"djmitche" githubnumber:"3.000000" githubrepo:"taskcluster\/taskcluster-worker-manager" githubtitle:"More changes" githubtype:"pull_request" githubupdatedat:"1541699191" githuburl:"https:\/\/github.com\/taskcluster\/taskcluster-worker-manager\/pull\/3" githubuser:"jhford" modified:"1541702602" priority:"H" project:"moz" status:"completed" tags:"respond" uuid:"2186f981-d1f5-4642-b833-5b16b3a2d334"]

taskchampion/tests/operation_transform_invariant.rs

@@ -1,85 +0,0 @@
-use chrono::Utc;
-use proptest::prelude::*;
-use taskchampion::{taskstorage, Operation, DB};
-use uuid::Uuid;
-
-fn newdb() -> DB {
-    DB::new(Box::new(taskstorage::InMemoryStorage::new()))
-}
-
-fn uuid_strategy() -> impl Strategy<Value = Uuid> {
-    prop_oneof![
-        Just(Uuid::parse_str("83a2f9ef-f455-4195-b92e-a54c161eebfc").unwrap()),
-        Just(Uuid::parse_str("56e0be07-c61f-494c-a54c-bdcfdd52d2a7").unwrap()),
-        Just(Uuid::parse_str("4b7ed904-f7b0-4293-8a10-ad452422c7b3").unwrap()),
-        Just(Uuid::parse_str("9bdd0546-07c8-4e1f-a9bc-9d6299f4773b").unwrap()),
-    ]
-}
-
-fn operation_strategy() -> impl Strategy<Value = Operation> {
-    prop_oneof![
-        uuid_strategy().prop_map(|uuid| Operation::Create { uuid }),
-        uuid_strategy().prop_map(|uuid| Operation::Delete { uuid }),
-        (uuid_strategy(), "(title|project|status)").prop_map(|(uuid, property)| {
-            Operation::Update {
-                uuid,
-                property,
-                value: Some("true".into()),
-                timestamp: Utc::now(),
-            }
-        }),
-    ]
-}
-
-proptest! {
-    #![proptest_config(ProptestConfig {
-      cases: 1024, .. ProptestConfig::default()
-    })]
-    #[test]
-    // check that the two operation sequences have the same effect, enforcing the invariant of
-    // the transform function.
-    fn transform_invariant_holds(o1 in operation_strategy(), o2 in operation_strategy()) {
-        let (o1p, o2p) = Operation::transform(o1.clone(), o2.clone());
-
-        let mut db1 = newdb();
-        let mut db2 = newdb();
-
-        // Ensure that any expected tasks already exist
-        if let Operation::Update{ ref uuid, .. } = o1 {
-            let _ = db1.apply(Operation::Create{uuid: uuid.clone()});
-            let _ = db2.apply(Operation::Create{uuid: uuid.clone()});
-        }
-
-        if let Operation::Update{ ref uuid, .. } = o2 {
-            let _ = db1.apply(Operation::Create{uuid: uuid.clone()});
-            let _ = db2.apply(Operation::Create{uuid: uuid.clone()});
-        }
-
-        if let Operation::Delete{ ref uuid } = o1 {
-            let _ = db1.apply(Operation::Create{uuid: uuid.clone()});
-            let _ = db2.apply(Operation::Create{uuid: uuid.clone()});
-        }
-
-        if let Operation::Delete{ ref uuid } = o2 {
-            let _ = db1.apply(Operation::Create{uuid: uuid.clone()});
-            let _ = db2.apply(Operation::Create{uuid: uuid.clone()});
-        }
-
-        // if applying the initial operations fail, that indicates the operation was invalid
-        // in the base state, so consider the case successful.
-        if let Err(_) = db1.apply(o1) {
-            return Ok(());
-        }
-        if let Err(_) = db2.apply(o2) {
-            return Ok(());
-        }
-
-        if let Some(o) = o2p {
-            db1.apply(o).map_err(|e| TestCaseError::Fail(format!("Applying to db1: {}", e).into()))?;
-        }
-        if let Some(o) = o1p {
-            db2.apply(o).map_err(|e| TestCaseError::Fail(format!("Applying to db2: {}", e).into()))?;
-        }
-        assert_eq!(db1.sorted_tasks(), db2.sorted_tasks());
-    }
-}

taskchampion/tests/shared/mod.rs

@@ -1,3 +0,0 @@
-mod testserver;
-
-pub use testserver::TestServer;

taskchampion/tests/sync.rs

@@ -1,123 +0,0 @@
-use chrono::Utc;
-use taskchampion::{taskstorage, Operation, DB};
-use uuid::Uuid;
-
-mod shared;
-use shared::TestServer;
-
-fn newdb() -> DB {
-    DB::new(Box::new(taskstorage::InMemoryStorage::new()))
-}
-
-#[test]
-fn test_sync() {
-    let mut server = TestServer::new();
-
-    let mut db1 = newdb();
-    db1.sync("me", &mut server).unwrap();
-
-    let mut db2 = newdb();
-    db2.sync("me", &mut server).unwrap();
-
-    // make some changes in parallel to db1 and db2..
-    let uuid1 = Uuid::new_v4();
-    db1.apply(Operation::Create { uuid: uuid1 }).unwrap();
-    db1.apply(Operation::Update {
-        uuid: uuid1,
-        property: "title".into(),
-        value: Some("my first task".into()),
-        timestamp: Utc::now(),
-    })
-    .unwrap();
-
-    let uuid2 = Uuid::new_v4();
-    db2.apply(Operation::Create { uuid: uuid2 }).unwrap();
-    db2.apply(Operation::Update {
-        uuid: uuid2,
-        property: "title".into(),
-        value: Some("my second task".into()),
-        timestamp: Utc::now(),
-    })
-    .unwrap();
-
-    // and synchronize those around
-    db1.sync("me", &mut server).unwrap();
-    db2.sync("me", &mut server).unwrap();
-    db1.sync("me", &mut server).unwrap();
-    assert_eq!(db1.sorted_tasks(), db2.sorted_tasks());
-
-    // now make updates to the same task on both sides
-    db1.apply(Operation::Update {
-        uuid: uuid2,
-        property: "priority".into(),
-        value: Some("H".into()),
-        timestamp: Utc::now(),
-    })
-    .unwrap();
-    db2.apply(Operation::Update {
-        uuid: uuid2,
-        property: "project".into(),
-        value: Some("personal".into()),
-        timestamp: Utc::now(),
-    })
-    .unwrap();
-
-    // and synchronize those around
-    db1.sync("me", &mut server).unwrap();
-    db2.sync("me", &mut server).unwrap();
-    db1.sync("me", &mut server).unwrap();
-    assert_eq!(db1.sorted_tasks(), db2.sorted_tasks());
-}
-
-#[test]
-fn test_sync_create_delete() {
-    let mut server = TestServer::new();
-
-    let mut db1 = newdb();
-    db1.sync("me", &mut server).unwrap();
-
-    let mut db2 = newdb();
-    db2.sync("me", &mut server).unwrap();
-
-    // create and update a task..
-    let uuid = Uuid::new_v4();
-    db1.apply(Operation::Create { uuid }).unwrap();
-    db1.apply(Operation::Update {
-        uuid: uuid,
-        property: "title".into(),
-        value: Some("my first task".into()),
-        timestamp: Utc::now(),
-    })
-    .unwrap();
-
-    // and synchronize those around
-    db1.sync("me", &mut server).unwrap();
-    db2.sync("me", &mut server).unwrap();
-    db1.sync("me", &mut server).unwrap();
-    assert_eq!(db1.sorted_tasks(), db2.sorted_tasks());
-
-    // delete and re-create the task on db1
-    db1.apply(Operation::Delete { uuid }).unwrap();
-    db1.apply(Operation::Create { uuid }).unwrap();
-    db1.apply(Operation::Update {
-        uuid: uuid,
-        property: "title".into(),
-        value: Some("my second task".into()),
-        timestamp: Utc::now(),
-    })
-    .unwrap();
-
-    // and on db2, update a property of the task
-    db2.apply(Operation::Update {
-        uuid: uuid,
-        property: "project".into(),
-        value: Some("personal".into()),
-        timestamp: Utc::now(),
-    })
-    .unwrap();
-
-    db1.sync("me", &mut server).unwrap();
-    db2.sync("me", &mut server).unwrap();
-    db1.sync("me", &mut server).unwrap();
-    assert_eq!(db1.sorted_tasks(), db2.sorted_tasks());
-}

taskchampion/tests/sync_action_sequences.rs

@@ -1,72 +0,0 @@
-use chrono::Utc;
-use proptest::prelude::*;
-use taskchampion::{taskstorage, Operation, DB};
-use uuid::Uuid;
-
-mod shared;
-use shared::TestServer;
-
-fn newdb() -> DB {
-    DB::new(Box::new(taskstorage::InMemoryStorage::new()))
-}
-
-#[derive(Debug)]
-enum Action {
-    Op(Operation),
-    Sync,
-}
-
-fn action_sequence_strategy() -> impl Strategy<Value = Vec<(Action, u8)>> {
-    // Create, Update, Delete, or Sync on client 1, 2, .., followed by a round of syncs
-    "([CUDS][123])*S1S2S3S1S2".prop_map(|seq| {
-        let uuid = Uuid::parse_str("83a2f9ef-f455-4195-b92e-a54c161eebfc").unwrap();
-        seq.as_bytes()
-            .chunks(2)
-            .map(|action_on| {
-                let action = match action_on[0] {
-                    b'C' => Action::Op(Operation::Create { uuid }),
-                    b'U' => Action::Op(Operation::Update {
-                        uuid,
-                        property: "title".into(),
-                        value: Some("foo".into()),
-                        timestamp: Utc::now(),
-                    }),
-                    b'D' => Action::Op(Operation::Delete { uuid }),
-                    b'S' => Action::Sync,
-                    _ => unreachable!(),
-                };
-                let acton = action_on[1] - b'1';
-                (action, acton)
-            })
-            .collect::<Vec<(Action, u8)>>()
-    })
-}
-
-proptest! {
-    #[test]
-    // check that various sequences of operations on mulitple db's do not get the db's into an
-    // incompatible state.  The main concern here is that there might be a sequence of create
-    // and delete operations that results in a task existing in one DB but not existing in
-    // another.  So, the generated sequences focus on a single task UUID.
-    fn transform_sequences_of_operations(action_sequence in action_sequence_strategy()) {
-        let mut server = TestServer::new();
-        let mut dbs = [newdb(), newdb(), newdb()];
-
-        for (action, db) in action_sequence {
-            println!("{:?} on db {}", action, db);
-
-            let db = &mut dbs[db as usize];
-            match action {
-                Action::Op(op) => {
-                    if let Err(e) = db.apply(op) {
-                        println!("  {:?} (ignored)", e);
-                    }
-                },
-                Action::Sync => db.sync("me", &mut server).unwrap(),
-            }
-        }
-
-        assert_eq!(dbs[0].sorted_tasks(), dbs[0].sorted_tasks());
-        assert_eq!(dbs[1].sorted_tasks(), dbs[2].sorted_tasks());
-    }
-}