GothenburgBitFactory/taskwarrior
1
0
Fork 0
mirror of https://github.com/GothenburgBitFactory/taskwarrior.git synced 2025-07-07 20:06:36 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Reorganize taskchampion crate for docs and tests

Browse source 
The public API of the taskchampion crate now contains the expected parts
and no more, and has some better documentation.

This moves the crate's external `tests/` into internal tests, as the
TaskDB is no longer exposed as part of the crate API.
This commit is contained in:
Dustin J. Mitchell 2020-11-23 15:59:37 -05:00
parent 8f4924f903
commit 8e2b4c3f6c
16 changed files with 395 additions and 347 deletions
Download patch file Download diff file Expand all files Collapse all files

9
cli/src/main.rs
View file

@ -1,6 +1,6 @@
use clap::{App, Arg, SubCommand};
use std::path::Path;
use taskchampion::{taskstorage, Replica, Status, DB};
use taskchampion::{taskstorage, Replica, Status};
use uuid::Uuid;
fn main() {
@ -20,12 +20,9 @@ fn main() {
.subcommand(SubCommand::with_name("gc").about("run garbage collection"))
.get_matches();
let mut replica = Replica::new(
DB::new(Box::new(
let mut replica = Replica::new(Box::new(
taskstorage::KVStorage::new(Path::new("/tmp/tasks")).unwrap(),
))
.into(),
);
));
match matches.subcommand() {
("add", Some(matches)) => {

32
taskchampion/src/lib.rs
View file

@ -1,14 +1,38 @@
/*!
This crate implements the core of TaskChampion, the [replica](crate::Replica).
A TaskChampion replica is a local copy of a user's task data. As the name suggests, several
replicas of the same data can exist (such as on a user's laptop and on their phone) and can
synchronize with one another.
# Task Storage
The [`taskstorage`](crate::taskstorage) module supports pluggable storage for a replica's data.
An implementation is provided, but users of this crate can provide their own implementation as well.
# Server
Replica synchronization takes place against a server.
The [`server`](crate::server) module defines the interface a server must meet.
# See Also
See the [TaskChampion Book](https://github.com/djmitche/taskchampion/blob/main/docs/src/SUMMARY.md)
for more information about the design and usage of the tool.
*/
mod errors;
mod operation;
mod replica;
pub mod server;
mod task;
mod taskdb;
pub mod taskstorage;
pub use operation::Operation;
pub use replica::Replica;
pub use task::Priority;
pub use task::Status;
pub use task::Task;
pub use taskdb::DB;
pub use task::{Task, TaskMut};
#[cfg(test)]
pub(crate) mod testing;

42
taskchampion/src/replica.rs
View file

@ -1,21 +1,29 @@
use crate::errors::Error;
use crate::operation::Operation;
use crate::server::Server;
use crate::task::{Status, Task};
use crate::taskdb::DB;
use crate::taskstorage::TaskMap;
use crate::taskstorage::{Operation, TaskMap, TaskStorage};
use chrono::Utc;
use failure::Fallible;
use std::collections::HashMap;
use uuid::Uuid;
/// A replica represents an instance of a user's task data.
/// A replica represents an instance of a user's task data, providing an easy interface
/// for querying and modifying that data.
pub struct Replica {
taskdb: Box<DB>,
taskdb: DB,
}
impl Replica {
pub fn new(taskdb: Box<DB>) -> Replica {
return Replica { taskdb };
pub fn new(storage: Box<dyn TaskStorage>) -> Replica {
return Replica {
taskdb: DB::new(storage),
};
}
#[cfg(test)]
pub fn new_inmemory() -> Replica {
Replica::new(Box::new(crate::taskstorage::InMemoryStorage::new()))
}
/// Update an existing task. If the value is Some, the property is added or updated. If the
@ -45,7 +53,7 @@ impl Replica {
}
/// Get all tasks represented as a map keyed by UUID
pub fn all_tasks<'a>(&'a mut self) -> Fallible<HashMap<Uuid, Task>> {
pub fn all_tasks(&mut self) -> Fallible<HashMap<Uuid, Task>> {
let mut res = HashMap::new();
for (uuid, tm) in self.taskdb.all_tasks()?.drain(..) {
res.insert(uuid.clone(), Task::new(uuid.clone(), tm));
@ -54,7 +62,7 @@ impl Replica {
}
/// Get the UUIDs of all tasks
pub fn all_task_uuids<'a>(&'a mut self) -> Fallible<Vec<Uuid>> {
pub fn all_task_uuids(&mut self) -> Fallible<Vec<Uuid>> {
self.taskdb.all_task_uuids()
}
@ -124,6 +132,11 @@ impl Replica {
self.taskdb.apply(Operation::Delete { uuid })
}
/// Synchronize this replica against the given server.
pub fn sync(&mut self, username: &str, server: &mut dyn Server) -> Fallible<()> {
self.taskdb.sync(username, server)
}
/// Perform "garbage collection" on this replica. In particular, this renumbers the working
/// set to contain only pending tasks.
pub fn gc(&mut self) -> Fallible<()> {
@ -138,12 +151,11 @@ impl Replica {
mod tests {
use super::*;
use crate::task::Status;
use crate::taskdb::DB;
use uuid::Uuid;
#[test]
fn new_task() {
let mut rep = Replica::new(DB::new_inmemory().into());
let mut rep = Replica::new_inmemory();
let uuid = Uuid::new_v4();
let t = rep
@ -156,7 +168,7 @@ mod tests {
#[test]
fn modify_task() {
let mut rep = Replica::new(DB::new_inmemory().into());
let mut rep = Replica::new_inmemory();
let uuid = Uuid::new_v4();
let t = rep
@ -183,7 +195,7 @@ mod tests {
#[test]
fn delete_task() {
let mut rep = Replica::new(DB::new_inmemory().into());
let mut rep = Replica::new_inmemory();
let uuid = Uuid::new_v4();
rep.new_task(uuid.clone(), Status::Pending, "a task".into())
@ -195,7 +207,7 @@ mod tests {
#[test]
fn get_and_modify() {
let mut rep = Replica::new(DB::new_inmemory().into());
let mut rep = Replica::new_inmemory();
let uuid = Uuid::new_v4();
rep.new_task(uuid.clone(), Status::Pending, "another task".into())
@ -215,7 +227,7 @@ mod tests {
#[test]
fn new_pending_adds_to_working_set() {
let mut rep = Replica::new(DB::new_inmemory().into());
let mut rep = Replica::new_inmemory();
let uuid = Uuid::new_v4();
rep.new_task(uuid.clone(), Status::Pending, "to-be-pending".into())
@ -233,7 +245,7 @@ mod tests {
#[test]
fn get_does_not_exist() {
let mut rep = Replica::new(DB::new_inmemory().into());
let mut rep = Replica::new_inmemory();
let uuid = Uuid::new_v4();
assert_eq!(rep.get_task(&uuid).unwrap(), None);
}

65
taskchampion/src/task.rs
View file

@ -1,34 +1,37 @@
use crate::replica::Replica;
use crate::taskstorage::TaskMap;
use chrono::prelude::*;
use failure::{format_err, Fallible};
use std::convert::TryFrom;
use failure::Fallible;
use uuid::Uuid;
pub type Timestamp = DateTime<Utc>;
/// The priority of a task
#[derive(Debug, PartialEq)]
pub enum Priority {
/// Low
L,
/// Medium
M,
/// High
H,
}
impl TryFrom<&str> for Priority {
type Error = failure::Error;
fn try_from(s: &str) -> Result<Self, Self::Error> {
#[allow(dead_code)]
impl Priority {
/// Get a Priority from the 1-character value in a TaskMap,
/// defaulting to M
pub(crate) fn from_taskmap(s: &str) -> Priority {
match s {
"L" => Ok(Priority::L),
"M" => Ok(Priority::M),
"H" => Ok(Priority::H),
_ => Err(format_err!("invalid status {}", s)),
}
"L" => Priority::L,
"M" => Priority::M,
"H" => Priority::H,
_ => Priority::M,
}
}
impl AsRef<str> for Priority {
fn as_ref(&self) -> &str {
/// Get the 1-character value for this priority to use in the TaskMap.
pub(crate) fn to_taskmap(&self) -> &str {
match self {
Priority::L => "L",
Priority::M => "M",
@ -36,6 +39,8 @@ impl AsRef<str> for Priority {
}
}
}
/// The status of a task. The default status in "Pending".
#[derive(Debug, PartialEq)]
pub enum Status {
Pending,
@ -86,8 +91,8 @@ pub struct Task {
taskmap: TaskMap,
}
/// A mutable task, with setter methods. Calling a setter will update the Replica, as well as the
/// included Task.
/// A mutable task, with setter methods. Most methods are simple setters and not further
/// described. Calling a setter will update the Replica, as well as the included Task.
pub struct TaskMut<'r> {
task: Task,
replica: &'r mut Replica,
@ -145,7 +150,8 @@ impl Task {
}
impl<'r> TaskMut<'r> {
/// Get the immutable task
/// Get the immutable version of this object. Note that TaskMut [`std::ops::Deref`]s to
/// [`crate::task::Task`], so all of that struct's getter methods can be used on TaskMut.
pub fn into_immut(self) -> Task {
self.task
}
@ -160,12 +166,10 @@ impl<'r> TaskMut<'r> {
self.set_string("status", Some(String::from(status.to_taskmap())))
}
/// Set the task's description
pub fn set_description(&mut self, description: String) -> Fallible<()> {
self.set_string("description", Some(description))
}
/// Set the task's description
pub fn set_modified(&mut self, modified: DateTime<Utc>) -> Fallible<()> {
self.set_timestamp("modified", Some(modified))
}
@ -213,3 +217,28 @@ impl<'r> std::ops::Deref for TaskMut<'r> {
&self.task
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn test_priority() {
assert_eq!(Priority::L.to_taskmap(), "L");
assert_eq!(Priority::M.to_taskmap(), "M");
assert_eq!(Priority::H.to_taskmap(), "H");
assert_eq!(Priority::from_taskmap("L"), Priority::L);
assert_eq!(Priority::from_taskmap("M"), Priority::M);
assert_eq!(Priority::from_taskmap("H"), Priority::H);
}
#[test]
fn test_status() {
assert_eq!(Status::Pending.to_taskmap(), "P");
assert_eq!(Status::Completed.to_taskmap(), "C");
assert_eq!(Status::Deleted.to_taskmap(), "D");
assert_eq!(Status::from_taskmap("P"), Status::Pending);
assert_eq!(Status::from_taskmap("C"), Status::Completed);
assert_eq!(Status::from_taskmap("D"), Status::Deleted);
}
}

190
taskchampion/src/taskdb.rs
View file

@ -1,7 +1,6 @@
use crate::errors::Error;
use crate::operation::Operation;
use crate::server::{Server, VersionAdd};
use crate::taskstorage::{TaskMap, TaskStorage, TaskStorageTxn};
use crate::taskstorage::{Operation, TaskMap, TaskStorage, TaskStorageTxn};
use failure::Fallible;
use serde::{Deserialize, Serialize};
use std::collections::HashSet;
@ -268,7 +267,8 @@ impl DB {
// functions for supporting tests
pub fn sorted_tasks(&mut self) -> Vec<(Uuid, Vec<(String, String)>)> {
#[cfg(test)]
pub(crate) fn sorted_tasks(&mut self) -> Vec<(Uuid, Vec<(String, String)>)> {
let mut res: Vec<(Uuid, Vec<(String, String)>)> = self
.all_tasks()
.unwrap()
@ -286,7 +286,8 @@ impl DB {
res
}
pub fn operations(&mut self) -> Vec<Operation> {
#[cfg(test)]
pub(crate) fn operations(&mut self) -> Vec<Operation> {
let mut txn = self.storage.txn().unwrap();
txn.operations()
.unwrap()
@ -299,7 +300,10 @@ impl DB {
#[cfg(test)]
mod tests {
use super::*;
use crate::taskstorage::InMemoryStorage;
use crate::testing::testserver::TestServer;
use chrono::Utc;
use proptest::prelude::*;
use std::collections::HashMap;
use uuid::Uuid;
@ -508,4 +512,182 @@ mod tests {
Ok(())
}
fn newdb() -> DB {
DB::new(Box::new(InMemoryStorage::new()))
}
#[test]
fn test_sync() {
let mut server = TestServer::new();
let mut db1 = newdb();
db1.sync("me", &mut server).unwrap();
let mut db2 = newdb();
db2.sync("me", &mut server).unwrap();
// make some changes in parallel to db1 and db2..
let uuid1 = Uuid::new_v4();
db1.apply(Operation::Create { uuid: uuid1 }).unwrap();
db1.apply(Operation::Update {
uuid: uuid1,
property: "title".into(),
value: Some("my first task".into()),
timestamp: Utc::now(),
})
.unwrap();
let uuid2 = Uuid::new_v4();
db2.apply(Operation::Create { uuid: uuid2 }).unwrap();
db2.apply(Operation::Update {
uuid: uuid2,
property: "title".into(),
value: Some("my second task".into()),
timestamp: Utc::now(),
})
.unwrap();
// and synchronize those around
db1.sync("me", &mut server).unwrap();
db2.sync("me", &mut server).unwrap();
db1.sync("me", &mut server).unwrap();
assert_eq!(db1.sorted_tasks(), db2.sorted_tasks());
// now make updates to the same task on both sides
db1.apply(Operation::Update {
uuid: uuid2,
property: "priority".into(),
value: Some("H".into()),
timestamp: Utc::now(),
})
.unwrap();
db2.apply(Operation::Update {
uuid: uuid2,
property: "project".into(),
value: Some("personal".into()),
timestamp: Utc::now(),
})
.unwrap();
// and synchronize those around
db1.sync("me", &mut server).unwrap();
db2.sync("me", &mut server).unwrap();
db1.sync("me", &mut server).unwrap();
assert_eq!(db1.sorted_tasks(), db2.sorted_tasks());
}
#[test]
fn test_sync_create_delete() {
let mut server = TestServer::new();
let mut db1 = newdb();
db1.sync("me", &mut server).unwrap();
let mut db2 = newdb();
db2.sync("me", &mut server).unwrap();
// create and update a task..
let uuid = Uuid::new_v4();
db1.apply(Operation::Create { uuid }).unwrap();
db1.apply(Operation::Update {
uuid: uuid,
property: "title".into(),
value: Some("my first task".into()),
timestamp: Utc::now(),
})
.unwrap();
// and synchronize those around
db1.sync("me", &mut server).unwrap();
db2.sync("me", &mut server).unwrap();
db1.sync("me", &mut server).unwrap();
assert_eq!(db1.sorted_tasks(), db2.sorted_tasks());
// delete and re-create the task on db1
db1.apply(Operation::Delete { uuid }).unwrap();
db1.apply(Operation::Create { uuid }).unwrap();
db1.apply(Operation::Update {
uuid: uuid,
property: "title".into(),
value: Some("my second task".into()),
timestamp: Utc::now(),
})
.unwrap();
// and on db2, update a property of the task
db2.apply(Operation::Update {
uuid: uuid,
property: "project".into(),
value: Some("personal".into()),
timestamp: Utc::now(),
})
.unwrap();
db1.sync("me", &mut server).unwrap();
db2.sync("me", &mut server).unwrap();
db1.sync("me", &mut server).unwrap();
assert_eq!(db1.sorted_tasks(), db2.sorted_tasks());
}
#[derive(Debug)]
enum Action {
Op(Operation),
Sync,
}
fn action_sequence_strategy() -> impl Strategy<Value = Vec<(Action, u8)>> {
// Create, Update, Delete, or Sync on client 1, 2, .., followed by a round of syncs
"([CUDS][123])*S1S2S3S1S2".prop_map(|seq| {
let uuid = Uuid::parse_str("83a2f9ef-f455-4195-b92e-a54c161eebfc").unwrap();
seq.as_bytes()
.chunks(2)
.map(|action_on| {
let action = match action_on[0] {
b'C' => Action::Op(Operation::Create { uuid }),
b'U' => Action::Op(Operation::Update {
uuid,
property: "title".into(),
value: Some("foo".into()),
timestamp: Utc::now(),
}),
b'D' => Action::Op(Operation::Delete { uuid }),
b'S' => Action::Sync,
_ => unreachable!(),
};
let acton = action_on[1] - b'1';
(action, acton)
})
.collect::<Vec<(Action, u8)>>()
})
}
proptest! {
#[test]
// check that various sequences of operations on mulitple db's do not get the db's into an
// incompatible state. The main concern here is that there might be a sequence of create
// and delete operations that results in a task existing in one DB but not existing in
// another. So, the generated sequences focus on a single task UUID.
fn transform_sequences_of_operations(action_sequence in action_sequence_strategy()) {
let mut server = TestServer::new();
let mut dbs = [newdb(), newdb(), newdb()];
for (action, db) in action_sequence {
println!("{:?} on db {}", action, db);
let db = &mut dbs[db as usize];
match action {
Action::Op(op) => {
if let Err(e) = db.apply(op) {
println!(" {:?} (ignored)", e);
}
},
Action::Sync => db.sync("me", &mut server).unwrap(),
}
}
assert_eq!(dbs[0].sorted_tasks(), dbs[0].sorted_tasks());
assert_eq!(dbs[1].sorted_tasks(), dbs[2].sorted_tasks());
}
}
}

5
taskchampion/src/taskstorage/inmemory.rs
View file

@ -1,5 +1,4 @@
use crate::operation::Operation;
use crate::taskstorage::{TaskMap, TaskStorage, TaskStorageTxn};
use crate::taskstorage::{Operation, TaskMap, TaskStorage, TaskStorageTxn};
use failure::{format_err, Fallible};
use std::collections::hash_map::Entry;
use std::collections::HashMap;
@ -139,6 +138,8 @@ impl<'t> TaskStorageTxn for Txn<'t> {
}
}
/// InMemoryStorage is a simple in-memory task storage implementation. It is not useful for
/// production data, but is useful for testing purposes.
#[derive(PartialEq, Debug, Clone)]
pub struct InMemoryStorage {
data: Data,

3
taskchampion/src/taskstorage/kv.rs
View file

@ -1,5 +1,4 @@
use crate::operation::Operation;
use crate::taskstorage::{TaskMap, TaskStorage, TaskStorageTxn};
use crate::taskstorage::{Operation, TaskMap, TaskStorage, TaskStorageTxn};
use failure::{format_err, Fallible};
use kv::msgpack::Msgpack;
use kv::{Bucket, Config, Error, Integer, Serde, Store, ValueBuf};

25
taskchampion/src/taskstorage/mod.rs
View file

@ -1,14 +1,16 @@
use crate::Operation;
use failure::Fallible;
use std::collections::HashMap;
use uuid::Uuid;
mod inmemory;
mod kv;
mod operation;
pub use self::kv::KVStorage;
pub use inmemory::InMemoryStorage;
pub use operation::Operation;
/// An in-memory representation of a task as a simple hashmap
pub type TaskMap = HashMap<String, String>;
@ -22,10 +24,18 @@ fn taskmap_with(mut properties: Vec<(String, String)>) -> TaskMap {
}
/// A TaskStorage transaction, in which storage operations are performed.
/// Serializable consistency is maintained, and implementations do not optimize
/// for concurrent access so some may simply apply a mutex to limit access to
/// one transaction at a time. Transactions are aborted if they are dropped.
/// It's safe to drop transactions that did not modify any data.
///
/// # Concurrency
///
/// Serializable consistency must be maintained. Concurrent access is unusual
/// and some implementations may simply apply a mutex to limit access to
/// one transaction at a time.
///
/// # Commiting and Aborting
///
/// A transaction is not visible to other readers until it is committed with
/// [`crate::taskstorage::TaskStorageTxn::commit`]. Transactions are aborted if they are dropped.
/// It is safe and performant to drop transactions that did not modify any data without committing.
pub trait TaskStorageTxn {
/// Get an (immutable) task, if it is in the storage
fn get_task(&mut self, uuid: &Uuid) -> Fallible<Option<TaskMap>>;
@ -83,9 +93,8 @@ pub trait TaskStorageTxn {
fn commit(&mut self) -> Fallible<()>;
}
/// A trait for objects able to act as backing storage for a DB. This API is optimized to be
/// easy to implement, with all of the semantic meaning of the data located in the DB
/// implementation, which is the sole consumer of this trait.
/// A trait for objects able to act as task storage. Most of the interesting behavior is in the
/// [`crate::taskstorage::TaskStorageTxn`] trait.
pub trait TaskStorage {
/// Begin a transaction
fn txn<'a>(&'a mut self) -> Fallible<Box<dyn TaskStorageTxn + 'a>>;

79
taskchampion/src/operation.rs → taskchampion/src/taskstorage/operation.rs
View file

@ -129,7 +129,9 @@ impl Operation {
mod test {
use super::*;
use crate::taskdb::DB;
use crate::taskstorage::InMemoryStorage;
use chrono::{Duration, Utc};
use proptest::prelude::*;
// note that `tests/operation_transform_invariant.rs` tests the transform function quite
// thoroughly, so this testing is light.
@ -273,4 +275,81 @@ mod test {
None,
);
}
fn uuid_strategy() -> impl Strategy<Value = Uuid> {
prop_oneof![
Just(Uuid::parse_str("83a2f9ef-f455-4195-b92e-a54c161eebfc").unwrap()),
Just(Uuid::parse_str("56e0be07-c61f-494c-a54c-bdcfdd52d2a7").unwrap()),
Just(Uuid::parse_str("4b7ed904-f7b0-4293-8a10-ad452422c7b3").unwrap()),
Just(Uuid::parse_str("9bdd0546-07c8-4e1f-a9bc-9d6299f4773b").unwrap()),
]
}
fn operation_strategy() -> impl Strategy<Value = Operation> {
prop_oneof![
uuid_strategy().prop_map(|uuid| Operation::Create { uuid }),
uuid_strategy().prop_map(|uuid| Operation::Delete { uuid }),
(uuid_strategy(), "(title|project|status)").prop_map(|(uuid, property)| {
Operation::Update {
uuid,
property,
value: Some("true".into()),
timestamp: Utc::now(),
}
}),
]
}
proptest! {
#![proptest_config(ProptestConfig {
cases: 1024, .. ProptestConfig::default()
})]
#[test]
// check that the two operation sequences have the same effect, enforcing the invariant of
// the transform function.
fn transform_invariant_holds(o1 in operation_strategy(), o2 in operation_strategy()) {
let (o1p, o2p) = Operation::transform(o1.clone(), o2.clone());
let mut db1 = DB::new(Box::new(InMemoryStorage::new()));
let mut db2 = DB::new(Box::new(InMemoryStorage::new()));
// Ensure that any expected tasks already exist
if let Operation::Update{ ref uuid, .. } = o1 {
let _ = db1.apply(Operation::Create{uuid: uuid.clone()});
let _ = db2.apply(Operation::Create{uuid: uuid.clone()});
}
if let Operation::Update{ ref uuid, .. } = o2 {
let _ = db1.apply(Operation::Create{uuid: uuid.clone()});
let _ = db2.apply(Operation::Create{uuid: uuid.clone()});
}
if let Operation::Delete{ ref uuid } = o1 {
let _ = db1.apply(Operation::Create{uuid: uuid.clone()});
let _ = db2.apply(Operation::Create{uuid: uuid.clone()});
}
if let Operation::Delete{ ref uuid } = o2 {
let _ = db1.apply(Operation::Create{uuid: uuid.clone()});
let _ = db2.apply(Operation::Create{uuid: uuid.clone()});
}
// if applying the initial operations fail, that indicates the operation was invalid
// in the base state, so consider the case successful.
if let Err(_) = db1.apply(o1) {
return Ok(());
}
if let Err(_) = db2.apply(o2) {
return Ok(());
}
if let Some(o) = o2p {
db1.apply(o).map_err(|e| TestCaseError::Fail(format!("Applying to db1: {}", e).into()))?;
}
if let Some(o) = o1p {
db2.apply(o).map_err(|e| TestCaseError::Fail(format!("Applying to db2: {}", e).into()))?;
}
assert_eq!(db1.sorted_tasks(), db2.sorted_tasks());
}
}
}

1
taskchampion/src/testing/mod.rs Normal file
View file

@ -0,0 +1 @@
pub mod testserver;

4
taskchampion/tests/shared/testserver.rs → taskchampion/src/testing/testserver.rs
View file

@ -1,7 +1,7 @@
use crate::server::{Blob, Server, VersionAdd};
use std::collections::HashMap;
use taskchampion::server::{Blob, Server, VersionAdd};
pub struct TestServer {
pub(crate) struct TestServer {
users: HashMap<String, User>,
}

2
taskchampion/tests/data/tdb2-test.data
View file

@ -1,2 +0,0 @@
[description:"https:\/\/phabricator.services.example.com\/D7364 &open;taskgraph&close; Download debian packages" end:"1541705209" entry:"1538520624" modified:"1541705209" phabricatorid:"D7364" priority:"M" project:"moz" status:"completed" tags:"phabricator,respond" uuid:"ca33f6d6-1688-4503-90be-3b3526a32b5a" wait:"1570118809"]
[annotation_1541461824:"https:\/\/github.com\/taskcluster\/taskcluster-worker-manager\/pull\/3" description:"https:\/\/github.com\/taskcluster\/taskcluster-worker-manager\/pull\/3 More changes" end:"1541702602" entry:"1541451283" githubbody:"some notes:\n\n1. This is a huge PR, so I'm not expecting a quick turn around at all. If you have questions, let me know and I can hope on Vidyo.\n1. Data persistence is written in a semi-janky way. My intention is to use the time while this is under review to make progress on postgres stuff so that the next review cycle will be using new Postgres things. Which means.... There's a lot of bugs in the concurrency because there's no synchronisation at all in this janky-ish model.\n1. The API is the minimum api required to get provisioning-ish things working\n1. I intend to write a system for automatically testing provider and bidding strategy implementations, so that you can do instantiate a provider\/strategy, stub\/spy it as needed then run a test suite against it and have it do its thing. The idea is that each provider will need to mock their underlying api system in their own way, but the set of tests we run for Provider API conformance would be pretty standardized. This should make writing tests for new providers a lot easier.\n1. The provider\/strategy loading system is intentionally simple. The idea is that these aren't general purpose plugins, but rather special ones. The idea is that the config files would essentially declare instances and then provide constructor arguments to initialize them all... This would make enabling\/disabling providers\/strategies fairly trivial\n1. I decided to drop fake implementations of providers and strategies for testing the provisioning logic and instead opt for Sinon stubs, which I think give us a better testing story\n1. I still intend to have fake providers and bidding strategies for doing API testing.\n\nLet me know, and again, I don't expect or need a quick turn around on this PR.\n" githubcreatedon:"1541451283" githubnamespace:"djmitche" githubnumber:"3.000000" githubrepo:"taskcluster\/taskcluster-worker-manager" githubtitle:"More changes" githubtype:"pull_request" githubupdatedat:"1541699191" githuburl:"https:\/\/github.com\/taskcluster\/taskcluster-worker-manager\/pull\/3" githubuser:"jhford" modified:"1541702602" priority:"H" project:"moz" status:"completed" tags:"respond" uuid:"2186f981-d1f5-4642-b833-5b16b3a2d334"]

85
taskchampion/tests/operation_transform_invariant.rs
View file

@ -1,85 +0,0 @@
use chrono::Utc;
use proptest::prelude::*;
use taskchampion::{taskstorage, Operation, DB};
use uuid::Uuid;
fn newdb() -> DB {
DB::new(Box::new(taskstorage::InMemoryStorage::new()))
}
fn uuid_strategy() -> impl Strategy<Value = Uuid> {
prop_oneof![
Just(Uuid::parse_str("83a2f9ef-f455-4195-b92e-a54c161eebfc").unwrap()),
Just(Uuid::parse_str("56e0be07-c61f-494c-a54c-bdcfdd52d2a7").unwrap()),
Just(Uuid::parse_str("4b7ed904-f7b0-4293-8a10-ad452422c7b3").unwrap()),
Just(Uuid::parse_str("9bdd0546-07c8-4e1f-a9bc-9d6299f4773b").unwrap()),
]
}
fn operation_strategy() -> impl Strategy<Value = Operation> {
prop_oneof![
uuid_strategy().prop_map(|uuid| Operation::Create { uuid }),
uuid_strategy().prop_map(|uuid| Operation::Delete { uuid }),
(uuid_strategy(), "(title|project|status)").prop_map(|(uuid, property)| {
Operation::Update {
uuid,
property,
value: Some("true".into()),
timestamp: Utc::now(),
}
}),
]
}
proptest! {
#![proptest_config(ProptestConfig {
cases: 1024, .. ProptestConfig::default()
})]
#[test]
// check that the two operation sequences have the same effect, enforcing the invariant of
// the transform function.
fn transform_invariant_holds(o1 in operation_strategy(), o2 in operation_strategy()) {
let (o1p, o2p) = Operation::transform(o1.clone(), o2.clone());
let mut db1 = newdb();
let mut db2 = newdb();
// Ensure that any expected tasks already exist
if let Operation::Update{ ref uuid, .. } = o1 {
let _ = db1.apply(Operation::Create{uuid: uuid.clone()});
let _ = db2.apply(Operation::Create{uuid: uuid.clone()});
}
if let Operation::Update{ ref uuid, .. } = o2 {
let _ = db1.apply(Operation::Create{uuid: uuid.clone()});
let _ = db2.apply(Operation::Create{uuid: uuid.clone()});
}
if let Operation::Delete{ ref uuid } = o1 {
let _ = db1.apply(Operation::Create{uuid: uuid.clone()});
let _ = db2.apply(Operation::Create{uuid: uuid.clone()});
}
if let Operation::Delete{ ref uuid } = o2 {
let _ = db1.apply(Operation::Create{uuid: uuid.clone()});
let _ = db2.apply(Operation::Create{uuid: uuid.clone()});
}
// if applying the initial operations fail, that indicates the operation was invalid
// in the base state, so consider the case successful.
if let Err(_) = db1.apply(o1) {
return Ok(());
}
if let Err(_) = db2.apply(o2) {
return Ok(());
}
if let Some(o) = o2p {
db1.apply(o).map_err(|e| TestCaseError::Fail(format!("Applying to db1: {}", e).into()))?;
}
if let Some(o) = o1p {
db2.apply(o).map_err(|e| TestCaseError::Fail(format!("Applying to db2: {}", e).into()))?;
}
assert_eq!(db1.sorted_tasks(), db2.sorted_tasks());
}
}

3
taskchampion/tests/shared/mod.rs
View file

@ -1,3 +0,0 @@
mod testserver;
pub use testserver::TestServer;

123
taskchampion/tests/sync.rs
View file

@ -1,123 +0,0 @@
use chrono::Utc;
use taskchampion::{taskstorage, Operation, DB};
use uuid::Uuid;
mod shared;
use shared::TestServer;
fn newdb() -> DB {
DB::new(Box::new(taskstorage::InMemoryStorage::new()))
}
#[test]
fn test_sync() {
let mut server = TestServer::new();
let mut db1 = newdb();
db1.sync("me", &mut server).unwrap();
let mut db2 = newdb();
db2.sync("me", &mut server).unwrap();
// make some changes in parallel to db1 and db2..
let uuid1 = Uuid::new_v4();
db1.apply(Operation::Create { uuid: uuid1 }).unwrap();
db1.apply(Operation::Update {
uuid: uuid1,
property: "title".into(),
value: Some("my first task".into()),
timestamp: Utc::now(),
})
.unwrap();
let uuid2 = Uuid::new_v4();
db2.apply(Operation::Create { uuid: uuid2 }).unwrap();
db2.apply(Operation::Update {
uuid: uuid2,
property: "title".into(),
value: Some("my second task".into()),
timestamp: Utc::now(),
})
.unwrap();
// and synchronize those around
db1.sync("me", &mut server).unwrap();
db2.sync("me", &mut server).unwrap();
db1.sync("me", &mut server).unwrap();
assert_eq!(db1.sorted_tasks(), db2.sorted_tasks());
// now make updates to the same task on both sides
db1.apply(Operation::Update {
uuid: uuid2,
property: "priority".into(),
value: Some("H".into()),
timestamp: Utc::now(),
})
.unwrap();
db2.apply(Operation::Update {
uuid: uuid2,
property: "project".into(),
value: Some("personal".into()),
timestamp: Utc::now(),
})
.unwrap();
// and synchronize those around
db1.sync("me", &mut server).unwrap();
db2.sync("me", &mut server).unwrap();
db1.sync("me", &mut server).unwrap();
assert_eq!(db1.sorted_tasks(), db2.sorted_tasks());
}
#[test]
fn test_sync_create_delete() {
let mut server = TestServer::new();
let mut db1 = newdb();
db1.sync("me", &mut server).unwrap();
let mut db2 = newdb();
db2.sync("me", &mut server).unwrap();
// create and update a task..
let uuid = Uuid::new_v4();
db1.apply(Operation::Create { uuid }).unwrap();
db1.apply(Operation::Update {
uuid: uuid,
property: "title".into(),
value: Some("my first task".into()),
timestamp: Utc::now(),
})
.unwrap();
// and synchronize those around
db1.sync("me", &mut server).unwrap();
db2.sync("me", &mut server).unwrap();
db1.sync("me", &mut server).unwrap();
assert_eq!(db1.sorted_tasks(), db2.sorted_tasks());
// delete and re-create the task on db1
db1.apply(Operation::Delete { uuid }).unwrap();
db1.apply(Operation::Create { uuid }).unwrap();
db1.apply(Operation::Update {
uuid: uuid,
property: "title".into(),
value: Some("my second task".into()),
timestamp: Utc::now(),
})
.unwrap();
// and on db2, update a property of the task
db2.apply(Operation::Update {
uuid: uuid,
property: "project".into(),
value: Some("personal".into()),
timestamp: Utc::now(),
})
.unwrap();
db1.sync("me", &mut server).unwrap();
db2.sync("me", &mut server).unwrap();
db1.sync("me", &mut server).unwrap();
assert_eq!(db1.sorted_tasks(), db2.sorted_tasks());
}

72
taskchampion/tests/sync_action_sequences.rs
View file

@ -1,72 +0,0 @@
use chrono::Utc;
use proptest::prelude::*;
use taskchampion::{taskstorage, Operation, DB};
use uuid::Uuid;
mod shared;
use shared::TestServer;
fn newdb() -> DB {
DB::new(Box::new(taskstorage::InMemoryStorage::new()))
}
#[derive(Debug)]
enum Action {
Op(Operation),
Sync,
}
fn action_sequence_strategy() -> impl Strategy<Value = Vec<(Action, u8)>> {
// Create, Update, Delete, or Sync on client 1, 2, .., followed by a round of syncs
"([CUDS][123])*S1S2S3S1S2".prop_map(|seq| {
let uuid = Uuid::parse_str("83a2f9ef-f455-4195-b92e-a54c161eebfc").unwrap();
seq.as_bytes()
.chunks(2)
.map(|action_on| {
let action = match action_on[0] {
b'C' => Action::Op(Operation::Create { uuid }),
b'U' => Action::Op(Operation::Update {
uuid,
property: "title".into(),
value: Some("foo".into()),
timestamp: Utc::now(),
}),
b'D' => Action::Op(Operation::Delete { uuid }),
b'S' => Action::Sync,
_ => unreachable!(),
};
let acton = action_on[1] - b'1';
(action, acton)
})
.collect::<Vec<(Action, u8)>>()
})
}
proptest! {
#[test]
// check that various sequences of operations on mulitple db's do not get the db's into an
// incompatible state. The main concern here is that there might be a sequence of create
// and delete operations that results in a task existing in one DB but not existing in
// another. So, the generated sequences focus on a single task UUID.
fn transform_sequences_of_operations(action_sequence in action_sequence_strategy()) {
let mut server = TestServer::new();
let mut dbs = [newdb(), newdb(), newdb()];
for (action, db) in action_sequence {
println!("{:?} on db {}", action, db);
let db = &mut dbs[db as usize];
match action {
Action::Op(op) => {
if let Err(e) = db.apply(op) {
println!(" {:?} (ignored)", e);
}
},
Action::Sync => db.sync("me", &mut server).unwrap(),
}
}
assert_eq!(dbs[0].sorted_tasks(), dbs[0].sorted_tasks());
assert_eq!(dbs[1].sorted_tasks(), dbs[2].sorted_tasks());
}
}