Add support for cloud sync, specifically GCP (#3223)

* Add support for cloud sync, specifically GCP

This adds generic support for sync to cloud services, with specific
spuport for GCP. Adding others -- so long as they support a
compare-and-set operation -- should be comparatively straightforward.

The cloud support includes cleanup of unnecessary data, and should keep
total space usage roughly proportional to the number of tasks.

Co-authored-by: ryneeverett <ryneeverett@gmail.com>
This commit is contained in:
Dustin J. Mitchell 2024-01-21 12:36:37 -05:00 committed by GitHub
parent 6f1c16fecd
commit 9566c929e2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
36 changed files with 4012 additions and 401 deletions

View file

@ -27,12 +27,13 @@ env_logger = "^0.10.0"
ffizz-header = "0.5"
flate2 = "1"
futures = "^0.3.25"
google-cloud-storage = { version = "0.15.0", default-features = false, features = ["rustls-tls", "auth"] }
lazy_static = "1"
libc = "0.2.136"
log = "^0.4.17"
pretty_assertions = "1"
proptest = "^1.4.0"
ring = "0.16"
ring = "0.17"
rstest = "0.17"
rusqlite = { version = "0.29", features = ["bundled"] }
serde_json = "^1.0"
@ -40,6 +41,7 @@ serde = { version = "^1.0.147", features = ["derive"] }
strum = "0.25"
strum_macros = "0.25"
tempfile = "3"
tokio = { version = "1", features = ["rt-multi-thread"] }
thiserror = "1.0"
ureq = "^2.9.0"
ureq = { version = "^2.9.0", features = ["tls"] }
uuid = { version = "^1.7.0", features = ["serde", "v4"] }