diff --git a/Cargo.lock b/Cargo.lock index 81e5a5b..4614ab8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -172,6 +172,7 @@ dependencies = [ "mime", "mime_guess", "notify", + "prost", "reqwest", "rusqlite", "serde", @@ -181,6 +182,7 @@ dependencies = [ "tempfile", "thiserror", "tokio", + "tokio-stream", "tokio-test", "tokio-util", "tower-http", @@ -300,6 +302,12 @@ dependencies = [ "syn", ] +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + [[package]] name = "encoding_rs" version = "0.8.35" @@ -913,6 +921,15 @@ dependencies = [ "serde", ] +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.17" @@ -1310,6 +1327,29 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pxfm" version = "0.1.28" @@ -1875,6 +1915,7 @@ dependencies = [ "futures-core", "pin-project-lite", "tokio", + "tokio-util", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 261423f..d09daa8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,12 @@ mime = "0.3" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } +# Protobuf (sync API) +prost = "0.13" + +# Stream utilities (SSE for sync events) +tokio-stream = { version = "0.1", features = ["sync"] } + # Utilities chrono = { version = "0.4", features = ["serde"] } anyhow = "1" diff --git a/config.yaml b/config.yaml index ac70e1b..485bdd4 100644 --- a/config.yaml +++ b/config.yaml @@ -3,3 +3,4 @@ admin_token: "super_secret_rebuild" enable_thumbnail_cache: true rebuild_error_threshold: 50 verify_interval_hours: 12 +sync_api_key: "can-sync-default-key" diff --git a/examples/can-sync/Cargo.lock b/examples/can-sync/Cargo.lock index c36ce67..6dd5f78 100644 --- a/examples/can-sync/Cargo.lock +++ b/examples/can-sync/Cargo.lock @@ -2,18 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "ahash" -version = "0.8.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" -dependencies = [ - "cfg-if", - "once_cell", - "version_check", - "zerocopy", -] - [[package]] name = "aho-corasick" version = "1.1.4" @@ -56,57 +44,6 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" -[[package]] -name = "asn1-rs" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56624a96882bb8c26d61312ae18cb45868e5a9992ea73c58e45c3101e56a1e60" -dependencies = [ - "asn1-rs-derive", - "asn1-rs-impl", - "displaydoc", - "nom", - "num-traits", - "rusticata-macros", - "thiserror 2.0.18", - "time", -] - -[[package]] -name = "asn1-rs-derive" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", - "synstructure", -] - -[[package]] -name = "asn1-rs-impl" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "async-channel" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" -dependencies = [ - "concurrent-queue", - "event-listener-strategy", - "futures-core", - "pin-project-lite", -] - [[package]] name = "async-compat" version = "0.2.5" @@ -128,7 +65,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -197,58 +134,6 @@ dependencies = [ "fs_extra", ] -[[package]] -name = "axum" -version = "0.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" -dependencies = [ - "axum-core", - "bytes", - "form_urlencoded", - "futures-util", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-util", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "serde_core", - "serde_json", - "serde_path_to_error", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tower", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "axum-core" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" -dependencies = [ - "bytes", - "futures-core", - "http", - "http-body", - "http-body-util", - "mime", - "pin-project-lite", - "sync_wrapper", - "tower-layer", - "tower-service", - "tracing", -] - [[package]] name = "backon" version = "1.6.0" @@ -260,25 +145,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "bao-tree" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06384416b1825e6e04fde63262fda2dc408f5b64c02d04e0d8b70ae72c17a52b" -dependencies = [ - "blake3", - "bytes", - "futures-lite", - "genawaiter", - "iroh-io", - "positioned-io", - "range-collections", - "self_cell", - "serde", - "smallvec", - "tokio", -] - [[package]] name = "base32" version = "0.5.1" @@ -297,12 +163,6 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" -[[package]] -name = "binary-merge" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597bb81c80a54b6a4381b23faba8d7774b144c94cbd1d6fe3f1329bd776554ab" - [[package]] name = "bitflags" version = "2.11.0" @@ -323,15 +183,6 @@ dependencies = [ "cpufeatures", ] -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - [[package]] name = "block-buffer" version = "0.11.0" @@ -373,32 +224,30 @@ dependencies = [ [[package]] name = "can-sync" -version = "0.1.0" +version = "0.2.0" dependencies = [ "anyhow", - "axum", + "blake3", "bytes", - "chrono", - "futures-lite", + "ed25519-dalek", + "futures-util", "hex", "iroh", - "iroh-blobs", - "iroh-docs", "iroh-gossip", - "open", - "postcard", + "n0-future 0.1.3", + "pkarr", + "prost", + "rand", "reqwest 0.12.28", - "rusqlite", "serde", "serde_json", "serde_yaml", - "sha2 0.10.9", + "simple-dns", + "tempfile", "tokio", - "tokio-util", - "tower-http", + "tokio-stream", "tracing", "tracing-subscriber", - "uuid", ] [[package]] @@ -438,10 +287,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" dependencies = [ "iana-time-zone", - "js-sys", "num-traits", "serde", - "wasm-bindgen", "windows-link", ] @@ -473,15 +320,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "concurrent-queue" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "const-oid" version = "0.10.2" @@ -578,16 +416,6 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" -[[package]] -name = "crypto-common" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" -dependencies = [ - "generic-array", - "typenum", -] - [[package]] name = "crypto-common" version = "0.2.1" @@ -606,7 +434,7 @@ dependencies = [ "cfg-if", "cpufeatures", "curve25519-dalek-derive", - "digest 0.11.0-rc.10", + "digest", "fiat-crypto", "rand_core", "rustc_version", @@ -623,7 +451,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -647,7 +475,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.117", + "syn", ] [[package]] @@ -658,7 +486,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -678,20 +506,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "der-parser" -version = "10.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" -dependencies = [ - "asn1-rs", - "displaydoc", - "nom", - "num-bigint", - "num-traits", - "rusticata-macros", -] - [[package]] name = "deranged" version = "0.5.8" @@ -719,7 +533,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -729,7 +543,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn 2.0.117", + "syn", +] + +[[package]] +name = "derive_more" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05" +dependencies = [ + "derive_more-impl 1.0.0", ] [[package]] @@ -738,7 +561,19 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" dependencies = [ - "derive_more-impl", + "derive_more-impl 2.1.1", +] + +[[package]] +name = "derive_more-impl" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", ] [[package]] @@ -751,7 +586,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.117", + "syn", "unicode-xid", ] @@ -761,25 +596,15 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab03c107fafeb3ee9f5925686dbb7a73bc76e3932abb0d2b365cb64b169cf04c" -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer 0.10.4", - "crypto-common 0.1.7", -] - [[package]] name = "digest" version = "0.11.0-rc.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "afa94b64bfc6549e6e4b5a3216f22593224174083da7a90db47e951c4fb31725" dependencies = [ - "block-buffer 0.11.0", + "block-buffer", "const-oid", - "crypto-common 0.2.1", + "crypto-common", ] [[package]] @@ -802,7 +627,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -858,12 +683,18 @@ dependencies = [ "ed25519", "rand_core", "serde", - "sha2 0.11.0-rc.2", + "sha2", "signature", "subtle", "zeroize", ] +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + [[package]] name = "embedded-io" version = "0.4.0" @@ -894,7 +725,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -905,7 +736,7 @@ checksum = "3ed8956bd5c1f0415200516e78ff07ec9e16415ade83c056c230d7b7ea0d55b7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -924,39 +755,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "event-listener" -version = "5.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" -dependencies = [ - "concurrent-queue", - "parking", - "pin-project-lite", -] - -[[package]] -name = "event-listener-strategy" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" -dependencies = [ - "event-listener", - "pin-project-lite", -] - -[[package]] -name = "fallible-iterator" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" - -[[package]] -name = "fallible-streaming-iterator" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" - [[package]] name = "fastbloom" version = "0.14.1" @@ -1136,7 +934,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -1168,37 +966,6 @@ dependencies = [ "slab", ] -[[package]] -name = "genawaiter" -version = "0.99.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c86bd0361bcbde39b13475e6e36cb24c329964aa2611be285289d1e4b751c1a0" -dependencies = [ - "futures-core", - "genawaiter-macro", - "genawaiter-proc-macro", - "proc-macro-hack", -] - -[[package]] -name = "genawaiter-macro" -version = "0.99.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b32dfe1fdfc0bbde1f22a5da25355514b5e450c33a6af6770884c8750aedfbc" - -[[package]] -name = "genawaiter-proc-macro" -version = "0.99.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784f84eebc366e15251c4a8c3acee82a6a6f427949776ecb88377362a9621738" -dependencies = [ - "proc-macro-error", - "proc-macro-hack", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "generator" version = "0.8.8" @@ -1214,16 +981,6 @@ dependencies = [ "windows-result", ] -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - [[package]] name = "getrandom" version = "0.2.17" @@ -1306,15 +1063,6 @@ dependencies = [ "byteorder", ] -[[package]] -name = "hashbrown" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" -dependencies = [ - "ahash", -] - [[package]] name = "hashbrown" version = "0.15.5" @@ -1335,15 +1083,6 @@ dependencies = [ "foldhash 0.2.0", ] -[[package]] -name = "hashlink" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" -dependencies = [ - "hashbrown 0.14.5", -] - [[package]] name = "heapless" version = "0.7.17" @@ -1735,15 +1474,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "inplace-vec-builder" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf64c2edc8226891a71f127587a2861b132d2b942310843814d5001d99a1d307" -dependencies = [ - "smallvec", -] - [[package]] name = "ipconfig" version = "0.3.2" @@ -1782,21 +1512,21 @@ dependencies = [ "bytes", "cfg_aliases", "data-encoding", - "derive_more", + "derive_more 2.1.1", "ed25519-dalek", "futures-util", "getrandom 0.3.4", "hickory-resolver", "http", "igd-next", - "iroh-base 0.96.1", + "iroh-base", "iroh-metrics", "iroh-quinn", "iroh-quinn-proto", "iroh-quinn-udp", "iroh-relay", "n0-error", - "n0-future", + "n0-future 0.3.2", "n0-watcher", "netdev", "netwatch", @@ -1813,7 +1543,7 @@ dependencies = [ "rustls-webpki", "serde", "smallvec", - "strum 0.27.2", + "strum", "sync_wrapper", "time", "tokio", @@ -1825,24 +1555,6 @@ dependencies = [ "webpki-roots", ] -[[package]] -name = "iroh-base" -version = "0.95.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25a8c5fb1cc65589f0d7ab44269a76f615a8c4458356952c9b0ef1c93ea45ff8" -dependencies = [ - "curve25519-dalek", - "data-encoding", - "derive_more", - "ed25519-dalek", - "n0-error", - "rand_core", - "serde", - "url", - "zeroize", - "zeroize_derive", -] - [[package]] name = "iroh-base" version = "0.96.1" @@ -1851,98 +1563,18 @@ checksum = "20c99d836a1c99e037e98d1bf3ef209c3a4df97555a00ce9510eb78eccdf5567" dependencies = [ "curve25519-dalek", "data-encoding", - "derive_more", - "digest 0.11.0-rc.10", + "derive_more 2.1.1", + "digest", "ed25519-dalek", "n0-error", "rand_core", "serde", - "sha2 0.11.0-rc.2", + "sha2", "url", "zeroize", "zeroize_derive", ] -[[package]] -name = "iroh-blobs" -version = "0.98.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1f253ea06293e51e166a88a3faa019b67e187d12bd7c6a04369a0ec86f53272" -dependencies = [ - "arrayvec", - "bao-tree", - "bytes", - "cfg_aliases", - "chrono", - "data-encoding", - "derive_more", - "futures-lite", - "genawaiter", - "hex", - "iroh", - "iroh-base 0.96.1", - "iroh-io", - "iroh-metrics", - "iroh-quinn", - "iroh-tickets 0.3.0", - "irpc", - "n0-error", - "n0-future", - "nested_enum_utils", - "postcard", - "rand", - "range-collections", - "redb", - "ref-cast", - "reflink-copy", - "self_cell", - "serde", - "smallvec", - "tokio", - "tracing", -] - -[[package]] -name = "iroh-docs" -version = "0.96.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42292da17d6be0b73c5897f1ff395ad7c6f858b107ff76a7605867fbdd6c2e72" -dependencies = [ - "anyhow", - "async-channel", - "blake3", - "bytes", - "derive_more", - "ed25519-dalek", - "futures-buffered", - "futures-lite", - "futures-util", - "hex", - "iroh", - "iroh-blobs", - "iroh-gossip", - "iroh-metrics", - "iroh-quinn", - "iroh-tickets 0.2.0", - "irpc", - "n0-error", - "n0-future", - "num_enum", - "postcard", - "rand", - "redb", - "self_cell", - "serde", - "serde-error", - "strum 0.26.3", - "tempfile", - "thiserror 2.0.18", - "tokio", - "tokio-stream", - "tokio-util", - "tracing", -] - [[package]] name = "iroh-gossip" version = "0.96.0" @@ -1952,7 +1584,7 @@ dependencies = [ "blake3", "bytes", "data-encoding", - "derive_more", + "derive_more 2.1.1", "ed25519-dalek", "futures-concurrency", "futures-lite", @@ -1960,11 +1592,11 @@ dependencies = [ "hex", "indexmap", "iroh", - "iroh-base 0.96.1", + "iroh-base", "iroh-metrics", "irpc", "n0-error", - "n0-future", + "n0-future 0.3.2", "postcard", "rand", "serde", @@ -1973,19 +1605,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "iroh-io" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a5feb781017b983ff1b155cd1faf8174da2acafd807aa482876da2d7e6577a" -dependencies = [ - "bytes", - "futures-lite", - "pin-project", - "smallvec", - "tokio", -] - [[package]] name = "iroh-metrics" version = "0.38.3" @@ -2011,7 +1630,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -2042,7 +1661,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0de99ad8adc878ee0e68509ad256152ce23b8bbe45f5539d04e179630aca40a9" dependencies = [ "bytes", - "derive_more", + "derive_more 2.1.1", "enum-assoc", "fastbloom", "getrandom 0.3.4", @@ -2053,7 +1672,6 @@ dependencies = [ "rustc-hash", "rustls", "rustls-pki-types", - "rustls-platform-verifier", "slab", "sorted-index-buffer", "thiserror 2.0.18", @@ -2085,20 +1703,20 @@ dependencies = [ "bytes", "cfg_aliases", "data-encoding", - "derive_more", + "derive_more 2.1.1", "getrandom 0.3.4", "hickory-resolver", "http", "http-body-util", "hyper", "hyper-util", - "iroh-base 0.96.1", + "iroh-base", "iroh-metrics", "iroh-quinn", "iroh-quinn-proto", "lru", "n0-error", - "n0-future", + "n0-future 0.3.2", "num_enum", "pin-project", "pkarr", @@ -2109,7 +1727,7 @@ dependencies = [ "rustls-pki-types", "serde", "serde_bytes", - "strum 0.27.2", + "strum", "tokio", "tokio-rustls", "tokio-util", @@ -2122,51 +1740,17 @@ dependencies = [ "z32", ] -[[package]] -name = "iroh-tickets" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a322053cacddeca222f0999ce3cf6aa45c64ae5ad8c8911eac9b66008ffbaa5" -dependencies = [ - "data-encoding", - "derive_more", - "iroh-base 0.95.1", - "n0-error", - "postcard", - "serde", -] - -[[package]] -name = "iroh-tickets" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09cd580bf680db919cbbce6886a47314acb0e9b4f7b639acebcea5e9f485d183" -dependencies = [ - "data-encoding", - "derive_more", - "iroh-base 0.96.1", - "n0-error", - "postcard", - "serde", -] - [[package]] name = "irpc" version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1bbc84aaeab13a6d7502bae4f40f2517b643924842e0230ea0bf807477cc208" dependencies = [ - "futures-buffered", "futures-util", - "iroh-quinn", "irpc-derive", "n0-error", - "n0-future", - "postcard", - "rcgen", - "rustls", + "n0-future 0.3.2", "serde", - "smallvec", "tokio", "tokio-util", "tracing", @@ -2180,26 +1764,16 @@ checksum = "58148196d2230183c9679431ac99b57e172000326d664e8456fa2cd27af6505a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] -name = "is-docker" -version = "0.2.0" +name = "itertools" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "928bae27f42bc99b60d9ac7334e3a21d10ad8f1835a4e12ec3ec0464765ed1b3" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" dependencies = [ - "once_cell", -] - -[[package]] -name = "is-wsl" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "173609498df190136aa7dea1a91db051746d339e18476eed5ca40521f02d7aa5" -dependencies = [ - "is-docker", - "once_cell", + "either", ] [[package]] @@ -2274,17 +1848,6 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" -[[package]] -name = "libsqlite3-sys" -version = "0.30.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" -dependencies = [ - "cc", - "pkg-config", - "vcpkg", -] - [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -2361,12 +1924,6 @@ dependencies = [ "regex-automata", ] -[[package]] -name = "matchit" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" - [[package]] name = "memchr" version = "2.8.0" @@ -2389,12 +1946,6 @@ dependencies = [ "unicase", ] -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - [[package]] name = "mio" version = "1.1.1" @@ -2442,7 +1993,28 @@ checksum = "03755949235714b2b307e5ae89dd8c1c2531fb127d9b8b7b4adf9c876cd3ed18" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", +] + +[[package]] +name = "n0-future" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb0e5d99e681ab3c938842b96fcb41bf8a7bb4bfdb11ccbd653a7e83e06c794" +dependencies = [ + "cfg_aliases", + "derive_more 1.0.0", + "futures-buffered", + "futures-lite", + "futures-util", + "js-sys", + "pin-project", + "send_wrapper", + "tokio", + "tokio-util", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-time", ] [[package]] @@ -2452,7 +2024,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2ab99dfb861450e68853d34ae665243a88b8c493d01ba957321a1e9b2312bbe" dependencies = [ "cfg_aliases", - "derive_more", + "derive_more 2.1.1", "futures-buffered", "futures-lite", "futures-util", @@ -2472,9 +2044,9 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38795f7932e6e9d1c6e989270ef5b3ff24ebb910e2c9d4bed2d28d8bae3007dc" dependencies = [ - "derive_more", + "derive_more 2.1.1", "n0-error", - "n0-future", + "n0-future 0.3.2", ] [[package]] @@ -2494,18 +2066,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "nested_enum_utils" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1d5475271bdd36a4a2769eac1ef88df0f99428ea43e52dfd8b0ee5cb674695f" -dependencies = [ - "proc-macro-crate", - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "netdev" version = "0.40.1" @@ -2597,12 +2157,12 @@ dependencies = [ "atomic-waker", "bytes", "cfg_aliases", - "derive_more", + "derive_more 2.1.1", "iroh-quinn-udp", "js-sys", "libc", "n0-error", - "n0-future", + "n0-future 0.3.2", "n0-watcher", "netdev", "netlink-packet-core", @@ -2624,16 +2184,6 @@ dependencies = [ "wmi", ] -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - [[package]] name = "ntimestamp" version = "1.0.0" @@ -2658,31 +2208,12 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "num-bigint" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" -dependencies = [ - "num-integer", - "num-traits", -] - [[package]] name = "num-conv" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" -[[package]] -name = "num-integer" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" -dependencies = [ - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.19" @@ -2711,7 +2242,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -2776,15 +2307,6 @@ dependencies = [ "objc2-security", ] -[[package]] -name = "oid-registry" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" -dependencies = [ - "asn1-rs", -] - [[package]] name = "once_cell" version = "1.21.3" @@ -2795,17 +2317,6 @@ dependencies = [ "portable-atomic", ] -[[package]] -name = "open" -version = "5.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43bb73a7fa3799b198970490a51174027ba0d4ec504b03cd08caf513d40024bc" -dependencies = [ - "is-wsl", - "libc", - "pathdiff", -] - [[package]] name = "openssl" version = "0.10.76" @@ -2829,7 +2340,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -2895,22 +2406,6 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" -[[package]] -name = "pathdiff" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" - -[[package]] -name = "pem" -version = "3.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" -dependencies = [ - "base64", - "serde_core", -] - [[package]] name = "pem-rfc7468" version = "1.0.0" @@ -2953,7 +2448,7 @@ checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -3045,7 +2540,7 @@ checksum = "7d2a8825353ace3285138da3378b1e21860d60351942f7aa3b99b13b41f80318" dependencies = [ "base64", "bytes", - "derive_more", + "derive_more 2.1.1", "futures-lite", "futures-util", "hyper-util", @@ -3067,16 +2562,6 @@ dependencies = [ "url", ] -[[package]] -name = "positioned-io" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4ec4b80060f033312b99b6874025d9503d2af87aef2dd4c516e253fbfcdada7" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "postcard" version = "1.1.3" @@ -3099,7 +2584,7 @@ checksum = "e0232bd009a197ceec9cc881ba46f727fcd8060a2d8d6a9dde7a69030a6fe2bb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -3133,7 +2618,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.117", + "syn", ] [[package]] @@ -3145,38 +2630,6 @@ dependencies = [ "toml_edit", ] -[[package]] -name = "proc-macro-error" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18f33027081eba0a6d8aba6d1b1c3a3be58cbb12106341c2d5759fcd9b5277e7" -dependencies = [ - "proc-macro-error-attr", - "proc-macro2", - "quote", - "syn 1.0.109", - "version_check", -] - -[[package]] -name = "proc-macro-error-attr" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a5b4b77fdb63c1eca72173d68d24501c54ab1269409f6b672c85deb18af69de" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", - "syn-mid", - "version_check", -] - -[[package]] -name = "proc-macro-hack" -version = "0.5.20+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" - [[package]] name = "proc-macro2" version = "1.0.106" @@ -3186,6 +2639,29 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "quick-xml" version = "0.38.4" @@ -3301,42 +2777,6 @@ dependencies = [ "getrandom 0.3.4", ] -[[package]] -name = "range-collections" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "861706ea9c4aded7584c5cd1d241cec2ea7f5f50999f236c22b65409a1f1a0d0" -dependencies = [ - "binary-merge", - "inplace-vec-builder", - "ref-cast", - "serde", - "smallvec", -] - -[[package]] -name = "rcgen" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10b99e0098aa4082912d4c649628623db6aba77335e4f4569ff5083a6448b32e" -dependencies = [ - "pem", - "ring", - "rustls-pki-types", - "time", - "x509-parser", - "yasna", -] - -[[package]] -name = "redb" -version = "2.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eca1e9d98d5a7e9002d0013e18d5a9b000aee942eb134883a82f06ebffb6c01" -dependencies = [ - "libc", -] - [[package]] name = "redox_syscall" version = "0.5.18" @@ -3346,38 +2786,6 @@ dependencies = [ "bitflags", ] -[[package]] -name = "ref-cast" -version = "1.0.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" -dependencies = [ - "ref-cast-impl", -] - -[[package]] -name = "ref-cast-impl" -version = "1.0.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "reflink-copy" -version = "0.1.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13362233b147e57674c37b802d216b7c5e3dcccbed8967c84f0d8d223868ae27" -dependencies = [ - "cfg-if", - "libc", - "rustix", - "windows", -] - [[package]] name = "regex-automata" version = "0.4.14" @@ -3404,6 +2812,7 @@ dependencies = [ "base64", "bytes", "encoding_rs", + "futures-channel", "futures-core", "futures-util", "h2", @@ -3498,20 +2907,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rusqlite" -version = "0.32.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" -dependencies = [ - "bitflags", - "fallible-iterator", - "fallible-streaming-iterator", - "hashlink", - "libsqlite3-sys", - "smallvec", -] - [[package]] name = "rustc-hash" version = "2.1.1" @@ -3527,15 +2922,6 @@ dependencies = [ "semver", ] -[[package]] -name = "rusticata-macros" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" -dependencies = [ - "nom", -] - [[package]] name = "rustix" version = "1.1.4" @@ -3729,15 +3115,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde-error" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "342110fb7a5d801060c885da03bf91bfa7c7ca936deafcc64bb6706375605d47" -dependencies = [ - "serde", -] - [[package]] name = "serde_bytes" version = "0.11.19" @@ -3765,7 +3142,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -3781,17 +3158,6 @@ dependencies = [ "zmij", ] -[[package]] -name = "serde_path_to_error" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" -dependencies = [ - "itoa", - "serde", - "serde_core", -] - [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -3823,17 +3189,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest 0.10.7", -] - [[package]] name = "sha2" version = "0.11.0-rc.2" @@ -3842,7 +3197,7 @@ checksum = "d1e3878ab0f98e35b2df35fe53201d088299b41a6bb63e3e34dada2ac4abd924" dependencies = [ "cfg-if", "cpufeatures", - "digest 0.11.0-rc.10", + "digest", ] [[package]] @@ -3908,9 +3263,6 @@ name = "smallvec" version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" -dependencies = [ - "serde", -] [[package]] name = "socket2" @@ -3946,7 +3298,7 @@ checksum = "c87e960f4dca2788eeb86bbdde8dd246be8948790b7618d656e68f9b720a86e8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -3986,35 +3338,13 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" -[[package]] -name = "strum" -version = "0.26.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" -dependencies = [ - "strum_macros 0.26.4", -] - [[package]] name = "strum" version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" dependencies = [ - "strum_macros 0.27.2", -] - -[[package]] -name = "strum_macros" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.117", + "strum_macros", ] [[package]] @@ -4026,7 +3356,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -4035,17 +3365,6 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - [[package]] name = "syn" version = "2.0.117" @@ -4057,17 +3376,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "syn-mid" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea305d57546cc8cd04feb14b62ec84bf17f50e3f7b12560d7bfa9265f39d9ed" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "sync_wrapper" version = "1.0.2" @@ -4085,7 +3393,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -4154,7 +3462,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -4165,7 +3473,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -4261,7 +3569,7 @@ checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -4375,7 +3683,6 @@ dependencies = [ "tokio", "tower-layer", "tower-service", - "tracing", ] [[package]] @@ -4428,7 +3735,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -4608,12 +3915,6 @@ dependencies = [ "rustversion", ] -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - [[package]] name = "walkdir" version = "2.5.0" @@ -4703,7 +4004,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.117", + "syn", "wasm-bindgen-shared", ] @@ -4891,7 +4192,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -4902,7 +4203,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -5305,7 +4606,7 @@ dependencies = [ "heck", "indexmap", "prettyplease", - "syn 2.0.117", + "syn", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -5321,7 +4622,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn 2.0.117", + "syn", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -5403,24 +4704,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "x509-parser" -version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" -dependencies = [ - "asn1-rs", - "data-encoding", - "der-parser", - "lazy_static", - "nom", - "oid-registry", - "ring", - "rusticata-macros", - "thiserror 2.0.18", - "time", -] - [[package]] name = "xml-rs" version = "0.8.28" @@ -5436,15 +4719,6 @@ dependencies = [ "xml-rs", ] -[[package]] -name = "yasna" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" -dependencies = [ - "time", -] - [[package]] name = "yoke" version = "0.8.1" @@ -5464,7 +4738,7 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", "synstructure", ] @@ -5491,7 +4765,7 @@ checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -5511,7 +4785,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", "synstructure", ] @@ -5532,7 +4806,7 @@ checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -5565,7 +4839,7 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] diff --git a/examples/can-sync/Cargo.toml b/examples/can-sync/Cargo.toml index 06660aa..4221293 100644 --- a/examples/can-sync/Cargo.toml +++ b/examples/can-sync/Cargo.toml @@ -1,44 +1,60 @@ [package] name = "can-sync" -version = "0.1.0" +version = "0.2.0" edition = "2021" -description = "P2P sync service for CAN content-addressable storage" +description = "P2P sync agent for CAN service — full mirror replication via iroh" [[bin]] name = "can-sync" path = "src/main.rs" +[[bin]] +name = "sync-test" +path = "tests/sync_test.rs" + [dependencies] -# P2P networking +# P2P networking (iroh for transport + gossip for discovery — NO iroh-docs) iroh = "0.96" -iroh-blobs = "0.98" -iroh-docs = "0.96" iroh-gossip = "0.96" -# HTTP server + client -axum = "0.8" -tokio = { version = "1", features = ["full"] } -reqwest = { version = "0.12", features = ["json", "multipart"] } -tower-http = { version = "0.6", features = ["cors"] } +# Protobuf (same message types as CAN service sync API) +prost = "0.13" + +# HTTP client for CAN service sync API +reqwest = { version = "0.12", features = ["json", "multipart", "blocking"] } # Serialization serde = { version = "1", features = ["derive"] } -serde_json = "1" serde_yaml = "0.9" -postcard = { version = "1", features = ["alloc"] } -# Storage -rusqlite = { version = "0.32", features = ["bundled"] } +# Crypto +blake3 = "1" +ed25519-dalek = "3.0.0-pre.1" -# Utilities +# Pkarr (internet rendezvous via relay servers — relay only, no DHT to avoid digest conflict) +pkarr = { version = "5", default-features = false, features = ["relays"] } + +# DNS record parsing (used by pkarr) +simple-dns = "0.9" + +# Async runtime +tokio = { version = "1", features = ["full"] } + +# Logging tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } + +# Stream utilities (needed for gossip event stream) +n0-future = "0.1" + +# SSE client (for real-time events from CAN service) +tokio-stream = "0.1" +futures-util = "0.3" + +# Utilities anyhow = "1" -open = "5" -sha2 = "0.10" -hex = "0.4" -uuid = { version = "1", features = ["v4"] } -chrono = { version = "0.4", features = ["serde"] } bytes = "1" -futures-lite = "2" -tokio-util = { version = "0.7", features = ["io"] } +hex = "0.4" +serde_json = "1" +tempfile = "3" +rand = "0.9" diff --git a/examples/can-sync/config.yaml b/examples/can-sync/config.yaml index c5b804c..5cce4e5 100644 --- a/examples/can-sync/config.yaml +++ b/examples/can-sync/config.yaml @@ -1,7 +1,20 @@ -# CAN Sync configuration -can_service_url: "http://127.0.0.1:3210/api/v1/can/0" -listen_addr: "127.0.0.1:3213" -data_dir: "./can_sync_data" -relay_url: null -poll_interval_secs: 5 -full_scan_interval_secs: 300 +# CAN Sync v2 configuration +# +# This config is used by the go_example_1.ps1 script. +# All machines that clone this repo and run the script will +# auto-discover each other via iroh's relay network as long +# as they share the same sync_passphrase. + +# URL of the local CAN Service (sync API is at /sync/*) +can_service_url: "http://127.0.0.1:3210" + +# API key for CAN service's sync endpoints (must match sync_api_key in CAN config) +sync_api_key: "can-sync-default-key" + +# Shared passphrase for peer discovery — all peers with the same passphrase +# find each other automatically over the internet via iroh relay servers. +# Change this to something unique to your team/project. +sync_passphrase: "duke-canman-sync" + +# Seconds between fallback polls (SSE handles instant sync, this is a safety net) +poll_interval_secs: 30 diff --git a/examples/can-sync/run-integration-test.ps1 b/examples/can-sync/run-integration-test.ps1 new file mode 100644 index 0000000..82db5aa --- /dev/null +++ b/examples/can-sync/run-integration-test.ps1 @@ -0,0 +1,75 @@ +#!/usr/bin/env pwsh +# CAN Sync v2 Integration Test Runner +# +# Usage: +# .\run-integration-test.ps1 # Build + run test +# .\run-integration-test.ps1 -NoBuild # Skip building, just run + +param( + [switch]$NoBuild +) + +$ErrorActionPreference = "Stop" +$canServiceRoot = Resolve-Path (Join-Path $PSScriptRoot "../..") +$canSyncRoot = $PSScriptRoot + +Write-Host "" +Write-Host "========================================" -ForegroundColor Cyan +Write-Host " CAN Sync v2 - Integration Test Runner" -ForegroundColor Cyan +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "" + +# Step 1: Build CAN service +if (-not $NoBuild) { + Write-Host "[1/3] Building CAN service..." -ForegroundColor Yellow + Push-Location $canServiceRoot + try { + cargo build 2>&1 | ForEach-Object { Write-Host " $_" -ForegroundColor DarkGray } + if ($LASTEXITCODE -ne 0) { + Write-Host "FAILED: CAN service build failed!" -ForegroundColor Red + exit 1 + } + Write-Host " CAN service built OK" -ForegroundColor Green + } finally { + Pop-Location + } + + # Step 2: Build can-sync + sync-test + Write-Host "" + Write-Host "[2/3] Building can-sync and sync-test..." -ForegroundColor Yellow + Push-Location $canSyncRoot + try { + cargo build --bin can-sync --bin sync-test 2>&1 | ForEach-Object { Write-Host " $_" -ForegroundColor DarkGray } + if ($LASTEXITCODE -ne 0) { + Write-Host "FAILED: can-sync build failed!" -ForegroundColor Red + exit 1 + } + Write-Host " can-sync built OK" -ForegroundColor Green + } finally { + Pop-Location + } +} else { + Write-Host "[SKIP] Builds skipped (-NoBuild)" -ForegroundColor DarkYellow +} + +# Step 3: Run integration test +Write-Host "" +Write-Host "[3/3] Running integration test..." -ForegroundColor Yellow +Write-Host "" + +Push-Location $canSyncRoot +try { + cargo run --bin sync-test + $testResult = $LASTEXITCODE +} finally { + Pop-Location +} + +Write-Host "" +if ($testResult -eq 0) { + Write-Host "ALL TESTS PASSED" -ForegroundColor Green +} else { + Write-Host "SOME TESTS FAILED (exit code: $testResult)" -ForegroundColor Red +} + +exit $testResult diff --git a/examples/can-sync/src/announcer.rs b/examples/can-sync/src/announcer.rs deleted file mode 100644 index 58b9b68..0000000 --- a/examples/can-sync/src/announcer.rs +++ /dev/null @@ -1,234 +0,0 @@ -use std::sync::Arc; -use std::time::Duration; - -use anyhow::Result; -use tracing::{debug, error, info, warn}; - -use crate::can_client::CanClient; -use crate::library::SyncState; -use crate::manifest::AssetSyncEntry; -use crate::node::SyncNode; - -/// The announcer periodically polls CAN service for new or changed assets -/// and writes matching entries into iroh library documents. -pub struct Announcer { - can: CanClient, - state: Arc, - node: Arc, - poll_interval: Duration, - full_scan_interval: Duration, -} - -impl Announcer { - pub fn new( - can: CanClient, - state: Arc, - node: Arc, - poll_interval_secs: u64, - full_scan_interval_secs: u64, - ) -> Self { - Self { - can, - state, - node, - poll_interval: Duration::from_secs(poll_interval_secs), - full_scan_interval: Duration::from_secs(full_scan_interval_secs), - } - } - - /// Run the announcer loop — fast polls + periodic full scans - pub async fn run(self) { - let mut fast_tick = tokio::time::interval(self.poll_interval); - let mut full_tick = tokio::time::interval(self.full_scan_interval); - // Skip the first immediate tick for full scan (let fast poll get first data) - full_tick.tick().await; - - info!( - "Announcer started (fast poll: {}s, full scan: {}s)", - self.poll_interval.as_secs(), - self.full_scan_interval.as_secs(), - ); - - loop { - tokio::select! { - _ = fast_tick.tick() => { - if let Err(e) = self.fast_poll().await { - warn!("Fast poll error: {:#}", e); - } - } - _ = full_tick.tick() => { - if let Err(e) = self.full_scan().await { - warn!("Full scan error: {:#}", e); - } - } - } - } - } - - /// Fast poll: check for recently ingested assets - async fn fast_poll(&self) -> Result<()> { - let last_ts = self - .state - .get_state("last_seen_timestamp")? - .and_then(|s| s.parse::().ok()) - .unwrap_or(0); - - // Get recent assets ordered newest first - let resp = self.can.list(50, 0, "desc", Some(last_ts)).await?; - - if resp.items.is_empty() { - return Ok(()); - } - - debug!("Fast poll found {} new assets since ts={}", resp.items.len(), last_ts); - - // Track the newest timestamp we see - let mut max_ts = last_ts; - for asset in &resp.items { - if asset.timestamp > max_ts { - max_ts = asset.timestamp; - } - } - - // Process assets against libraries - let libraries = self.state.list_libraries()?; - - for asset in &resp.items { - for lib in &libraries { - if lib.filter.matches(asset) { - self.announce_asset(lib, asset).await?; - } - } - } - - // Update last seen timestamp - self.state.set_state("last_seen_timestamp", &max_ts.to_string())?; - - Ok(()) - } - - /// Full scan: paginate through all assets, checking for metadata changes - async fn full_scan(&self) -> Result<()> { - info!("Starting full scan..."); - - let libraries = self.state.list_libraries()?; - - if libraries.is_empty() { - debug!("No libraries configured, skipping full scan"); - return Ok(()); - } - - let page_size = 100; - let mut offset = 0; - let mut total_scanned = 0; - let mut total_announced = 0; - - loop { - let resp = self.can.list_all(page_size, offset, true).await?; - let count = resp.items.len(); - total_scanned += count; - - for asset in &resp.items { - for lib in &libraries { - if lib.filter.matches(asset) { - let was_new = self.announce_asset(lib, asset).await?; - if was_new { - total_announced += 1; - } - } - } - } - - if (count as i64) < page_size { - break; - } - offset += page_size; - } - - info!( - "Full scan complete: scanned {}, announced {} new/updated", - total_scanned, total_announced - ); - Ok(()) - } - - /// Announce a single asset to a library's iroh document. - /// Returns true if the asset was newly announced or updated. - async fn announce_asset( - &self, - lib: &crate::library::Library, - asset: &crate::can_client::AssetMeta, - ) -> Result { - let doc_id = match &lib.doc_id { - Some(id) => id.clone(), - None => { - debug!("Library '{}' has no doc_id yet, skipping", lib.name); - return Ok(false); - } - }; - - // Check if already announced at current version - if self.state.is_announced(&lib.id, &asset.hash)? { - // Already announced — skip unless metadata changed - // (full scan handles re-announcement on metadata change) - return Ok(false); - } - - // Download file content from CAN service and add as iroh blob - let iroh_blob_hash = match self.can.get_asset(&asset.hash).await { - Ok(content) => { - // Add to iroh blob store so remote peers can download it - match self.node.blobs.add_bytes(content).await { - Ok(tag_info) => Some(tag_info.hash.to_string()), - Err(e) => { - warn!( - "Failed to add blob for asset {}: {:#}", - &asset.hash[..12], - e - ); - None - } - } - } - Err(e) => { - warn!( - "Failed to download asset {} from CAN service: {:#}", - &asset.hash[..12], - e - ); - None - } - }; - - // Create sync entry with the iroh blob hash - let mut entry = AssetSyncEntry::from_asset_meta(asset, &self.node.peer_id()); - entry.iroh_blob_hash = iroh_blob_hash; - let entry_bytes = entry.to_bytes(); - - // Write to iroh document (CRDT — concurrent writes merge automatically) - if let Err(e) = self - .node - .write_to_doc(&doc_id, asset.hash.as_bytes(), &entry_bytes) - .await - { - error!( - "Failed to write asset {} to doc {}: {:#}", - &asset.hash[..12], - &doc_id[..12], - e - ); - return Ok(false); - } - - // Mark as announced in local state - self.state.mark_announced(&lib.id, &asset.hash, entry.version)?; - - debug!( - "Announced asset {} to library '{}' (doc {})", - &asset.hash[..12], - lib.name, - &doc_id[..12] - ); - Ok(true) - } -} diff --git a/examples/can-sync/src/can_client.rs b/examples/can-sync/src/can_client.rs index 12d1af6..9e634a6 100644 --- a/examples/can-sync/src/can_client.rs +++ b/examples/can-sync/src/can_client.rs @@ -1,291 +1,244 @@ +//! HTTP client for CAN service's private sync API (protobuf-encoded). +//! +//! Includes SSE subscription for real-time ingest notifications and +//! incremental hash queries via `?since=` parameter. + use anyhow::{Context, Result}; -use bytes::Bytes; -use reqwest::multipart; -use serde::{Deserialize, Serialize}; +use futures_util::StreamExt; +use prost::Message; +use tokio::sync::mpsc; +use tracing::{debug, info, warn}; -/// HTTP client for CAN service API +use crate::protocol::*; + +/// Event received from the CAN service SSE stream. #[derive(Debug, Clone)] -pub struct CanClient { - client: reqwest::Client, +pub struct SyncEvent { + pub hash: String, + pub timestamp: i64, +} + +/// Client for CAN service's /sync/* endpoints. +#[derive(Clone)] +pub struct CanSyncClient { + http: reqwest::Client, base_url: String, + sync_key: String, } -// ── API response types (mirror CAN service) ── - -#[derive(Debug, Deserialize)] -pub struct ApiResponse { - pub status: String, - pub data: T, -} - -#[derive(Debug, Deserialize)] -pub struct ErrorResponse { - pub status: String, - pub error: String, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AssetMeta { - pub hash: String, - pub mime_type: String, - pub application: Option, - pub user: Option, - pub tags: Vec, - pub description: Option, - pub human_filename: Option, - pub human_path: Option, - pub timestamp: i64, - pub is_trashed: bool, - #[serde(default)] - pub is_corrupted: bool, - pub size: i64, -} - -#[derive(Debug, Deserialize)] -pub struct ListResponse { - pub items: Vec, - pub pagination: Pagination, -} - -#[derive(Debug, Deserialize)] -pub struct Pagination { - pub limit: i64, - pub offset: i64, - pub total: i64, -} - -#[derive(Debug, Deserialize)] -pub struct IngestResult { - pub timestamp: i64, - pub hash: String, - pub filename: String, -} - -// ── Search parameters ── - -#[derive(Debug, Default, Serialize)] -pub struct SearchParams { - #[serde(skip_serializing_if = "Option::is_none")] - pub hash: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub start_time: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub end_time: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub tags: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub mime_type: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub user: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub application: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub limit: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub offset: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub order: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub include_trashed: Option, -} - -// ── Ingest metadata ── - -#[derive(Debug, Default)] -pub struct IngestMeta { - pub mime_type: Option, - pub human_file_name: Option, - pub human_readable_path: Option, - pub application: Option, - pub user: Option, - pub tags: Option, - pub description: Option, -} - -// ── Client implementation ── - -impl CanClient { - pub fn new(base_url: &str) -> Self { +impl CanSyncClient { + /// Create a new client pointed at the given CAN service URL, authenticated with the sync API key. + pub fn new(base_url: &str, sync_key: &str) -> Self { Self { - client: reqwest::Client::new(), + http: reqwest::Client::new(), base_url: base_url.trim_end_matches('/').to_string(), + sync_key: sync_key.to_string(), } } - /// List assets with pagination and ordering - pub async fn list( - &self, - limit: i64, - offset: i64, - order: &str, - offset_time: Option, - ) -> Result { - let mut url = format!("{}/list?limit={}&offset={}&order={}", self.base_url, limit, offset, order); - if let Some(ts) = offset_time { - url.push_str(&format!("&offset_time={}", ts)); - } - let resp = self.client.get(&url).send().await.context("list request failed")?; - let status = resp.status(); - if !status.is_success() { - let text = resp.text().await.unwrap_or_default(); - anyhow::bail!("CAN list failed ({}): {}", status, text); - } - let api: ApiResponse = resp.json().await.context("parse list response")?; - Ok(api.data) - } - - /// List all assets (paginated, including trashed for full sync) - pub async fn list_all( - &self, - limit: i64, - offset: i64, - include_trashed: bool, - ) -> Result { - let mut url = format!("{}/list?limit={}&offset={}&order=asc", self.base_url, limit, offset); - if include_trashed { - url.push_str("&include_trashed=true"); - } - let resp = self.client.get(&url).send().await.context("list_all request failed")?; - let status = resp.status(); - if !status.is_success() { - let text = resp.text().await.unwrap_or_default(); - anyhow::bail!("CAN list_all failed ({}): {}", status, text); - } - let api: ApiResponse = resp.json().await.context("parse list_all response")?; - Ok(api.data) - } - - /// Search assets by filters - pub async fn search(&self, params: &SearchParams) -> Result { + /// POST protobuf request, return protobuf response bytes + async fn post_proto(&self, path: &str, body: Vec) -> Result { + let url = format!("{}{}", self.base_url, path); let resp = self - .client - .get(&format!("{}/search", self.base_url)) - .query(params) + .http + .post(&url) + .header("X-Sync-Key", &self.sync_key) + .header("Content-Type", "application/x-protobuf") + .body(body) .send() .await - .context("search request failed")?; - let status = resp.status(); - if !status.is_success() { + .with_context(|| format!("POST {}", url))?; + + if !resp.status().is_success() { + let status = resp.status(); let text = resp.text().await.unwrap_or_default(); - anyhow::bail!("CAN search failed ({}): {}", status, text); + anyhow::bail!("{} returned {}: {}", path, status, text); } - let api: ApiResponse = resp.json().await.context("parse search response")?; - Ok(api.data) + + resp.bytes().await.with_context(|| format!("reading body from {}", path)) } - /// Download asset content by hash - pub async fn get_asset(&self, hash: &str) -> Result { + /// Get all asset digests (full list — use for initial reconciliation only). + pub async fn get_hashes(&self) -> Result { + let req = HashListRequest {}; + let mut buf = Vec::with_capacity(req.encoded_len()); + req.encode(&mut buf)?; + + let resp_bytes = self.post_proto("/sync/hashes", buf).await?; + HashListResponse::decode(resp_bytes).context("decode HashListResponse") + } + + /// Get only asset digests newer than `since` timestamp (incremental query). + pub async fn get_hashes_since(&self, since: i64) -> Result { + let req = HashListRequest {}; + let mut buf = Vec::with_capacity(req.encoded_len()); + req.encode(&mut buf)?; + + let url = format!("{}/sync/hashes?since={}", self.base_url, since); let resp = self - .client - .get(&format!("{}/asset/{}", self.base_url, hash)) + .http + .post(&url) + .header("X-Sync-Key", &self.sync_key) + .header("Content-Type", "application/x-protobuf") + .body(buf) .send() .await - .context("get_asset request failed")?; - let status = resp.status(); - if !status.is_success() { + .with_context(|| format!("POST {}", url))?; + + if !resp.status().is_success() { + let status = resp.status(); let text = resp.text().await.unwrap_or_default(); - anyhow::bail!("CAN get_asset failed ({}): {}", status, text); + anyhow::bail!("/sync/hashes?since={} returned {}: {}", since, status, text); } - resp.bytes().await.context("read asset bytes") + + let resp_bytes = resp.bytes().await?; + HashListResponse::decode(resp_bytes).context("decode HashListResponse") } - /// Get asset metadata by hash - pub async fn get_meta(&self, hash: &str) -> Result { - let resp = self - .client - .get(&format!("{}/asset/{}/meta", self.base_url, hash)) - .send() - .await - .context("get_meta request failed")?; - let status = resp.status(); - if !status.is_success() { - let text = resp.text().await.unwrap_or_default(); - anyhow::bail!("CAN get_meta failed ({}): {}", status, text); - } - let api: ApiResponse = resp.json().await.context("parse meta response")?; - Ok(api.data) + /// Pull full assets by hash. + pub async fn pull(&self, hashes: Vec) -> Result { + let req = PullRequest { hashes }; + let mut buf = Vec::with_capacity(req.encoded_len()); + req.encode(&mut buf)?; + + let resp_bytes = self.post_proto("/sync/pull", buf).await?; + PullResponse::decode(resp_bytes).context("decode PullResponse") } - /// Ingest a file into CAN service via multipart upload - pub async fn ingest(&self, content: Bytes, meta: IngestMeta) -> Result { - let file_part = multipart::Part::bytes(content.to_vec()) - .file_name(meta.human_file_name.clone().unwrap_or_else(|| "file".to_string())) - .mime_str(meta.mime_type.as_deref().unwrap_or("application/octet-stream"))?; + /// Push a single asset bundle. + pub async fn push(&self, bundle: AssetBundle) -> Result { + let req = PushRequest { + bundle: Some(bundle), + }; + let mut buf = Vec::with_capacity(req.encoded_len()); + req.encode(&mut buf)?; - let mut form = multipart::Form::new().part("file", file_part); - - if let Some(ref v) = meta.mime_type { - form = form.text("mime_type", v.clone()); - } - if let Some(ref v) = meta.human_file_name { - form = form.text("human_file_name", v.clone()); - } - if let Some(ref v) = meta.human_readable_path { - form = form.text("human_readable_path", v.clone()); - } - if let Some(ref v) = meta.application { - form = form.text("application", v.clone()); - } - if let Some(ref v) = meta.user { - form = form.text("user", v.clone()); - } - if let Some(ref v) = meta.tags { - form = form.text("tags", v.clone()); - } - if let Some(ref v) = meta.description { - form = form.text("description", v.clone()); - } - - let resp = self - .client - .post(&format!("{}/ingest", self.base_url)) - .multipart(form) - .send() - .await - .context("ingest request failed")?; - let status = resp.status(); - if !status.is_success() { - let text = resp.text().await.unwrap_or_default(); - anyhow::bail!("CAN ingest failed ({}): {}", status, text); - } - let api: ApiResponse = resp.json().await.context("parse ingest response")?; - Ok(api.data) + let resp_bytes = self.post_proto("/sync/push", buf).await?; + PushResponse::decode(resp_bytes).context("decode PushResponse") } - /// Update asset metadata (tags, description) + /// Update metadata for an existing asset. pub async fn update_meta( &self, - hash: &str, - tags: Option>, + hash: String, description: Option, - ) -> Result<()> { - #[derive(Serialize)] - struct MetadataUpdate { - #[serde(skip_serializing_if = "Option::is_none")] - tags: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - description: Option, - } - let resp = self - .client - .patch(&format!("{}/asset/{}", self.base_url, hash)) - .json(&MetadataUpdate { tags, description }) - .send() - .await - .context("update_meta request failed")?; - let status = resp.status(); - if !status.is_success() { - let text = resp.text().await.unwrap_or_default(); - anyhow::bail!("CAN update_meta failed ({}): {}", status, text); - } - Ok(()) + tags: Vec, + is_trashed: bool, + ) -> Result { + let req = MetaUpdateRequest { + hash, + description, + tags, + is_trashed, + }; + let mut buf = Vec::with_capacity(req.encoded_len()); + req.encode(&mut buf)?; + + let resp_bytes = self.post_proto("/sync/meta", buf).await?; + MetaUpdateResponse::decode(resp_bytes).context("decode MetaUpdateResponse") } - /// Check if CAN service is reachable - pub async fn health_check(&self) -> Result { - match self.list(1, 0, "desc", None).await { - Ok(_) => Ok(true), - Err(_) => Ok(false), + /// Health check: try to get hashes (will fail if sync API disabled). + pub async fn health_check(&self) -> bool { + self.get_hashes().await.is_ok() + } + + /// Subscribe to SSE events from CAN service. Sends `SyncEvent` on the + /// returned channel whenever the CAN service ingests a new asset. + /// + /// Automatically reconnects on disconnect (with incremental catch-up). + /// Returns a channel receiver that yields events. + pub fn subscribe_events(&self) -> mpsc::UnboundedReceiver { + let (tx, rx) = mpsc::unbounded_channel(); + let url = format!( + "{}/sync/events?key={}", + self.base_url, self.sync_key + ); + let http = self.http.clone(); + + tokio::spawn(async move { + loop { + info!("Connecting to SSE stream: {}", url.split('?').next().unwrap_or(&url)); + match Self::run_sse_stream(&http, &url, &tx).await { + Ok(()) => { + info!("SSE stream ended cleanly"); + } + Err(e) => { + warn!("SSE stream error: {:#}", e); + } + } + + // If the receiver is dropped, stop reconnecting + if tx.is_closed() { + debug!("SSE subscriber dropped, stopping reconnect loop"); + break; + } + + // Reconnect after a short delay + info!("Reconnecting SSE in 2s..."); + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + } + }); + + rx + } + + // Connect to the SSE endpoint and forward parsed events to the channel + // until the stream ends or an error occurs. + async fn run_sse_stream( + http: &reqwest::Client, + url: &str, + tx: &mpsc::UnboundedSender, + ) -> Result<()> { + let resp = http + .get(url) + .header("Accept", "text/event-stream") + .send() + .await + .context("SSE connect")?; + + if !resp.status().is_success() { + anyhow::bail!("SSE returned status {}", resp.status()); } + + let mut stream = resp.bytes_stream(); + let mut buffer = String::new(); + + while let Some(chunk) = stream.next().await { + let chunk = chunk.context("reading SSE chunk")?; + buffer.push_str(&String::from_utf8_lossy(&chunk)); + + // Process complete SSE messages (separated by double newlines) + while let Some(pos) = buffer.find("\n\n") { + let message = buffer[..pos].to_string(); + buffer = buffer[pos + 2..].to_string(); + + // Parse SSE message: look for "data: {...}" lines + for line in message.lines() { + if let Some(data) = line.strip_prefix("data:") { + let data = data.trim(); + if data == "ping" || data.is_empty() { + continue; + } + + // Parse JSON: {"hash":"...","timestamp":...} + if let Ok(value) = serde_json::from_str::(data) { + if let (Some(hash), Some(ts)) = ( + value["hash"].as_str(), + value["timestamp"].as_i64(), + ) { + debug!("SSE event: new_asset hash={}", &hash[..hash.len().min(12)]); + let _ = tx.send(SyncEvent { + hash: hash.to_string(), + timestamp: ts, + }); + } + } + } + } + } + } + + Ok(()) } } diff --git a/examples/can-sync/src/config.rs b/examples/can-sync/src/config.rs index d889d1b..65bd283 100644 --- a/examples/can-sync/src/config.rs +++ b/examples/can-sync/src/config.rs @@ -1,78 +1,42 @@ -use anyhow::{Context, Result}; -use serde::{Deserialize, Serialize}; -use std::path::{Path, PathBuf}; +use serde::Deserialize; +use std::path::Path; -#[derive(Debug, Clone, Serialize, Deserialize)] +/// All settings needed to run the sync agent, loaded from a YAML file. +#[derive(Debug, Clone, Deserialize)] pub struct SyncConfig { - /// Base URL for the CAN service API (e.g. "http://127.0.0.1:3210/api/v1/can/0") + /// Base URL of the local CAN service (e.g. "http://127.0.0.1:3210") pub can_service_url: String, - /// Address for the CAN Sync HTTP API (e.g. "127.0.0.1:3213") - pub listen_addr: String, + /// API key for CAN service's sync endpoints (must match config.sync_api_key) + pub sync_api_key: String, - /// Directory for persistent data (peer key, sync state DB) - pub data_dir: String, + /// Shared passphrase for peer discovery (all peers must use the same one) + pub sync_passphrase: String, - /// Optional custom relay URL; null uses iroh's public relay - pub relay_url: Option, - - /// Seconds between fast polls for new assets + /// Seconds between polls for new local assets #[serde(default = "default_poll_interval")] pub poll_interval_secs: u64, - /// Seconds between full scans of all assets - #[serde(default = "default_full_scan_interval")] - pub full_scan_interval_secs: u64, + /// Optional: path to write this node's ticket to (for direct connection) + #[serde(default)] + pub ticket_file: Option, + + /// Optional: path to a file containing a peer's node ticket (for direct connection). + /// If set, the agent will read this ticket and connect directly instead of waiting + /// for gossip discovery. The file is polled until it exists and is non-empty. + #[serde(default)] + pub connect_ticket_file: Option, } fn default_poll_interval() -> u64 { - 5 -} - -fn default_full_scan_interval() -> u64 { - 300 + 3 } impl SyncConfig { - /// Load config from a YAML file, falling back to defaults if not found - pub fn load(path: &Path) -> Result { - if path.exists() { - let contents = - std::fs::read_to_string(path).context("Failed to read config file")?; - let config: SyncConfig = - serde_yaml::from_str(&contents).context("Failed to parse config YAML")?; - Ok(config) - } else { - tracing::warn!("Config file not found at {}, using defaults", path.display()); - Ok(Self::default()) - } - } - - /// Resolved data directory path - pub fn data_path(&self) -> PathBuf { - PathBuf::from(&self.data_dir) - } - - /// Path to the peer keypair file - pub fn peer_key_path(&self) -> PathBuf { - self.data_path().join("peer_key") - } - - /// Path to the sync state SQLite database - pub fn db_path(&self) -> PathBuf { - self.data_path().join("can_sync.db") - } -} - -impl Default for SyncConfig { - fn default() -> Self { - Self { - can_service_url: "http://127.0.0.1:3210/api/v1/can/0".to_string(), - listen_addr: "127.0.0.1:3213".to_string(), - data_dir: "./can_sync_data".to_string(), - relay_url: None, - poll_interval_secs: default_poll_interval(), - full_scan_interval_secs: default_full_scan_interval(), - } + /// Read a YAML config file from disk and parse it into a SyncConfig. + pub fn load(path: &Path) -> anyhow::Result { + let contents = std::fs::read_to_string(path)?; + let config: Self = serde_yaml::from_str(&contents)?; + Ok(config) } } diff --git a/examples/can-sync/src/discovery.rs b/examples/can-sync/src/discovery.rs new file mode 100644 index 0000000..f19b180 --- /dev/null +++ b/examples/can-sync/src/discovery.rs @@ -0,0 +1,110 @@ +//! Peer discovery via iroh-gossip using a shared passphrase. +//! +//! All CAN sync agents with the same `sync_passphrase` derive the same +//! BLAKE3 gossip topic and discover each other automatically. + +use std::collections::HashSet; + +use anyhow::Result; +use iroh::{Endpoint, EndpointId}; +use iroh_gossip::net::Gossip; +use iroh_gossip::proto::TopicId; +use n0_future::StreamExt; +use tokio::sync::mpsc; +use tracing::{debug, info, warn}; + +/// Derive a deterministic gossip TopicId from a shared passphrase. +pub fn derive_topic(passphrase: &str) -> TopicId { + let hash = blake3::hash(format!("can-sync-v1:{}", passphrase).as_bytes()); + TopicId::from_bytes(*hash.as_bytes()) +} + +/// Manages peer discovery via gossip announcements. +pub struct Discovery { + gossip: Gossip, + topic: TopicId, + endpoint: Endpoint, +} + +impl Discovery { + /// Create a new Discovery that listens on a gossip topic derived from the shared passphrase. + pub fn new(endpoint: Endpoint, gossip: Gossip, passphrase: &str) -> Self { + let topic = derive_topic(passphrase); + info!("Gossip topic: {}", hex::encode(topic.as_bytes())); + Self { + gossip, + topic, + endpoint, + } + } + + /// Subscribe to the gossip topic and yield newly discovered peer EndpointIds. + /// + /// Sends discovered EndpointIds on the channel. Runs forever. + pub async fn run(self, tx: mpsc::Sender) -> Result<()> { + info!("Joining gossip topic for peer discovery..."); + + // Subscribe to the topic with no bootstrap peers (we discover via gossip) + let mut topic = self + .gossip + .subscribe(self.topic, vec![]) + .await + .map_err(|e| anyhow::anyhow!("gossip subscribe failed: {}", e))?; + + // Wait until we have at least one neighbor + info!("Waiting for gossip neighbors..."); + + // Broadcast our EndpointId periodically + let our_id = self.endpoint.id(); + let (sender, mut receiver) = topic.split(); + let sender_clone = sender.clone(); + tokio::spawn(async move { + let msg = our_id.as_bytes().to_vec(); + loop { + if let Err(e) = sender_clone.broadcast(msg.clone().into()).await { + warn!("Failed to broadcast discovery: {}", e); + } + tokio::time::sleep(std::time::Duration::from_secs(10)).await; + } + }); + + // Listen for peer announcements + let mut known_peers: HashSet = HashSet::new(); + + while let Some(event) = receiver.next().await { + match event { + Ok(iroh_gossip::api::Event::Received(msg)) => { + if msg.content.len() == 32 { + if let Ok(bytes) = <[u8; 32]>::try_from(msg.content.as_ref()) { + if let Ok(peer_id) = EndpointId::from_bytes(&bytes) { + if peer_id != our_id && known_peers.insert(peer_id) { + info!("Discovered new peer: {}", peer_id.fmt_short()); + let _ = tx.send(peer_id).await; + } + } + } + } + } + Ok(iroh_gossip::api::Event::NeighborUp(peer_id)) => { + if peer_id != our_id && known_peers.insert(peer_id) { + info!("Gossip neighbor up: {}", peer_id.fmt_short()); + let _ = tx.send(peer_id).await; + } + } + Ok(iroh_gossip::api::Event::NeighborDown(peer_id)) => { + info!("Gossip neighbor down: {}", peer_id.fmt_short()); + known_peers.remove(&peer_id); + } + Ok(iroh_gossip::api::Event::Lagged) => { + warn!("Gossip receiver lagged, may have missed messages"); + } + Err(e) => { + warn!("Gossip receive error: {}", e); + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + } + } + } + + Ok(()) + } +} diff --git a/examples/can-sync/src/fetcher.rs b/examples/can-sync/src/fetcher.rs deleted file mode 100644 index 84f1dc1..0000000 --- a/examples/can-sync/src/fetcher.rs +++ /dev/null @@ -1,352 +0,0 @@ -use std::sync::Arc; - -use anyhow::Result; -use futures_lite::StreamExt; -use sha2::{Digest, Sha256}; -use tokio::io::AsyncReadExt; -use tracing::{debug, error, info, warn}; - -use crate::can_client::{CanClient, IngestMeta}; -use crate::library::SyncState; -use crate::manifest::AssetSyncEntry; -use crate::node::SyncNode; - -/// The fetcher receives remote asset entries from iroh documents -/// and ingests them into the local CAN service. -pub struct Fetcher { - can: CanClient, - state: Arc, - node: Arc, -} - -impl Fetcher { - pub fn new(can: CanClient, state: Arc, node: Arc) -> Self { - Self { can, state, node } - } - - /// Run the fetcher — subscribes to library document events for real-time sync, - /// falls back to periodic polling for documents without active subscriptions - pub async fn run(self) { - info!("Fetcher started — watching for remote asset entries"); - - // Run two loops concurrently: - // 1. Subscription watcher — subscribes to active library docs - // 2. Periodic checker — catches anything missed - let poll_interval = tokio::time::interval(std::time::Duration::from_secs(10)); - let sub_interval = tokio::time::interval(std::time::Duration::from_secs(5)); - - tokio::pin!(poll_interval); - tokio::pin!(sub_interval); - - loop { - tokio::select! { - _ = poll_interval.tick() => { - if let Err(e) = self.check_for_new_entries().await { - warn!("Fetcher poll error: {:#}", e); - } - } - _ = sub_interval.tick() => { - // Try to subscribe to any library docs that we haven't subscribed to yet - if let Err(e) = self.subscribe_to_libraries().await { - debug!("Fetcher subscription check: {:#}", e); - } - } - } - } - } - - /// Subscribe to document events for all libraries that have doc_ids - async fn subscribe_to_libraries(&self) -> Result<()> { - let libraries = self.state.list_libraries()?; - - for lib in &libraries { - if let Some(ref doc_id_hex) = lib.doc_id { - // Open the doc and subscribe to events - let doc = match self.node.open_doc(doc_id_hex).await { - Ok(d) => d, - Err(_) => continue, - }; - - let mut events = match doc.subscribe().await { - Ok(e) => e, - Err(_) => continue, - }; - - // Spawn a task to process events from this doc - let can = self.can.clone(); - let node_peer_id = self.node.peer_id(); - let node = self.node.clone(); - let lib_name = lib.name.clone(); - - tokio::spawn(async move { - while let Some(event) = events.next().await { - match event { - Ok(iroh_docs::engine::LiveEvent::InsertRemote { - entry, - content_status, - .. - }) => { - let key = entry.key().to_vec(); - let can_hash = String::from_utf8_lossy(&key).to_string(); - - if content_status == iroh_docs::ContentStatus::Complete { - // The entry value (our AssetSyncEntry) is available - // Read the entry content from the blob store - let content_hash = entry.content_hash(); - let mut reader = node.blobs.reader(content_hash); - let mut buf = Vec::new(); - if reader.read_to_end(&mut buf).await.is_ok() { - if let Ok(sync_entry) = AssetSyncEntry::from_bytes(&buf) { - if sync_entry.last_modified_by == node_peer_id { - continue; // Skip our own entries - } - info!( - "Received remote entry for {} in library '{}'", - &can_hash[..can_hash.len().min(12)], - lib_name - ); - if let Err(e) = process_remote_entry( - &can, - &node, - &node_peer_id, - &can_hash, - sync_entry, - ) - .await - { - error!( - "Error processing remote entry {}: {:#}", - &can_hash[..can_hash.len().min(12)], - e - ); - } - } - } - } - } - Ok(iroh_docs::engine::LiveEvent::NeighborUp(peer)) => { - info!("Peer connected: {}", peer.fmt_short()); - } - Ok(iroh_docs::engine::LiveEvent::NeighborDown(peer)) => { - info!("Peer disconnected: {}", peer.fmt_short()); - } - Ok(_) => {} // Ignore other events - Err(e) => { - warn!("Document event error: {:#}", e); - break; - } - } - } - }); - - // Only subscribe to one doc per tick to avoid overwhelming - return Ok(()); - } - } - Ok(()) - } - - /// Check all library documents for entries we don't have locally (polling fallback) - async fn check_for_new_entries(&self) -> Result<()> { - let libraries = self.state.list_libraries()?; - - for lib in &libraries { - if let Some(ref doc_id_hex) = lib.doc_id { - let doc = match self.node.open_doc(doc_id_hex).await { - Ok(d) => d, - Err(_) => continue, - }; - - // Query all entries (latest per key) - let query = iroh_docs::store::Query::single_latest_per_key().build(); - let entries = match doc.get_many(query).await { - Ok(e) => e, - Err(_) => continue, - }; - tokio::pin!(entries); - - while let Some(Ok(entry)) = entries.next().await { - let key = entry.key().to_vec(); - let can_hash = String::from_utf8_lossy(&key).to_string(); - - // Read the entry value (AssetSyncEntry) - let content_hash = entry.content_hash(); - let mut reader = self.node.blobs.reader(content_hash); - let mut buf = Vec::new(); - if reader.read_to_end(&mut buf).await.is_err() { - continue; - } - - let sync_entry = match AssetSyncEntry::from_bytes(&buf) { - Ok(e) => e, - Err(_) => continue, - }; - - // Skip our own entries - if sync_entry.last_modified_by == self.node.peer_id() { - continue; - } - - // Check if already processed - if self.state.is_announced(&lib.id, &can_hash).unwrap_or(false) { - continue; - } - - info!( - "Polling found remote entry for {} in library '{}'", - &can_hash[..can_hash.len().min(12)], - lib.name - ); - - if let Err(e) = process_remote_entry( - &self.can, - &self.node, - &self.node.peer_id(), - &can_hash, - sync_entry, - ) - .await - { - error!( - "Error processing remote entry {}: {:#}", - &can_hash[..can_hash.len().min(12)], - e - ); - } - - // Mark as processed - let _ = self.state.mark_announced(&lib.id, &can_hash, 1); - } - } - } - Ok(()) - } -} - -/// Process a remote asset entry — download blob and ingest into CAN service -async fn process_remote_entry( - can: &CanClient, - node: &SyncNode, - local_peer_id: &str, - can_hash: &str, - entry: AssetSyncEntry, -) -> Result<()> { - // Skip if this is our own entry - if entry.last_modified_by == local_peer_id { - return Ok(()); - } - - // Check if already in local CAN service - match can.get_meta(can_hash).await { - Ok(existing) => { - // Asset exists — check if metadata needs updating - if entry.tags != existing.tags - || entry.description != existing.description - || entry.is_trashed != existing.is_trashed - { - info!("Updating metadata for {} from remote peer", &can_hash[..12]); - can.update_meta( - can_hash, - Some(entry.tags.clone()), - entry.description.clone(), - ) - .await?; - } - return Ok(()); - } - Err(_) => { - // Asset not found locally — need to fetch and ingest - } - } - - info!( - "Fetching remote asset {} ({}B) from peer {}", - &can_hash[..12], - entry.size, - &entry.last_modified_by[..entry.last_modified_by.len().min(12)] - ); - - // Download blob via iroh - let content = download_blob(node, &entry).await?; - - if content.is_empty() { - warn!("Downloaded empty blob for {} — skipping", &can_hash[..12]); - return Ok(()); - } - - // Verify CAN hash: SHA256(timestamp_bytes + content) - if !verify_can_hash(can_hash, entry.timestamp, &content) { - error!( - "CAN hash verification failed for {} — rejecting", - &can_hash[..12] - ); - return Ok(()); - } - - // Ingest into local CAN service - let meta = IngestMeta { - mime_type: Some(entry.mime_type.clone()), - human_file_name: entry.human_filename.clone(), - human_readable_path: entry.human_path.clone(), - application: entry.application.clone(), - user: entry.user.clone(), - tags: if entry.tags.is_empty() { - None - } else { - Some(entry.tags.join(",")) - }, - description: entry.description.clone(), - }; - - match can.ingest(content.into(), meta).await { - Ok(result) => { - info!( - "Ingested remote asset: hash={}, filename={}", - &result.hash[..12], - result.filename - ); - } - Err(e) => { - error!("Failed to ingest remote asset {}: {:#}", &can_hash[..12], e); - } - } - - Ok(()) -} - -/// Download a blob via iroh using the blob hash from the sync entry -async fn download_blob(node: &SyncNode, entry: &AssetSyncEntry) -> Result> { - let blob_hash_str = match &entry.iroh_blob_hash { - Some(h) => h, - None => { - warn!("No iroh blob hash in sync entry — cannot download"); - return Ok(Vec::new()); - } - }; - - // Parse the BLAKE3 hash - let blob_hash: iroh_blobs::Hash = blob_hash_str - .parse() - .map_err(|_| anyhow::anyhow!("Invalid iroh blob hash: {}", &blob_hash_str[..12]))?; - - // Read from the local blob store (iroh-docs should have synced it) - let mut reader = node.blobs.reader(blob_hash); - let mut buf = Vec::with_capacity(entry.size as usize); - reader.read_to_end(&mut buf).await?; - - debug!( - "Downloaded blob {} ({} bytes)", - &blob_hash_str[..12], - buf.len() - ); - Ok(buf) -} - -/// Verify CAN hash: SHA256(timestamp_string + content) matches expected hash -fn verify_can_hash(expected_hash: &str, timestamp: i64, content: &[u8]) -> bool { - let mut hasher = Sha256::new(); - hasher.update(timestamp.to_string().as_bytes()); - hasher.update(content); - let computed = hex::encode(hasher.finalize()); - computed == expected_hash -} diff --git a/examples/can-sync/src/library.rs b/examples/can-sync/src/library.rs deleted file mode 100644 index 1c0f985..0000000 --- a/examples/can-sync/src/library.rs +++ /dev/null @@ -1,288 +0,0 @@ -use anyhow::{Context, Result}; -use rusqlite::Connection; -use serde::{Deserialize, Serialize}; - -use crate::can_client::AssetMeta; - -/// Filter criteria that determines which CAN assets belong to a library -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct LibraryFilter { - /// Match assets with this application tag - #[serde(skip_serializing_if = "Option::is_none")] - pub application: Option, - /// Match assets with any of these tags - #[serde(skip_serializing_if = "Option::is_none")] - pub tags: Option>, - /// Match assets from this user - #[serde(skip_serializing_if = "Option::is_none")] - pub user: Option, - /// Match assets with MIME type prefix (e.g. "image/") - #[serde(skip_serializing_if = "Option::is_none")] - pub mime_prefix: Option, - /// Manual list of specific hashes to include - #[serde(skip_serializing_if = "Option::is_none")] - pub hashes: Option>, -} - -impl LibraryFilter { - /// Check if an asset matches this filter - pub fn matches(&self, asset: &AssetMeta) -> bool { - // If hashes list is set, only match those exact hashes - if let Some(ref hashes) = self.hashes { - return hashes.contains(&asset.hash); - } - - // All set criteria must match (AND logic) - if let Some(ref app) = self.application { - if asset.application.as_deref() != Some(app.as_str()) { - return false; - } - } - - if let Some(ref required_tags) = self.tags { - // Asset must have at least one of the required tags - if !required_tags.iter().any(|t| asset.tags.contains(t)) { - return false; - } - } - - if let Some(ref user) = self.user { - if asset.user.as_deref() != Some(user.as_str()) { - return false; - } - } - - if let Some(ref prefix) = self.mime_prefix { - if !asset.mime_type.starts_with(prefix.as_str()) { - return false; - } - } - - true - } -} - -/// A library definition stored locally -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Library { - /// Unique library ID (UUID) - pub id: String, - /// Human-readable name - pub name: String, - /// Filter criteria - pub filter: LibraryFilter, - /// iroh document ID (namespace) — set after creation - pub doc_id: Option, - /// Whether this library was created locally or joined from remote - pub is_local: bool, - /// Creation timestamp - pub created_at: i64, -} - -/// Tracks which assets have been announced to which libraries. -/// Uses std::sync::Mutex because rusqlite::Connection is !Send, -/// so tokio::sync::RwLock won't work across .await points. -pub struct SyncState { - db: std::sync::Mutex, -} - -impl SyncState { - /// Open or create the sync state database - pub fn open(path: &std::path::Path) -> Result { - let db = Connection::open(path).context("open sync state DB")?; - db.execute_batch( - " - CREATE TABLE IF NOT EXISTS libraries ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL, - filter_json TEXT NOT NULL, - doc_id TEXT, - is_local INTEGER NOT NULL DEFAULT 1, - created_at INTEGER NOT NULL - ); - - CREATE TABLE IF NOT EXISTS announced_assets ( - library_id TEXT NOT NULL, - hash TEXT NOT NULL, - version INTEGER NOT NULL DEFAULT 1, - announced_at INTEGER NOT NULL, - PRIMARY KEY (library_id, hash), - FOREIGN KEY (library_id) REFERENCES libraries(id) ON DELETE CASCADE - ); - - CREATE TABLE IF NOT EXISTS sync_state ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL - ); - ", - ) - .context("init sync state tables")?; - Ok(Self { - db: std::sync::Mutex::new(db), - }) - } - - fn lock_db(&self) -> std::sync::MutexGuard<'_, Connection> { - self.db.lock().expect("sync state DB lock poisoned") - } - - // ── Library CRUD ── - - pub fn save_library(&self, lib: &Library) -> Result<()> { - let db = self.lock_db(); - let filter_json = serde_json::to_string(&lib.filter)?; - db.execute( - "INSERT OR REPLACE INTO libraries (id, name, filter_json, doc_id, is_local, created_at) - VALUES (?1, ?2, ?3, ?4, ?5, ?6)", - rusqlite::params![ - lib.id, - lib.name, - filter_json, - lib.doc_id, - lib.is_local as i32, - lib.created_at, - ], - )?; - Ok(()) - } - - pub fn list_libraries(&self) -> Result> { - let db = self.lock_db(); - let mut stmt = - db.prepare("SELECT id, name, filter_json, doc_id, is_local, created_at FROM libraries")?; - let libs = stmt - .query_map([], |row| { - let filter_json: String = row.get(2)?; - Ok(Library { - id: row.get(0)?, - name: row.get(1)?, - filter: serde_json::from_str(&filter_json).unwrap_or(LibraryFilter { - application: None, - tags: None, - user: None, - mime_prefix: None, - hashes: None, - }), - doc_id: row.get(3)?, - is_local: row.get::<_, i32>(4)? != 0, - created_at: row.get(5)?, - }) - })? - .collect::, _>>()?; - Ok(libs) - } - - pub fn get_library(&self, id: &str) -> Result> { - let db = self.lock_db(); - let mut stmt = db.prepare( - "SELECT id, name, filter_json, doc_id, is_local, created_at FROM libraries WHERE id = ?1", - )?; - let mut rows = stmt.query_map([id], |row| { - let filter_json: String = row.get(2)?; - Ok(Library { - id: row.get(0)?, - name: row.get(1)?, - filter: serde_json::from_str(&filter_json).unwrap_or(LibraryFilter { - application: None, - tags: None, - user: None, - mime_prefix: None, - hashes: None, - }), - doc_id: row.get(3)?, - is_local: row.get::<_, i32>(4)? != 0, - created_at: row.get(5)?, - }) - })?; - match rows.next() { - Some(Ok(lib)) => Ok(Some(lib)), - Some(Err(e)) => Err(e.into()), - None => Ok(None), - } - } - - pub fn delete_library(&self, id: &str) -> Result<()> { - let db = self.lock_db(); - db.execute("DELETE FROM announced_assets WHERE library_id = ?1", [id])?; - db.execute("DELETE FROM libraries WHERE id = ?1", [id])?; - Ok(()) - } - - pub fn update_library_doc_id(&self, id: &str, doc_id: &str) -> Result<()> { - let db = self.lock_db(); - db.execute( - "UPDATE libraries SET doc_id = ?1 WHERE id = ?2", - [doc_id, id], - )?; - Ok(()) - } - - // ── Asset announcement tracking ── - - pub fn is_announced(&self, library_id: &str, hash: &str) -> Result { - let db = self.lock_db(); - let count: i64 = db.query_row( - "SELECT COUNT(*) FROM announced_assets WHERE library_id = ?1 AND hash = ?2", - [library_id, hash], - |row| row.get(0), - )?; - Ok(count > 0) - } - - pub fn get_announced_version(&self, library_id: &str, hash: &str) -> Result> { - let db = self.lock_db(); - let mut stmt = db.prepare( - "SELECT version FROM announced_assets WHERE library_id = ?1 AND hash = ?2", - )?; - let mut rows = stmt.query_map(rusqlite::params![library_id, hash], |row| { - row.get::<_, i64>(0) - })?; - match rows.next() { - Some(Ok(v)) => Ok(Some(v as u64)), - Some(Err(e)) => Err(e.into()), - None => Ok(None), - } - } - - pub fn mark_announced(&self, library_id: &str, hash: &str, version: u64) -> Result<()> { - let db = self.lock_db(); - let now = chrono::Utc::now().timestamp_millis(); - db.execute( - "INSERT OR REPLACE INTO announced_assets (library_id, hash, version, announced_at) - VALUES (?1, ?2, ?3, ?4)", - rusqlite::params![library_id, hash, version as i64, now], - )?; - Ok(()) - } - - pub fn remove_announced(&self, library_id: &str, hash: &str) -> Result<()> { - let db = self.lock_db(); - db.execute( - "DELETE FROM announced_assets WHERE library_id = ?1 AND hash = ?2", - [library_id, hash], - )?; - Ok(()) - } - - // ── General state ── - - pub fn get_state(&self, key: &str) -> Result> { - let db = self.lock_db(); - let mut stmt = db.prepare("SELECT value FROM sync_state WHERE key = ?1")?; - let mut rows = stmt.query_map([key], |row| row.get::<_, String>(0))?; - match rows.next() { - Some(Ok(v)) => Ok(Some(v)), - Some(Err(e)) => Err(e.into()), - None => Ok(None), - } - } - - pub fn set_state(&self, key: &str, value: &str) -> Result<()> { - let db = self.lock_db(); - db.execute( - "INSERT OR REPLACE INTO sync_state (key, value) VALUES (?1, ?2)", - [key, value], - )?; - Ok(()) - } -} diff --git a/examples/can-sync/src/main.rs b/examples/can-sync/src/main.rs index 4dc31ad..1fc84d4 100644 --- a/examples/can-sync/src/main.rs +++ b/examples/can-sync/src/main.rs @@ -1,121 +1,231 @@ -#![allow(dead_code)] +//! CAN Sync — P2P full-mirror replication agent for CAN Service. +//! +//! Uses iroh for encrypted QUIC transport + NAT traversal, +//! and iroh-gossip for peer discovery via a shared passphrase. +//! +//! Each instance talks to its local CAN Service via the private +//! protobuf sync API (/sync/*), authenticated with an API key. -mod announcer; mod can_client; mod config; -mod fetcher; -mod library; -mod manifest; -mod node; -mod routes; +mod discovery; +mod peer; +mod protocol; +mod rendezvous; -use std::path::PathBuf; -use std::sync::Arc; +use std::path::Path; use anyhow::{Context, Result}; -use tracing::info; +use iroh::endpoint::presets::N0; +use iroh::{Endpoint, EndpointAddr, EndpointId}; +use iroh_gossip::net::Gossip; +use tokio::sync::mpsc; +use tracing::{error, info, warn}; -use crate::announcer::Announcer; -use crate::can_client::CanClient; +use crate::can_client::CanSyncClient; use crate::config::SyncConfig; -use crate::fetcher::Fetcher; -use crate::library::SyncState; -use crate::node::SyncNode; -use crate::routes::AppState; +use crate::discovery::Discovery; +use crate::rendezvous::Rendezvous; +/// ALPN protocol identifier for CAN sync peer connections. +const SYNC_ALPN: &[u8] = b"can-sync/1"; + +/// Entry point: loads config, connects to the local CAN service, sets up +/// encrypted P2P networking (iroh), and discovers + syncs with peers. #[tokio::main] async fn main() -> Result<()> { - // Initialize tracing + // Initialize logging tracing_subscriber::fmt() .with_env_filter( tracing_subscriber::EnvFilter::try_from_default_env() - .unwrap_or_else(|_| "can_sync=info,iroh=warn".parse().unwrap()), + .unwrap_or_else(|_| "can_sync=info,iroh=warn,iroh_gossip=warn".parse().unwrap()), ) .init(); // Load config let config_path = std::env::args() .nth(1) - .map(PathBuf::from) - .unwrap_or_else(|| PathBuf::from("config.yaml")); + .unwrap_or_else(|| "config.yaml".to_string()); + let config = SyncConfig::load(Path::new(&config_path)) + .with_context(|| format!("loading config from {}", config_path))?; - let config = SyncConfig::load(&config_path)?; - info!("CAN Sync starting..."); - info!(" CAN service: {}", config.can_service_url); - info!(" Listen addr: {}", config.listen_addr); - info!(" Data dir: {}", config.data_dir); + info!("CAN Sync v2 starting"); + info!("CAN service: {}", config.can_service_url); + info!("Poll interval: {}s", config.poll_interval_secs); - // Ensure data directory exists - std::fs::create_dir_all(config.data_path()) - .context("Failed to create data directory")?; + // Create HTTP client for local CAN service's sync API + let can = CanSyncClient::new(&config.can_service_url, &config.sync_api_key); - // Initialize CAN service client - let can = CanClient::new(&config.can_service_url); + // Verify CAN service is reachable + if can.health_check().await { + info!("CAN service sync API is healthy"); + } else { + warn!("CAN service sync API not reachable — will retry on sync"); + } - // Check CAN service health - match can.health_check().await { - Ok(true) => info!("CAN service is reachable"), - Ok(false) | Err(_) => { - tracing::warn!( - "CAN service at {} is not reachable — will retry on each poll", - config.can_service_url - ); + // Create iroh endpoint for QUIC transport with n0 defaults (relay + discovery) + let endpoint = Endpoint::builder() + .preset(N0) + .alpns(vec![SYNC_ALPN.to_vec()]) + .bind() + .await + .context("creating iroh endpoint")?; + + let node_id = endpoint.id(); + info!("Node ID: {}", node_id); + + let addrs = endpoint.bound_sockets(); + if let Some(addr) = addrs.first() { + info!("Listening on {}", addr); + } + + // Write our EndpointAddr to file if configured (for direct peer connection in tests) + if let Some(ref ticket_path) = config.ticket_file { + // Wait briefly for the endpoint to register with relay + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + + let addr = endpoint.addr(); + let addr_json = serde_json::to_string(&addr) + .context("serializing EndpointAddr")?; + std::fs::write(ticket_path, &addr_json) + .with_context(|| format!("writing addr to {}", ticket_path))?; + info!("Wrote EndpointAddr to {}", ticket_path); + } + + // Create gossip instance for peer discovery (not async — returns directly) + let gossip = Gossip::builder().spawn(endpoint.clone()); + + // Channel for discovered peers + let (peer_tx, mut peer_rx) = mpsc::channel::(32); + + // Spawn discovery via gossip (works once bootstrap peers are known) + let disc = Discovery::new(endpoint.clone(), gossip.clone(), &config.sync_passphrase); + let peer_tx_gossip = peer_tx.clone(); + tokio::spawn(async move { + if let Err(e) = disc.run(peer_tx_gossip).await { + error!("Gossip discovery failed: {:#}", e); } + }); + + // Spawn internet rendezvous via pkarr relay (discovers peers worldwide) + let rendezvous = Rendezvous::new(&config.sync_passphrase, node_id) + .context("creating rendezvous")?; + let peer_tx_rdv = peer_tx.clone(); + tokio::spawn(async move { + if let Err(e) = rendezvous.run(peer_tx_rdv).await { + error!("Rendezvous discovery failed: {:#}", e); + } + }); + + // If a direct connect ticket file is specified, spawn a task to read it and connect + if let Some(ref ticket_path) = config.connect_ticket_file { + let ticket_path = ticket_path.clone(); + let endpoint_direct = endpoint.clone(); + let can_direct = can.clone(); + + tokio::spawn(async move { + info!("Waiting for peer addr file: {}", ticket_path); + + // Poll until the file exists and is non-empty + let addr_json = loop { + match std::fs::read_to_string(&ticket_path) { + Ok(s) if !s.trim().is_empty() => break s.trim().to_string(), + _ => tokio::time::sleep(std::time::Duration::from_millis(200)).await, + } + }; + + info!("Read peer addr from {}", ticket_path); + + let peer_addr: EndpointAddr = match serde_json::from_str(&addr_json) { + Ok(a) => a, + Err(e) => { + error!("Invalid EndpointAddr JSON: {:#}", e); + return; + } + }; + + let peer_id = peer_addr.id; + let short = peer_id.fmt_short().to_string(); + info!("Direct connecting to peer: {} (from addr file)", short); + + match endpoint_direct.connect(peer_addr, SYNC_ALPN).await { + Ok(conn) => { + info!("Direct connection to {} established!", short); + + // Initial reconciliation + if let Err(e) = peer::run_sync_session(conn.clone(), can_direct.clone(), true).await { + error!("Initial sync with {} failed: {:#}", short, e); + return; + } + + info!("Initial sync with {} complete, starting live sync", short); + + // Live sync: SSE-driven push + accept incoming streams + peer::run_live_sync(conn, can_direct).await; + } + Err(e) => { + error!("Failed to connect to {}: {:#}", short, e); + } + } + }); } - // Open sync state database - let state = SyncState::open(&config.db_path())?; - let state = Arc::new(state); - info!("Sync state DB opened at {}", config.db_path().display()); - - // Start iroh P2P node - let node = SyncNode::spawn(&config).await?; - let node = Arc::new(node); - info!("iroh node ID: {}", node.peer_id()); - - // Build shared app state - let app_state = Arc::new(AppState { - node: node.clone(), - state: state.clone(), - can: can.clone(), - }); - - // Start the announcer (polls CAN service for new assets) - let announcer = Announcer::new( - can.clone(), - state.clone(), - node.clone(), - config.poll_interval_secs, - config.full_scan_interval_secs, - ); + // Spawn incoming connection handler + let endpoint_accept = endpoint.clone(); + let can_accept = can.clone(); tokio::spawn(async move { - announcer.run().await; + loop { + match endpoint_accept.accept().await { + Some(incoming) => { + let can_clone = can_accept.clone(); + tokio::spawn(async move { + match incoming.await { + Ok(conn) => { + info!("Accepted incoming connection from {}", conn.remote_id().fmt_short()); + peer::handle_incoming(conn, can_clone, std::time::Duration::from_secs(0)).await; + } + Err(e) => { + warn!("Failed to accept connection: {:#}", e); + } + } + }); + } + None => { + info!("Endpoint closed, stopping accept loop"); + break; + } + } + } }); - // Start the fetcher (receives remote assets and ingests them) - let fetcher = Fetcher::new(can.clone(), state.clone(), node.clone()); - tokio::spawn(async move { - fetcher.run().await; - }); + // Main loop: connect to discovered peers (from gossip) and sync + info!("Waiting for peers..."); - // Build HTTP router - let router = routes::build_router(app_state); + while let Some(peer_id) = peer_rx.recv().await { + let short = peer_id.fmt_short(); + info!("Connecting to discovered peer: {}", short); - // Start HTTP server - let listener = tokio::net::TcpListener::bind(&config.listen_addr) - .await - .context("Failed to bind HTTP listener")?; - info!("CAN Sync API listening on http://{}", config.listen_addr); + let endpoint_clone = endpoint.clone(); + let can_clone = can.clone(); - // Open browser to status page - let status_url = format!("http://{}/status", config.listen_addr); - if open::that(&status_url).is_err() { - info!("Open {} in your browser to check status", status_url); + tokio::spawn(async move { + let conn = match endpoint_clone.connect(peer_id, SYNC_ALPN).await { + Ok(c) => c, + Err(e) => { + error!("Failed to connect to {}: {:#}", short, e); + return; + } + }; + + if let Err(e) = peer::run_sync_session(conn.clone(), can_clone.clone(), true).await { + error!("Initial sync with {} failed: {:#}", short, e); + return; + } + + peer::run_live_sync(conn, can_clone).await; + }); } - axum::serve(listener, router) - .await - .context("HTTP server error")?; - + info!("CAN Sync shutting down"); Ok(()) } diff --git a/examples/can-sync/src/manifest.rs b/examples/can-sync/src/manifest.rs deleted file mode 100644 index b8210ed..0000000 --- a/examples/can-sync/src/manifest.rs +++ /dev/null @@ -1,75 +0,0 @@ -use serde::{Deserialize, Serialize}; - -use crate::can_client::AssetMeta; - -/// Entry stored in iroh documents for each synced asset. -/// Key = CAN hash, Value = serialized AssetSyncEntry -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AssetSyncEntry { - /// CAN timestamp (milliseconds since epoch) - pub timestamp: i64, - /// MIME type - pub mime_type: String, - /// Application tag - pub application: Option, - /// User identity - pub user: Option, - /// Tags list - pub tags: Vec, - /// Description - pub description: Option, - /// Original human-readable filename - pub human_filename: Option, - /// Original human-readable path - pub human_path: Option, - /// File size in bytes - pub size: i64, - /// Whether the asset is trashed - pub is_trashed: bool, - /// iroh blob hash (BLAKE3) for downloading via iroh - pub iroh_blob_hash: Option, - /// Version counter for conflict resolution (higher wins) - pub version: u64, - /// Peer ID that last modified this entry - pub last_modified_by: String, -} - -impl AssetSyncEntry { - /// Create from CAN service asset metadata - pub fn from_asset_meta(meta: &AssetMeta, peer_id: &str) -> Self { - Self { - timestamp: meta.timestamp, - mime_type: meta.mime_type.clone(), - application: meta.application.clone(), - user: meta.user.clone(), - tags: meta.tags.clone(), - description: meta.description.clone(), - human_filename: meta.human_filename.clone(), - human_path: meta.human_path.clone(), - size: meta.size, - is_trashed: meta.is_trashed, - iroh_blob_hash: None, - version: 1, - last_modified_by: peer_id.to_string(), - } - } - - /// Serialize to bytes for storage in iroh document - pub fn to_bytes(&self) -> Vec { - postcard::to_allocvec(self).expect("serialize AssetSyncEntry") - } - - /// Deserialize from bytes - pub fn from_bytes(bytes: &[u8]) -> anyhow::Result { - Ok(postcard::from_bytes(bytes)?) - } - - /// Check if metadata differs from a CAN asset (indicates update needed) - pub fn metadata_differs(&self, meta: &AssetMeta) -> bool { - self.tags != meta.tags - || self.description != meta.description - || self.is_trashed != meta.is_trashed - || self.human_filename != meta.human_filename - || self.human_path != meta.human_path - } -} diff --git a/examples/can-sync/src/node.rs b/examples/can-sync/src/node.rs deleted file mode 100644 index 4a32d61..0000000 --- a/examples/can-sync/src/node.rs +++ /dev/null @@ -1,150 +0,0 @@ -use anyhow::{Context, Result}; -use iroh::protocol::Router; -use iroh::Endpoint; -use iroh_blobs::store::mem::MemStore; -use iroh_blobs::{BlobsProtocol, ALPN as BLOBS_ALPN}; -use iroh_docs::api::protocol::{AddrInfoOptions, ShareMode}; -use iroh_docs::protocol::Docs; -use iroh_docs::{AuthorId, DocTicket, NamespaceId, ALPN as DOCS_ALPN}; -use iroh_gossip::net::Gossip; -use iroh_gossip::ALPN as GOSSIP_ALPN; -use tokio::sync::OnceCell; - -use crate::config::SyncConfig; - -/// Holds all iroh subsystems for the P2P node -pub struct SyncNode { - pub endpoint: Endpoint, - pub blobs: BlobsProtocol, - pub docs: Docs, - pub gossip: Gossip, - pub router: Router, - /// Cached default author ID (created once on startup) - author_id: OnceCell, -} - -impl SyncNode { - /// Start the iroh node with all protocol handlers - pub async fn spawn(_config: &SyncConfig) -> Result { - // Build endpoint (Ed25519 keypair auto-generated and cached) - let endpoint = Endpoint::bind() - .await - .map_err(|e| anyhow::anyhow!("Failed to bind iroh endpoint: {}", e))?; - - tracing::info!( - "iroh node started — EndpointID: {}", - endpoint.id() - ); - - // Gossip for peer communication - let gossip = Gossip::builder().spawn(endpoint.clone()); - - // Blob store (in-memory — blobs are transient, CAN service is authoritative) - let mem_store = MemStore::default(); - let blobs_store: &iroh_blobs::api::Store = &mem_store; - let blobs = BlobsProtocol::new(blobs_store, None); - - // Document sync (CRDT-replicated key-value store) - let docs = Docs::memory() - .spawn(endpoint.clone(), blobs_store.clone(), gossip.clone()) - .await - .context("Failed to spawn iroh-docs")?; - - // Router accepts incoming connections and dispatches to handlers - let router = Router::builder(endpoint.clone()) - .accept(BLOBS_ALPN, blobs.clone()) - .accept(GOSSIP_ALPN, gossip.clone()) - .accept(DOCS_ALPN, docs.clone()) - .spawn(); - - Ok(Self { - endpoint, - blobs, - docs, - gossip, - router, - author_id: OnceCell::new(), - }) - } - - /// Get this node's peer ID as a string - pub fn peer_id(&self) -> String { - self.endpoint.id().to_string() - } - - /// Get the node's endpoint address info for sharing - pub fn endpoint_addr(&self) -> iroh::EndpointAddr { - self.endpoint.addr() - } - - /// Get or create the default author for writing to documents - pub async fn author(&self) -> Result { - self.author_id - .get_or_try_init(|| async { - self.docs.author_default().await - }) - .await - .copied() - } - - /// Create a new iroh document and return its NamespaceId as a hex string - pub async fn create_doc(&self) -> Result { - let doc = self.docs.create().await?; - let ns_id = doc.id(); - Ok(hex::encode(ns_id.to_bytes())) - } - - /// Open an existing document by its hex-encoded namespace ID - pub async fn open_doc(&self, doc_id_hex: &str) -> Result { - let ns_id = parse_namespace_id(doc_id_hex)?; - self.docs - .open(ns_id) - .await? - .ok_or_else(|| anyhow::anyhow!("Document {} not found", &doc_id_hex[..12])) - } - - /// Write a key-value entry to a document - pub async fn write_to_doc( - &self, - doc_id_hex: &str, - key: &[u8], - value: &[u8], - ) -> Result<()> { - let doc = self.open_doc(doc_id_hex).await?; - let author = self.author().await?; - doc.set_bytes(author, key.to_vec(), value.to_vec()).await?; - Ok(()) - } - - /// Generate a share ticket (DocTicket) for a document - pub async fn share_doc(&self, doc_id_hex: &str) -> Result { - let doc = self.open_doc(doc_id_hex).await?; - let ticket = doc - .share(ShareMode::Write, AddrInfoOptions::RelayAndAddresses) - .await?; - Ok(ticket) - } - - /// Import a document from a DocTicket, returns the namespace ID as hex - pub async fn import_doc(&self, ticket: DocTicket) -> Result { - let doc = self.docs.import(ticket).await?; - let ns_id = doc.id(); - Ok(hex::encode(ns_id.to_bytes())) - } - - /// Graceful shutdown - pub async fn shutdown(self) -> Result<()> { - tracing::info!("Shutting down iroh node..."); - self.router.shutdown().await?; - Ok(()) - } -} - -/// Parse a hex-encoded NamespaceId -pub fn parse_namespace_id(hex_str: &str) -> Result { - let bytes: [u8; 32] = hex::decode(hex_str) - .context("Invalid hex in doc_id")? - .try_into() - .map_err(|_| anyhow::anyhow!("doc_id must be 32 bytes (64 hex chars)"))?; - Ok(NamespaceId::from(bytes)) -} diff --git a/examples/can-sync/src/peer.rs b/examples/can-sync/src/peer.rs new file mode 100644 index 0000000..9663c35 --- /dev/null +++ b/examples/can-sync/src/peer.rs @@ -0,0 +1,469 @@ +//! Per-peer sync: reconciliation and live bidirectional asset transfer. +//! +//! When two sync agents connect, they: +//! 1. Exchange hash lists (from their local CAN services) +//! 2. Compute the diff (what each side is missing) +//! 3. Send/receive missing assets concurrently (avoids deadlock) +//! 4. Subscribe to SSE events from local CAN for instant push on new assets +//! +//! The live sync uses: +//! - **SSE events** from local CAN service to detect new assets instantly +//! (replaces the old polling loop — no more wasted hash-list queries) +//! - An unbounded channel to share received hashes from the receive loop +//! to the push loop, preventing "echo" where an asset received from a +//! peer gets pushed right back to them. +//! - A fallback incremental poll on timeout for catch-up if SSE was briefly down. + +use std::collections::{HashMap, HashSet}; + +use anyhow::{Context, Result}; +use iroh::endpoint::Connection; +use prost::Message; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::sync::mpsc; +use tracing::{debug, error, info, warn}; + +use crate::can_client::{CanSyncClient, SyncEvent}; +use crate::protocol::*; + +// Message type tags for QUIC stream framing +const MSG_HASH_SET: u8 = 0x01; +const MSG_ASSET_BUNDLE: u8 = 0x02; +const MSG_META_UPDATE: u8 = 0x03; +const MSG_DONE: u8 = 0x04; + +/// Frame a protobuf message with a type tag and length prefix. +fn encode_frame(msg_type: u8, payload: &[u8]) -> Vec { + let len = payload.len() as u32; + let mut frame = Vec::with_capacity(5 + payload.len()); + frame.push(msg_type); + frame.extend_from_slice(&len.to_be_bytes()); + frame.extend_from_slice(payload); + frame +} + +/// Read a single framed message from a QUIC recv stream. +/// Returns (msg_type, payload_bytes). +async fn read_frame(recv: &mut iroh::endpoint::RecvStream) -> Result<(u8, Vec)> { + let msg_type = recv.read_u8().await.context("reading message type")?; + let len = recv.read_u32().await.context("reading message length")?; + + if len > 256 * 1024 * 1024 { + anyhow::bail!("Message too large: {} bytes", len); + } + + let mut payload = vec![0u8; len as usize]; + recv.read_exact(&mut payload) + .await + .context("reading message payload")?; + Ok((msg_type, payload)) +} + +/// Run a full sync session with a connected peer. +/// +/// This handles initial reconciliation: exchange hash lists, compute diffs, +/// then send/receive missing assets **concurrently** to avoid deadlock when +/// both sides have large amounts of data to transfer. +pub async fn run_sync_session( + conn: Connection, + can: CanSyncClient, + is_initiator: bool, +) -> Result<()> { + let peer_id = conn.remote_id(); + let short_id = peer_id.fmt_short().to_string(); + info!("Starting sync session with {} (initiator={})", short_id, is_initiator); + + // Initiator opens the stream, responder accepts it + let (mut send, mut recv) = if is_initiator { + conn.open_bi().await.context("opening bi stream")? + } else { + conn.accept_bi().await.context("accepting bi stream")? + }; + + // Step 1: Get our local hash list from CAN service + let our_hashes = can.get_hashes().await.context("getting local hashes")?; + let our_hash_map: HashMap = our_hashes + .assets + .iter() + .map(|a| (a.hash.clone(), a)) + .collect(); + + info!( + "Local state: {} assets, sending to peer {}", + our_hashes.assets.len(), + short_id + ); + + // Step 2: Send our hash set to peer + let hash_set_msg = PeerHashSet { + assets: our_hashes.assets.clone(), + }; + let mut buf = Vec::with_capacity(hash_set_msg.encoded_len()); + hash_set_msg.encode(&mut buf)?; + let frame = encode_frame(MSG_HASH_SET, &buf); + send.write_all(&frame).await.context("sending hash set")?; + send.flush().await?; + + // Step 3: Receive peer's hash set + let (msg_type, payload) = read_frame(&mut recv).await.context("reading peer hash set")?; + if msg_type != MSG_HASH_SET { + anyhow::bail!("Expected hash set message, got type {}", msg_type); + } + let peer_hash_set = PeerHashSet::decode(payload.as_slice()).context("decoding peer hash set")?; + + let peer_hash_map: HashMap = peer_hash_set + .assets + .iter() + .map(|a| (a.hash.clone(), a)) + .collect(); + + info!( + "Peer {} has {} assets", + short_id, + peer_hash_set.assets.len() + ); + + // Step 4: Compute diffs + let our_hashes_set: HashSet<&String> = our_hash_map.keys().collect(); + let peer_hashes_set: HashSet<&String> = peer_hash_map.keys().collect(); + + let we_need: Vec = peer_hashes_set + .difference(&our_hashes_set) + .map(|h| (*h).clone()) + .collect(); + let they_need: Vec = our_hashes_set + .difference(&peer_hashes_set) + .map(|h| (*h).clone()) + .collect(); + + info!( + "Diff with {}: we need {}, they need {}", + short_id, + we_need.len(), + they_need.len() + ); + + // Step 5+6: Send and receive assets CONCURRENTLY to avoid deadlock. + let send_fut = async { + if !they_need.is_empty() { + send_assets(&can, &mut send, &they_need, &short_id).await?; + } + let done_frame = encode_frame(MSG_DONE, &[]); + send.write_all(&done_frame).await.context("sending DONE")?; + send.flush().await.context("flushing after DONE")?; + Ok::<_, anyhow::Error>(()) + }; + + let recv_fut = receive_assets(&can, &mut recv, &short_id); + + let (send_result, recv_result) = tokio::join!(send_fut, recv_fut); + send_result.context("sending assets to peer")?; + recv_result.context("receiving assets from peer")?; + + info!("Sync session with {} complete", short_id); + Ok(()) +} + +/// Pull assets from local CAN service and send them to the peer. +async fn send_assets( + can: &CanSyncClient, + send: &mut iroh::endpoint::SendStream, + hashes: &[String], + peer_short: &str, +) -> Result<()> { + for chunk in hashes.chunks(10) { + let pull_resp = can + .pull(chunk.to_vec()) + .await + .context("pulling assets from CAN")?; + + for bundle in pull_resp.bundles { + let hash_short = &bundle.hash[..bundle.hash.len().min(12)]; + info!("Sending asset {} to peer {}", hash_short, peer_short); + + let mut buf = Vec::with_capacity(bundle.encoded_len()); + bundle.encode(&mut buf)?; + let frame = encode_frame(MSG_ASSET_BUNDLE, &buf); + send.write_all(&frame).await?; + send.flush().await?; + } + } + Ok(()) +} + +/// Receive assets from peer and push them to local CAN service. +/// Returns the list of hashes that were successfully ingested. +async fn receive_assets( + can: &CanSyncClient, + recv: &mut iroh::endpoint::RecvStream, + peer_short: &str, +) -> Result> { + let mut received = Vec::new(); + + loop { + let (msg_type, payload) = read_frame(recv).await.context("reading asset from peer")?; + + match msg_type { + MSG_DONE => { + debug!("Peer {} finished sending assets", peer_short); + break; + } + MSG_ASSET_BUNDLE => { + let bundle = + AssetBundle::decode(payload.as_slice()).context("decoding asset bundle")?; + let hash = bundle.hash.clone(); + let hash_short = hash[..hash.len().min(12)].to_string(); + info!("Received asset {} from peer {}", hash_short, peer_short); + + match can.push(bundle).await { + Ok(resp) => { + if resp.already_existed { + debug!("Asset {} already existed locally", hash_short); + } else { + info!("Ingested asset {} from peer {}", resp.hash, peer_short); + } + received.push(hash); + } + Err(e) => { + error!("Failed to push asset {} to CAN: {:#}", hash_short, e); + } + } + } + MSG_META_UPDATE => { + let meta = MetaUpdateRequest::decode(payload.as_slice()) + .context("decoding meta update")?; + let hash_short = meta.hash[..meta.hash.len().min(12)].to_string(); + debug!( + "Received meta update for {} from peer {}", + hash_short, peer_short + ); + + if let Err(e) = can + .update_meta( + meta.hash.clone(), + meta.description.clone(), + meta.tags.clone(), + meta.is_trashed, + ) + .await + { + error!("Failed to update meta for {}: {:#}", hash_short, e); + } + } + other => { + warn!("Unknown message type {} from peer {}", other, peer_short); + } + } + } + Ok(received) +} + +/// Handle an incoming connection from a peer who connected to us. +pub async fn handle_incoming( + conn: Connection, + can: CanSyncClient, + _poll_interval: std::time::Duration, +) { + let peer_id = conn.remote_id(); + let short_id = peer_id.fmt_short().to_string(); + info!("Incoming sync connection from {}", short_id); + + if let Err(e) = run_sync_session(conn.clone(), can.clone(), false).await { + error!("Sync session with {} failed: {:#}", short_id, e); + return; + } + + info!("Initial sync with {} complete, starting live sync", short_id); + run_live_sync(conn, can).await; +} + +/// Run both live sync loops (push + receive) concurrently. +/// +/// Uses SSE events from CAN service for instant push (no polling). +/// Uses an unbounded channel to prevent the "echo" problem. +pub async fn run_live_sync( + conn: Connection, + can: CanSyncClient, +) { + let short_id = conn.remote_id().fmt_short().to_string(); + + // Channel for receive loop to notify push loop about received hashes + let (received_tx, received_rx) = mpsc::unbounded_channel::(); + + // Subscribe to SSE events from local CAN service + let sse_rx = can.subscribe_events(); + + // Run push loop and receive loop concurrently — when either ends, we're done + tokio::select! { + result = live_push_loop(conn.clone(), can.clone(), received_rx, sse_rx) => { + if let Err(e) = result { + warn!("Live push loop with {} ended: {:#}", short_id, e); + } + } + result = live_receive_loop(conn, can, received_tx) => { + if let Err(e) = result { + warn!("Live receive loop with {} ended: {:#}", short_id, e); + } + } + } +} + +/// Wait for SSE events from local CAN service and push new assets to the peer. +/// +/// Drains the `received_rx` channel to learn about hashes that arrived from +/// the peer, so we don't echo them back. +/// +/// Falls back to incremental poll if no SSE events arrive within 30s. +async fn live_push_loop( + conn: Connection, + can: CanSyncClient, + mut received_rx: mpsc::UnboundedReceiver, + mut sse_rx: mpsc::UnboundedReceiver, +) -> Result<()> { + let peer_id = conn.remote_id(); + let short_id = peer_id.fmt_short().to_string(); + info!("Starting live push loop with {} (SSE-driven)", short_id); + + // Track what we've already synced (local + received from peer) + let resp = can.get_hashes().await?; + let mut max_timestamp: i64 = resp.assets.iter().map(|a| a.timestamp).max().unwrap_or(0); + let mut known_hashes: HashSet = resp.assets.into_iter().map(|a| a.hash).collect(); + + // Fallback: if no SSE event in 30s, do an incremental poll to catch gaps + let fallback_interval = std::time::Duration::from_secs(30); + + loop { + // Wait for SSE event, or fallback timeout + let new_hashes: Vec = tokio::select! { + event = sse_rx.recv() => { + match event { + Some(evt) => { + // Drain any additional events that arrived at the same time + let mut batch = vec![evt]; + while let Ok(more) = sse_rx.try_recv() { + batch.push(more); + } + + // Drain received-from-peer hashes (echo prevention) + while let Ok(hash) = received_rx.try_recv() { + known_hashes.insert(hash); + } + + // Filter to only truly new hashes + batch + .into_iter() + .filter(|e| { + if e.timestamp > max_timestamp { + max_timestamp = e.timestamp; + } + !known_hashes.contains(&e.hash) + }) + .map(|e| e.hash) + .collect() + } + None => { + warn!("SSE channel closed, stopping push loop"); + break; + } + } + } + + // Fallback: periodic incremental poll + _ = tokio::time::sleep(fallback_interval) => { + debug!("Fallback incremental poll (no SSE events in {}s)", fallback_interval.as_secs()); + + while let Ok(hash) = received_rx.try_recv() { + known_hashes.insert(hash); + } + + match can.get_hashes_since(max_timestamp).await { + Ok(resp) => { + resp.assets + .into_iter() + .filter(|a| { + if a.timestamp > max_timestamp { + max_timestamp = a.timestamp; + } + !known_hashes.contains(&a.hash) + }) + .map(|a| a.hash) + .collect() + } + Err(e) => { + warn!("Fallback poll failed: {:#}", e); + continue; + } + } + } + }; + + if new_hashes.is_empty() { + continue; + } + + info!( + "Pushing {} new assets to peer {}", + new_hashes.len(), + short_id + ); + + // Open a new QUIC stream for this batch + match conn.open_bi().await { + Ok((mut send, _recv)) => { + if let Err(e) = send_assets(&can, &mut send, &new_hashes, &short_id).await { + error!("Failed to push new assets to {}: {:#}", short_id, e); + } + let done_frame = encode_frame(MSG_DONE, &[]); + let _ = send.write_all(&done_frame).await; + let _ = send.flush().await; + let _ = send.finish(); + } + Err(e) => { + warn!("Failed to open stream to {}: {:#}", short_id, e); + break; // Connection probably dead + } + } + + // Update known set + for h in new_hashes { + known_hashes.insert(h); + } + } + + Ok(()) +} + +/// Accept incoming QUIC bi-streams from the peer and receive assets. +async fn live_receive_loop( + conn: Connection, + can: CanSyncClient, + received_tx: mpsc::UnboundedSender, +) -> Result<()> { + let peer_id = conn.remote_id(); + let short_id = peer_id.fmt_short().to_string(); + info!("Starting live receive loop with {}", short_id); + + loop { + match conn.accept_bi().await { + Ok((_send, mut recv)) => { + info!("Accepted live sync stream from peer {}", short_id); + match receive_assets(&can, &mut recv, &short_id).await { + Ok(received_hashes) => { + for hash in received_hashes { + let _ = received_tx.send(hash); + } + } + Err(e) => { + warn!("Error receiving live assets from {}: {:#}", short_id, e); + } + } + } + Err(e) => { + info!("Live receive loop: connection to {} closed: {:#}", short_id, e); + break; + } + } + } + + Ok(()) +} diff --git a/examples/can-sync/src/protocol.rs b/examples/can-sync/src/protocol.rs new file mode 100644 index 0000000..d96645f --- /dev/null +++ b/examples/can-sync/src/protocol.rs @@ -0,0 +1,123 @@ +//! Protobuf message types for CAN sync API + peer-to-peer protocol. +//! +//! These match the types in CAN service's routes/sync.rs exactly. + +use prost::Message; + +// ── CAN Sync API messages (protobuf, same as CAN service) ─────────────── + +#[derive(Clone, PartialEq, Message)] +pub struct HashListRequest {} + +#[derive(Clone, PartialEq, Message)] +pub struct HashListResponse { + #[prost(message, repeated, tag = "1")] + pub assets: Vec, +} + +#[derive(Clone, PartialEq, Message)] +pub struct AssetDigest { + #[prost(string, tag = "1")] + pub hash: String, + #[prost(int64, tag = "2")] + pub timestamp: i64, + #[prost(int64, tag = "3")] + pub size: i64, + #[prost(bool, tag = "4")] + pub is_trashed: bool, +} + +#[derive(Clone, PartialEq, Message)] +pub struct PullRequest { + #[prost(string, repeated, tag = "1")] + pub hashes: Vec, +} + +#[derive(Clone, PartialEq, Message)] +pub struct PullResponse { + #[prost(message, repeated, tag = "1")] + pub bundles: Vec, +} + +#[derive(Clone, PartialEq, Message)] +pub struct AssetBundle { + #[prost(string, tag = "1")] + pub hash: String, + #[prost(int64, tag = "2")] + pub timestamp: i64, + #[prost(string, tag = "3")] + pub mime_type: String, + #[prost(string, optional, tag = "4")] + pub application: Option, + #[prost(string, optional, tag = "5")] + pub user_identity: Option, + #[prost(string, optional, tag = "6")] + pub description: Option, + #[prost(string, optional, tag = "7")] + pub human_filename: Option, + #[prost(string, optional, tag = "8")] + pub human_path: Option, + #[prost(bool, tag = "9")] + pub is_trashed: bool, + #[prost(int64, tag = "10")] + pub size: i64, + #[prost(string, repeated, tag = "11")] + pub tags: Vec, + #[prost(bytes = "vec", tag = "12")] + pub content: Vec, +} + +#[derive(Clone, PartialEq, Message)] +pub struct PushRequest { + #[prost(message, optional, tag = "1")] + pub bundle: Option, +} + +#[derive(Clone, PartialEq, Message)] +pub struct PushResponse { + #[prost(string, tag = "1")] + pub hash: String, + #[prost(bool, tag = "2")] + pub already_existed: bool, +} + +#[derive(Clone, PartialEq, Message)] +pub struct MetaUpdateRequest { + #[prost(string, tag = "1")] + pub hash: String, + #[prost(string, optional, tag = "2")] + pub description: Option, + #[prost(string, repeated, tag = "3")] + pub tags: Vec, + #[prost(bool, tag = "4")] + pub is_trashed: bool, +} + +#[derive(Clone, PartialEq, Message)] +pub struct MetaUpdateResponse { + #[prost(bool, tag = "1")] + pub success: bool, +} + +// ── Peer-to-peer messages (sent over QUIC streams between sync agents) ── + +/// Sent between peers during reconciliation: "here are all the hashes I have" +#[derive(Clone, PartialEq, Message)] +pub struct PeerHashSet { + #[prost(message, repeated, tag = "1")] + pub assets: Vec, +} + +/// Request from peer: "please send me these assets" +#[derive(Clone, PartialEq, Message)] +pub struct PeerPullRequest { + #[prost(string, repeated, tag = "1")] + pub hashes: Vec, +} + +/// A single asset being sent from one peer to another +#[derive(Clone, PartialEq, Message)] +pub struct PeerAssetTransfer { + #[prost(message, optional, tag = "1")] + pub bundle: Option, +} diff --git a/examples/can-sync/src/rendezvous.rs b/examples/can-sync/src/rendezvous.rs new file mode 100644 index 0000000..c2a329f --- /dev/null +++ b/examples/can-sync/src/rendezvous.rs @@ -0,0 +1,195 @@ +//! Internet peer discovery via pkarr relay servers. +//! +//! Derives deterministic keypair "slots" from the shared passphrase. +//! Each peer claims a slot by publishing its EndpointId as a TXT record. +//! All peers scan all slots periodically to discover each other. +//! +//! This works over the internet — no LAN, no port forwarding needed. +//! Uses n0's public pkarr relay servers (same infrastructure as iroh). + +use std::collections::HashSet; +use std::time::Duration; + +use anyhow::{Context, Result}; +use iroh::EndpointId; +use pkarr::{Client as PkarrClient, Keypair, SignedPacket}; +use simple_dns::rdata::RData; +use tokio::sync::mpsc; +use tracing::{debug, info, warn}; + +const NUM_SLOTS: usize = 8; +const PUBLISH_INTERVAL: Duration = Duration::from_secs(60); +const SCAN_INTERVAL: Duration = Duration::from_secs(15); +const RECORD_NAME: &str = "_can_sync"; + +/// Derive a deterministic pkarr keypair for a given slot index. +fn derive_slot_keypair(passphrase: &str, slot: usize) -> Keypair { + let seed = blake3::hash( + format!("can-sync-rendezvous:{}:{}", passphrase, slot).as_bytes(), + ); + let seed_bytes: [u8; 32] = *seed.as_bytes(); + let secret = ed25519_dalek::SecretKey::from(seed_bytes); + Keypair::from_secret_key(&secret) +} + +/// Internet peer discovery via pkarr relay. +pub struct Rendezvous { + slots: Vec, + our_id: EndpointId, + client: PkarrClient, +} + +impl Rendezvous { + /// Create a new Rendezvous by deriving keypairs for all slots from the passphrase. + pub fn new(passphrase: &str, our_id: EndpointId) -> Result { + let slots: Vec = (0..NUM_SLOTS) + .map(|i| derive_slot_keypair(passphrase, i)) + .collect(); + + let client = PkarrClient::builder() + .build() + .context("creating pkarr client")?; + + Ok(Self { + slots, + our_id, + client, + }) + } + + /// Run the rendezvous loop: claim a slot, periodically re-publish and scan. + pub async fn run(self, tx: mpsc::Sender) -> Result<()> { + let our_id_hex = hex::encode(self.our_id.as_bytes()); + info!( + "Rendezvous: starting internet discovery ({} slots, publish every {}s, scan every {}s)", + NUM_SLOTS, + PUBLISH_INTERVAL.as_secs(), + SCAN_INTERVAL.as_secs(), + ); + + // Claim our slot (first empty, or hash-based fallback) + let our_slot = self.claim_slot(&our_id_hex).await; + info!("Rendezvous: claimed slot {}", our_slot); + + let mut known_peers: HashSet = HashSet::new(); + let mut publish_tick = tokio::time::interval(PUBLISH_INTERVAL); + let mut scan_tick = tokio::time::interval(SCAN_INTERVAL); + + // Do an initial scan immediately + self.scan_all_slots(&mut known_peers, &tx).await; + + loop { + tokio::select! { + _ = publish_tick.tick() => { + if let Err(e) = self.publish_slot(our_slot, &our_id_hex).await { + warn!("Rendezvous: failed to re-publish slot {}: {:#}", our_slot, e); + } + } + _ = scan_tick.tick() => { + self.scan_all_slots(&mut known_peers, &tx).await; + } + } + } + } + + // Read every slot and report any newly discovered peer IDs. + async fn scan_all_slots( + &self, + known_peers: &mut HashSet, + tx: &mpsc::Sender, + ) { + for i in 0..NUM_SLOTS { + match self.read_slot(i).await { + Some(peer_id) if peer_id != self.our_id && known_peers.insert(peer_id) => { + info!( + "Rendezvous: discovered peer {} in slot {}", + peer_id.fmt_short(), + i + ); + let _ = tx.send(peer_id).await; + } + _ => {} + } + } + } + + // Pick an available slot for this peer: reuse our old slot, take an empty one, + // or fall back to a deterministic slot based on our ID. + async fn claim_slot(&self, our_id_hex: &str) -> usize { + // Check if we already own a slot (from a previous run) + for i in 0..NUM_SLOTS { + if let Some(peer_id) = self.read_slot(i).await { + if peer_id == self.our_id { + debug!("Rendezvous: already own slot {}", i); + return i; + } + } + } + + // Claim first empty slot + for i in 0..NUM_SLOTS { + if self.read_slot(i).await.is_none() { + if let Err(e) = self.publish_slot(i, our_id_hex).await { + warn!("Rendezvous: failed to claim slot {}: {:#}", i, e); + continue; + } + return i; + } + } + + // All slots occupied — use deterministic slot based on our ID + let slot = { + let h = blake3::hash(self.our_id.as_bytes()); + let bytes: [u8; 8] = h.as_bytes()[..8].try_into().unwrap(); + u64::from_le_bytes(bytes) as usize % NUM_SLOTS + }; + let _ = self.publish_slot(slot, our_id_hex).await; + slot + } + + // Write our EndpointId into the given slot's DNS TXT record via the pkarr relay. + async fn publish_slot(&self, slot: usize, our_id_hex: &str) -> Result<()> { + let keypair = &self.slots[slot]; + let packet = SignedPacket::builder() + .txt( + RECORD_NAME.try_into().context("invalid record name")?, + our_id_hex.try_into().context("invalid txt value")?, + 300, // 5 min TTL + ) + .sign(keypair) + .context("signing pkarr packet")?; + + self.client + .publish(&packet, None) + .await + .context("publishing to pkarr relay")?; + + debug!("Rendezvous: published slot {}", slot); + Ok(()) + } + + // Look up a slot's DNS TXT record and parse the EndpointId stored there, if any. + async fn read_slot(&self, slot: usize) -> Option { + let public_key = self.slots[slot].public_key(); + let packet = self.client.resolve(&public_key).await?; + + // Use pkarr's resource_records iterator to find our TXT record + for record in packet.resource_records(RECORD_NAME) { + if let RData::TXT(txt) = &record.rdata { + // Try to extract the hex-encoded EndpointId from TXT attributes + if let Ok(txt_string) = String::try_from(txt.clone()) { + let hex_str = txt_string.trim(); + if let Ok(bytes) = hex::decode(hex_str) { + if bytes.len() == 32 { + if let Ok(arr) = <[u8; 32]>::try_from(bytes.as_slice()) { + return EndpointId::from_bytes(&arr).ok(); + } + } + } + } + } + } + + None + } +} diff --git a/examples/can-sync/src/routes.rs b/examples/can-sync/src/routes.rs deleted file mode 100644 index 3175b16..0000000 --- a/examples/can-sync/src/routes.rs +++ /dev/null @@ -1,430 +0,0 @@ -use std::sync::Arc; - -use axum::{ - extract::{Path, State}, - http::StatusCode, - response::IntoResponse, - routing::{get, post}, - Json, Router, -}; -use serde::{Deserialize, Serialize}; - -use crate::can_client::CanClient; -use crate::library::{Library, LibraryFilter, SyncState}; -use crate::node::SyncNode; - -/// Shared application state for route handlers -pub struct AppState { - pub node: Arc, - pub state: Arc, - pub can: CanClient, -} - -// ── Request/Response types ── - -#[derive(Serialize)] -struct StatusResponse { - peer_id: String, - can_service_healthy: bool, - library_count: usize, -} - -#[derive(Serialize)] -struct PeerInfo { - peer_id: String, -} - -#[derive(Deserialize)] -pub struct CreateLibraryRequest { - pub name: String, - pub filter: LibraryFilter, -} - -#[derive(Serialize)] -struct LibraryResponse { - id: String, - name: String, - filter: LibraryFilter, - doc_id: Option, - is_local: bool, - created_at: i64, -} - -#[derive(Serialize)] -struct InviteResponse { - ticket: String, -} - -#[derive(Deserialize)] -pub struct JoinRequest { - pub ticket: String, -} - -#[derive(Serialize)] -struct JoinResponse { - library_id: String, - message: String, -} - -#[derive(Serialize)] -struct ApiResp { - status: String, - data: T, -} - -#[derive(Serialize)] -struct ApiErr { - status: String, - error: String, -} - -fn ok_json(data: T) -> Json> { - Json(ApiResp { - status: "success".to_string(), - data, - }) -} - -fn err_resp(status: StatusCode, msg: &str) -> (StatusCode, Json) { - ( - status, - Json(ApiErr { - status: "error".to_string(), - error: msg.to_string(), - }), - ) -} - -// ── Routes ── - -pub fn build_router(app_state: Arc) -> Router { - Router::new() - .route("/status", get(get_status)) - .route("/peers", get(get_peers)) - .route("/libraries", post(create_library).get(list_libraries)) - .route( - "/libraries/{id}", - get(get_library).delete(delete_library), - ) - .route("/libraries/{id}/invite", post(create_invite)) - .route("/join", post(join_library)) - .with_state(app_state) -} - -// ── Handlers ── - -async fn get_status(State(app): State>) -> impl IntoResponse { - let can_healthy = app.can.health_check().await.unwrap_or(false); - let lib_count = app.state.list_libraries().unwrap_or_default().len(); - - ok_json(StatusResponse { - peer_id: app.node.peer_id(), - can_service_healthy: can_healthy, - library_count: lib_count, - }) - .into_response() -} - -async fn get_peers(State(app): State>) -> impl IntoResponse { - let peers: Vec = vec![PeerInfo { - peer_id: app.node.peer_id(), - }]; - ok_json(peers).into_response() -} - -async fn create_library( - State(app): State>, - Json(req): Json, -) -> impl IntoResponse { - // Create an iroh document for this library - let doc_id = match app.node.create_doc().await { - Ok(id) => Some(id), - Err(e) => { - tracing::warn!("Failed to create iroh document for library: {:#}", e); - None - } - }; - - let lib = Library { - id: uuid::Uuid::new_v4().to_string(), - name: req.name, - filter: req.filter, - doc_id, - is_local: true, - created_at: chrono::Utc::now().timestamp_millis(), - }; - - if let Err(e) = app.state.save_library(&lib) { - return err_resp( - StatusCode::INTERNAL_SERVER_ERROR, - &format!("save failed: {}", e), - ) - .into_response(); - } - - tracing::info!( - "Created library '{}' (id={}, doc_id={:?})", - lib.name, - &lib.id[..8], - lib.doc_id.as_deref().map(|d| &d[..12.min(d.len())]) - ); - - ok_json(LibraryResponse { - id: lib.id, - name: lib.name, - filter: lib.filter, - doc_id: lib.doc_id, - is_local: lib.is_local, - created_at: lib.created_at, - }) - .into_response() -} - -async fn list_libraries(State(app): State>) -> impl IntoResponse { - match app.state.list_libraries() { - Ok(libs) => { - let responses: Vec = libs - .into_iter() - .map(|lib| LibraryResponse { - id: lib.id, - name: lib.name, - filter: lib.filter, - doc_id: lib.doc_id, - is_local: lib.is_local, - created_at: lib.created_at, - }) - .collect(); - ok_json(responses).into_response() - } - Err(e) => { - err_resp(StatusCode::INTERNAL_SERVER_ERROR, &format!("{}", e)).into_response() - } - } -} - -async fn get_library( - State(app): State>, - Path(id): Path, -) -> impl IntoResponse { - match app.state.get_library(&id) { - Ok(Some(lib)) => ok_json(LibraryResponse { - id: lib.id, - name: lib.name, - filter: lib.filter, - doc_id: lib.doc_id, - is_local: lib.is_local, - created_at: lib.created_at, - }) - .into_response(), - Ok(None) => err_resp(StatusCode::NOT_FOUND, "Library not found").into_response(), - Err(e) => { - err_resp(StatusCode::INTERNAL_SERVER_ERROR, &format!("{}", e)).into_response() - } - } -} - -async fn delete_library( - State(app): State>, - Path(id): Path, -) -> impl IntoResponse { - match app.state.delete_library(&id) { - Ok(()) => ok_json("deleted").into_response(), - Err(e) => { - err_resp(StatusCode::INTERNAL_SERVER_ERROR, &format!("{}", e)).into_response() - } - } -} - -async fn create_invite( - State(app): State>, - Path(id): Path, -) -> impl IntoResponse { - match app.state.get_library(&id) { - Ok(Some(lib)) => { - let doc_id = match &lib.doc_id { - Some(d) => d, - None => { - return err_resp( - StatusCode::BAD_REQUEST, - "Library has no iroh document — cannot create invite", - ) - .into_response() - } - }; - - // Generate a real DocTicket via iroh - match app.node.share_doc(doc_id).await { - Ok(ticket) => { - // DocTicket implements Display via iroh's Ticket trait (base32 serialization) - let ticket_str = ticket.to_string(); - - // Wrap with library metadata so the joiner knows the name and filter - let invite_data = serde_json::json!({ - "ticket": ticket_str, - "library_name": lib.name, - "filter": lib.filter, - }); - let invite_b64 = base64_encode( - &serde_json::to_vec(&invite_data).unwrap(), - ); - - ok_json(InviteResponse { ticket: invite_b64 }).into_response() - } - Err(e) => err_resp( - StatusCode::INTERNAL_SERVER_ERROR, - &format!("Failed to create invite: {}", e), - ) - .into_response(), - } - } - Ok(None) => err_resp(StatusCode::NOT_FOUND, "Library not found").into_response(), - Err(e) => { - err_resp(StatusCode::INTERNAL_SERVER_ERROR, &format!("{}", e)).into_response() - } - } -} - -async fn join_library( - State(app): State>, - Json(req): Json, -) -> impl IntoResponse { - // Decode our envelope - let ticket_bytes = match base64_decode(&req.ticket) { - Ok(b) => b, - Err(_) => { - return err_resp(StatusCode::BAD_REQUEST, "Invalid ticket encoding").into_response() - } - }; - - let ticket_data: serde_json::Value = match serde_json::from_slice(&ticket_bytes) { - Ok(v) => v, - Err(_) => { - return err_resp(StatusCode::BAD_REQUEST, "Invalid ticket data").into_response() - } - }; - - // Extract the real DocTicket string - let ticket_str = match ticket_data["ticket"].as_str() { - Some(s) => s, - None => { - return err_resp(StatusCode::BAD_REQUEST, "Missing 'ticket' field in invite") - .into_response() - } - }; - - // Parse DocTicket from the serialized string - let doc_ticket: iroh_docs::DocTicket = match ticket_str.parse() { - Ok(t) => t, - Err(e) => { - return err_resp( - StatusCode::BAD_REQUEST, - &format!("Invalid DocTicket: {}", e), - ) - .into_response() - } - }; - - // Import the document via iroh (starts sync with remote peers) - let doc_id_hex = match app.node.import_doc(doc_ticket).await { - Ok(id) => id, - Err(e) => { - return err_resp( - StatusCode::INTERNAL_SERVER_ERROR, - &format!("Failed to join document: {}", e), - ) - .into_response() - } - }; - - let name = ticket_data["library_name"] - .as_str() - .unwrap_or("remote library") - .to_string(); - - let filter: LibraryFilter = serde_json::from_value(ticket_data["filter"].clone()) - .unwrap_or(LibraryFilter { - application: None, - tags: None, - user: None, - mime_prefix: None, - hashes: None, - }); - - let lib = Library { - id: uuid::Uuid::new_v4().to_string(), - name: name.clone(), - filter, - doc_id: Some(doc_id_hex), - is_local: false, - created_at: chrono::Utc::now().timestamp_millis(), - }; - - if let Err(e) = app.state.save_library(&lib) { - return err_resp( - StatusCode::INTERNAL_SERVER_ERROR, - &format!("save failed: {}", e), - ) - .into_response(); - } - - tracing::info!( - "Joined library '{}' (id={}, doc_id={:?})", - name, - &lib.id[..8], - lib.doc_id.as_deref().map(|d| &d[..12.min(d.len())]) - ); - - ok_json(JoinResponse { - library_id: lib.id, - message: "Joined library successfully".to_string(), - }) - .into_response() -} - -// ── Base64 helpers ── - -fn base64_encode(data: &[u8]) -> String { - const CHARS: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - let mut result = Vec::new(); - for chunk in data.chunks(3) { - let b0 = chunk[0] as u32; - let b1 = if chunk.len() > 1 { chunk[1] as u32 } else { 0 }; - let b2 = if chunk.len() > 2 { chunk[2] as u32 } else { 0 }; - let triple = (b0 << 16) | (b1 << 8) | b2; - result.push(CHARS[((triple >> 18) & 0x3F) as usize]); - result.push(CHARS[((triple >> 12) & 0x3F) as usize]); - if chunk.len() > 1 { - result.push(CHARS[((triple >> 6) & 0x3F) as usize]); - } else { - result.push(b'='); - } - if chunk.len() > 2 { - result.push(CHARS[(triple & 0x3F) as usize]); - } else { - result.push(b'='); - } - } - String::from_utf8(result).unwrap() -} - -fn base64_decode(input: &str) -> Result, &'static str> { - const CHARS: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - let input = input.trim_end_matches('='); - let bytes: Vec = input - .bytes() - .filter_map(|b| CHARS.iter().position(|&c| c == b).map(|p| p as u8)) - .collect(); - let mut buf = Vec::new(); - for chunk in bytes.chunks(4) { - if chunk.len() >= 2 { - buf.push((chunk[0] << 2) | (chunk[1] >> 4)); - } - if chunk.len() >= 3 { - buf.push((chunk[1] << 4) | (chunk[2] >> 2)); - } - if chunk.len() >= 4 { - buf.push((chunk[2] << 6) | chunk[3]); - } - } - Ok(buf) -} diff --git a/examples/can-sync/tests/sync_test.rs b/examples/can-sync/tests/sync_test.rs new file mode 100644 index 0000000..872f228 --- /dev/null +++ b/examples/can-sync/tests/sync_test.rs @@ -0,0 +1,1044 @@ +//! Integration + stress test for CAN Sync v2 +//! +//! Starts two CAN service instances + two sync agents, then runs increasingly +//! aggressive sync scenarios: +//! 1. Single file A→B +//! 2. Single file B→A +//! 3. Rapid burst 25 files A→B (mixed sizes, some 10MB+) +//! 4. Rapid burst 25 files B→A (mixed sizes, some 10MB+) +//! 5. Simultaneous burst: 25 files on EACH side at the same time +//! 6. Final full-mirror verification +//! +//! Usage: +//! cargo run --bin sync-test +//! +//! Prerequisites: +//! CAN service must be built: `cargo build` in the CanService root + +use std::collections::HashSet; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use rand::Rng; +use serde_json::Value; +use tempfile::TempDir; + +// ── Configuration ──────────────────────────────────────────────────────── + +const CAN_A_PORT: u16 = 13210; +const CAN_B_PORT: u16 = 13220; +const SYNC_KEY: &str = "test-sync-key-42"; +const SYNC_PASSPHRASE: &str = "integration-test-passphrase"; +const SYNC_TIMEOUT: Duration = Duration::from_secs(120); // longer for large files +const POLL_INTERVAL: Duration = Duration::from_millis(500); + +// Stress test tuning +const BURST_COUNT: usize = 25; +const LARGE_FILE_SIZE: usize = 12 * 1024 * 1024; // 12 MB +const MEDIUM_FILE_SIZE: usize = 2 * 1024 * 1024; // 2 MB +const SMALL_FILE_SIZE: usize = 4096; // 4 KB + +// ── Process management ─────────────────────────────────────────────────── + +struct ManagedProcess { + child: Child, + name: String, +} + +impl ManagedProcess { + fn spawn( + name: &str, + cmd: &str, + args: &[&str], + envs: &[(&str, &str)], + log_dir: &Path, + ) -> Self { + println!(" Starting {} ...", name); + let mut command = Command::new(cmd); + + let log_file = std::fs::File::create(log_dir.join(format!("{}.log", name))) + .expect("create log file"); + let log_file_clone = log_file.try_clone().expect("clone log file handle"); + command + .args(args) + .stdout(Stdio::from(log_file)) + .stderr(Stdio::from(log_file_clone)) + .env("RUST_LOG", "can_sync=debug,can_service=debug,iroh=info,iroh_gossip=info"); + + for (k, v) in envs { + command.env(k, v); + } + let child = command + .spawn() + .unwrap_or_else(|e| panic!("Failed to start {}: {}", name, e)); + println!(" {} started (pid={})", name, child.id()); + Self { + child, + name: name.to_string(), + } + } + + fn kill(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + println!(" {} stopped", self.name); + } +} + +impl Drop for ManagedProcess { + fn drop(&mut self) { + self.kill(); + } +} + +// ── Test harness ───────────────────────────────────────────────────────── + +struct TestHarness { + _can_a: ManagedProcess, + _can_b: ManagedProcess, + _sync_a: ManagedProcess, + _sync_b: ManagedProcess, + _tmp_a: TempDir, + _tmp_b: TempDir, + _tmp_sync_a: TempDir, + _tmp_sync_b: TempDir, + log_dir: TempDir, + can_a_url: String, + can_b_url: String, +} + +impl TestHarness { + fn new(can_service_bin: &Path) -> Self { + println!("\n=== Setting up test harness ===\n"); + + // Create temp directories + let tmp_a = TempDir::new().expect("create temp dir A"); + let tmp_b = TempDir::new().expect("create temp dir B"); + let tmp_sync_a = TempDir::new().expect("create temp dir sync A"); + let tmp_sync_b = TempDir::new().expect("create temp dir sync B"); + let log_dir = TempDir::new().expect("create log dir"); + + println!(" Logs: {}", log_dir.path().display()); + println!(" CAN A storage: {}", tmp_a.path().display()); + println!(" CAN B storage: {}", tmp_b.path().display()); + + // Write CAN service configs + let config_a = tmp_a.path().join("config.yaml"); + let config_b = tmp_b.path().join("config.yaml"); + write_can_config(&config_a, tmp_a.path(), SYNC_KEY); + write_can_config(&config_b, tmp_b.path(), SYNC_KEY); + + // Start CAN services + let can_a = ManagedProcess::spawn( + "CAN-A", + can_service_bin.to_str().unwrap(), + &[config_a.to_str().unwrap()], + &[("CAN_PORT", &CAN_A_PORT.to_string())], + log_dir.path(), + ); + let can_b = ManagedProcess::spawn( + "CAN-B", + can_service_bin.to_str().unwrap(), + &[config_b.to_str().unwrap()], + &[("CAN_PORT", &CAN_B_PORT.to_string())], + log_dir.path(), + ); + + let can_a_url = format!("http://127.0.0.1:{}", CAN_A_PORT); + let can_b_url = format!("http://127.0.0.1:{}", CAN_B_PORT); + + // Wait for CAN services to be ready + println!("\n Waiting for CAN services to start..."); + wait_for_http(&can_a_url, Duration::from_secs(10), log_dir.path(), "CAN-A"); + wait_for_http(&can_b_url, Duration::from_secs(10), log_dir.path(), "CAN-B"); + println!(" Both CAN services ready!"); + + // Find can-sync binary + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let can_sync_bin = manifest_dir + .join("target") + .join("debug") + .join("can-sync") + .with_extension(std::env::consts::EXE_EXTENSION); + assert!( + can_sync_bin.exists(), + "can-sync binary not found at {}", + can_sync_bin.display() + ); + println!(" Using can-sync: {}", can_sync_bin.display()); + + // Ticket file paths for direct peer connection + let ticket_a = tmp_sync_a.path().join("ticket_a.json"); + let ticket_a_str = ticket_a.to_str().unwrap().replace('\\', "/"); + + // Write sync agent configs with ticket file exchange + let sync_config_a = tmp_sync_a.path().join("config.yaml"); + let sync_config_b = tmp_sync_b.path().join("config.yaml"); + + write_sync_config_with_tickets( + &sync_config_a, + &can_a_url, + SYNC_KEY, + SYNC_PASSPHRASE, + Some(&ticket_a_str), + None, + ); + + write_sync_config_with_tickets( + &sync_config_b, + &can_b_url, + SYNC_KEY, + SYNC_PASSPHRASE, + None, + Some(&ticket_a_str), + ); + + // Start Sync-A first (it writes the ticket) + let sync_a = ManagedProcess::spawn( + "Sync-A", + can_sync_bin.to_str().unwrap(), + &[sync_config_a.to_str().unwrap()], + &[], + log_dir.path(), + ); + + // Wait for Sync-A to write its ticket file + println!(" Waiting for Sync-A to write ticket..."); + let ticket_start = Instant::now(); + loop { + if ticket_start.elapsed() > Duration::from_secs(15) { + print_log(log_dir.path(), "Sync-A"); + panic!("Timeout waiting for Sync-A ticket file"); + } + if let Ok(contents) = std::fs::read_to_string(&ticket_a) { + if !contents.trim().is_empty() { + println!(" Sync-A ticket ready ({} bytes)", contents.len()); + break; + } + } + std::thread::sleep(Duration::from_millis(100)); + } + + // Start Sync-B + let sync_b = ManagedProcess::spawn( + "Sync-B", + can_sync_bin.to_str().unwrap(), + &[sync_config_b.to_str().unwrap()], + &[], + log_dir.path(), + ); + + // Wait for peers to connect + println!(" Waiting for sync agents to connect..."); + std::thread::sleep(Duration::from_secs(5)); + + println!(" Test harness ready!\n"); + + Self { + _can_a: can_a, + _can_b: can_b, + _sync_a: sync_a, + _sync_b: sync_b, + _tmp_a: tmp_a, + _tmp_b: tmp_b, + _tmp_sync_a: tmp_sync_a, + _tmp_sync_b: tmp_sync_b, + log_dir, + can_a_url, + can_b_url, + } + } + + fn print_logs(&self) { + println!("\n=== Process Logs ==="); + for name in &["Sync-A", "Sync-B", "CAN-A", "CAN-B"] { + print_log(self.log_dir.path(), name); + } + } +} + +// ── Config writers ─────────────────────────────────────────────────────── + +fn write_can_config(path: &Path, storage_root: &Path, sync_key: &str) { + let storage_str = storage_root.to_str().unwrap().replace('\\', "/"); + let content = format!( + r#"storage_root: "{}" +admin_token: "test-admin-token" +enable_thumbnail_cache: false +sync_api_key: "{}" +"#, + storage_str, sync_key + ); + std::fs::write(path, content).expect("write CAN config"); +} + +fn write_sync_config_with_tickets( + path: &Path, + can_url: &str, + sync_key: &str, + passphrase: &str, + ticket_file: Option<&str>, + connect_ticket_file: Option<&str>, +) { + let mut content = format!( + r#"can_service_url: "{}" +sync_api_key: "{}" +sync_passphrase: "{}" +poll_interval_secs: 2 +"#, + can_url, sync_key, passphrase + ); + + if let Some(tf) = ticket_file { + content.push_str(&format!("ticket_file: \"{}\"\n", tf)); + } + if let Some(ctf) = connect_ticket_file { + content.push_str(&format!("connect_ticket_file: \"{}\"\n", ctf)); + } + + std::fs::write(path, content).expect("write sync config"); +} + +// ── Logging helpers ───────────────────────────────────────────────────── + +fn print_log(log_dir: &Path, name: &str) { + let log_path = log_dir.join(format!("{}.log", name)); + if let Ok(contents) = std::fs::read_to_string(&log_path) { + let lines: Vec<&str> = contents.lines().collect(); + let show = if lines.len() > 80 { &lines[lines.len() - 80..] } else { &lines }; + println!("\n--- {} (last {} of {} lines) ---", name, show.len(), lines.len()); + for line in show { + println!(" {}", line); + } + } else { + println!("\n--- {} (no log file) ---", name); + } +} + +// ── HTTP helpers ───────────────────────────────────────────────────────── + +fn wait_for_http(base_url: &str, timeout: Duration, log_dir: &Path, name: &str) { + let client = reqwest::blocking::Client::new(); + let start = Instant::now(); + let url = format!("{}/api/v1/can/0/list?limit=1", base_url); + + loop { + if start.elapsed() > timeout { + print_log(log_dir, name); + panic!("Timeout waiting for {} to become ready", base_url); + } + match client.get(&url).timeout(Duration::from_secs(1)).send() { + Ok(resp) if resp.status().is_success() => return, + _ => std::thread::sleep(Duration::from_millis(200)), + } + } +} + +/// Ingest a file into a CAN service instance. Returns the asset hash. +fn ingest_file(base_url: &str, filename: &str, content: &[u8], mime_type: &str) -> String { + let client = reqwest::blocking::Client::new(); + let url = format!("{}/api/v1/can/0/ingest", base_url); + + let part = reqwest::blocking::multipart::Part::bytes(content.to_vec()) + .file_name(filename.to_string()) + .mime_str(mime_type) + .unwrap(); + + let form = reqwest::blocking::multipart::Form::new() + .part("file", part) + .text("mime_type", mime_type.to_string()); + + let resp = client + .post(&url) + .multipart(form) + .timeout(Duration::from_secs(30)) + .send() + .expect("ingest request failed"); + + assert!( + resp.status().is_success(), + "Ingest failed with status {}", + resp.status() + ); + + let body: Value = resp.json().expect("parse ingest response"); + body["data"]["hash"] + .as_str() + .expect("no hash in response") + .to_string() +} + +/// List all assets from a CAN service. Returns list of hashes. +fn list_hashes(base_url: &str) -> Vec { + let client = reqwest::blocking::Client::new(); + let url = format!("{}/api/v1/can/0/list?limit=10000", base_url); + + let resp = client + .get(&url) + .timeout(Duration::from_secs(5)) + .send() + .expect("list request failed"); + + if !resp.status().is_success() { + return vec![]; + } + + let body: Value = resp.json().expect("parse list response"); + body["data"]["items"] + .as_array() + .unwrap_or(&vec![]) + .iter() + .filter_map(|item| item["hash"].as_str().map(String::from)) + .collect() +} + +/// Wait for a specific hash to appear on a CAN service. +fn wait_for_hash(base_url: &str, hash: &str, timeout: Duration) -> bool { + let start = Instant::now(); + while start.elapsed() < timeout { + let hashes = list_hashes(base_url); + if hashes.contains(&hash.to_string()) { + return true; + } + std::thread::sleep(POLL_INTERVAL); + } + false +} + +/// Wait for ALL given hashes to appear on a CAN service. +/// Returns (found_count, total, elapsed). +fn wait_for_all_hashes( + base_url: &str, + expected: &[String], + timeout: Duration, +) -> (usize, usize, Duration) { + let start = Instant::now(); + let expected_set: HashSet<&String> = expected.iter().collect(); + + loop { + let current = list_hashes(base_url); + let current_set: HashSet = current.into_iter().collect(); + let found = expected_set + .iter() + .filter(|h| current_set.contains(**h)) + .count(); + + if found == expected.len() { + return (found, expected.len(), start.elapsed()); + } + + if start.elapsed() >= timeout { + return (found, expected.len(), start.elapsed()); + } + + std::thread::sleep(POLL_INTERVAL); + } +} + +/// Generate random file content of given size. +fn random_content(size: usize) -> Vec { + let mut rng = rand::rng(); + let mut buf = vec![0u8; size]; + rng.fill(&mut buf[..]); + buf +} + +/// Human-readable byte size. +fn human_size(bytes: usize) -> String { + if bytes >= 1024 * 1024 { + format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)) + } else if bytes >= 1024 { + format!("{:.1} KB", bytes as f64 / 1024.0) + } else { + format!("{} B", bytes) + } +} + +/// Generate a mixed-size file list for stress tests. +/// Returns Vec<(filename, content)> with a mix of small, medium, and large files. +fn generate_burst_files(prefix: &str, count: usize) -> Vec<(String, Vec)> { + let mut files = Vec::with_capacity(count); + let mut total_bytes: usize = 0; + + for i in 0..count { + let size = match i % 5 { + 0 => LARGE_FILE_SIZE + (i * 1024 * 100), // ~12-14 MB (every 5th file) + 1 => MEDIUM_FILE_SIZE + (i * 1024 * 50), // ~2-3 MB + 2 => SMALL_FILE_SIZE + (i * 100), // ~4-6 KB + 3 => 512 * 1024 + (i * 1024 * 10), // ~500 KB - 750 KB + _ => 64 * 1024 + (i * 1024), // ~64-90 KB + }; + + let filename = format!("{}_{:03}.bin", prefix, i); + let content = random_content(size); + total_bytes += content.len(); + files.push((filename, content)); + } + + println!( + " Generated {} files, total {}", + count, + human_size(total_bytes) + ); + + // Print size breakdown + let large = files.iter().filter(|(_, c)| c.len() >= 10 * 1024 * 1024).count(); + let medium = files.iter().filter(|(_, c)| c.len() >= 1024 * 1024 && c.len() < 10 * 1024 * 1024).count(); + let small = files.iter().filter(|(_, c)| c.len() < 1024 * 1024).count(); + println!( + " Breakdown: {} large (10MB+), {} medium (1-10MB), {} small (<1MB)", + large, medium, small + ); + + files +} + +/// Ingest a batch of files as fast as possible (sequentially, but no delays). +/// Returns list of hashes. +fn rapid_ingest(base_url: &str, files: &[(String, Vec)]) -> Vec { + let mut hashes = Vec::with_capacity(files.len()); + let start = Instant::now(); + + for (i, (filename, content)) in files.iter().enumerate() { + let hash = ingest_file(base_url, filename, content, "application/octet-stream"); + if i < 3 || i == files.len() - 1 || content.len() >= 10 * 1024 * 1024 { + println!( + " [{}/{}] Ingested {} ({}) → {}", + i + 1, + files.len(), + filename, + human_size(content.len()), + &hash[..16] + ); + } else if i == 3 { + println!(" ... ingesting remaining files ..."); + } + hashes.push(hash); + } + + let elapsed = start.elapsed(); + let total_bytes: usize = files.iter().map(|(_, c)| c.len()).sum(); + println!( + " Ingested {} files ({}) in {:.1}s ({}/s)", + files.len(), + human_size(total_bytes), + elapsed.as_secs_f64(), + human_size((total_bytes as f64 / elapsed.as_secs_f64()) as usize) + ); + + hashes +} + +/// Ingest files from two threads simultaneously, return (hashes_a, hashes_b). +fn parallel_ingest( + url_a: &str, + files_a: &[(String, Vec)], + url_b: &str, + files_b: &[(String, Vec)], +) -> (Vec, Vec) { + let url_a = url_a.to_string(); + let url_b = url_b.to_string(); + + // Clone the file data for the threads + let files_a: Vec<(String, Vec)> = files_a.to_vec(); + let files_b: Vec<(String, Vec)> = files_b.to_vec(); + + let hashes_a = Arc::new(Mutex::new(Vec::new())); + let hashes_b = Arc::new(Mutex::new(Vec::new())); + + let ha = hashes_a.clone(); + let hb = hashes_b.clone(); + + // Spawn two threads that ingest simultaneously + let thread_a = std::thread::spawn(move || { + let mut results = Vec::with_capacity(files_a.len()); + for (filename, content) in &files_a { + let hash = ingest_file(&url_a, filename, content, "application/octet-stream"); + results.push(hash); + } + *ha.lock().unwrap() = results; + }); + + let thread_b = std::thread::spawn(move || { + let mut results = Vec::with_capacity(files_b.len()); + for (filename, content) in &files_b { + let hash = ingest_file(&url_b, filename, content, "application/octet-stream"); + results.push(hash); + } + *hb.lock().unwrap() = results; + }); + + thread_a.join().expect("ingest thread A panicked"); + thread_b.join().expect("ingest thread B panicked"); + + let a = hashes_a.lock().unwrap().clone(); + let b = hashes_b.lock().unwrap().clone(); + (a, b) +} + +// ── Test runner ────────────────────────────────────────────────────────── + +fn find_can_service_bin() -> PathBuf { + let self_exe = std::env::current_exe().expect("get current exe path"); + let target_dir = self_exe.parent().unwrap(); + + let bin_name = if cfg!(windows) { + "can-service.exe" + } else { + "can-service" + }; + + let candidate = target_dir.join(bin_name); + if candidate.exists() { + return candidate; + } + + let candidate = target_dir.parent().unwrap().join("debug").join(bin_name); + if candidate.exists() { + return candidate; + } + + let workspace_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() + .to_path_buf(); + let candidate = workspace_root.join("target").join("debug").join(bin_name); + if candidate.exists() { + return candidate; + } + + panic!( + "Cannot find {} binary. Build it first:\n cd {} && cargo build", + bin_name, + workspace_root.display() + ); +} + +fn main() { + println!("╔══════════════════════════════════════════════════╗"); + println!("║ CAN Sync v2 — Integration & Stress Test ║"); + println!("╚══════════════════════════════════════════════════╝"); + + let can_service_bin = find_can_service_bin(); + println!("\nUsing CAN service: {}", can_service_bin.display()); + + // Build can-sync if needed + println!("\nBuilding can-sync..."); + let build_status = Command::new("cargo") + .args(["build", "--bin", "can-sync"]) + .current_dir(env!("CARGO_MANIFEST_DIR")) + .status() + .expect("cargo build failed"); + assert!(build_status.success(), "Failed to build can-sync"); + + // Set up harness (starts all processes) + let harness = TestHarness::new(&can_service_bin); + + let mut passed = 0u32; + let mut failed = 0u32; + let mut results: Vec<(String, bool, String)> = vec![]; + + // ── Test 1: Single file A→B ────────────────────────────────────── + print_test_header("Test 1: Single file A→B"); + { + let content = random_content(4096); + let hash = ingest_file( + &harness.can_a_url, + "test1.bin", + &content, + "application/octet-stream", + ); + println!(" Ingested on A: hash={}", &hash[..16]); + + let found = wait_for_hash(&harness.can_b_url, &hash, Duration::from_secs(30)); + if found { + println!(" ✓ File appeared on CAN-B"); + results.push(("A→B single".into(), true, "ok".into())); + passed += 1; + } else { + println!(" ✗ File NOT found on CAN-B after 30s"); + results.push(("A→B single".into(), false, "timeout".into())); + failed += 1; + } + } + + // ── Test 2: Single file B→A ────────────────────────────────────── + print_test_header("Test 2: Single file B→A"); + { + let content = random_content(8192); + let hash = ingest_file( + &harness.can_b_url, + "test2.dat", + &content, + "application/octet-stream", + ); + println!(" Ingested on B: hash={}", &hash[..16]); + + let found = wait_for_hash(&harness.can_a_url, &hash, Duration::from_secs(30)); + if found { + println!(" ✓ File appeared on CAN-A"); + results.push(("B→A single".into(), true, "ok".into())); + passed += 1; + } else { + println!(" ✗ File NOT found on CAN-A after 30s"); + results.push(("B→A single".into(), false, "timeout".into())); + failed += 1; + } + } + + // ── Test 3: Rapid burst A→B (25 files, mixed sizes, some 10MB+) ─ + print_test_header(&format!( + "Test 3: Rapid burst {} files A→B (mixed sizes, some 10MB+)", + BURST_COUNT + )); + { + println!(" Generating files..."); + let files = generate_burst_files("burst_a2b", BURST_COUNT); + + println!(" Ingesting rapidly on CAN-A..."); + let hashes = rapid_ingest(&harness.can_a_url, &files); + + println!(" Waiting for all {} files to sync to CAN-B...", hashes.len()); + let (found, total, elapsed) = wait_for_all_hashes(&harness.can_b_url, &hashes, SYNC_TIMEOUT); + + if found == total { + println!( + " ✓ All {} files synced to B in {:.1}s", + total, + elapsed.as_secs_f64() + ); + let total_bytes: usize = files.iter().map(|(_, c)| c.len()).sum(); + println!( + " Throughput: {}/s", + human_size((total_bytes as f64 / elapsed.as_secs_f64()) as usize) + ); + results.push(( + format!("Burst A→B ({})", total), + true, + format!("{:.1}s", elapsed.as_secs_f64()), + )); + passed += 1; + } else { + println!( + " ✗ Only {}/{} files synced after {:.1}s", + found, + total, + elapsed.as_secs_f64() + ); + // Report which ones are missing + let b_hashes: HashSet = list_hashes(&harness.can_b_url).into_iter().collect(); + let missing: Vec<_> = hashes + .iter() + .enumerate() + .filter(|(_, h)| !b_hashes.contains(*h)) + .map(|(i, h)| format!("#{} {} ({})", i, &h[..12], human_size(files[i].1.len()))) + .collect(); + if missing.len() <= 10 { + for m in &missing { + println!(" MISSING: {}", m); + } + } else { + println!(" {} files missing (showing first 10):", missing.len()); + for m in &missing[..10] { + println!(" MISSING: {}", m); + } + } + results.push(( + format!("Burst A→B ({})", total), + false, + format!("{}/{}", found, total), + )); + failed += 1; + } + } + + // Small pause to let things settle + println!("\n (pause 3s between tests)"); + std::thread::sleep(Duration::from_secs(3)); + + // ── Test 4: Rapid burst B→A (25 files, mixed sizes, some 10MB+) ─ + print_test_header(&format!( + "Test 4: Rapid burst {} files B→A (mixed sizes, some 10MB+)", + BURST_COUNT + )); + { + println!(" Generating files..."); + let files = generate_burst_files("burst_b2a", BURST_COUNT); + + println!(" Ingesting rapidly on CAN-B..."); + let hashes = rapid_ingest(&harness.can_b_url, &files); + + println!(" Waiting for all {} files to sync to CAN-A...", hashes.len()); + let (found, total, elapsed) = wait_for_all_hashes(&harness.can_a_url, &hashes, SYNC_TIMEOUT); + + if found == total { + println!( + " ✓ All {} files synced to A in {:.1}s", + total, + elapsed.as_secs_f64() + ); + let total_bytes: usize = files.iter().map(|(_, c)| c.len()).sum(); + println!( + " Throughput: {}/s", + human_size((total_bytes as f64 / elapsed.as_secs_f64()) as usize) + ); + results.push(( + format!("Burst B→A ({})", total), + true, + format!("{:.1}s", elapsed.as_secs_f64()), + )); + passed += 1; + } else { + println!( + " ✗ Only {}/{} files synced after {:.1}s", + found, + total, + elapsed.as_secs_f64() + ); + let a_hashes: HashSet = list_hashes(&harness.can_a_url).into_iter().collect(); + let missing: Vec<_> = hashes + .iter() + .enumerate() + .filter(|(_, h)| !a_hashes.contains(*h)) + .map(|(i, h)| format!("#{} {} ({})", i, &h[..12], human_size(files[i].1.len()))) + .collect(); + if missing.len() <= 10 { + for m in &missing { + println!(" MISSING: {}", m); + } + } else { + println!(" {} files missing (showing first 10):", missing.len()); + for m in &missing[..10] { + println!(" MISSING: {}", m); + } + } + results.push(( + format!("Burst B→A ({})", total), + false, + format!("{}/{}", found, total), + )); + failed += 1; + } + } + + println!("\n (pause 3s between tests)"); + std::thread::sleep(Duration::from_secs(3)); + + // ── Test 5: Simultaneous burst — 25 files on EACH side at once ─── + print_test_header(&format!( + "Test 5: Simultaneous burst — {} files on EACH side at once", + BURST_COUNT + )); + { + println!(" Generating files for BOTH sides..."); + let files_for_a = generate_burst_files("simul_onA", BURST_COUNT); + let files_for_b = generate_burst_files("simul_onB", BURST_COUNT); + + let total_bytes: usize = files_for_a.iter().map(|(_, c)| c.len()).sum::() + + files_for_b.iter().map(|(_, c)| c.len()).sum::(); + println!( + " Total data: {} across {} files", + human_size(total_bytes), + BURST_COUNT * 2 + ); + + println!(" Ingesting on BOTH sides simultaneously..."); + let start = Instant::now(); + let (hashes_a, hashes_b) = parallel_ingest( + &harness.can_a_url, + &files_for_a, + &harness.can_b_url, + &files_for_b, + ); + let ingest_elapsed = start.elapsed(); + println!( + " Parallel ingest done in {:.1}s ({} on A, {} on B)", + ingest_elapsed.as_secs_f64(), + hashes_a.len(), + hashes_b.len() + ); + + // Now wait for cross-sync: files from A should appear on B, files from B on A + println!( + " Waiting for A's {} files to appear on B...", + hashes_a.len() + ); + let (found_on_b, total_a, elapsed_b) = + wait_for_all_hashes(&harness.can_b_url, &hashes_a, SYNC_TIMEOUT); + + println!( + " Waiting for B's {} files to appear on A...", + hashes_b.len() + ); + let (found_on_a, total_b, elapsed_a) = + wait_for_all_hashes(&harness.can_a_url, &hashes_b, SYNC_TIMEOUT); + + let a_ok = found_on_b == total_a; + let b_ok = found_on_a == total_b; + + if a_ok && b_ok { + let max_elapsed = elapsed_a.max(elapsed_b); + println!( + " ✓ Bidirectional sync complete! A→B: {}/{} in {:.1}s, B→A: {}/{} in {:.1}s", + found_on_b, + total_a, + elapsed_b.as_secs_f64(), + found_on_a, + total_b, + elapsed_a.as_secs_f64() + ); + println!( + " Effective throughput: {}/s (both directions)", + human_size((total_bytes as f64 / max_elapsed.as_secs_f64()) as usize) + ); + results.push(( + format!("Simul {}+{}", BURST_COUNT, BURST_COUNT), + true, + format!("{:.1}s", max_elapsed.as_secs_f64()), + )); + passed += 1; + } else { + println!( + " ✗ A→B: {}/{}, B→A: {}/{}", + found_on_b, total_a, found_on_a, total_b + ); + if !a_ok { + let b_hashes: HashSet = + list_hashes(&harness.can_b_url).into_iter().collect(); + let missing_count = hashes_a.iter().filter(|h| !b_hashes.contains(*h)).count(); + println!(" A→B: {} files missing on B", missing_count); + } + if !b_ok { + let a_hashes: HashSet = + list_hashes(&harness.can_a_url).into_iter().collect(); + let missing_count = hashes_b.iter().filter(|h| !a_hashes.contains(*h)).count(); + println!(" B→A: {} files missing on A", missing_count); + } + results.push(( + format!("Simul {}+{}", BURST_COUNT, BURST_COUNT), + false, + format!( + "A→B {}/{} B→A {}/{}", + found_on_b, total_a, found_on_a, total_b + ), + )); + failed += 1; + } + } + + // ── Test 6: Final full-mirror verification ─────────────────────── + print_test_header("Test 6: Final full-mirror verification"); + { + // Give a final settlement window + println!(" Waiting 10s for any stragglers..."); + std::thread::sleep(Duration::from_secs(10)); + + let a_hashes = list_hashes(&harness.can_a_url); + let b_hashes = list_hashes(&harness.can_b_url); + + println!(" CAN-A has {} assets", a_hashes.len()); + println!(" CAN-B has {} assets", b_hashes.len()); + + let a_set: HashSet<&String> = a_hashes.iter().collect(); + let b_set: HashSet<&String> = b_hashes.iter().collect(); + + let only_a: Vec<_> = a_set.difference(&b_set).collect(); + let only_b: Vec<_> = b_set.difference(&a_set).collect(); + + if only_a.is_empty() && only_b.is_empty() && a_hashes.len() == b_hashes.len() { + println!( + " ✓ Perfect mirror! {} assets identical on both sides", + a_hashes.len() + ); + results.push(( + "Full mirror".into(), + true, + format!("{} assets", a_hashes.len()), + )); + passed += 1; + } else { + if !only_a.is_empty() { + println!(" ✗ {} assets only on A:", only_a.len()); + for h in only_a.iter().take(5) { + println!(" {}", &h[..16]); + } + if only_a.len() > 5 { + println!(" ... and {} more", only_a.len() - 5); + } + } + if !only_b.is_empty() { + println!(" ✗ {} assets only on B:", only_b.len()); + for h in only_b.iter().take(5) { + println!(" {}", &h[..16]); + } + if only_b.len() > 5 { + println!(" ... and {} more", only_b.len() - 5); + } + } + results.push(( + "Full mirror".into(), + false, + format!( + "A={} B={} onlyA={} onlyB={}", + a_hashes.len(), + b_hashes.len(), + only_a.len(), + only_b.len() + ), + )); + failed += 1; + } + } + + // ── Results ────────────────────────────────────────────────────── + let expected_total = 2 + BURST_COUNT * 2 + BURST_COUNT * 2; // singles + bursts + simul + let total_large = BURST_COUNT * 3 / 5; // roughly every 5th file is large, across 3 batches + + println!("\n╔════════════════════════════════════════════════════╗"); + println!("║ Test Results ║"); + println!("╠════════════════════════════════════════════════════╣"); + for (name, pass, detail) in &results { + let icon = if *pass { "✓" } else { "✗" }; + let detail_trunc = if detail.len() > 20 { + &detail[..20] + } else { + detail + }; + println!("║ {} {:<28} {}", icon, name, detail_trunc); + } + println!("╠════════════════════════════════════════════════════╣"); + println!( + "║ Passed: {} Failed: {} ║", + passed, failed + ); + println!( + "║ Files: ~{} total, ~{} large (10MB+) ║", + expected_total + 2, // +2 for the single file tests + total_large + ); + println!("╚════════════════════════════════════════════════════╝"); + + // Print logs on failure + if failed > 0 { + harness.print_logs(); + } + + // Clean up + println!("\n=== Cleaning up ===\n"); + drop(harness); + println!(" All temp directories removed."); + + if failed > 0 { + std::process::exit(1); + } +} + +fn print_test_header(name: &str) { + println!("\n╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌"); + println!(" {}", name); + println!("╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌\n"); +} diff --git a/examples/canfs/src/api.rs b/examples/canfs/src/api.rs index 87b45d3..d574145 100644 --- a/examples/canfs/src/api.rs +++ b/examples/canfs/src/api.rs @@ -49,6 +49,7 @@ pub struct CanClient { } impl CanClient { + /// Create a new client pointed at the given CAN service base URL. pub fn new(base_url: &str) -> Self { Self { client: reqwest::blocking::Client::new(), diff --git a/examples/canfs/src/fs.rs b/examples/canfs/src/fs.rs index 3853d78..742866c 100644 --- a/examples/canfs/src/fs.rs +++ b/examples/canfs/src/fs.rs @@ -26,6 +26,7 @@ const FILE_ATTRIBUTE_DIRECTORY: u32 = 0x10; const FILE_ATTRIBUTE_READONLY: u32 = 0x01; const FILE_ATTRIBUTE_ARCHIVE: u32 = 0x20; +// Wrap a raw NTSTATUS error code into WinFSP's error type. fn ntstatus(code: i32) -> FspError { FspError::NTSTATUS(code) } @@ -54,6 +55,7 @@ pub struct CanFileContext { impl FileSystemContext for CanFs { type FileContext = CanFileContext; + /// Called by Windows to check if a file/folder exists and get its basic attributes before opening it. fn get_security_by_name( &self, file_name: &U16CStr, @@ -83,6 +85,7 @@ impl FileSystemContext for CanFs { }) } + /// Called when a file or directory is opened; returns a context handle and fills in size/timestamps. fn open( &self, file_name: &U16CStr, @@ -142,8 +145,10 @@ impl FileSystemContext for CanFs { }) } + /// Called when a handle is closed; nothing to clean up since content is dropped automatically. fn close(&self, _context: Self::FileContext) {} + /// Returns up-to-date size and attribute info for an already-opened file or directory. fn get_file_info( &self, context: &Self::FileContext, @@ -194,6 +199,7 @@ impl FileSystemContext for CanFs { Ok(()) } + /// Reads file bytes at the given offset; downloads the asset from the CAN service on first access. fn read( &self, context: &Self::FileContext, @@ -233,6 +239,7 @@ impl FileSystemContext for CanFs { Ok(count as u32) } + /// Lists the contents of a directory, including "." and ".." entries, for Windows Explorer and dir commands. fn read_directory( &self, context: &Self::FileContext, @@ -308,6 +315,7 @@ impl FileSystemContext for CanFs { Ok(context.dir_buffer.read(marker, buffer)) } + /// Reports the virtual drive's total and free space (shows as a 1 GB read-only volume). fn get_volume_info(&self, out_volume_info: &mut VolumeInfo) -> winfsp::Result<()> { out_volume_info.total_size = 1024 * 1024 * 1024; // 1 GB out_volume_info.free_size = 0; diff --git a/examples/canfs/src/main.rs b/examples/canfs/src/main.rs index b552a2a..7e71cb8 100644 --- a/examples/canfs/src/main.rs +++ b/examples/canfs/src/main.rs @@ -17,6 +17,7 @@ use crate::api::CanClient; use crate::fs::{CacheState, CanFs}; use crate::tree::VirtualTree; +/// Command-line arguments for mounting CAN service assets as a virtual Windows drive using WinFSP. #[derive(Parser)] #[command(name = "canfs", about = "Mount CAN service assets as a virtual drive")] struct Args { @@ -33,6 +34,7 @@ struct Args { refresh_secs: u64, } +/// Entry point: connects to the CAN service, builds a virtual file tree, and mounts it as a read-only Windows drive. fn main() { tracing_subscriber::fmt() .with_env_filter( diff --git a/examples/canfs/src/tree.rs b/examples/canfs/src/tree.rs index d8698f6..6c2e394 100644 --- a/examples/canfs/src/tree.rs +++ b/examples/canfs/src/tree.rs @@ -131,6 +131,7 @@ struct TreeBuilder { } impl TreeBuilder { + // Create a new tree builder with an empty root directory node. fn new() -> Self { let root = VNode { name: String::new(), diff --git a/examples/filemanager/src/html.rs b/examples/filemanager/src/html.rs index 154ee69..dfa8f47 100644 --- a/examples/filemanager/src/html.rs +++ b/examples/filemanager/src/html.rs @@ -531,6 +531,24 @@ function mimeToExt(mime) { return map[mime] || mime.split('/').pop() || 'bin'; } +function mimeToTypeCategory(mime) { + if (mime.startsWith('image/')) return 'images'; + if (mime === 'application/pdf') return 'pdf'; + if (mime.startsWith('video/')) return 'video'; + if (mime.startsWith('audio/')) return 'audio'; + if (mime.startsWith('text/') + || mime === 'application/json' + || mime === 'application/xml' + || mime === 'application/msword' + || mime === 'application/rtf' + || mime.startsWith('application/vnd.openxmlformats') + || mime.startsWith('application/vnd.ms-') + || mime === 'application/vnd.oasis.opendocument.text' + || mime === 'application/vnd.oasis.opendocument.spreadsheet') + return 'documents'; + return 'others'; +} + function buildVirtualTree(assets) { const root = { name: '', type: 'dir', children: {}, items: [] }; @@ -591,6 +609,12 @@ function buildVirtualTree(assets) { addFile(tagDir, friendlyName, asset); } } + + // TYPE/ + const typeRoot = ensureDir(root, 'TYPE'); + const typeCat = mimeToTypeCategory(asset.mime_type); + const typeDir = ensureDir(typeRoot, typeCat); + addFile(typeDir, friendlyName, asset); } return root; diff --git a/examples/filemanager/src/main.rs b/examples/filemanager/src/main.rs index cbdb6b6..82f4f84 100644 --- a/examples/filemanager/src/main.rs +++ b/examples/filemanager/src/main.rs @@ -9,11 +9,13 @@ use std::collections::HashMap; const CAN_API: &str = "http://127.0.0.1:3210/api/v1/can/0"; +// Shared state passed to every request handler; holds a reusable HTTP client. #[derive(Clone)] struct AppState { client: reqwest::Client, } +/// Web-based file manager UI that proxies requests to the CAN service API. #[tokio::main] async fn main() { tracing_subscriber::fmt() @@ -47,6 +49,7 @@ async fn main() { axum::serve(listener, app).await.unwrap(); } +// Return the single-page HTML UI for the file manager. async fn serve_index() -> Html<&'static str> { Html(html::INDEX_HTML) } @@ -86,6 +89,7 @@ fn build_qs(params: &HashMap) -> String { format!("?{}", qs.join("&")) } +// Percent-encode a string for use in URL query parameters. fn urlencoding(s: &str) -> String { s.chars() .map(|c| match c { diff --git a/examples/paste/Cargo.lock b/examples/paste/Cargo.lock index 4834dad..174121e 100644 --- a/examples/paste/Cargo.lock +++ b/examples/paste/Cargo.lock @@ -241,6 +241,23 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "futures-sink" version = "0.3.32" @@ -260,7 +277,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ "futures-core", + "futures-io", + "futures-macro", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "slab", ] @@ -850,11 +871,13 @@ name = "paste" version = "0.1.0" dependencies = [ "axum", + "futures-util", "open", "reqwest", "serde", "serde_json", "tokio", + "tokio-stream", "tracing", "tracing-subscriber", ] @@ -991,12 +1014,14 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-native-tls", + "tokio-util", "tower", "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", ] @@ -1379,6 +1404,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.18" @@ -1680,6 +1716,19 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmparser" version = "0.244.0" diff --git a/examples/paste/Cargo.toml b/examples/paste/Cargo.toml index d019dbd..2a6d1a7 100644 --- a/examples/paste/Cargo.toml +++ b/examples/paste/Cargo.toml @@ -12,9 +12,11 @@ path = "src/main.rs" [dependencies] axum = { version = "0.8", features = ["multipart"] } tokio = { version = "1", features = ["full"] } -reqwest = { version = "0.12", features = ["multipart", "json"] } +reqwest = { version = "0.12", features = ["multipart", "json", "stream"] } serde = { version = "1", features = ["derive"] } serde_json = "1" open = "5" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } +tokio-stream = "0.1" +futures-util = "0.3" diff --git a/examples/paste/src/html.rs b/examples/paste/src/html.rs index c641bd1..85db561 100644 --- a/examples/paste/src/html.rs +++ b/examples/paste/src/html.rs @@ -376,6 +376,14 @@ fileInput.addEventListener('change', () => { // Initial load loadItems(); + +// Live updates via SSE — auto-refresh when assets arrive from sync or other sources +const evtSource = new EventSource('/paste/events'); +evtSource.addEventListener('new_asset', () => loadItems()); +evtSource.onerror = () => { + // EventSource auto-reconnects; just log for debugging + console.debug('SSE connection lost, reconnecting...'); +}; diff --git a/examples/paste/src/main.rs b/examples/paste/src/main.rs index 0c54e9b..9b56a33 100644 --- a/examples/paste/src/main.rs +++ b/examples/paste/src/main.rs @@ -2,19 +2,23 @@ mod html; use axum::extract::{DefaultBodyLimit, Multipart, Path, State}; use axum::http::{header, StatusCode}; +use axum::response::sse::{Event, Sse}; use axum::response::{Html, IntoResponse, Response}; use axum::routing::{get, post}; use axum::{Json, Router}; use serde::Deserialize; +use std::convert::Infallible; use std::net::SocketAddr; const CAN_API: &str = "http://127.0.0.1:3210/api/v1/can/0"; +/// Shared HTTP client for talking to the CAN service backend. #[derive(Clone)] struct AppState { client: reqwest::Client, } +/// JSON body for the text paste endpoint. #[derive(Deserialize)] struct PasteTextRequest { text: String, @@ -60,6 +64,7 @@ async fn forward(resp: Result) -> Response { // ── Handlers ───────────────────────────────────────────────────────────── +/// Serve the single-page HTML frontend. async fn serve_index() -> Html<&'static str> { Html(html::INDEX_HTML) } @@ -225,8 +230,70 @@ async fn proxy_thumb( forward(resp).await } +/// Proxy SSE (Server-Sent Events) from the CAN service to the browser so +/// the frontend auto-refreshes when new pastes arrive. +async fn paste_events( + State(state): State, +) -> Sse>> { + let (tx, rx) = tokio::sync::mpsc::channel::>(32); + + let client = state.client.clone(); + tokio::spawn(async move { + loop { + match client.get(format!("{CAN_API}/events")).send().await { + Ok(resp) => { + use futures_util::StreamExt; + let mut stream = resp.bytes_stream(); + let mut buf = String::new(); + while let Some(chunk) = stream.next().await { + let Ok(bytes) = chunk else { break }; + buf.push_str(&String::from_utf8_lossy(&bytes)); + // Parse SSE frames (double-newline delimited) + while let Some(pos) = buf.find("\n\n") { + let frame = buf[..pos].to_string(); + buf = buf[pos + 2..].to_string(); + let mut event_type = None; + let mut data = None; + for line in frame.lines() { + if let Some(v) = line.strip_prefix("event: ") { + event_type = Some(v.to_string()); + } else if let Some(v) = line.strip_prefix("data: ") { + data = Some(v.to_string()); + } + // lines starting with ':' are SSE comments (keepalive) — skip + } + if let Some(d) = data { + let mut evt = Event::default().data(d); + if let Some(t) = event_type { + evt = evt.event(t); + } + if tx.send(Ok(evt)).await.is_err() { + return; // client disconnected + } + } + } + } + } + Err(e) => { + tracing::warn!("SSE proxy connect failed: {e}"); + } + } + // Reconnect after a short delay + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + } + }); + + let stream = tokio_stream::wrappers::ReceiverStream::new(rx); + Sse::new(stream).keep_alive( + axum::response::sse::KeepAlive::new() + .interval(std::time::Duration::from_secs(15)) + .text("ping"), + ) +} + // ── Main ───────────────────────────────────────────────────────────────── +/// Start the Paste web app: a simple pastebin that stores text and images in CAN service. #[tokio::main] async fn main() { tracing_subscriber::fmt() @@ -247,6 +314,7 @@ async fn main() { .route("/paste/list", get(paste_list)) .route("/paste/asset/{hash}", get(proxy_asset)) .route("/paste/thumb/{hash}", get(proxy_thumb)) + .route("/paste/events", get(paste_events)) .layer(DefaultBodyLimit::max(100 * 1024 * 1024)) // 100 MB .with_state(state); diff --git a/go_example_1.ps1 b/go_example_1.ps1 index 172ad11..6e1e123 100644 --- a/go_example_1.ps1 +++ b/go_example_1.ps1 @@ -1,4 +1,7 @@ -# go_example_1.ps1 — Start CanService + Paste example, open browser +# go_example_1.ps1 — Start CanService + Paste + Sync agent, open browser +# +# Run on multiple machines (after git clone) and they will auto-sync +# all ingested assets via iroh's relay network. No port forwarding needed. $ErrorActionPreference = "Stop" $root = $PSScriptRoot @@ -11,13 +14,19 @@ Get-NetTCPConnection -LocalPort 3211 -ErrorAction SilentlyContinue | ForEach-Object { Stop-Process -Id $_.OwningProcess -Force -ErrorAction SilentlyContinue } Start-Sleep -Milliseconds 500 +# --- Build everything --- + Write-Host "Building CanService..." -ForegroundColor Cyan cargo build --manifest-path "$root\Cargo.toml" Write-Host "Building Paste example..." -ForegroundColor Cyan cargo build --manifest-path "$root\examples\paste\Cargo.toml" -# Start CanService in background +Write-Host "Building CAN Sync agent..." -ForegroundColor Cyan +cargo build --manifest-path "$root\examples\can-sync\Cargo.toml" --bin can-sync + +# --- Start CanService --- + Write-Host "Starting CanService on port 3210..." -ForegroundColor Green $canService = Start-Process -FilePath "cargo" ` -ArgumentList "run --manifest-path `"$root\Cargo.toml`"" ` @@ -43,7 +52,20 @@ if (-not $ready) { } Write-Host "CanService ready." -ForegroundColor Green -# Start Paste example (it opens the browser itself) +# --- Start Sync agent --- + +$syncConfig = "$root\examples\can-sync\config.yaml" +Write-Host "Starting CAN Sync agent (P2P replication)..." -ForegroundColor Green +$syncAgent = Start-Process -FilePath "cargo" ` + -ArgumentList "run --manifest-path `"$root\examples\can-sync\Cargo.toml`" --bin can-sync -- `"$syncConfig`"" ` + -WorkingDirectory $root ` + -PassThru -NoNewWindow + +# Give sync agent a moment to connect to CAN service +Start-Sleep -Seconds 2 + +# --- Start Paste example --- + Write-Host "Starting Paste on port 3211..." -ForegroundColor Green $paste = Start-Process -FilePath "cargo" ` -ArgumentList "run --manifest-path `"$root\examples\paste\Cargo.toml`"" ` @@ -54,16 +76,22 @@ Write-Host "" Write-Host "Running:" -ForegroundColor Cyan Write-Host " CanService -> http://127.0.0.1:3210" Write-Host " Paste UI -> http://127.0.0.1:3211" +Write-Host " CAN Sync -> P2P replication active (iroh relay)" -ForegroundColor Magenta Write-Host "" -Write-Host "Press Ctrl+C to stop both." -ForegroundColor Yellow +Write-Host "Sync passphrase: 'duke-canman-sync'" -ForegroundColor Magenta +Write-Host "Any other machine running this script with the same passphrase" -ForegroundColor Magenta +Write-Host "will automatically discover this instance and sync all assets." -ForegroundColor Magenta +Write-Host "" +Write-Host "Press Ctrl+C to stop all services." -ForegroundColor Yellow -# Wait for either process to exit, then clean up both +# Wait for any process to exit, then clean up all try { - while (-not $canService.HasExited -and -not $paste.HasExited) { + while (-not $canService.HasExited -and -not $paste.HasExited -and -not $syncAgent.HasExited) { Start-Sleep -Seconds 1 } } finally { Write-Host "Shutting down..." -ForegroundColor Yellow Stop-Process -Id $canService.Id -Force -ErrorAction SilentlyContinue Stop-Process -Id $paste.Id -Force -ErrorAction SilentlyContinue + Stop-Process -Id $syncAgent.Id -Force -ErrorAction SilentlyContinue } diff --git a/go_example_1.sh b/go_example_1.sh new file mode 100644 index 0000000..fe92a58 --- /dev/null +++ b/go_example_1.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# go_example_1.sh — Start CanService + Paste + Sync agent +# +# Run on multiple machines (after git clone) and they will auto-sync +# all ingested assets via iroh's relay network. No port forwarding needed. + +set -e +ROOT="$(cd "$(dirname "$0")" && pwd)" + +cleanup() { + echo "" + echo "Shutting down..." + [ -n "$CAN_PID" ] && kill "$CAN_PID" 2>/dev/null + [ -n "$SYNC_PID" ] && kill "$SYNC_PID" 2>/dev/null + [ -n "$PASTE_PID" ] && kill "$PASTE_PID" 2>/dev/null + wait 2>/dev/null + exit 0 +} +trap cleanup INT TERM EXIT + +# Kill anything on our ports +echo "Cleaning up stale processes..." +lsof -ti:3210 2>/dev/null | xargs kill -9 2>/dev/null || true +lsof -ti:3211 2>/dev/null | xargs kill -9 2>/dev/null || true +sleep 0.5 + +# --- Build everything --- + +echo "Building CanService..." +cargo build --manifest-path "$ROOT/Cargo.toml" + +echo "Building Paste example..." +cargo build --manifest-path "$ROOT/examples/paste/Cargo.toml" + +echo "Building CAN Sync agent..." +cargo build --manifest-path "$ROOT/examples/can-sync/Cargo.toml" --bin can-sync + +# --- Start CanService --- + +echo "Starting CanService on port 3210..." +cargo run --manifest-path "$ROOT/Cargo.toml" & +CAN_PID=$! + +# Wait for CanService to be ready +echo "Waiting for CanService..." +for i in $(seq 1 30); do + if curl -sf http://127.0.0.1:3210/api/v1/can/0/list >/dev/null 2>&1; then + break + fi + if [ "$i" -eq 30 ]; then + echo "CanService failed to start within 15s" + exit 1 + fi + sleep 0.5 +done +echo "CanService ready." + +# --- Start Sync agent --- + +SYNC_CONFIG="$ROOT/examples/can-sync/config.yaml" +echo "Starting CAN Sync agent (P2P replication)..." +cargo run --manifest-path "$ROOT/examples/can-sync/Cargo.toml" --bin can-sync -- "$SYNC_CONFIG" & +SYNC_PID=$! +sleep 2 + +# --- Start Paste example --- + +echo "Starting Paste on port 3211..." +cargo run --manifest-path "$ROOT/examples/paste/Cargo.toml" & +PASTE_PID=$! + +echo "" +echo "Running:" +echo " CanService -> http://127.0.0.1:3210" +echo " Paste UI -> http://127.0.0.1:3211" +echo " CAN Sync -> P2P replication active (iroh relay)" +echo "" +echo "Sync passphrase: 'duke-canman-sync'" +echo "Any other machine running this script with the same passphrase" +echo "will automatically discover this instance and sync all assets." +echo "" +echo "Press Ctrl+C to stop all services." + +# Wait for any process to exit +wait -n "$CAN_PID" "$PASTE_PID" "$SYNC_PID" 2>/dev/null || true diff --git a/src/config.rs b/src/config.rs index 8feef56..e39dc4e 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,6 +1,7 @@ use serde::Deserialize; use std::path::{Path, PathBuf}; +/// Application settings loaded from config.yaml at startup. #[derive(Debug, Clone, Deserialize)] pub struct Config { pub storage_root: PathBuf, @@ -12,8 +13,13 @@ pub struct Config { pub rebuild_error_threshold: u32, #[serde(default = "default_verify_interval")] pub verify_interval_hours: u64, + /// Optional API key for the private sync endpoints (/sync/*). + /// If not set, sync endpoints are disabled (return 404). + #[serde(default)] + pub sync_api_key: Option, } +// Default values used when a field is missing from config.yaml. fn default_admin_token() -> String { "changeme".to_string() } @@ -28,24 +34,29 @@ fn default_verify_interval() -> u64 { } impl Config { + /// Read and parse the YAML config file from disk. pub fn load(path: &Path) -> anyhow::Result { let contents = std::fs::read_to_string(path)?; let config: Config = serde_yaml::from_str(&contents)?; Ok(config) } + /// Returns the path to the SQLite database file inside storage_root. pub fn db_path(&self) -> PathBuf { self.storage_root.join(".can.db") } + /// Returns the path to the trash folder for soft-deleted files. pub fn trash_dir(&self) -> PathBuf { self.storage_root.join(".trash") } + /// Returns the path to the cached thumbnail images folder. pub fn thumbs_dir(&self) -> PathBuf { self.storage_root.join(".thumbs") } + /// Create the storage, trash, and thumbnail directories if they don't exist yet. pub fn ensure_dirs(&self) -> anyhow::Result<()> { std::fs::create_dir_all(&self.storage_root)?; std::fs::create_dir_all(self.trash_dir())?; diff --git a/src/db.rs b/src/db.rs index 97fb192..e787054 100644 --- a/src/db.rs +++ b/src/db.rs @@ -4,8 +4,11 @@ use std::sync::{Arc, Mutex}; use crate::models::{Asset, AssetMeta, ListParams, SearchParams}; +/// Thread-safe handle to the SQLite database (wrapped in Arc so multiple +/// threads can share it safely). pub type Db = Arc>; +/// Open (or create) the SQLite database file and set up tables. pub fn open(path: &Path) -> anyhow::Result { let conn = Connection::open(path)?; conn.execute_batch("PRAGMA journal_mode=WAL; PRAGMA foreign_keys=ON;")?; @@ -13,6 +16,7 @@ pub fn open(path: &Path) -> anyhow::Result { Ok(Arc::new(Mutex::new(conn))) } +/// Open a temporary in-memory database (used for tests). pub fn open_in_memory() -> anyhow::Result { let conn = Connection::open_in_memory()?; conn.execute_batch("PRAGMA foreign_keys=ON;")?; @@ -20,6 +24,8 @@ pub fn open_in_memory() -> anyhow::Result { Ok(Arc::new(Mutex::new(conn))) } +/// Create the assets, tags, and asset_tags tables if they don't already exist, +/// and run any pending migrations. fn init_schema(conn: &Connection) -> rusqlite::Result<()> { conn.execute_batch( " @@ -66,7 +72,7 @@ fn init_schema(conn: &Connection) -> rusqlite::Result<()> { Ok(()) } -/// Insert a new asset. Returns the row id. +/// Save a new asset record to the database. Returns the auto-generated row id. pub fn insert_asset(conn: &Connection, asset: &Asset) -> rusqlite::Result { conn.execute( "INSERT INTO assets (timestamp, hash, mime_type, application, user_identity, description, actual_filename, human_filename, human_path, size) @@ -87,7 +93,7 @@ pub fn insert_asset(conn: &Connection, asset: &Asset) -> rusqlite::Result { Ok(conn.last_insert_rowid()) } -/// Look up an asset by its hash. +/// Find an asset by its unique SHA-256 hash. Returns None if not found. pub fn get_asset_by_hash(conn: &Connection, hash: &str) -> rusqlite::Result> { conn.query_row( "SELECT id, timestamp, hash, mime_type, application, user_identity, description, @@ -115,7 +121,7 @@ pub fn get_asset_by_hash(conn: &Connection, hash: &str) -> rusqlite::Result rusqlite::Result> { let mut stmt = conn.prepare( "SELECT t.name FROM tags t @@ -127,7 +133,7 @@ pub fn get_asset_tags(conn: &Connection, asset_id: i64) -> rusqlite::Result rusqlite::Result { conn.execute( "INSERT OR IGNORE INTO tags (name) VALUES (?1)", @@ -138,7 +144,7 @@ pub fn upsert_tag(conn: &Connection, name: &str) -> rusqlite::Result { }) } -/// Replace all tags for an asset within a transaction. +/// Remove all existing tags for an asset and assign the new ones. pub fn set_asset_tags(conn: &Connection, asset_id: i64, tags: &[String]) -> rusqlite::Result<()> { conn.execute( "DELETE FROM asset_tags WHERE asset_id = ?1", @@ -154,7 +160,8 @@ pub fn set_asset_tags(conn: &Connection, asset_id: i64, tags: &[String]) -> rusq Ok(()) } -/// Build an AssetMeta from an Asset row + tags. +/// Convert an internal Asset database row into the API-friendly AssetMeta format +/// (includes tags fetched from the join table). pub fn asset_to_meta(conn: &Connection, asset: &Asset) -> rusqlite::Result { let tags = get_asset_tags(conn, asset.id)?; Ok(AssetMeta { @@ -173,7 +180,7 @@ pub fn asset_to_meta(conn: &Connection, asset: &Asset) -> rusqlite::Result rusqlite::Result<()> { conn.execute( "UPDATE assets SET is_corrupted = ?1 WHERE hash = ?2", @@ -204,7 +211,8 @@ pub fn flag_corrupted(conn: &Connection, hash: &str, corrupted: bool) -> rusqlit Ok(()) } -/// Update file size for an asset (used by verifier to backfill). +/// Store the file size in bytes for an asset (used by the verifier to fill in +/// sizes for assets that were created before the size column existed). pub fn update_asset_size(conn: &Connection, hash: &str, size: i64) -> rusqlite::Result<()> { conn.execute( "UPDATE assets SET size = ?1 WHERE hash = ?2", @@ -213,7 +221,7 @@ pub fn update_asset_size(conn: &Connection, hash: &str, size: i64) -> rusqlite:: Ok(()) } -/// Soft-delete: mark as trashed. +/// Soft-delete an asset by marking it as trashed (the file is moved to .trash/). pub fn trash_asset(conn: &Connection, hash: &str) -> rusqlite::Result<()> { conn.execute( "UPDATE assets SET is_trashed = 1 WHERE hash = ?1", @@ -222,7 +230,8 @@ pub fn trash_asset(conn: &Connection, hash: &str) -> rusqlite::Result<()> { Ok(()) } -/// List assets with pagination and filtering. +/// Fetch a page of assets with optional filters (application, trashed, etc.). +/// Returns the matching assets and the total count for pagination. pub fn list_assets(conn: &Connection, params: &ListParams) -> rusqlite::Result<(Vec, i64)> { let limit = params.limit.unwrap_or(50); let offset = params.offset.unwrap_or(0); @@ -301,7 +310,8 @@ pub fn list_assets(conn: &Connection, params: &ListParams) -> rusqlite::Result<( Ok((assets, total)) } -/// Search assets with various filters. +/// Search assets with multiple filters (hash prefix, time range, MIME type, tags, etc.). +/// Returns matching assets and total count for pagination. pub fn search_assets( conn: &Connection, params: &SearchParams, @@ -428,7 +438,69 @@ pub fn search_assets( Ok((assets, total)) } -/// Get all non-trashed asset records (for verifier startup scan). +/// Get every asset record in the database, including trashed ones. +/// Used by the sync system to compare what two peers have. +pub fn get_all_assets(conn: &Connection) -> rusqlite::Result> { + let mut stmt = conn.prepare( + "SELECT id, timestamp, hash, mime_type, application, user_identity, description, + actual_filename, human_filename, human_path, is_trashed, is_corrupted, size + FROM assets", + )?; + let assets = stmt + .query_map([], |row| { + Ok(Asset { + id: row.get(0)?, + timestamp: row.get(1)?, + hash: row.get(2)?, + mime_type: row.get(3)?, + application: row.get(4)?, + user_identity: row.get(5)?, + description: row.get(6)?, + actual_filename: row.get(7)?, + human_filename: row.get(8)?, + human_path: row.get(9)?, + is_trashed: row.get(10)?, + is_corrupted: row.get(11)?, + size: row.get(12)?, + }) + })? + .collect::>>()?; + Ok(assets) +} + +/// Get only assets added after a given timestamp (for incremental sync -- +/// "what's new since last time I checked?"). +pub fn get_assets_since(conn: &Connection, since: i64) -> rusqlite::Result> { + let mut stmt = conn.prepare( + "SELECT id, timestamp, hash, mime_type, application, user_identity, description, + actual_filename, human_filename, human_path, is_trashed, is_corrupted, size + FROM assets WHERE timestamp > ?1 + ORDER BY timestamp ASC", + )?; + let assets = stmt + .query_map([since], |row| { + Ok(Asset { + id: row.get(0)?, + timestamp: row.get(1)?, + hash: row.get(2)?, + mime_type: row.get(3)?, + application: row.get(4)?, + user_identity: row.get(5)?, + description: row.get(6)?, + actual_filename: row.get(7)?, + human_filename: row.get(8)?, + human_path: row.get(9)?, + is_trashed: row.get(10)?, + is_corrupted: row.get(11)?, + size: row.get(12)?, + }) + })? + .collect::>>()?; + Ok(assets) +} + +/// Get all non-trashed assets (used by the background verifier to check +/// file integrity on startup). pub fn get_all_active_assets(conn: &Connection) -> rusqlite::Result> { let mut stmt = conn.prepare( "SELECT id, timestamp, hash, mime_type, application, user_identity, description, diff --git a/src/error.rs b/src/error.rs index 8728c1d..1a17495 100644 --- a/src/error.rs +++ b/src/error.rs @@ -2,6 +2,7 @@ use axum::http::StatusCode; use axum::response::{IntoResponse, Response}; use crate::models::ErrorResponse; +/// All the error types the API can return. Each variant maps to an HTTP status code. #[derive(Debug, thiserror::Error)] pub enum AppError { #[error("Not found: {0}")] @@ -23,6 +24,7 @@ pub enum AppError { Internal(String), } +/// Converts an AppError into an HTTP response with the right status code and a JSON body. impl IntoResponse for AppError { fn into_response(self) -> Response { let (status, message) = match &self { diff --git a/src/lib.rs b/src/lib.rs index 199acf1..3f04866 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,20 +1,26 @@ -pub mod config; -pub mod db; -pub mod error; -pub mod hash; -pub mod models; -pub mod routes; -pub mod storage; -pub mod verifier; -pub mod xattr; +pub mod config; // Configuration loading from YAML +pub mod db; // SQLite database access (CRUD for assets and tags) +pub mod error; // Centralized error types and HTTP error responses +pub mod hash; // SHA-256 content hashing +pub mod models; // Data structures shared across the codebase +pub mod routes; // HTTP API route handlers +pub mod storage; // File I/O: reading, writing, and trashing asset files +pub mod verifier; // Background integrity checker and file-attribute syncer +pub mod xattr; // OS-level file metadata (xattr on Unix, NTFS ADS on Windows) use std::sync::Arc; use crate::config::Config; use crate::db::Db; +/// Broadcast channel for notifying sync subscribers about new assets. +/// Each message is `"hash:timestamp"` (e.g. `"abc123def456:1710000000000"`). +pub type SyncEventSender = tokio::sync::broadcast::Sender; + +/// Shared application state passed to every HTTP handler. #[derive(Clone)] pub struct AppState { pub config: Arc, pub db: Db, + pub sync_events: SyncEventSender, } diff --git a/src/main.rs b/src/main.rs index 05fdab6..cca77de 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,6 +10,8 @@ use tower_http::trace::TraceLayer; use can_service::config::Config; use can_service::{db, routes, verifier, AppState}; +/// Entry point: loads config, opens the database, starts background services, +/// and launches the HTTP server. #[tokio::main] async fn main() -> anyhow::Result<()> { // Initialize tracing @@ -41,9 +43,14 @@ async fn main() -> anyhow::Result<()> { // Start background verifier verifier::start((*config).clone(), db.clone()); + // Broadcast channel for SSE sync events (capacity doesn't matter much — + // slow receivers just miss events and do a full reconciliation on reconnect) + let (sync_events, _) = tokio::sync::broadcast::channel::(256); + let state = AppState { config: config.clone(), db, + sync_events, }; // Build router @@ -54,7 +61,11 @@ async fn main() -> anyhow::Result<()> { .layer(CorsLayer::permissive()) .with_state(state); - let addr = SocketAddr::from(([0, 0, 0, 0], 3210)); + let port: u16 = std::env::var("CAN_PORT") + .ok() + .and_then(|p| p.parse().ok()) + .unwrap_or(3210); + let addr = SocketAddr::from(([0, 0, 0, 0], port)); tracing::info!("CAN service listening on {}", addr); let listener = tokio::net::TcpListener::bind(addr).await?; diff --git a/src/models.rs b/src/models.rs index 392153c..e4c32a3 100644 --- a/src/models.rs +++ b/src/models.rs @@ -1,6 +1,7 @@ use serde::{Deserialize, Serialize}; -/// Database representation of an asset. +/// Internal database row for a stored file. Contains all metadata fields +/// that are persisted in SQLite. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Asset { pub id: i64, @@ -18,7 +19,8 @@ pub struct Asset { pub size: i64, } -/// API-facing asset metadata response. +/// The public-facing version of an asset's metadata, returned by the API. +/// Includes resolved tags and omits internal fields like `id` and `actual_filename`. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct AssetMeta { pub hash: String, @@ -35,7 +37,7 @@ pub struct AssetMeta { pub size: i64, } -/// Standard API response wrapper. +/// Wraps every successful API response in `{ "status": "success", "data": ... }`. #[derive(Debug, Serialize, Deserialize)] pub struct ApiResponse { pub status: String, @@ -43,6 +45,7 @@ pub struct ApiResponse { } impl ApiResponse { + /// Create a success response wrapping the given data. pub fn success(data: T) -> Self { Self { status: "success".to_string(), @@ -51,7 +54,7 @@ impl ApiResponse { } } -/// Error response body. +/// JSON body for error responses: `{ "status": "error", "error": "..." }`. #[derive(Debug, Serialize, Deserialize)] pub struct ErrorResponse { pub status: String, @@ -67,7 +70,7 @@ impl ErrorResponse { } } -/// Ingest success response data. +/// Returned after a successful file upload: the timestamp, hash, and on-disk filename. #[derive(Debug, Serialize, Deserialize)] pub struct IngestResult { pub timestamp: i64, @@ -97,7 +100,9 @@ pub struct MetadataUpdate { pub description: Option, } -/// OS-level file attribute metadata (for xattr / NTFS ADS). +/// Metadata stored directly on the file via OS-level attributes +/// (xattr on macOS/Linux, NTFS Alternate Data Streams on Windows). +/// This lets external tools read CAN metadata without hitting the database. #[derive(Debug, Clone, Default, PartialEq)] pub struct FileAttributes { pub mime_type: Option, diff --git a/src/routes/asset.rs b/src/routes/asset.rs index b4d329a..023f628 100644 --- a/src/routes/asset.rs +++ b/src/routes/asset.rs @@ -17,7 +17,8 @@ pub fn router() -> Router { .route("/api/v1/can/0/asset/{hash}", patch(patch_asset)) } -/// GET /api/v1/can/0/asset/{hash} - Stream the physical file. +/// Download an asset's file by its hash. Streams the raw bytes back to the +/// client with the correct MIME type and a suggested filename. async fn get_asset( State(state): State, Path(hash): Path, @@ -59,7 +60,8 @@ async fn get_asset( .into_response()) } -/// PATCH /api/v1/can/0/asset/{hash} - Update metadata (tags, description). +/// Update an asset's tags and/or description. Saves changes to both the +/// database and the OS-level file attributes. async fn patch_asset( State(state): State, Path(hash): Path, diff --git a/src/routes/events.rs b/src/routes/events.rs new file mode 100644 index 0000000..5589edb --- /dev/null +++ b/src/routes/events.rs @@ -0,0 +1,39 @@ +//! Public SSE endpoint for real-time asset notifications. +//! +//! `GET /api/v1/can/0/events` — no authentication required. +//! Streams `new_asset` events whenever an asset is ingested or synced. +//! Used by frontends (e.g. Paste) to auto-refresh when content arrives. + +use std::convert::Infallible; + +use axum::extract::State; +use axum::response::sse::{Event, Sse}; +use axum::routing::get; +use axum::Router; +use tokio_stream::wrappers::BroadcastStream; +use tokio_stream::StreamExt; + +use crate::AppState; + +pub fn router() -> Router { + Router::new().route("/api/v1/can/0/events", get(asset_events)) +} + +/// Public SSE stream of new asset events. +/// +/// Each event is `event: new_asset` with `data: {"hash":"...","timestamp":...}`. +async fn asset_events( + State(state): State, +) -> Sse>> { + let rx = state.sync_events.subscribe(); + let stream = BroadcastStream::new(rx).filter_map(|result| match result { + Ok(data) => Some(Ok(Event::default().event("new_asset").data(data))), + Err(_) => None, // lagged — skip, client will see the data on next loadItems() + }); + + Sse::new(stream).keep_alive( + axum::response::sse::KeepAlive::new() + .interval(std::time::Duration::from_secs(15)) + .text("ping"), + ) +} diff --git a/src/routes/ingest.rs b/src/routes/ingest.rs index 8bd6151..594541c 100644 --- a/src/routes/ingest.rs +++ b/src/routes/ingest.rs @@ -7,6 +7,9 @@ use crate::error::AppError; use crate::models::{ApiResponse, Asset, DataIngestRequest, FileAttributes, IngestResult}; use crate::{db, hash, storage, xattr, AppState}; +/// Register the two upload endpoints: +/// - POST /ingest (multipart file upload) +/// - POST /ingest/data (JSON body upload, agent-friendly) pub fn router() -> Router { Router::new() .route("/api/v1/can/0/ingest", post(ingest_multipart)) @@ -27,7 +30,9 @@ struct IngestInput { description: Option, } -/// Common pipeline: timestamp → hash → write file → xattr → DB insert. +/// Core ingest pipeline shared by both upload endpoints. +/// Steps: generate timestamp -> hash content -> write file to disk -> +/// save OS-level metadata -> insert into database -> notify SSE subscribers. fn do_ingest(state: &AppState, input: IngestInput) -> Result { let timestamp = SystemTime::now() .duration_since(UNIX_EPOCH) @@ -85,6 +90,13 @@ fn do_ingest(state: &AppState, input: IngestInput) -> Result Result) -> Vec { raw.unwrap_or("") .split(',') @@ -103,6 +115,8 @@ fn parse_tags(raw: Option<&str>) -> Vec { // ── POST /api/v1/can/0/ingest (multipart — file uploads) ────────────── +/// Handle multipart file upload. Reads the "file" field plus optional metadata +/// fields (tags, application, user, etc.) and runs the ingest pipeline. async fn ingest_multipart( State(state): State, mut multipart: Multipart, diff --git a/src/routes/list.rs b/src/routes/list.rs index 30283ad..fa5808b 100644 --- a/src/routes/list.rs +++ b/src/routes/list.rs @@ -10,6 +10,8 @@ pub fn router() -> Router { Router::new().route("/api/v1/can/0/list", get(list_assets)) } +/// GET /api/v1/can/0/list - Return a paginated list of assets with their metadata. +/// Supports query params: limit, offset, order (asc/desc), application filter. async fn list_assets( State(state): State, Query(params): Query, diff --git a/src/routes/meta.rs b/src/routes/meta.rs index bdfc1b8..889fdc2 100644 --- a/src/routes/meta.rs +++ b/src/routes/meta.rs @@ -10,6 +10,9 @@ pub fn router() -> Router { Router::new().route("/api/v1/can/0/asset/{hash}/meta", get(get_meta)) } +/// GET /api/v1/can/0/asset/{hash}/meta - Return an asset's metadata as JSON +/// (hash, MIME type, tags, description, timestamps, etc.) without downloading +/// the actual file. async fn get_meta( State(state): State, Path(hash): Path, diff --git a/src/routes/mod.rs b/src/routes/mod.rs index 0a784fe..f01013f 100644 --- a/src/routes/mod.rs +++ b/src/routes/mod.rs @@ -1,13 +1,16 @@ -pub mod ingest; -pub mod asset; -pub mod meta; -pub mod list; -pub mod search; -pub mod thumb; +pub mod ingest; // POST endpoints for uploading files and JSON data +pub mod asset; // GET/PATCH endpoints for downloading files and updating metadata +pub mod meta; // GET endpoint for reading asset metadata as JSON +pub mod list; // GET endpoint for paginated asset listing +pub mod search; // GET endpoint for searching/filtering assets +pub mod thumb; // GET endpoint for generating resized thumbnail images +pub mod sync; // Private P2P sync endpoints (protobuf, requires API key) +pub mod events; // Public SSE endpoint for real-time "new asset" notifications use axum::Router; use crate::AppState; +/// Combine all route modules into one router. Called once at startup. pub fn router() -> Router { Router::new() .merge(ingest::router()) @@ -16,4 +19,6 @@ pub fn router() -> Router { .merge(list::router()) .merge(search::router()) .merge(thumb::router()) + .merge(sync::router()) + .merge(events::router()) } diff --git a/src/routes/search.rs b/src/routes/search.rs index f671246..93d47c8 100644 --- a/src/routes/search.rs +++ b/src/routes/search.rs @@ -10,6 +10,8 @@ pub fn router() -> Router { Router::new().route("/api/v1/can/0/search", get(search_assets)) } +/// GET /api/v1/can/0/search - Search assets by hash prefix, time range, +/// MIME type, user, application, or tags. Returns paginated results. async fn search_assets( State(state): State, Query(params): Query, diff --git a/src/routes/sync.rs b/src/routes/sync.rs new file mode 100644 index 0000000..ac7d9f1 --- /dev/null +++ b/src/routes/sync.rs @@ -0,0 +1,481 @@ +//! Private sync API endpoints (protobuf-encoded). +//! +//! All endpoints require `X-Sync-Key` header matching `config.sync_api_key`. +//! If `sync_api_key` is not configured, all endpoints return 404. +//! +//! Includes an SSE endpoint (`GET /sync/events`) that streams real-time +//! notifications when new assets are ingested. + +use std::convert::Infallible; + +use axum::body::Bytes; +use axum::extract::{Query, State}; +use axum::http::{HeaderMap, StatusCode}; +use axum::response::sse::{Event, Sse}; +use axum::response::IntoResponse; +use axum::routing::{get, post}; +use axum::Router; +use prost::Message; +use tokio_stream::wrappers::BroadcastStream; +use tokio_stream::StreamExt; + +use crate::models::{Asset, FileAttributes}; +use crate::{db, hash, storage, xattr, AppState}; + +// ── Protobuf message types ─────────────────────────────────────────────── +// These structs are serialized/deserialized as protobuf using the `prost` crate. +// They define the wire format for peer-to-peer sync communication. + +#[derive(Clone, PartialEq, Message)] +pub struct HashListRequest {} + +#[derive(Clone, PartialEq, Message)] +pub struct HashListResponse { + #[prost(message, repeated, tag = "1")] + pub assets: Vec, +} + +#[derive(Clone, PartialEq, Message)] +pub struct AssetDigest { + #[prost(string, tag = "1")] + pub hash: String, + #[prost(int64, tag = "2")] + pub timestamp: i64, + #[prost(int64, tag = "3")] + pub size: i64, + #[prost(bool, tag = "4")] + pub is_trashed: bool, +} + +#[derive(Clone, PartialEq, Message)] +pub struct PullRequest { + #[prost(string, repeated, tag = "1")] + pub hashes: Vec, +} + +#[derive(Clone, PartialEq, Message)] +pub struct PullResponse { + #[prost(message, repeated, tag = "1")] + pub bundles: Vec, +} + +#[derive(Clone, PartialEq, Message)] +pub struct AssetBundle { + #[prost(string, tag = "1")] + pub hash: String, + #[prost(int64, tag = "2")] + pub timestamp: i64, + #[prost(string, tag = "3")] + pub mime_type: String, + #[prost(string, optional, tag = "4")] + pub application: Option, + #[prost(string, optional, tag = "5")] + pub user_identity: Option, + #[prost(string, optional, tag = "6")] + pub description: Option, + #[prost(string, optional, tag = "7")] + pub human_filename: Option, + #[prost(string, optional, tag = "8")] + pub human_path: Option, + #[prost(bool, tag = "9")] + pub is_trashed: bool, + #[prost(int64, tag = "10")] + pub size: i64, + #[prost(string, repeated, tag = "11")] + pub tags: Vec, + #[prost(bytes = "vec", tag = "12")] + pub content: Vec, +} + +#[derive(Clone, PartialEq, Message)] +pub struct PushRequest { + #[prost(message, optional, tag = "1")] + pub bundle: Option, +} + +#[derive(Clone, PartialEq, Message)] +pub struct PushResponse { + #[prost(string, tag = "1")] + pub hash: String, + #[prost(bool, tag = "2")] + pub already_existed: bool, +} + +#[derive(Clone, PartialEq, Message)] +pub struct MetaUpdateRequest { + #[prost(string, tag = "1")] + pub hash: String, + #[prost(string, optional, tag = "2")] + pub description: Option, + #[prost(string, repeated, tag = "3")] + pub tags: Vec, + #[prost(bool, tag = "4")] + pub is_trashed: bool, +} + +#[derive(Clone, PartialEq, Message)] +pub struct MetaUpdateResponse { + #[prost(bool, tag = "1")] + pub success: bool, +} + +// ── Router ────────────────────────────────────────────────────────────── + +pub fn router() -> Router { + Router::new() + .route("/sync/hashes", post(sync_hashes)) + .route("/sync/pull", post(sync_pull)) + .route("/sync/push", post(sync_push)) + .route("/sync/meta", post(sync_meta)) + .route("/sync/events", get(sync_events)) +} + +/// Query params for /sync/hashes (optional `since` timestamp for incremental queries). +#[derive(serde::Deserialize, Default)] +struct HashesQuery { + /// Only return assets with `timestamp > since`. Omit or 0 for full list. + since: Option, +} + +// ── Auth ──────────────────────────────────────────────────────────────── + +/// Verify the X-Sync-Key header matches the configured API key. +/// Returns 404 if sync is not configured, 401 if the key is wrong. +fn check_sync_key(state: &AppState, headers: &HeaderMap) -> Result<(), (StatusCode, String)> { + let expected = match &state.config.sync_api_key { + Some(key) if !key.is_empty() => key, + _ => return Err((StatusCode::NOT_FOUND, "Sync API not enabled".into())), + }; + + let provided = headers + .get("X-Sync-Key") + .and_then(|v| v.to_str().ok()) + .unwrap_or(""); + + if provided != expected { + return Err((StatusCode::UNAUTHORIZED, "Invalid sync key".into())); + } + + Ok(()) +} + +// ── Helpers ───────────────────────────────────────────────────────────── + +/// Serialize a protobuf message into bytes. +fn encode_proto(msg: &M) -> Result, (StatusCode, String)> { + let mut buf = Vec::with_capacity(msg.encoded_len()); + msg.encode(&mut buf) + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("Encode error: {}", e)))?; + Ok(buf) +} + +/// Wrap protobuf bytes into an HTTP 200 response with the right content type. +fn proto_response(buf: Vec) -> (StatusCode, [(&'static str, &'static str); 1], Vec) { + (StatusCode::OK, [("content-type", "application/x-protobuf")], buf) +} + +// ── POST /sync/hashes ─────────────────────────────────────────────────── + +/// Return a compact list of all known asset hashes + timestamps. +/// A remote peer calls this first to figure out which assets it's missing. +/// Supports `?since=` for incremental queries. +async fn sync_hashes( + State(state): State, + headers: HeaderMap, + query: Query, + _body: Bytes, +) -> Result { + check_sync_key(&state, &headers)?; + + let since = query.since.unwrap_or(0); + + let assets = { + let conn = state.db.lock().unwrap(); + if since > 0 { + db::get_assets_since(&conn, since) + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("DB error: {}", e)))? + } else { + db::get_all_assets(&conn) + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("DB error: {}", e)))? + } + }; + + let resp = HashListResponse { + assets: assets + .iter() + .map(|a| AssetDigest { + hash: a.hash.clone(), + timestamp: a.timestamp, + size: a.size, + is_trashed: a.is_trashed, + }) + .collect(), + }; + + Ok(proto_response(encode_proto(&resp)?)) +} + +// ── POST /sync/pull ───────────────────────────────────────────────────── + +/// Download full asset bundles (metadata + file content) for a list of hashes. +/// A remote peer calls this to fetch assets it doesn't have yet. +async fn sync_pull( + State(state): State, + headers: HeaderMap, + body: Bytes, +) -> Result { + check_sync_key(&state, &headers)?; + + let req = PullRequest::decode(body) + .map_err(|e| (StatusCode::BAD_REQUEST, format!("Decode error: {}", e)))?; + + let mut bundles = Vec::new(); + + for hash_str in &req.hashes { + let (asset, tags) = { + let conn = state.db.lock().unwrap(); + let asset = match db::get_asset_by_hash(&conn, hash_str) { + Ok(Some(a)) => a, + _ => continue, + }; + let tags = db::get_asset_tags(&conn, asset.id).unwrap_or_default(); + (asset, tags) + }; + + let content = + match storage::read_asset(&state.config.storage_root, &asset.actual_filename) { + Ok(c) => c, + Err(e) => { + tracing::warn!("Failed to read {}: {}", &asset.actual_filename, e); + continue; + } + }; + + bundles.push(AssetBundle { + hash: asset.hash, + timestamp: asset.timestamp, + mime_type: asset.mime_type, + application: asset.application, + user_identity: asset.user_identity, + description: asset.description, + human_filename: asset.human_filename, + human_path: asset.human_path, + is_trashed: asset.is_trashed, + size: asset.size, + tags, + content, + }); + } + + Ok(proto_response(encode_proto(&PullResponse { bundles })?)) +} + +// ── POST /sync/push ───────────────────────────────────────────────────── + +/// Receive and store a new asset pushed from a remote peer. +/// Verifies the hash, writes the file, and inserts the DB record. +/// Returns early if the asset already exists locally. +async fn sync_push( + State(state): State, + headers: HeaderMap, + body: Bytes, +) -> Result { + check_sync_key(&state, &headers)?; + + let req = PushRequest::decode(body) + .map_err(|e| (StatusCode::BAD_REQUEST, format!("Decode error: {}", e)))?; + + let bundle = req + .bundle + .ok_or_else(|| (StatusCode::BAD_REQUEST, "Missing bundle".into()))?; + + // 1. Verify hash + let computed = hash::compute_hash(bundle.timestamp, &bundle.content); + if computed != bundle.hash { + return Err(( + StatusCode::BAD_REQUEST, + format!( + "Hash mismatch: computed {} vs provided {}", + &computed[..12], + &bundle.hash[..12.min(bundle.hash.len())] + ), + )); + } + + // 2. Check if already exists + { + let conn = state.db.lock().unwrap(); + if let Ok(Some(_)) = db::get_asset_by_hash(&conn, &bundle.hash) { + return Ok(proto_response(encode_proto(&PushResponse { + hash: bundle.hash, + already_existed: true, + })?)); + } + } + + // 3. Write file + let actual_filename = + storage::build_filename(bundle.timestamp, &bundle.hash, &bundle.tags, &bundle.mime_type); + + let file_path = + storage::write_asset(&state.config.storage_root, &actual_filename, &bundle.content) + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("Write error: {}", e)))?; + + // 4. OS attributes (best-effort) + let attrs = FileAttributes { + mime_type: Some(bundle.mime_type.clone()), + application: bundle.application.clone(), + user: bundle.user_identity.clone(), + tags: if bundle.tags.is_empty() { + None + } else { + Some(bundle.tags.join(",")) + }, + description: bundle.description.clone(), + human_filename: bundle.human_filename.clone(), + human_path: bundle.human_path.clone(), + }; + if let Err(e) = xattr::write_attributes(&file_path, &attrs) { + tracing::warn!("Failed to write OS attributes: {}", e); + } + + // 5. DB insert + let asset = Asset { + id: 0, + timestamp: bundle.timestamp, + hash: bundle.hash.clone(), + mime_type: bundle.mime_type, + application: bundle.application, + user_identity: bundle.user_identity, + description: bundle.description, + actual_filename, + human_filename: bundle.human_filename, + human_path: bundle.human_path, + is_trashed: false, + is_corrupted: false, + size: bundle.content.len() as i64, + }; + + { + let conn = state.db.lock().unwrap(); + let asset_id = db::insert_asset(&conn, &asset) + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("DB error: {}", e)))?; + if !bundle.tags.is_empty() { + db::set_asset_tags(&conn, asset_id, &bundle.tags) + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("Tag error: {}", e)))?; + } + if bundle.is_trashed { + let _ = db::trash_asset(&conn, &bundle.hash); + } + } + + tracing::info!("Sync push: ingested {} ({}B)", &bundle.hash[..12], bundle.content.len()); + + // Notify SSE subscribers about the new asset + let event_data = format!( + r#"{{"hash":"{}","timestamp":{}}}"#, + bundle.hash, bundle.timestamp + ); + let _ = state.sync_events.send(event_data); + + Ok(proto_response(encode_proto(&PushResponse { + hash: bundle.hash, + already_existed: false, + })?)) +} + +// ── POST /sync/meta ───────────────────────────────────────────────────── + +/// Receive a metadata update from a remote peer (description, tags, trash status). +async fn sync_meta( + State(state): State, + headers: HeaderMap, + body: Bytes, +) -> Result { + check_sync_key(&state, &headers)?; + + let req = MetaUpdateRequest::decode(body) + .map_err(|e| (StatusCode::BAD_REQUEST, format!("Decode error: {}", e)))?; + + let conn = state.db.lock().unwrap(); + + let asset = db::get_asset_by_hash(&conn, &req.hash) + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("DB error: {}", e)))? + .ok_or_else(|| (StatusCode::NOT_FOUND, "Asset not found".into()))?; + + if let Some(ref desc) = req.description { + db::update_asset_metadata(&conn, &req.hash, Some(desc), None) + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("Update error: {}", e)))?; + } + + if !req.tags.is_empty() { + db::set_asset_tags(&conn, asset.id, &req.tags) + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("Tag error: {}", e)))?; + } + + if req.is_trashed && !asset.is_trashed { + db::trash_asset(&conn, &req.hash) + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("Trash error: {}", e)))?; + let _ = storage::trash_asset_file(&state.config.storage_root, &asset.actual_filename); + } + + tracing::info!("Sync meta update for {}", &req.hash[..12.min(req.hash.len())]); + + Ok(proto_response(encode_proto(&MetaUpdateResponse { + success: true, + })?)) +} + +// ── GET /sync/events (SSE) ──────────────────────────────────────────── + +/// Server-Sent Events endpoint. Streams `new_asset` events whenever a file is +/// ingested (via public API or sync push). Requires `X-Sync-Key` as a query +/// param (`?key=...`) since SSE/EventSource doesn't support custom headers. +/// +/// Each event is: +/// ```text +/// event: new_asset +/// data: {"hash":"abc...","timestamp":1710000000000} +/// ``` +async fn sync_events( + State(state): State, + headers: HeaderMap, + query: Query, +) -> Result>>, (StatusCode, String)> +{ + // SSE clients (EventSource) can't set custom headers, so accept key from query param too + let key_ok = check_sync_key(&state, &headers).is_ok() + || query + .key + .as_deref() + .map(|k| { + state + .config + .sync_api_key + .as_deref() + .map(|expected| k == expected) + .unwrap_or(false) + }) + .unwrap_or(false); + + if !key_ok { + return Err((StatusCode::UNAUTHORIZED, "Invalid sync key".into())); + } + + let rx = state.sync_events.subscribe(); + let stream = BroadcastStream::new(rx).filter_map(|result| match result { + Ok(data) => Some(Ok(Event::default().event("new_asset").data(data))), + Err(_) => None, // lagged — skip missed events, client will reconcile + }); + + Ok(Sse::new(stream).keep_alive( + axum::response::sse::KeepAlive::new() + .interval(std::time::Duration::from_secs(15)) + .text("ping"), + )) +} + +#[derive(serde::Deserialize, Default)] +struct SseQuery { + key: Option, +} diff --git a/src/routes/thumb.rs b/src/routes/thumb.rs index 19236b7..066ef27 100644 --- a/src/routes/thumb.rs +++ b/src/routes/thumb.rs @@ -18,12 +18,15 @@ pub fn router() -> Router { ) } -/// Static fallback SVG icon for non-image assets. +/// A simple "?" placeholder icon returned when the asset isn't a resizable image. const FALLBACK_SVG: &str = r##" ? "##; +/// GET /api/v1/can/0/asset/{hash}/thumb/{width}/{height} +/// Generate (or serve from cache) a resized JPEG thumbnail for image assets. +/// Non-image assets get a placeholder SVG icon instead. async fn get_thumb( State(state): State, Path((hash, max_width, max_height)): Path<(String, u32, u32)>, diff --git a/src/storage.rs b/src/storage.rs index b819387..ffd16cf 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -1,7 +1,8 @@ use std::path::{Path, PathBuf}; -/// Build the physical filename per the spec: -/// `{timestamp}_{sha256}_{truncated_tags}.{extension}` +/// Build the on-disk filename for a new asset. +/// Format: `{timestamp}_{sha256hash}_{tags}.{extension}` +/// Tags are sanitized (alphanumeric only) and truncated to fit filesystem limits. pub fn build_filename( timestamp: i64, hash: &str, @@ -41,7 +42,8 @@ pub fn build_filename( .replace(". ", ".") } -/// Derive file extension from MIME type. +/// Convert a MIME type string (like "image/png") into a file extension (like "png"). +/// Falls back to "bin" for unknown types. pub fn mime_to_extension(mime: &str) -> &str { match mime { "application/pdf" => "pdf", @@ -75,20 +77,20 @@ pub fn mime_to_extension(mime: &str) -> &str { } } -/// Write asset bytes to the storage root. Returns the full path. +/// Save a file's raw bytes to the storage directory. Returns the full path on disk. pub fn write_asset(root: &Path, filename: &str, data: &[u8]) -> std::io::Result { let path = root.join(filename); std::fs::write(&path, data)?; Ok(path) } -/// Read asset bytes from the storage root. +/// Load the raw bytes of a stored file from the storage directory. pub fn read_asset(root: &Path, filename: &str) -> std::io::Result> { let path = root.join(filename); std::fs::read(path) } -/// Move an asset file to the .trash directory. +/// Move a file from the storage directory into the .trash/ folder (soft delete). pub fn trash_asset_file(root: &Path, filename: &str) -> std::io::Result<()> { let src = root.join(filename); let trash_dir = root.join(".trash"); @@ -98,8 +100,9 @@ pub fn trash_asset_file(root: &Path, filename: &str) -> std::io::Result<()> { Ok(()) } -/// Parse a physical filename to extract the hash component. -/// Format: `{timestamp}_{sha256}_{tags}.{ext}` or `{timestamp}_{sha256}.{ext}` +/// Extract the SHA-256 hash from a CAN filename. +/// Expects format: `{timestamp}_{sha256hash}_{tags}.{ext}` +/// Returns None if the filename doesn't match the expected pattern. pub fn parse_hash_from_filename(filename: &str) -> Option { // Remove extension let stem = filename.rsplit_once('.')?.0; @@ -112,7 +115,8 @@ pub fn parse_hash_from_filename(filename: &str) -> Option { } } -/// Parse a physical filename to extract the timestamp component. +/// Extract the millisecond timestamp from a CAN filename. +/// Returns None if the filename doesn't match the expected pattern. pub fn parse_timestamp_from_filename(filename: &str) -> Option { let stem = filename.rsplit_once('.')?.0; let ts_str = stem.split('_').next()?; diff --git a/src/verifier.rs b/src/verifier.rs index 3c91ab2..899bc09 100644 --- a/src/verifier.rs +++ b/src/verifier.rs @@ -11,10 +11,10 @@ use crate::models::FileAttributes; use crate::storage::{parse_hash_from_filename, parse_timestamp_from_filename}; use crate::xattr; -/// Start the background verifier subsystem. -/// - Runs an initial full scrub -/// - Watches for filesystem changes -/// - Runs periodic scrubs +/// Launch the background integrity checker. It does three things: +/// 1. Immediately scans all files to detect corruption or missing data. +/// 2. Watches the storage folder for file changes and re-checks them in real time. +/// 3. Re-runs the full scan on a timer (configurable in config.yaml). pub fn start(config: Config, db: Db) { let config2 = config.clone(); let db2 = db.clone(); @@ -58,6 +58,7 @@ fn config3_for_watcher(config: Config) -> Config { config } +/// Watch the storage directory for file changes and verify each changed file. async fn run_watcher(config: Config, db: Db) -> anyhow::Result<()> { let (tx, mut rx) = mpsc::channel::(100); let storage_root = config.storage_root.clone(); @@ -114,7 +115,9 @@ async fn run_watcher(config: Config, db: Db) -> anyhow::Result<()> { Ok(()) } -/// Run a full scrub: verify every active asset's hash. +/// Full integrity scan: re-hashes every active file on disk and compares it +/// to the expected hash in the database. Also syncs OS-level file attributes +/// and backfills missing file sizes. async fn run_scrub(config: &Config, db: &Db) -> anyhow::Result<()> { let assets = { let conn = db.lock().unwrap(); @@ -276,7 +279,8 @@ async fn run_scrub(config: &Config, db: &Db) -> anyhow::Result<()> { Ok(()) } -/// Verify a single file by its physical filename. +/// Re-hash a single file and flag it as corrupted if the hash doesn't match. +/// Called when the filesystem watcher detects a change. async fn verify_single_file( config: &Config, db: &Db, diff --git a/src/xattr.rs b/src/xattr.rs index 62490c0..ec777da 100644 --- a/src/xattr.rs +++ b/src/xattr.rs @@ -27,7 +27,8 @@ pub fn read_attributes(path: &Path) -> std::io::Result { } } -// ── Unix implementation using xattr crate ── +// ── Unix implementation ── +// Stores each metadata field as an extended attribute (e.g. "user.can.mime_type"). #[cfg(unix)] fn write_xattr(path: &Path, attrs: &FileAttributes) -> std::io::Result<()> { @@ -58,6 +59,7 @@ fn write_xattr(path: &Path, attrs: &FileAttributes) -> std::io::Result<()> { Ok(()) } +/// Read all CAN metadata from Unix extended attributes on a file. #[cfg(unix)] fn read_xattr(path: &Path) -> std::io::Result { use xattr::FileExt; @@ -81,8 +83,10 @@ fn read_xattr(path: &Path) -> std::io::Result { }) } -// ── Windows implementation using NTFS Alternate Data Streams ── +// ── Windows implementation ── +// Stores each metadata field as an NTFS Alternate Data Stream (e.g. "file.txt:can.mime_type"). +/// Write CAN metadata fields as NTFS Alternate Data Streams on a file. #[cfg(windows)] fn write_ntfs_ads(path: &Path, attrs: &FileAttributes) -> std::io::Result<()> { let base = path.to_string_lossy(); @@ -111,6 +115,7 @@ fn write_ntfs_ads(path: &Path, attrs: &FileAttributes) -> std::io::Result<()> { Ok(()) } +/// Read all CAN metadata from NTFS Alternate Data Streams on a file. #[cfg(windows)] fn read_ntfs_ads(path: &Path) -> std::io::Result { let base = path.to_string_lossy();