Add SSE events and incremental hash queries for live sync
Replace polling-based sync detection with SSE (Server-Sent Events) from CAN service for instant push notifications on new asset ingests. Add incremental hash queries via ?since=timestamp parameter to avoid transferring full hash lists on every sync cycle. CAN service changes: - Add broadcast channel (SyncEventSender) in AppState for SSE events - Add GET /sync/events SSE endpoint with auth via header or query param - Fire broadcast events on both ingest and sync push - Add db::get_assets_since() for incremental queries - Support ?since= parameter on POST /sync/hashes can-sync agent changes: - Add SSE subscription with auto-reconnect in can_client - Add get_hashes_since() for incremental catch-up - Rewrite live push loop: SSE-driven with 30s fallback poll - Remove poll_interval parameter from live sync functions All 6 stress tests pass (102 assets, 63 MB/s bidirectional). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
4dc0f8c12d
commit
7b54bbe892
2
Cargo.lock
generated
2
Cargo.lock
generated
@ -182,6 +182,7 @@ dependencies = [
|
|||||||
"tempfile",
|
"tempfile",
|
||||||
"thiserror",
|
"thiserror",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
"tokio-stream",
|
||||||
"tokio-test",
|
"tokio-test",
|
||||||
"tokio-util",
|
"tokio-util",
|
||||||
"tower-http",
|
"tower-http",
|
||||||
@ -1914,6 +1915,7 @@ dependencies = [
|
|||||||
"futures-core",
|
"futures-core",
|
||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
"tokio-util",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|||||||
@ -40,6 +40,9 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
|||||||
# Protobuf (sync API)
|
# Protobuf (sync API)
|
||||||
prost = "0.13"
|
prost = "0.13"
|
||||||
|
|
||||||
|
# Stream utilities (SSE for sync events)
|
||||||
|
tokio-stream = { version = "0.1", features = ["sync"] }
|
||||||
|
|
||||||
# Utilities
|
# Utilities
|
||||||
chrono = { version = "0.4", features = ["serde"] }
|
chrono = { version = "0.4", features = ["serde"] }
|
||||||
anyhow = "1"
|
anyhow = "1"
|
||||||
|
|||||||
2
examples/can-sync/Cargo.lock
generated
2
examples/can-sync/Cargo.lock
generated
@ -229,6 +229,7 @@ dependencies = [
|
|||||||
"anyhow",
|
"anyhow",
|
||||||
"blake3",
|
"blake3",
|
||||||
"bytes",
|
"bytes",
|
||||||
|
"futures-util",
|
||||||
"hex",
|
"hex",
|
||||||
"iroh",
|
"iroh",
|
||||||
"iroh-gossip",
|
"iroh-gossip",
|
||||||
@ -241,6 +242,7 @@ dependencies = [
|
|||||||
"serde_yaml",
|
"serde_yaml",
|
||||||
"tempfile",
|
"tempfile",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
"tokio-stream",
|
||||||
"tracing",
|
"tracing",
|
||||||
"tracing-subscriber",
|
"tracing-subscriber",
|
||||||
]
|
]
|
||||||
|
|||||||
@ -40,6 +40,10 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
|||||||
# Stream utilities (needed for gossip event stream)
|
# Stream utilities (needed for gossip event stream)
|
||||||
n0-future = "0.1"
|
n0-future = "0.1"
|
||||||
|
|
||||||
|
# SSE client (for real-time events from CAN service)
|
||||||
|
tokio-stream = "0.1"
|
||||||
|
futures-util = "0.3"
|
||||||
|
|
||||||
# Utilities
|
# Utilities
|
||||||
anyhow = "1"
|
anyhow = "1"
|
||||||
bytes = "1"
|
bytes = "1"
|
||||||
|
|||||||
@ -1,10 +1,23 @@
|
|||||||
//! HTTP client for CAN service's private sync API (protobuf-encoded).
|
//! HTTP client for CAN service's private sync API (protobuf-encoded).
|
||||||
|
//!
|
||||||
|
//! Includes SSE subscription for real-time ingest notifications and
|
||||||
|
//! incremental hash queries via `?since=` parameter.
|
||||||
|
|
||||||
use anyhow::{Context, Result};
|
use anyhow::{Context, Result};
|
||||||
|
use futures_util::StreamExt;
|
||||||
use prost::Message;
|
use prost::Message;
|
||||||
|
use tokio::sync::mpsc;
|
||||||
|
use tracing::{debug, info, warn};
|
||||||
|
|
||||||
use crate::protocol::*;
|
use crate::protocol::*;
|
||||||
|
|
||||||
|
/// Event received from the CAN service SSE stream.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct SyncEvent {
|
||||||
|
pub hash: String,
|
||||||
|
pub timestamp: i64,
|
||||||
|
}
|
||||||
|
|
||||||
/// Client for CAN service's /sync/* endpoints.
|
/// Client for CAN service's /sync/* endpoints.
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct CanSyncClient {
|
pub struct CanSyncClient {
|
||||||
@ -44,7 +57,7 @@ impl CanSyncClient {
|
|||||||
resp.bytes().await.with_context(|| format!("reading body from {}", path))
|
resp.bytes().await.with_context(|| format!("reading body from {}", path))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get all asset digests for reconciliation.
|
/// Get all asset digests (full list — use for initial reconciliation only).
|
||||||
pub async fn get_hashes(&self) -> Result<HashListResponse> {
|
pub async fn get_hashes(&self) -> Result<HashListResponse> {
|
||||||
let req = HashListRequest {};
|
let req = HashListRequest {};
|
||||||
let mut buf = Vec::with_capacity(req.encoded_len());
|
let mut buf = Vec::with_capacity(req.encoded_len());
|
||||||
@ -54,6 +67,33 @@ impl CanSyncClient {
|
|||||||
HashListResponse::decode(resp_bytes).context("decode HashListResponse")
|
HashListResponse::decode(resp_bytes).context("decode HashListResponse")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Get only asset digests newer than `since` timestamp (incremental query).
|
||||||
|
pub async fn get_hashes_since(&self, since: i64) -> Result<HashListResponse> {
|
||||||
|
let req = HashListRequest {};
|
||||||
|
let mut buf = Vec::with_capacity(req.encoded_len());
|
||||||
|
req.encode(&mut buf)?;
|
||||||
|
|
||||||
|
let url = format!("{}/sync/hashes?since={}", self.base_url, since);
|
||||||
|
let resp = self
|
||||||
|
.http
|
||||||
|
.post(&url)
|
||||||
|
.header("X-Sync-Key", &self.sync_key)
|
||||||
|
.header("Content-Type", "application/x-protobuf")
|
||||||
|
.body(buf)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.with_context(|| format!("POST {}", url))?;
|
||||||
|
|
||||||
|
if !resp.status().is_success() {
|
||||||
|
let status = resp.status();
|
||||||
|
let text = resp.text().await.unwrap_or_default();
|
||||||
|
anyhow::bail!("/sync/hashes?since={} returned {}: {}", since, status, text);
|
||||||
|
}
|
||||||
|
|
||||||
|
let resp_bytes = resp.bytes().await?;
|
||||||
|
HashListResponse::decode(resp_bytes).context("decode HashListResponse")
|
||||||
|
}
|
||||||
|
|
||||||
/// Pull full assets by hash.
|
/// Pull full assets by hash.
|
||||||
pub async fn pull(&self, hashes: Vec<String>) -> Result<PullResponse> {
|
pub async fn pull(&self, hashes: Vec<String>) -> Result<PullResponse> {
|
||||||
let req = PullRequest { hashes };
|
let req = PullRequest { hashes };
|
||||||
@ -101,4 +141,102 @@ impl CanSyncClient {
|
|||||||
pub async fn health_check(&self) -> bool {
|
pub async fn health_check(&self) -> bool {
|
||||||
self.get_hashes().await.is_ok()
|
self.get_hashes().await.is_ok()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Subscribe to SSE events from CAN service. Sends `SyncEvent` on the
|
||||||
|
/// returned channel whenever the CAN service ingests a new asset.
|
||||||
|
///
|
||||||
|
/// Automatically reconnects on disconnect (with incremental catch-up).
|
||||||
|
/// Returns a channel receiver that yields events.
|
||||||
|
pub fn subscribe_events(&self) -> mpsc::UnboundedReceiver<SyncEvent> {
|
||||||
|
let (tx, rx) = mpsc::unbounded_channel();
|
||||||
|
let url = format!(
|
||||||
|
"{}/sync/events?key={}",
|
||||||
|
self.base_url, self.sync_key
|
||||||
|
);
|
||||||
|
let http = self.http.clone();
|
||||||
|
|
||||||
|
tokio::spawn(async move {
|
||||||
|
loop {
|
||||||
|
info!("Connecting to SSE stream: {}", url.split('?').next().unwrap_or(&url));
|
||||||
|
match Self::run_sse_stream(&http, &url, &tx).await {
|
||||||
|
Ok(()) => {
|
||||||
|
info!("SSE stream ended cleanly");
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
warn!("SSE stream error: {:#}", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// If the receiver is dropped, stop reconnecting
|
||||||
|
if tx.is_closed() {
|
||||||
|
debug!("SSE subscriber dropped, stopping reconnect loop");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reconnect after a short delay
|
||||||
|
info!("Reconnecting SSE in 2s...");
|
||||||
|
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
rx
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Internal: connect to SSE and forward events until the stream ends or errors.
|
||||||
|
async fn run_sse_stream(
|
||||||
|
http: &reqwest::Client,
|
||||||
|
url: &str,
|
||||||
|
tx: &mpsc::UnboundedSender<SyncEvent>,
|
||||||
|
) -> Result<()> {
|
||||||
|
let resp = http
|
||||||
|
.get(url)
|
||||||
|
.header("Accept", "text/event-stream")
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.context("SSE connect")?;
|
||||||
|
|
||||||
|
if !resp.status().is_success() {
|
||||||
|
anyhow::bail!("SSE returned status {}", resp.status());
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut stream = resp.bytes_stream();
|
||||||
|
let mut buffer = String::new();
|
||||||
|
|
||||||
|
while let Some(chunk) = stream.next().await {
|
||||||
|
let chunk = chunk.context("reading SSE chunk")?;
|
||||||
|
buffer.push_str(&String::from_utf8_lossy(&chunk));
|
||||||
|
|
||||||
|
// Process complete SSE messages (separated by double newlines)
|
||||||
|
while let Some(pos) = buffer.find("\n\n") {
|
||||||
|
let message = buffer[..pos].to_string();
|
||||||
|
buffer = buffer[pos + 2..].to_string();
|
||||||
|
|
||||||
|
// Parse SSE message: look for "data: {...}" lines
|
||||||
|
for line in message.lines() {
|
||||||
|
if let Some(data) = line.strip_prefix("data:") {
|
||||||
|
let data = data.trim();
|
||||||
|
if data == "ping" || data.is_empty() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse JSON: {"hash":"...","timestamp":...}
|
||||||
|
if let Ok(value) = serde_json::from_str::<serde_json::Value>(data) {
|
||||||
|
if let (Some(hash), Some(ts)) = (
|
||||||
|
value["hash"].as_str(),
|
||||||
|
value["timestamp"].as_i64(),
|
||||||
|
) {
|
||||||
|
debug!("SSE event: new_asset hash={}", &hash[..hash.len().min(12)]);
|
||||||
|
let _ = tx.send(SyncEvent {
|
||||||
|
hash: hash.to_string(),
|
||||||
|
timestamp: ts,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -107,7 +107,6 @@ async fn main() -> Result<()> {
|
|||||||
let ticket_path = ticket_path.clone();
|
let ticket_path = ticket_path.clone();
|
||||||
let endpoint_direct = endpoint.clone();
|
let endpoint_direct = endpoint.clone();
|
||||||
let can_direct = can.clone();
|
let can_direct = can.clone();
|
||||||
let poll_interval = std::time::Duration::from_secs(config.poll_interval_secs);
|
|
||||||
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
info!("Waiting for peer addr file: {}", ticket_path);
|
info!("Waiting for peer addr file: {}", ticket_path);
|
||||||
@ -146,8 +145,8 @@ async fn main() -> Result<()> {
|
|||||||
|
|
||||||
info!("Initial sync with {} complete, starting live sync", short);
|
info!("Initial sync with {} complete, starting live sync", short);
|
||||||
|
|
||||||
// Live sync: push new local assets + accept incoming streams
|
// Live sync: SSE-driven push + accept incoming streams
|
||||||
peer::run_live_sync(conn, can_direct, poll_interval).await;
|
peer::run_live_sync(conn, can_direct).await;
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
error!("Failed to connect to {}: {:#}", short, e);
|
error!("Failed to connect to {}: {:#}", short, e);
|
||||||
@ -159,18 +158,16 @@ async fn main() -> Result<()> {
|
|||||||
// Spawn incoming connection handler
|
// Spawn incoming connection handler
|
||||||
let endpoint_accept = endpoint.clone();
|
let endpoint_accept = endpoint.clone();
|
||||||
let can_accept = can.clone();
|
let can_accept = can.clone();
|
||||||
let accept_poll_interval = std::time::Duration::from_secs(config.poll_interval_secs);
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
loop {
|
loop {
|
||||||
match endpoint_accept.accept().await {
|
match endpoint_accept.accept().await {
|
||||||
Some(incoming) => {
|
Some(incoming) => {
|
||||||
let can_clone = can_accept.clone();
|
let can_clone = can_accept.clone();
|
||||||
let poll_dur = accept_poll_interval;
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
match incoming.await {
|
match incoming.await {
|
||||||
Ok(conn) => {
|
Ok(conn) => {
|
||||||
info!("Accepted incoming connection from {}", conn.remote_id().fmt_short());
|
info!("Accepted incoming connection from {}", conn.remote_id().fmt_short());
|
||||||
peer::handle_incoming(conn, can_clone, poll_dur).await;
|
peer::handle_incoming(conn, can_clone, std::time::Duration::from_secs(0)).await;
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
warn!("Failed to accept connection: {:#}", e);
|
warn!("Failed to accept connection: {:#}", e);
|
||||||
@ -187,8 +184,6 @@ async fn main() -> Result<()> {
|
|||||||
});
|
});
|
||||||
|
|
||||||
// Main loop: connect to discovered peers (from gossip) and sync
|
// Main loop: connect to discovered peers (from gossip) and sync
|
||||||
let poll_interval = std::time::Duration::from_secs(config.poll_interval_secs);
|
|
||||||
|
|
||||||
info!("Waiting for peers...");
|
info!("Waiting for peers...");
|
||||||
|
|
||||||
while let Some(peer_id) = peer_rx.recv().await {
|
while let Some(peer_id) = peer_rx.recv().await {
|
||||||
@ -197,7 +192,6 @@ async fn main() -> Result<()> {
|
|||||||
|
|
||||||
let endpoint_clone = endpoint.clone();
|
let endpoint_clone = endpoint.clone();
|
||||||
let can_clone = can.clone();
|
let can_clone = can.clone();
|
||||||
let poll_dur = poll_interval;
|
|
||||||
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
let conn = match endpoint_clone.connect(peer_id, SYNC_ALPN).await {
|
let conn = match endpoint_clone.connect(peer_id, SYNC_ALPN).await {
|
||||||
@ -213,7 +207,7 @@ async fn main() -> Result<()> {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
peer::run_live_sync(conn, can_clone, poll_dur).await;
|
peer::run_live_sync(conn, can_clone).await;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -4,11 +4,15 @@
|
|||||||
//! 1. Exchange hash lists (from their local CAN services)
|
//! 1. Exchange hash lists (from their local CAN services)
|
||||||
//! 2. Compute the diff (what each side is missing)
|
//! 2. Compute the diff (what each side is missing)
|
||||||
//! 3. Send/receive missing assets concurrently (avoids deadlock)
|
//! 3. Send/receive missing assets concurrently (avoids deadlock)
|
||||||
//! 4. Continue polling for new assets and pushing them
|
//! 4. Subscribe to SSE events from local CAN for instant push on new assets
|
||||||
//!
|
//!
|
||||||
//! The live sync uses an unbounded channel to share received hashes
|
//! The live sync uses:
|
||||||
//! from the receive loop to the push loop, preventing "echo" where
|
//! - **SSE events** from local CAN service to detect new assets instantly
|
||||||
//! an asset received from a peer gets pushed right back to them.
|
//! (replaces the old polling loop — no more wasted hash-list queries)
|
||||||
|
//! - An unbounded channel to share received hashes from the receive loop
|
||||||
|
//! to the push loop, preventing "echo" where an asset received from a
|
||||||
|
//! peer gets pushed right back to them.
|
||||||
|
//! - A fallback incremental poll on timeout for catch-up if SSE was briefly down.
|
||||||
|
|
||||||
use std::collections::{HashMap, HashSet};
|
use std::collections::{HashMap, HashSet};
|
||||||
|
|
||||||
@ -19,7 +23,7 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
|||||||
use tokio::sync::mpsc;
|
use tokio::sync::mpsc;
|
||||||
use tracing::{debug, error, info, warn};
|
use tracing::{debug, error, info, warn};
|
||||||
|
|
||||||
use crate::can_client::CanSyncClient;
|
use crate::can_client::{CanSyncClient, SyncEvent};
|
||||||
use crate::protocol::*;
|
use crate::protocol::*;
|
||||||
|
|
||||||
// Message type tags for QUIC stream framing
|
// Message type tags for QUIC stream framing
|
||||||
@ -140,10 +144,6 @@ pub async fn run_sync_session(
|
|||||||
);
|
);
|
||||||
|
|
||||||
// Step 5+6: Send and receive assets CONCURRENTLY to avoid deadlock.
|
// Step 5+6: Send and receive assets CONCURRENTLY to avoid deadlock.
|
||||||
//
|
|
||||||
// If both sides have many large assets, doing send-then-receive sequentially
|
|
||||||
// can deadlock: QUIC flow control stalls the writer because the reader hasn't
|
|
||||||
// drained its buffer (it's busy writing). Running both in parallel avoids this.
|
|
||||||
let send_fut = async {
|
let send_fut = async {
|
||||||
if !they_need.is_empty() {
|
if !they_need.is_empty() {
|
||||||
send_assets(&can, &mut send, &they_need, &short_id).await?;
|
send_assets(&can, &mut send, &they_need, &short_id).await?;
|
||||||
@ -171,7 +171,6 @@ async fn send_assets(
|
|||||||
hashes: &[String],
|
hashes: &[String],
|
||||||
peer_short: &str,
|
peer_short: &str,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
// Pull assets in batches to avoid huge single requests
|
|
||||||
for chunk in hashes.chunks(10) {
|
for chunk in hashes.chunks(10) {
|
||||||
let pull_resp = can
|
let pull_resp = can
|
||||||
.pull(chunk.to_vec())
|
.pull(chunk.to_vec())
|
||||||
@ -263,7 +262,7 @@ async fn receive_assets(
|
|||||||
pub async fn handle_incoming(
|
pub async fn handle_incoming(
|
||||||
conn: Connection,
|
conn: Connection,
|
||||||
can: CanSyncClient,
|
can: CanSyncClient,
|
||||||
poll_interval: std::time::Duration,
|
_poll_interval: std::time::Duration,
|
||||||
) {
|
) {
|
||||||
let peer_id = conn.remote_id();
|
let peer_id = conn.remote_id();
|
||||||
let short_id = peer_id.fmt_short().to_string();
|
let short_id = peer_id.fmt_short().to_string();
|
||||||
@ -275,29 +274,28 @@ pub async fn handle_incoming(
|
|||||||
}
|
}
|
||||||
|
|
||||||
info!("Initial sync with {} complete, starting live sync", short_id);
|
info!("Initial sync with {} complete, starting live sync", short_id);
|
||||||
|
run_live_sync(conn, can).await;
|
||||||
// Run both live loops concurrently
|
|
||||||
run_live_sync(conn, can, poll_interval).await;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Run both live sync loops (push + receive) concurrently.
|
/// Run both live sync loops (push + receive) concurrently.
|
||||||
///
|
///
|
||||||
/// Uses an unbounded channel to prevent the "echo" problem: when we receive
|
/// Uses SSE events from CAN service for instant push (no polling).
|
||||||
/// an asset from the peer, we tell the push loop about it so it doesn't
|
/// Uses an unbounded channel to prevent the "echo" problem.
|
||||||
/// push the same asset right back.
|
|
||||||
pub async fn run_live_sync(
|
pub async fn run_live_sync(
|
||||||
conn: Connection,
|
conn: Connection,
|
||||||
can: CanSyncClient,
|
can: CanSyncClient,
|
||||||
poll_interval: std::time::Duration,
|
|
||||||
) {
|
) {
|
||||||
let short_id = conn.remote_id().fmt_short().to_string();
|
let short_id = conn.remote_id().fmt_short().to_string();
|
||||||
|
|
||||||
// Channel for receive loop to notify push loop about received hashes
|
// Channel for receive loop to notify push loop about received hashes
|
||||||
let (received_tx, received_rx) = mpsc::unbounded_channel::<String>();
|
let (received_tx, received_rx) = mpsc::unbounded_channel::<String>();
|
||||||
|
|
||||||
|
// Subscribe to SSE events from local CAN service
|
||||||
|
let sse_rx = can.subscribe_events();
|
||||||
|
|
||||||
// Run push loop and receive loop concurrently — when either ends, we're done
|
// Run push loop and receive loop concurrently — when either ends, we're done
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
result = live_push_loop(conn.clone(), can.clone(), poll_interval, received_rx) => {
|
result = live_push_loop(conn.clone(), can.clone(), received_rx, sse_rx) => {
|
||||||
if let Err(e) = result {
|
if let Err(e) = result {
|
||||||
warn!("Live push loop with {} ended: {:#}", short_id, e);
|
warn!("Live push loop with {} ended: {:#}", short_id, e);
|
||||||
}
|
}
|
||||||
@ -310,81 +308,124 @@ pub async fn run_live_sync(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Poll for new local assets and push them to the peer via new QUIC streams.
|
/// Wait for SSE events from local CAN service and push new assets to the peer.
|
||||||
///
|
///
|
||||||
/// Drains the `received_rx` channel each tick to learn about hashes that arrived
|
/// Drains the `received_rx` channel to learn about hashes that arrived from
|
||||||
/// from the peer, so we don't echo them back.
|
/// the peer, so we don't echo them back.
|
||||||
|
///
|
||||||
|
/// Falls back to incremental poll if no SSE events arrive within 30s.
|
||||||
async fn live_push_loop(
|
async fn live_push_loop(
|
||||||
conn: Connection,
|
conn: Connection,
|
||||||
can: CanSyncClient,
|
can: CanSyncClient,
|
||||||
poll_interval: std::time::Duration,
|
|
||||||
mut received_rx: mpsc::UnboundedReceiver<String>,
|
mut received_rx: mpsc::UnboundedReceiver<String>,
|
||||||
|
mut sse_rx: mpsc::UnboundedReceiver<SyncEvent>,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
let peer_id = conn.remote_id();
|
let peer_id = conn.remote_id();
|
||||||
let short_id = peer_id.fmt_short().to_string();
|
let short_id = peer_id.fmt_short().to_string();
|
||||||
info!("Starting live push loop with {}", short_id);
|
info!("Starting live push loop with {} (SSE-driven)", short_id);
|
||||||
|
|
||||||
// Track what we've already synced (local + received from peer)
|
// Track what we've already synced (local + received from peer)
|
||||||
let mut known_hashes: HashSet<String> = {
|
let resp = can.get_hashes().await?;
|
||||||
let resp = can.get_hashes().await?;
|
let mut max_timestamp: i64 = resp.assets.iter().map(|a| a.timestamp).max().unwrap_or(0);
|
||||||
resp.assets.into_iter().map(|a| a.hash).collect()
|
let mut known_hashes: HashSet<String> = resp.assets.into_iter().map(|a| a.hash).collect();
|
||||||
};
|
|
||||||
|
|
||||||
let mut interval = tokio::time::interval(poll_interval);
|
// Fallback: if no SSE event in 30s, do an incremental poll to catch gaps
|
||||||
|
let fallback_interval = std::time::Duration::from_secs(30);
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
interval.tick().await;
|
// Wait for SSE event, or fallback timeout
|
||||||
|
let new_hashes: Vec<String> = tokio::select! {
|
||||||
|
event = sse_rx.recv() => {
|
||||||
|
match event {
|
||||||
|
Some(evt) => {
|
||||||
|
// Drain any additional events that arrived at the same time
|
||||||
|
let mut batch = vec![evt];
|
||||||
|
while let Ok(more) = sse_rx.try_recv() {
|
||||||
|
batch.push(more);
|
||||||
|
}
|
||||||
|
|
||||||
// Drain any hashes that the receive loop got from the peer.
|
// Drain received-from-peer hashes (echo prevention)
|
||||||
// This prevents us from pushing them right back (echo prevention).
|
while let Ok(hash) = received_rx.try_recv() {
|
||||||
while let Ok(hash) = received_rx.try_recv() {
|
known_hashes.insert(hash);
|
||||||
known_hashes.insert(hash);
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Poll for new assets
|
// Filter to only truly new hashes
|
||||||
let resp = match can.get_hashes().await {
|
batch
|
||||||
Ok(r) => r,
|
.into_iter()
|
||||||
Err(e) => {
|
.filter(|e| {
|
||||||
warn!("Failed to poll CAN service: {:#}", e);
|
if e.timestamp > max_timestamp {
|
||||||
continue;
|
max_timestamp = e.timestamp;
|
||||||
|
}
|
||||||
|
!known_hashes.contains(&e.hash)
|
||||||
|
})
|
||||||
|
.map(|e| e.hash)
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
warn!("SSE channel closed, stopping push loop");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback: periodic incremental poll
|
||||||
|
_ = tokio::time::sleep(fallback_interval) => {
|
||||||
|
debug!("Fallback incremental poll (no SSE events in {}s)", fallback_interval.as_secs());
|
||||||
|
|
||||||
|
while let Ok(hash) = received_rx.try_recv() {
|
||||||
|
known_hashes.insert(hash);
|
||||||
|
}
|
||||||
|
|
||||||
|
match can.get_hashes_since(max_timestamp).await {
|
||||||
|
Ok(resp) => {
|
||||||
|
resp.assets
|
||||||
|
.into_iter()
|
||||||
|
.filter(|a| {
|
||||||
|
if a.timestamp > max_timestamp {
|
||||||
|
max_timestamp = a.timestamp;
|
||||||
|
}
|
||||||
|
!known_hashes.contains(&a.hash)
|
||||||
|
})
|
||||||
|
.map(|a| a.hash)
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
warn!("Fallback poll failed: {:#}", e);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let current_hashes: HashSet<String> =
|
if new_hashes.is_empty() {
|
||||||
resp.assets.iter().map(|a| a.hash.clone()).collect();
|
continue;
|
||||||
let new_hashes: Vec<String> = current_hashes
|
}
|
||||||
.difference(&known_hashes)
|
|
||||||
.cloned()
|
|
||||||
.collect();
|
|
||||||
|
|
||||||
if !new_hashes.is_empty() {
|
info!(
|
||||||
info!(
|
"Pushing {} new assets to peer {}",
|
||||||
"Detected {} new local assets, pushing to {}",
|
new_hashes.len(),
|
||||||
new_hashes.len(),
|
short_id
|
||||||
short_id
|
);
|
||||||
);
|
|
||||||
|
|
||||||
// Open a new stream for this batch
|
// Open a new QUIC stream for this batch
|
||||||
match conn.open_bi().await {
|
match conn.open_bi().await {
|
||||||
Ok((mut send, _recv)) => {
|
Ok((mut send, _recv)) => {
|
||||||
if let Err(e) = send_assets(&can, &mut send, &new_hashes, &short_id).await {
|
if let Err(e) = send_assets(&can, &mut send, &new_hashes, &short_id).await {
|
||||||
error!("Failed to push new assets to {}: {:#}", short_id, e);
|
error!("Failed to push new assets to {}: {:#}", short_id, e);
|
||||||
}
|
|
||||||
// Send done marker
|
|
||||||
let done_frame = encode_frame(MSG_DONE, &[]);
|
|
||||||
let _ = send.write_all(&done_frame).await;
|
|
||||||
let _ = send.flush().await;
|
|
||||||
let _ = send.finish();
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
warn!("Failed to open stream to {}: {:#}", short_id, e);
|
|
||||||
break; // Connection probably dead
|
|
||||||
}
|
}
|
||||||
|
let done_frame = encode_frame(MSG_DONE, &[]);
|
||||||
|
let _ = send.write_all(&done_frame).await;
|
||||||
|
let _ = send.flush().await;
|
||||||
|
let _ = send.finish();
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
warn!("Failed to open stream to {}: {:#}", short_id, e);
|
||||||
|
break; // Connection probably dead
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update our known set (union with current so we keep peer-received hashes too)
|
// Update known set
|
||||||
for h in current_hashes {
|
for h in new_hashes {
|
||||||
known_hashes.insert(h);
|
known_hashes.insert(h);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -393,10 +434,6 @@ async fn live_push_loop(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Accept incoming QUIC bi-streams from the peer and receive assets.
|
/// Accept incoming QUIC bi-streams from the peer and receive assets.
|
||||||
///
|
|
||||||
/// The peer's live_push_loop opens new bi-streams for each batch of assets.
|
|
||||||
/// This loop accepts those streams, ingests assets into the local CAN service,
|
|
||||||
/// and notifies the push loop via channel to prevent echo.
|
|
||||||
async fn live_receive_loop(
|
async fn live_receive_loop(
|
||||||
conn: Connection,
|
conn: Connection,
|
||||||
can: CanSyncClient,
|
can: CanSyncClient,
|
||||||
@ -412,7 +449,6 @@ async fn live_receive_loop(
|
|||||||
info!("Accepted live sync stream from peer {}", short_id);
|
info!("Accepted live sync stream from peer {}", short_id);
|
||||||
match receive_assets(&can, &mut recv, &short_id).await {
|
match receive_assets(&can, &mut recv, &short_id).await {
|
||||||
Ok(received_hashes) => {
|
Ok(received_hashes) => {
|
||||||
// Notify push loop about received hashes to prevent echo
|
|
||||||
for hash in received_hashes {
|
for hash in received_hashes {
|
||||||
let _ = received_tx.send(hash);
|
let _ = received_tx.send(hash);
|
||||||
}
|
}
|
||||||
|
|||||||
30
src/db.rs
30
src/db.rs
@ -457,6 +457,36 @@ pub fn get_all_assets(conn: &Connection) -> rusqlite::Result<Vec<Asset>> {
|
|||||||
Ok(assets)
|
Ok(assets)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Get assets with `timestamp > since` (for incremental sync queries).
|
||||||
|
pub fn get_assets_since(conn: &Connection, since: i64) -> rusqlite::Result<Vec<Asset>> {
|
||||||
|
let mut stmt = conn.prepare(
|
||||||
|
"SELECT id, timestamp, hash, mime_type, application, user_identity, description,
|
||||||
|
actual_filename, human_filename, human_path, is_trashed, is_corrupted, size
|
||||||
|
FROM assets WHERE timestamp > ?1
|
||||||
|
ORDER BY timestamp ASC",
|
||||||
|
)?;
|
||||||
|
let assets = stmt
|
||||||
|
.query_map([since], |row| {
|
||||||
|
Ok(Asset {
|
||||||
|
id: row.get(0)?,
|
||||||
|
timestamp: row.get(1)?,
|
||||||
|
hash: row.get(2)?,
|
||||||
|
mime_type: row.get(3)?,
|
||||||
|
application: row.get(4)?,
|
||||||
|
user_identity: row.get(5)?,
|
||||||
|
description: row.get(6)?,
|
||||||
|
actual_filename: row.get(7)?,
|
||||||
|
human_filename: row.get(8)?,
|
||||||
|
human_path: row.get(9)?,
|
||||||
|
is_trashed: row.get(10)?,
|
||||||
|
is_corrupted: row.get(11)?,
|
||||||
|
size: row.get(12)?,
|
||||||
|
})
|
||||||
|
})?
|
||||||
|
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||||
|
Ok(assets)
|
||||||
|
}
|
||||||
|
|
||||||
/// Get all non-trashed asset records (for verifier startup scan).
|
/// Get all non-trashed asset records (for verifier startup scan).
|
||||||
pub fn get_all_active_assets(conn: &Connection) -> rusqlite::Result<Vec<Asset>> {
|
pub fn get_all_active_assets(conn: &Connection) -> rusqlite::Result<Vec<Asset>> {
|
||||||
let mut stmt = conn.prepare(
|
let mut stmt = conn.prepare(
|
||||||
|
|||||||
@ -13,8 +13,13 @@ use std::sync::Arc;
|
|||||||
use crate::config::Config;
|
use crate::config::Config;
|
||||||
use crate::db::Db;
|
use crate::db::Db;
|
||||||
|
|
||||||
|
/// Broadcast channel for notifying sync subscribers about new assets.
|
||||||
|
/// Each message is `"hash:timestamp"` (e.g. `"abc123def456:1710000000000"`).
|
||||||
|
pub type SyncEventSender = tokio::sync::broadcast::Sender<String>;
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct AppState {
|
pub struct AppState {
|
||||||
pub config: Arc<Config>,
|
pub config: Arc<Config>,
|
||||||
pub db: Db,
|
pub db: Db,
|
||||||
|
pub sync_events: SyncEventSender,
|
||||||
}
|
}
|
||||||
|
|||||||
@ -41,9 +41,14 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
// Start background verifier
|
// Start background verifier
|
||||||
verifier::start((*config).clone(), db.clone());
|
verifier::start((*config).clone(), db.clone());
|
||||||
|
|
||||||
|
// Broadcast channel for SSE sync events (capacity doesn't matter much —
|
||||||
|
// slow receivers just miss events and do a full reconciliation on reconnect)
|
||||||
|
let (sync_events, _) = tokio::sync::broadcast::channel::<String>(256);
|
||||||
|
|
||||||
let state = AppState {
|
let state = AppState {
|
||||||
config: config.clone(),
|
config: config.clone(),
|
||||||
db,
|
db,
|
||||||
|
sync_events,
|
||||||
};
|
};
|
||||||
|
|
||||||
// Build router
|
// Build router
|
||||||
|
|||||||
@ -85,6 +85,13 @@ fn do_ingest(state: &AppState, input: IngestInput) -> Result<IngestResult, AppEr
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Notify SSE subscribers about the new asset
|
||||||
|
let event_data = format!(
|
||||||
|
r#"{{"hash":"{}","timestamp":{}}}"#,
|
||||||
|
file_hash, timestamp
|
||||||
|
);
|
||||||
|
let _ = state.sync_events.send(event_data);
|
||||||
|
|
||||||
Ok(IngestResult {
|
Ok(IngestResult {
|
||||||
timestamp,
|
timestamp,
|
||||||
hash: file_hash,
|
hash: file_hash,
|
||||||
|
|||||||
@ -2,14 +2,22 @@
|
|||||||
//!
|
//!
|
||||||
//! All endpoints require `X-Sync-Key` header matching `config.sync_api_key`.
|
//! All endpoints require `X-Sync-Key` header matching `config.sync_api_key`.
|
||||||
//! If `sync_api_key` is not configured, all endpoints return 404.
|
//! If `sync_api_key` is not configured, all endpoints return 404.
|
||||||
|
//!
|
||||||
|
//! Includes an SSE endpoint (`GET /sync/events`) that streams real-time
|
||||||
|
//! notifications when new assets are ingested.
|
||||||
|
|
||||||
|
use std::convert::Infallible;
|
||||||
|
|
||||||
use axum::body::Bytes;
|
use axum::body::Bytes;
|
||||||
use axum::extract::State;
|
use axum::extract::{Query, State};
|
||||||
use axum::http::{HeaderMap, StatusCode};
|
use axum::http::{HeaderMap, StatusCode};
|
||||||
|
use axum::response::sse::{Event, Sse};
|
||||||
use axum::response::IntoResponse;
|
use axum::response::IntoResponse;
|
||||||
use axum::routing::post;
|
use axum::routing::{get, post};
|
||||||
use axum::Router;
|
use axum::Router;
|
||||||
use prost::Message;
|
use prost::Message;
|
||||||
|
use tokio_stream::wrappers::BroadcastStream;
|
||||||
|
use tokio_stream::StreamExt;
|
||||||
|
|
||||||
use crate::models::{Asset, FileAttributes};
|
use crate::models::{Asset, FileAttributes};
|
||||||
use crate::{db, hash, storage, xattr, AppState};
|
use crate::{db, hash, storage, xattr, AppState};
|
||||||
@ -117,6 +125,14 @@ pub fn router() -> Router<AppState> {
|
|||||||
.route("/sync/pull", post(sync_pull))
|
.route("/sync/pull", post(sync_pull))
|
||||||
.route("/sync/push", post(sync_push))
|
.route("/sync/push", post(sync_push))
|
||||||
.route("/sync/meta", post(sync_meta))
|
.route("/sync/meta", post(sync_meta))
|
||||||
|
.route("/sync/events", get(sync_events))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Query params for /sync/hashes (optional `since` timestamp for incremental queries).
|
||||||
|
#[derive(serde::Deserialize, Default)]
|
||||||
|
struct HashesQuery {
|
||||||
|
/// Only return assets with `timestamp > since`. Omit or 0 for full list.
|
||||||
|
since: Option<i64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Auth ────────────────────────────────────────────────────────────────
|
// ── Auth ────────────────────────────────────────────────────────────────
|
||||||
@ -157,14 +173,22 @@ fn proto_response(buf: Vec<u8>) -> (StatusCode, [(&'static str, &'static str); 1
|
|||||||
async fn sync_hashes(
|
async fn sync_hashes(
|
||||||
State(state): State<AppState>,
|
State(state): State<AppState>,
|
||||||
headers: HeaderMap,
|
headers: HeaderMap,
|
||||||
|
query: Query<HashesQuery>,
|
||||||
_body: Bytes,
|
_body: Bytes,
|
||||||
) -> Result<impl IntoResponse, (StatusCode, String)> {
|
) -> Result<impl IntoResponse, (StatusCode, String)> {
|
||||||
check_sync_key(&state, &headers)?;
|
check_sync_key(&state, &headers)?;
|
||||||
|
|
||||||
|
let since = query.since.unwrap_or(0);
|
||||||
|
|
||||||
let assets = {
|
let assets = {
|
||||||
let conn = state.db.lock().unwrap();
|
let conn = state.db.lock().unwrap();
|
||||||
db::get_all_assets(&conn)
|
if since > 0 {
|
||||||
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("DB error: {}", e)))?
|
db::get_assets_since(&conn, since)
|
||||||
|
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("DB error: {}", e)))?
|
||||||
|
} else {
|
||||||
|
db::get_all_assets(&conn)
|
||||||
|
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("DB error: {}", e)))?
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let resp = HashListResponse {
|
let resp = HashListResponse {
|
||||||
@ -333,6 +357,13 @@ async fn sync_push(
|
|||||||
|
|
||||||
tracing::info!("Sync push: ingested {} ({}B)", &bundle.hash[..12], bundle.content.len());
|
tracing::info!("Sync push: ingested {} ({}B)", &bundle.hash[..12], bundle.content.len());
|
||||||
|
|
||||||
|
// Notify SSE subscribers about the new asset
|
||||||
|
let event_data = format!(
|
||||||
|
r#"{{"hash":"{}","timestamp":{}}}"#,
|
||||||
|
bundle.hash, bundle.timestamp
|
||||||
|
);
|
||||||
|
let _ = state.sync_events.send(event_data);
|
||||||
|
|
||||||
Ok(proto_response(encode_proto(&PushResponse {
|
Ok(proto_response(encode_proto(&PushResponse {
|
||||||
hash: bundle.hash,
|
hash: bundle.hash,
|
||||||
already_existed: false,
|
already_existed: false,
|
||||||
@ -379,3 +410,57 @@ async fn sync_meta(
|
|||||||
success: true,
|
success: true,
|
||||||
})?))
|
})?))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── GET /sync/events (SSE) ────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// Server-Sent Events endpoint. Streams `new_asset` events whenever a file is
|
||||||
|
/// ingested (via public API or sync push). Requires `X-Sync-Key` as a query
|
||||||
|
/// param (`?key=...`) since SSE/EventSource doesn't support custom headers.
|
||||||
|
///
|
||||||
|
/// Each event is:
|
||||||
|
/// ```text
|
||||||
|
/// event: new_asset
|
||||||
|
/// data: {"hash":"abc...","timestamp":1710000000000}
|
||||||
|
/// ```
|
||||||
|
async fn sync_events(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
headers: HeaderMap,
|
||||||
|
query: Query<SseQuery>,
|
||||||
|
) -> Result<Sse<impl tokio_stream::Stream<Item = Result<Event, Infallible>>>, (StatusCode, String)>
|
||||||
|
{
|
||||||
|
// SSE clients (EventSource) can't set custom headers, so accept key from query param too
|
||||||
|
let key_ok = check_sync_key(&state, &headers).is_ok()
|
||||||
|
|| query
|
||||||
|
.key
|
||||||
|
.as_deref()
|
||||||
|
.map(|k| {
|
||||||
|
state
|
||||||
|
.config
|
||||||
|
.sync_api_key
|
||||||
|
.as_deref()
|
||||||
|
.map(|expected| k == expected)
|
||||||
|
.unwrap_or(false)
|
||||||
|
})
|
||||||
|
.unwrap_or(false);
|
||||||
|
|
||||||
|
if !key_ok {
|
||||||
|
return Err((StatusCode::UNAUTHORIZED, "Invalid sync key".into()));
|
||||||
|
}
|
||||||
|
|
||||||
|
let rx = state.sync_events.subscribe();
|
||||||
|
let stream = BroadcastStream::new(rx).filter_map(|result| match result {
|
||||||
|
Ok(data) => Some(Ok(Event::default().event("new_asset").data(data))),
|
||||||
|
Err(_) => None, // lagged — skip missed events, client will reconcile
|
||||||
|
});
|
||||||
|
|
||||||
|
Ok(Sse::new(stream).keep_alive(
|
||||||
|
axum::response::sse::KeepAlive::new()
|
||||||
|
.interval(std::time::Duration::from_secs(15))
|
||||||
|
.text("ping"),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(serde::Deserialize, Default)]
|
||||||
|
struct SseQuery {
|
||||||
|
key: Option<String>,
|
||||||
|
}
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user