Skip to main content

sioul_sync/
remote.rs

1// SPDX-License-Identifier: GPL-3.0-or-later
2// Copyright © 2026 Aurélien Pierre
3
4//! The other devices' files fetched from the server too (docs/database.md,
5//! "Fetched from the server too"). The sharing folder stays the one a sync app
6//! carries, whichever it is; when that folder lives on a WebDAV server Sioul
7//! has an account for (Nextcloud, Murena, ownCloud), Sioul also reads it there,
8//! in case the sync app is late bringing the others' files, or stops: Murena's
9//! eDrive was seen bringing a computer's records an hour and a half late, and
10//! its entry among the devices never.
11//!
12//! - **Found** on the servers of the contacts-and-calendars accounts that keep
13//!   files Nextcloud's way (`…/remote.php/dav/files/<user>/`, the user as the
14//!   server names it): the place given by hand, else the folder's path from a
15//!   phone's storage (eDrive carries it as the cloud's root), from a folder the
16//!   Nextcloud client carries (`localPath` → `targetPath`), or its last names.
17//! - **Confirmed** only where the server's `seal.toml` is this folder's, byte
18//!   for byte: a folder sealed otherwise is never read. Read again when it
19//!   changes there, and every six hours.
20//! - **Fetched** when the window or the phone's background step asks
21//!   (`pull`): `PROPFIND` (`Depth: 1`) on the folder, `devices/`, `leases/` and
22//!   each part's claims, a folder whose ETag did not change skipped for ten
23//!   minutes; of the other devices' files, only those newer than both the
24//!   synced folder's copy and what was fetched before come down, into this
25//!   device's own state (`$XDG_STATE_HOME/sioul/share/remote/`, their paths as
26//!   in the folder): records from where the copy here ends (its last line
27//!   asked again and checked: a copy that does not join comes whole), only
28//!   whole lines kept; the rest whole, a few hundred bytes each, kept only when
29//!   it came whole. Ten seconds for each wait of the network, thirty for a pull.
30//! - **Read** (`overlay`): the sharing reads, for each other device's file, the
31//!   newer of the two copies: records the longer (they only grow), notes to
32//!   each other, entries and claims by the times they hold. The folder's copy
33//!   wins a tie. What was fetched goes once the folder's copy caught up.
34//! - **Sent too** (`send`, docs/database.md, "Sent to the server too"): this
35//!   device's own files, beside the sync app, each over what is there as it
36//!   was seen; found there the same (the sync app sent it), not sent twice;
37//!   records never shorter; never another device's file.
38//! - **Never**: another device's file written, moved or deleted on the
39//!   server; anything written into the synced folder; this device's own files
40//!   fetched or read from there; plain HTTP (a test's own stand-in apart); the
41//!   password kept or said (it comes from the keyring, as the calendars' sync
42//!   has it, for the requests only).
43
44use crate::SyncError;
45use crate::dav::{self, DAV};
46use serde::{Deserialize, Serialize};
47use std::collections::{BTreeMap, BTreeSet};
48use std::io::{Read, Seek, SeekFrom, Write};
49use std::path::{Path, PathBuf};
50use std::sync::Mutex;
51use std::time::{Duration, Instant, SystemTime};
52
53/// How long the network may take: each wait (to connect, for the next
54/// bytes); a listing or a small file in all; a records' file in all, as long
55/// as it moves; a whole pull.
56#[derive(Debug, Clone, Copy)]
57pub(crate) struct Limits {
58    pub(crate) wait: Duration,
59    pub(crate) small: Duration,
60    pub(crate) large: Duration,
61    pub(crate) pull: Duration,
62    /// A sealed note or paper, as long as it moves.
63    pub(crate) blob: Duration,
64    /// A file sent (`Server::put_file`): given `large`, and as long more as
65    /// it takes at `floor` bytes a second; cut when nothing moved for `stall`.
66    pub(crate) floor: u64,
67    pub(crate) stall: Duration,
68}
69
70pub(crate) const LIMITS: Limits = Limits { wait: Duration::from_secs(10), small: Duration::from_secs(10), large: Duration::from_secs(60), pull: Duration::from_secs(30), blob: Duration::from_secs(15 * 60), floor: FLOOR, stall: Duration::from_secs(30) };
71
72/// The slowest upload Sioul waits for (bytes a second): a phone's mobile
73/// data on 3G sends faster. A 2.5 MB file is given 60 s and 156 s more.
74pub const FLOOR: u64 = 16 << 10;
75
76/// Sending this device's files (`send`): a step that sends a large file on a
77/// slow line is not cut after thirty seconds, as a pull is; each file is
78/// given the time its size asks (`FLOOR`).
79pub(crate) const SEND_LIMITS: Limits = Limits { wait: Duration::from_secs(20), small: Duration::from_secs(30), large: Duration::from_secs(60), pull: Duration::from_secs(20 * 60), blob: Duration::from_secs(15 * 60), floor: FLOOR, stall: Duration::from_secs(30) };
80
81/// Looked for again this long after the last look, when not found (seconds).
82pub const LOOK_AGAIN: i64 = 6 * 3600;
83/// The server's seal read again this long after the last time, even unchanged (seconds).
84const SEAL_AGAIN: i64 = 6 * 3600;
85/// A folder whose ETag did not change is looked through again after this long all the same (seconds).
86const LIST_AGAIN: i64 = 10 * 60;
87/// A records' file is never fetched past this.
88const LARGEST: u64 = 64 << 20;
89/// A device's texts to send (`texts/send/<id>.jsonl`) are never fetched past this.
90const TEXTS_LARGEST: u64 = 4 << 20;
91/// A listing is never read past this.
92const LISTING: u64 = 4 << 20;
93/// The last line asked again, to check the copy here joins what is there, when shorter than this.
94const OVERLAP: u64 = 1 << 20;
95
96const XML: &str = "application/xml; charset=utf-8";
97const LIST: &str = r#"<?xml version="1.0" encoding="utf-8"?><d:propfind xmlns:d="DAV:" xmlns:oc="http://owncloud.org/ns"><d:prop><d:resourcetype/><d:getetag/><d:getcontentlength/><oc:checksums/></d:prop></d:propfind>"#;
98/// Nextcloud's own properties: the checksums a file was sent with (`oc:checksums`).
99const OC: &str = "http://owncloud.org/ns";
100const PRINCIPAL: &str = r#"<?xml version="1.0" encoding="utf-8"?><d:propfind xmlns:d="DAV:"><d:prop><d:current-user-principal/></d:prop></d:propfind>"#;
101
102// ---------------------------------------------------------------- where
103
104/// Where the backup keeps what it fetched, beside the sharing's memory: `<state>/share/remote/`.
105pub fn cache_of(memory: &Path) -> PathBuf {
106    memory.with_file_name("remote")
107}
108
109/// What the backup knows, beside it: `<state>/share/remote.toml`.
110fn state_path(memory: &Path) -> PathBuf {
111    memory.with_file_name("remote.toml")
112}
113
114fn shown(folder: &Path) -> String {
115    folder.display().to_string()
116}
117
118/// A server's name, to say where: "murena.io".
119pub fn host_of(url: &str) -> String {
120    let rest = url.split_once("://").map_or(url, |(_, rest)| rest);
121    let authority = rest.split(['/', '?', '#']).next().unwrap_or("");
122    authority.rsplit('@').next().unwrap_or("").to_string()
123}
124
125// ---------------------------------------------------------------- the state
126
127/// What the backup knows, kept privately beside the sharing's memory
128/// (`remote.toml`), never shared. `on`, `given` and `asked` are yours,
129/// written only through `State::choose`; the rest is the backup's.
130#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
131pub struct State {
132    /// Switched on or off here; unsaid, on.
133    #[serde(default, skip_serializing_if = "Option::is_none")]
134    pub on: Option<bool>,
135    /// The folder's place on the server, given by hand: a path in an
136    /// account's files ("Documents/Sioul"), or an address; "" to look by itself.
137    #[serde(default, skip_serializing_if = "String::is_empty")]
138    pub given: String,
139    /// When you asked to look again (Unix seconds).
140    #[serde(default)]
141    pub asked: i64,
142    /// This device's own files sent there too, beside the sync app (`send`):
143    /// as switched here, else on wherever the backup is.
144    #[serde(default, skip_serializing_if = "Option::is_none")]
145    pub send: Option<bool>,
146    /// The sharing folder here it backs.
147    #[serde(default, skip_serializing_if = "String::is_empty")]
148    pub folder: String,
149    /// The account whose server holds it, by its id, and the folder there (an address ending with "/").
150    #[serde(default, skip_serializing_if = "String::is_empty")]
151    pub account: String,
152    #[serde(default, skip_serializing_if = "String::is_empty")]
153    pub url: String,
154    /// Since when the server's seal is found to be this folder's (Unix seconds); 0: not confirmed.
155    #[serde(default)]
156    pub confirmed: i64,
157    /// When the server's seal was last read and found the same (Unix seconds).
158    #[serde(default)]
159    pub sealed: i64,
160    /// When the folder was last looked for (Unix seconds).
161    #[serde(default)]
162    pub looked: i64,
163    /// When a pull last went through, and was last tried (Unix seconds).
164    #[serde(default)]
165    pub last: i64,
166    #[serde(default)]
167    pub tried: i64,
168    /// Why nothing is fetched, or the last pull failed, in a code
169    /// ("not-found:murena.io", "network:murena.io"); "" when all is well.
170    #[serde(default, skip_serializing_if = "String::is_empty")]
171    pub said: String,
172    /// The other devices' records on the server past what a pull brings
173    /// (`LARGEST`), longer there than here, by name, as the last look through
174    /// the folder found them: only the sync app brings them, and the pull
175    /// alone is not trusted meanwhile (`pulls_well`).
176    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
177    pub left: BTreeMap<String, u64>,
178    /// This device's entry on the server (`devices/<id>.device`) as a pull
179    /// last listed it: its size, and when (Unix seconds). Of another size than
180    /// this device last sent, listed after that send, it is an older copy put
181    /// back there: looked at again by the next send, and sent over it (`send`).
182    #[serde(default, skip_serializing_if = "Option::is_none")]
183    pub own_entry: Option<(u64, i64)>,
184    /// `MIRROR` when Sioul keeps the folder in step itself, no sync app: the
185    /// folder here is its own copy (`mirror_of`), the server's is the one the
186    /// devices share; "" beside a sync app's folder.
187    #[serde(default, skip_serializing_if = "String::is_empty")]
188    pub mode: String,
189    /// The folder's place in the account's files, as given ("Documents/Sioul"): to say where.
190    #[serde(default, skip_serializing_if = "String::is_empty")]
191    pub place: String,
192    /// When this device's own files last went up (`MIRROR`, Unix seconds).
193    #[serde(default)]
194    pub pushed: i64,
195    /// The server's files as last fetched, or found here already, by their path in the folder.
196    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
197    pub files: BTreeMap<String, Fetched>,
198    /// The server's folders as last looked through, by their path ("" the folder itself, "devices/").
199    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
200    pub folders: BTreeMap<String, Looked>,
201    /// This device's own files as they last went up (`MIRROR`), by their path in the folder.
202    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
203    pub sent: BTreeMap<String, Sent>,
204}
205
206/// A file of the server, as last fetched: its ETag and size there.
207#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
208pub struct Fetched {
209    #[serde(default)]
210    pub etag: String,
211    #[serde(default)]
212    pub size: u64,
213}
214
215/// A folder of the server, as last looked through: its ETag, and when (Unix seconds).
216#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
217pub struct Looked {
218    #[serde(default)]
219    pub etag: String,
220    #[serde(default)]
221    pub at: i64,
222}
223
224/// One of this device's own files as it last went up (`MIRROR`): its size and
225/// time here (nanoseconds), its ETag there ("" when the server did not say).
226#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
227pub struct Sent {
228    #[serde(default)]
229    pub size: u64,
230    #[serde(default)]
231    pub modified: u64,
232    #[serde(default)]
233    pub etag: String,
234}
235
236impl State {
237    pub fn load(memory: &Path) -> State {
238        std::fs::read_to_string(state_path(memory)).ok().and_then(|text| toml::from_str(&text).ok()).unwrap_or_default()
239    }
240
241    fn write(&self, memory: &Path) -> Result<(), String> {
242        let text = toml::to_string(self).map_err(|e| e.to_string())?;
243        crate::share::write_atomically(&state_path(memory), format!("# Sioul: the sharing folder fetched from its server too (docs/database.md). Never shared.\n{text}").as_bytes())
244    }
245
246    /// Written as the backup's work left it, under its lock: what you chose
247    /// meanwhile (`on`, `given`, `asked`) kept as the file has it.
248    fn save(&self, memory: &Path) -> Result<(), String> {
249        let path = state_path(memory);
250        sioul_core::filelock::with_lock(&path, || {
251            let there = State::load(memory);
252            State { on: there.on, given: there.given, asked: there.asked, send: there.send, ..self.clone() }.write(memory)
253        })
254    }
255
256    /// What a pull found, written over the state as it is now, under its lock:
257    /// two processes pull (the window's or an alarm's, and the background
258    /// step's), and neither undoes the other. The pull begun later says when
259    /// a pull was tried and how it went; the latest that went through is kept;
260    /// what was fetched and looked through, each file's and folder's latest;
261    /// what is left to the sync app (`left`): as it is now, with what this
262    /// pull found left added (new, or at another size) and what it found
263    /// brought taken out, from what it knew when it began (`loaded_left`):
264    /// another pull's finding meanwhile is
265    /// never erased, only a look that finds it brought clears it; the rest
266    /// (your choices, the place) as it is now. A place changed meanwhile
267    /// (found elsewhere): what this pull found is not that place's, left out.
268    /// `ended`: this pull's last moment by the clock it began by (Unix
269    /// seconds); one said begun after that began after this pull ended,
270    /// which no pull of the other process can: the clock was set back since,
271    /// and this pull is the later.
272    fn save_pulled(&self, memory: &Path, ended: i64, loaded_left: &BTreeMap<String, u64>) -> Result<(), String> {
273        let path = state_path(memory);
274        sioul_core::filelock::with_lock(&path, || {
275            let mut merged = State::load(memory);
276            if (merged.url.as_str(), merged.folder.as_str(), merged.account.as_str()) != (self.url.as_str(), self.folder.as_str(), self.account.as_str()) && !merged.url.is_empty() {
277                return Ok(());
278            }
279            let ours_later = self.tried >= merged.tried || merged.tried > ended;
280            let (later, earlier) = if ours_later { (self.clone(), merged.clone()) } else { (merged.clone(), self.clone()) };
281            merged.url = later.url.clone();
282            merged.folder = later.folder.clone();
283            merged.account = later.account.clone();
284            merged.tried = later.tried;
285            // A time later than this pull's end dates from before the clock was set back: not a later pull.
286            merged.last = [later.last, earlier.last].into_iter().filter(|last| *last <= ended).max().unwrap_or(self.last);
287            merged.said = later.said.clone();
288            merged.confirmed = later.confirmed;
289            merged.sealed = later.sealed.max(earlier.sealed);
290            merged.files = earlier.files.into_iter().chain(later.files).collect();
291            merged.folders = earlier.folders.into_iter().chain(later.folders).collect();
292            let brought: Vec<&String> = loaded_left.keys().filter(|name| !self.left.contains_key(*name)).collect();
293            merged.left.retain(|name, _| !brought.contains(&name));
294            // Found by this pull: new, or found again at another size (grown there since).
295            merged.left.extend(self.left.iter().filter(|(name, size)| loaded_left.get(*name) != Some(*size)).map(|(name, size)| (name.clone(), *size)));
296            merged.own_entry = [earlier.own_entry, later.own_entry].into_iter().flatten().max_by_key(|(_, at)| *at);
297            merged.write(memory)
298        })
299    }
300
301    /// What you choose (the switch, the place given by hand), changed under its lock.
302    pub fn choose(memory: &Path, change: impl FnOnce(&mut State)) -> Result<State, String> {
303        let path = state_path(memory);
304        sioul_core::filelock::with_lock(&path, || {
305            let mut state = State::load(memory);
306            change(&mut state);
307            state.write(memory)?;
308            Ok(state)
309        })
310    }
311
312    /// Its server's seal was found to be this folder's.
313    pub fn confirmed_for(&self, folder: &Path) -> bool {
314        self.confirmed > 0 && !self.url.is_empty() && self.folder == shown(folder)
315    }
316
317    /// Fetching: as switched here, else on.
318    pub fn fetching(&self) -> bool {
319        self.on != Some(false)
320    }
321
322    /// Pulling from the server works for `folder` beside a sync app: switched
323    /// on, the folder confirmed there, a pull gone through and the last one
324    /// too (`said` empty). A phone's background step then reads the other
325    /// devices' news from its pull alone, the sync app neither asked to look
326    /// nor waited for (docs/android.md, "In the background"); otherwise, as
327    /// before: the sync app asked, and given twenty seconds.
328    pub fn pulls_well(&self, folder: &Path) -> bool {
329        self.mode != MIRROR && self.fetching() && self.confirmed_for(folder) && self.last > 0 && self.said.is_empty() && self.left.is_empty()
330    }
331
332    /// A pull begun at or after `since` and not after `now` (Unix seconds)
333    /// went through, with nothing left on the server that only the sync app
334    /// brings: what a dose's or a waking's alarm trusts alone (`alarm_news`).
335    /// A pull said begun after now is one from before the clock was set back:
336    /// not since, pulled again.
337    pub fn went_through_since(&self, folder: &Path, since: i64, now: i64) -> bool {
338        self.pulls_well(folder) && (since..=now).contains(&self.last)
339    }
340
341    /// Sending this device's own files there too (`send`): with the backup
342    /// on, as switched here, else on.
343    pub fn sending(&self) -> bool {
344        self.fetching() && self.send != Some(false)
345    }
346
347    /// The server, by its name: "murena.io".
348    pub fn host(&self) -> String {
349        host_of(&self.url)
350    }
351
352    /// Whether to look for the folder now: asked to (a place given by hand,
353    /// the switch turned on), another folder shared here, or not confirmed
354    /// and never looked for, or looked for `LOOK_AGAIN` ago.
355    pub fn look_due(&self, folder: &Path, now: i64) -> bool {
356        self.folder != shown(folder) || self.asked > self.looked || (!self.confirmed_for(folder) && (self.looked == 0 || now - self.looked >= LOOK_AGAIN))
357    }
358}
359
360// ---------------------------------------------------------------- read here
361
362/// The cache attached to each sharing folder in this process (`attach`), and
363/// what that was decided on: the backup's state, this folder's seal, the
364/// cache's (their sizes and times).
365type Stamp = [Option<(u64, SystemTime)>; 3];
366static ATTACHED: Mutex<BTreeMap<PathBuf, (Stamp, Option<PathBuf>)>> = Mutex::new(BTreeMap::new());
367
368fn stamp_of(path: &Path) -> Option<(u64, SystemTime)> {
369    std::fs::metadata(path).ok().map(|m| (m.len(), m.modified().unwrap_or(SystemTime::UNIX_EPOCH)))
370}
371
372/// The copy fetched for the sharing folder `folder`, read beside it from now
373/// on in this process (`overlay`), when there is one for it: its state names
374/// this folder, and what was fetched was fetched under this folder's seal
375/// (the cache's `seal.toml`, written when the server's was found the same).
376/// Cheap when nothing changed; nothing of it needs the network. Returns it.
377pub fn attach(folder: &Path, memory: &Path) -> Option<PathBuf> {
378    let cache = cache_of(memory);
379    let stamp = [stamp_of(&state_path(memory)), stamp_of(&folder.join("seal.toml")), stamp_of(&cache.join("seal.toml"))];
380    let mut attached = ATTACHED.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
381    if let Some((was, chosen)) = attached.get(folder)
382        && *was == stamp
383    {
384        return chosen.clone();
385    }
386    let chosen = usable(folder, memory).then_some(cache);
387    attached.insert(folder.to_path_buf(), (stamp, chosen.clone()));
388    chosen
389}
390
391/// The copy fetched for `folder`, when one is attached in this process: the
392/// sharing reads, for each other device's file, the newer of the two.
393pub fn overlay(folder: &Path) -> Option<PathBuf> {
394    ATTACHED.lock().ok()?.get(folder).and_then(|(_, chosen)| chosen.clone())
395}
396
397/// The seal of a folder, its bytes: at most a small file's.
398fn seal_of(folder: &Path) -> Option<Vec<u8>> {
399    let path = folder.join("seal.toml");
400    let size = std::fs::metadata(&path).ok()?.len();
401    (size <= crate::share::SMALL_FILE).then(|| std::fs::read(&path).ok()).flatten()
402}
403
404/// What was fetched for this folder may be read: fetched for it, under its seal.
405fn usable(folder: &Path, memory: &Path) -> bool {
406    let state = State::load(memory);
407    state.mode != MIRROR && state.folder == shown(folder) && seal_of(folder).is_some_and(|here| seal_of(&cache_of(memory)).is_some_and(|fetched| fetched == here))
408}
409
410/// Sharing stopped here: what was fetched and known goes, and nothing is read beside the folder.
411pub fn forget(memory: &Path, folder: &Path) {
412    let _ = std::fs::remove_dir_all(cache_of(memory));
413    let _ = std::fs::remove_file(state_path(memory));
414    let _ = std::fs::remove_file(crate::blobs::Wanted::path(memory));
415    let _ = std::fs::remove_file(Lane::Main.memory(memory));
416    let _ = std::fs::remove_file(Lane::Urgent.memory(memory));
417    if let Ok(mut attached) = ATTACHED.lock() {
418        attached.remove(folder);
419    }
420}
421
422// ---------------------------------------------------------------- the server
423
424/// An account to fetch with: its id in the configuration, where its server
425/// was found (`Account::url`), its login, and its password from the keyring
426/// (none kept there): used for the requests only, never kept nor said.
427pub struct Login {
428    pub account: String,
429    pub url: String,
430    pub user: String,
431    pub password: Option<String>,
432}
433
434/// One server, signed in: requests with the account's login, over TLS (plain
435/// HTTP only to a test's own stand-in, `dav::allowed`), never redirected. Its
436/// requests go through the process's one agent (`dav::shared_agent`): a pull,
437/// the sends after it and the next pull reuse one connection while it is
438/// open, each request with its own budget (`limits`).
439pub(crate) struct Server {
440    agent: ureq::Agent,
441    authorization: String,
442    started: Instant,
443    limits: Limits,
444}
445
446/// A folder of the server, as listed: its own ETag, and what it holds by name.
447#[derive(Debug, Default)]
448pub(crate) struct Listing {
449    pub(crate) etag: String,
450    pub(crate) items: BTreeMap<String, Item>,
451}
452
453#[derive(Debug, Default, Clone)]
454pub(crate) struct Item {
455    pub(crate) dir: bool,
456    pub(crate) etag: String,
457    pub(crate) size: u64,
458    /// The checksums the server keeps of it, as it says them ("SHA1:… MD5:…"); "" when none.
459    pub(crate) checksum: String,
460}
461
462/// What came of a file asked from a place in it: where it starts, its bytes,
463/// whether all came, and why not.
464pub(crate) struct Got {
465    start: u64,
466    bytes: Vec<u8>,
467    error: Option<SyncError>,
468}
469
470/// A file that did not go (`Server::put_file`): the error, what kind of
471/// failure ("timeout", "stalled", "unreachable", "tls", "login", "network"),
472/// how long the try lasted (seconds), how many of its bytes went, of how many.
473#[derive(Debug)]
474pub(crate) struct PutFailed {
475    pub(crate) error: SyncError,
476    pub(crate) kind: &'static str,
477    pub(crate) words: String,
478    pub(crate) seconds: u64,
479    pub(crate) sent: u64,
480    pub(crate) total: u64,
481}
482
483/// A body read from memory, counting what went: what a failure says it sent.
484struct Counted {
485    inner: std::io::Cursor<Vec<u8>>,
486    moved: std::sync::Arc<std::sync::atomic::AtomicU64>,
487}
488
489impl Read for Counted {
490    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
491        let n = self.inner.read(buf)?;
492        self.moved.fetch_add(n as u64, std::sync::atomic::Ordering::Relaxed);
493        Ok(n)
494    }
495}
496
497fn failed(e: &ureq::Error) -> SyncError {
498    match e {
499        ureq::Error::Tls(_) => SyncError::Tls(e.to_string()),
500        _ if format!("{e:?}").starts_with("Rustls") => SyncError::Tls(e.to_string()),
501        _ => SyncError::Network(e.to_string()),
502    }
503}
504
505fn header(response: &ureq::http::Response<ureq::Body>, name: &str) -> Option<String> {
506    response.headers().get(name).and_then(|v| v.to_str().ok()).map(str::to_string)
507}
508
509impl Server {
510    pub(crate) fn new(login: &Login, limits: Limits) -> Result<Server, SyncError> {
511        let password = login.password.as_deref().ok_or(SyncError::NoPassword)?;
512        Ok(Server {
513            agent: dav::shared_agent(),
514            authorization: format!("Basic {}", sioul_core::lines::base64_encode(format!("{}:{password}", login.user).as_bytes())),
515            started: Instant::now(),
516            limits,
517        })
518    }
519
520    pub(crate) fn request(&self, large: bool, method: &str, url: &str, headers: &[(&str, &str)], body: Vec<u8>) -> Result<ureq::http::Response<ureq::Body>, SyncError> {
521        self.request_with(if large { self.limits.large } else { self.limits.small }, method, url, headers, body)
522    }
523
524    /// A request run with `whole` for all of it, `limits.wait` to connect and
525    /// for each read or write (`stalls::ask`, its answer's body read after
526    /// included).
527    fn request_with(&self, whole: Duration, method: &str, url: &str, headers: &[(&str, &str)], body: Vec<u8>) -> Result<ureq::http::Response<ureq::Body>, SyncError> {
528        dav::allowed(url)?;
529        if self.started.elapsed() > self.limits.pull {
530            return Err(SyncError::Network("out of time".into()));
531        }
532        let mut request = ureq::http::Request::builder().method(method).uri(url).header("Authorization", &self.authorization).header("User-Agent", "Sioul");
533        for (name, value) in headers {
534            request = request.header(*name, *value);
535        }
536        let request = request.body(body).map_err(|e| SyncError::Server(e.to_string()))?;
537        let request = self.agent.configure_request(request).timeout_connect(Some(self.limits.wait)).timeout_global(Some(whole)).build();
538        crate::stalls::ask(self.limits.wait, false);
539        let response = self.agent.run(request).map_err(|e| failed(&e))?;
540        if response.status().as_u16() == 401 {
541            return Err(SyncError::Login(format!("{method}: 401")));
542        }
543        Ok(response)
544    }
545
546    /// A folder's own ETag and what it holds (`Depth: 1`), by name.
547    pub(crate) fn list(&self, url: &str) -> Result<Listing, SyncError> {
548        let mut response = self.request(false, "PROPFIND", url, &[("Depth", "1"), ("Content-Type", XML)], LIST.as_bytes().to_vec())?;
549        match response.status().as_u16() {
550            207 => {}
551            404 => return Err(SyncError::NotFound(String::new())),
552            status => return Err(SyncError::Server(format!("PROPFIND: {status}"))),
553        }
554        let body = response.body_mut().with_config().limit(LISTING).read_to_string().map_err(|e| failed(&e))?;
555        let (responses, _) = dav::multistatus(&body)?;
556        let sums = checksums(&body, url);
557        let me = dav::path_key(url);
558        let mut listing = Listing::default();
559        for response in responses {
560            let path = dav::path_key(&dav::absolute(url, &response.href));
561            let etag = response.text(DAV, "getetag").unwrap_or_default();
562            if path == me {
563                listing.etag = etag;
564                continue;
565            }
566            let Some(name) = path.strip_prefix(&me).and_then(|rest| rest.strip_prefix('/')).filter(|n| !n.is_empty() && !n.contains('/')) else { continue };
567            let size = response.text(DAV, "getcontentlength").and_then(|t| t.parse().ok()).unwrap_or(0);
568            listing.items.insert(name.to_string(), Item { dir: response.is(DAV, "collection"), etag, size, checksum: sums.get(&path).cloned().unwrap_or_default() });
569        }
570        Ok(listing)
571    }
572
573    /// One file as the server has it now (`Depth: 0`): its ETag, size and
574    /// checksums; none when it is not there.
575    pub(crate) fn stat(&self, url: &str) -> Result<Option<Item>, SyncError> {
576        let mut response = self.request(false, "PROPFIND", url, &[("Depth", "0"), ("Content-Type", XML)], LIST.as_bytes().to_vec())?;
577        match response.status().as_u16() {
578            207 => {}
579            404 => return Ok(None),
580            status => return Err(SyncError::Server(format!("PROPFIND: {status}"))),
581        }
582        let body = response.body_mut().with_config().limit(LISTING).read_to_string().map_err(|e| failed(&e))?;
583        let (responses, _) = dav::multistatus(&body)?;
584        let sums = checksums(&body, url);
585        Ok(responses.first().map(|r| Item {
586            dir: r.is(DAV, "collection"),
587            etag: r.text(DAV, "getetag").unwrap_or_default(),
588            size: r.text(DAV, "getcontentlength").and_then(|t| t.parse().ok()).unwrap_or(0),
589            checksum: sums.get(&dav::path_key(&dav::absolute(url, &r.href))).cloned().unwrap_or_default(),
590        }))
591    }
592
593    /// One of this device's files sent whole beside a sync app (`send`): over
594    /// what is there as it was seen (`If-Match` its ETag, `If-None-Match: *`
595    /// when none was there), dated as the file here (`X-OC-Mtime`), with its
596    /// SHA-1 (`OC-Checksum`): a Nextcloud client finding the same file there
597    /// as here, same size, date and checksum, takes it as its own, without a
598    /// download and without a conflicted copy. Its status, and its ETag there.
599    ///
600    /// Given the time its size asks: `limits.large`, and as long more as it
601    /// takes at `limits.floor` bytes a second (a file still moving is not cut
602    /// at a flat minute); cut when nothing moved for `limits.stall`. Failed:
603    /// why, how long it lasted and how many of its bytes went (`PutFailed`).
604    pub(crate) fn put_file(&self, url: &str, body: Vec<u8>, seen: Option<&str>, modified_s: u64, sha1: &str) -> Result<(u16, Option<String>), PutFailed> {
605        let total = body.len() as u64;
606        let started = Instant::now();
607        let gave_up = |error: SyncError, kind: &'static str, sent: u64| PutFailed { words: error.to_string(), error, kind, seconds: started.elapsed().as_millis().div_ceil(1000) as u64, sent, total };
608        dav::allowed(url).map_err(|e| gave_up(e, "refused", 0))?;
609        if self.started.elapsed() > self.limits.pull {
610            return Err(gave_up(SyncError::Network("out of time".into()), "out-of-time", 0));
611        }
612        let precondition = match seen.filter(|etag| !etag.is_empty()) {
613            Some(etag) => ("If-Match", etag),
614            None => ("If-None-Match", "*"),
615        };
616        let whole = self.limits.large + Duration::from_secs(total / self.limits.floor.max(1));
617        let moved = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0));
618        let reader = Counted { inner: std::io::Cursor::new(body), moved: std::sync::Arc::clone(&moved) };
619        let request = ureq::http::Request::builder()
620            .method("PUT")
621            .uri(url)
622            .header("Authorization", &self.authorization)
623            .header("User-Agent", "Sioul")
624            .header(precondition.0, precondition.1)
625            .header("Content-Type", "application/octet-stream")
626            .header("Content-Length", total.to_string())
627            .header("X-OC-Mtime", modified_s.to_string())
628            .header("OC-Checksum", format!("SHA1:{sha1}"))
629            .body(ureq::SendBody::from_owned_reader(reader))
630            .map_err(|e| gave_up(SyncError::Server(e.to_string()), "server", 0))?;
631        // Given `whole`, each write `limits.stall` at most; the answer awaited
632        // within `whole` alone while the last bytes drain from the system's
633        // buffers on a slow uplink.
634        let request = self.agent.configure_request(request).timeout_connect(Some(self.limits.wait)).timeout_global(Some(whole)).build();
635        crate::stalls::ask(self.limits.stall, true);
636        match self.agent.run(request) {
637            Ok(response) if response.status().as_u16() == 401 => Err(gave_up(SyncError::Login("PUT: 401".into()), "login", moved.load(std::sync::atomic::Ordering::Relaxed))),
638            Ok(response) => Ok((response.status().as_u16(), header(&response, "ETag").or_else(|| header(&response, "OC-ETag")))),
639            Err(e) => {
640                let sent = moved.load(std::sync::atomic::Ordering::Relaxed);
641                let kind = match &e {
642                    ureq::Error::Timeout(ureq::Timeout::Connect | ureq::Timeout::Resolve) | ureq::Error::ConnectionFailed | ureq::Error::HostNotFound => "unreachable",
643                    // Nothing moved for `stall`, well before the time given.
644                    ureq::Error::Timeout(_) | ureq::Error::Io(_) if started.elapsed() + Duration::from_secs(2) < whole && sent > 0 => "stalled",
645                    ureq::Error::Timeout(_) => "timeout",
646                    ureq::Error::Tls(_) => "tls",
647                    _ if format!("{e:?}").starts_with("Rustls") => "tls",
648                    _ => "network",
649                };
650                Err(gave_up(failed(&e), kind, sent))
651            }
652        }
653    }
654
655    /// A small file whole (`share::SMALL_FILE` at most) and its ETag; none
656    /// when it is not there, or larger than any such file. A body that came
657    /// cut, or whose end cannot be told, is an error: never taken as whole.
658    pub(crate) fn small(&self, url: &str) -> Result<Option<(Vec<u8>, Option<String>)>, SyncError> {
659        self.small_up_to(url, crate::share::SMALL_FILE)
660    }
661
662    /// The same, at most `cap` bytes (a texts' file may grow past a small file's).
663    pub(crate) fn small_up_to(&self, url: &str, cap: u64) -> Result<Option<(Vec<u8>, Option<String>)>, SyncError> {
664        let mut response = self.request(false, "GET", url, &[("Accept-Encoding", "identity")], Vec::new())?;
665        match response.status().as_u16() {
666            200 => {}
667            404 | 410 => return Ok(None),
668            status => return Err(SyncError::Server(format!("GET: {status}"))),
669        }
670        let etag = header(&response, "ETag");
671        let length = response.body().content_length();
672        let chunked = header(&response, "Transfer-Encoding").is_some_and(|t| t.to_ascii_lowercase().contains("chunked"));
673        if length.is_some_and(|l| l > cap) {
674            return Ok(None);
675        }
676        let bytes = match response.body_mut().with_config().limit(cap).read_to_vec() {
677            Ok(bytes) => bytes,
678            Err(ureq::Error::BodyExceedsLimit(_)) => return Ok(None),
679            Err(e) => return Err(failed(&e)),
680        };
681        if !(length == Some(bytes.len() as u64) || (length.is_none() && chunked)) {
682            return Err(SyncError::Network("GET: cut".into()));
683        }
684        Ok(Some((bytes, etag)))
685    }
686
687    /// A file from `from` on (all of it from 0): where what came starts (0
688    /// when the server sent it all), what came as it came, and why the rest
689    /// did not; none when it is not there, or shorter than `from` now.
690    fn tail(&self, url: &str, from: u64) -> Result<Option<Got>, SyncError> {
691        let range = format!("bytes={from}-");
692        let mut headers = vec![("Accept-Encoding", "identity")];
693        if from > 0 {
694            headers.push(("Range", range.as_str()));
695        }
696        let mut response = self.request(true, "GET", url, &headers, Vec::new())?;
697        let start = match response.status().as_u16() {
698            200 => 0,
699            206 => {
700                // "bytes 100-199/200".
701                let said = header(&response, "Content-Range").unwrap_or_default();
702                said.trim().strip_prefix("bytes").and_then(|r| r.trim().split('-').next()).and_then(|s| s.trim().parse().ok()).ok_or_else(|| SyncError::Server(format!("GET: Content-Range {said}")))?
703            }
704            404 | 410 | 416 => return Ok(None),
705            status => return Err(SyncError::Server(format!("GET: {status}"))),
706        };
707        let length = response.body().content_length();
708        let mut reader = response.body_mut().with_config().limit(LARGEST).reader();
709        let mut bytes = Vec::new();
710        let mut piece = vec![0u8; 64 << 10];
711        let mut error = loop {
712            match reader.read(&mut piece) {
713                Ok(0) => break None,
714                Ok(n) => bytes.extend_from_slice(&piece[..n]),
715                Err(e) => break Some(SyncError::Network(e.to_string())),
716            }
717        };
718        if error.is_none() && length.is_some_and(|l| l != bytes.len() as u64) {
719            error = Some(SyncError::Network("GET: cut".into()));
720        }
721        Ok(Some(Got { start, bytes, error }))
722    }
723
724    /// A file sent whole (`PUT`), over what is there as it was seen: `If-Match`
725    /// its ETag, or `If-None-Match: *` when none was there. Its status, and its
726    /// ETag there when the server says it.
727    pub(crate) fn put(&self, url: &str, body: Vec<u8>, seen: Option<&str>) -> Result<(u16, Option<String>), SyncError> {
728        let precondition = match seen {
729            Some(etag) => ("If-Match", etag),
730            None => ("If-None-Match", "*"),
731        };
732        let response = self.request(true, "PUT", url, &[precondition, ("Content-Type", "application/octet-stream")], body)?;
733        Ok((response.status().as_u16(), header(&response, "ETag").or_else(|| header(&response, "OC-ETag"))))
734    }
735
736    /// A folder made (`MKCOL`): 201 made, 405 there already.
737    pub(crate) fn mkcol(&self, url: &str) -> Result<u16, SyncError> {
738        Ok(self.request(false, "MKCOL", url, &[], Vec::new())?.status().as_u16())
739    }
740
741    /// One of this device's files taken out (`DELETE`), only as it was seen (`If-Match`) when its ETag is known.
742    pub(crate) fn delete(&self, url: &str, seen: Option<&str>) -> Result<u16, SyncError> {
743        let headers: Vec<(&str, &str)> = seen.map(|etag| vec![("If-Match", etag)]).unwrap_or_default();
744        Ok(self.request(false, "DELETE", url, &headers, Vec::new())?.status().as_u16())
745    }
746
747    /// A file's ETag there now (`Depth: 0`); none when it is not there.
748    pub(crate) fn etag(&self, url: &str) -> Result<Option<String>, SyncError> {
749        let mut response = self.request(false, "PROPFIND", url, &[("Depth", "0"), ("Content-Type", XML)], LIST.as_bytes().to_vec())?;
750        match response.status().as_u16() {
751            207 => {}
752            404 => return Ok(None),
753            status => return Err(SyncError::Server(format!("PROPFIND: {status}"))),
754        }
755        let body = response.body_mut().with_config().limit(LISTING).read_to_string().map_err(|e| failed(&e))?;
756        let (responses, _) = dav::multistatus(&body)?;
757        Ok(responses.first().map(|r| r.text(DAV, "getetag").unwrap_or_default()))
758    }
759
760    /// A file fetched whole into `path`, under a hidden name beside it, then
761    /// renamed once all of it came (its length said, or chunked to its end),
762    /// at most `limit` bytes, as long as it moves (`Limits::blob`). Its size;
763    /// none when it is not there, or larger.
764    pub(crate) fn download(&self, url: &str, path: &Path, limit: u64) -> Result<Option<u64>, SyncError> {
765        let mut response = self.request_with(self.limits.blob, "GET", url, &[("Accept-Encoding", "identity")], Vec::new())?;
766        match response.status().as_u16() {
767            200 => {}
768            404 | 410 => return Ok(None),
769            status => return Err(SyncError::Server(format!("GET: {status}"))),
770        }
771        let length = response.body().content_length();
772        let chunked = header(&response, "Transfer-Encoding").is_some_and(|t| t.to_ascii_lowercase().contains("chunked"));
773        if length.is_some_and(|l| l > limit) {
774            return Ok(None);
775        }
776        if length.is_none() && !chunked {
777            return Err(SyncError::Network("GET: its end cannot be told".into()));
778        }
779        let temporary = crate::share::temporary(path);
780        let disk = |e: std::io::Error| SyncError::Disk(format!("{}: {e}", path.display()));
781        let written = (|| -> Result<u64, SyncError> {
782            if let Some(parent) = path.parent() {
783                crate::share::private_dirs(parent).map_err(disk)?;
784            }
785            let mut out = crate::share::new_private(&temporary).map_err(disk)?;
786            let mut reader = response.body_mut().with_config().limit(limit).reader();
787            let copied = std::io::copy(&mut reader, &mut out).map_err(|e| SyncError::Network(e.to_string()))?;
788            if length.is_some_and(|l| l != copied) {
789                return Err(SyncError::Network("GET: cut".into()));
790            }
791            out.sync_all().map_err(disk)?;
792            std::fs::rename(&temporary, path).map_err(disk)?;
793            Ok(copied)
794        })();
795        if written.is_err() {
796            let _ = std::fs::remove_file(&temporary);
797        }
798        written.map(Some)
799    }
800}
801
802/// A path in a folder of the server as an address's: each name percent-encoded ("Mes documents/Sioul" → "Mes%20documents/Sioul").
803pub(crate) fn encode_path(path: &str) -> String {
804    path.split('/')
805        .map(|name| name.bytes().map(|b| if b.is_ascii_alphanumeric() || b"-._~".contains(&b) { (b as char).to_string() } else { format!("%{b:02X}") }).collect::<String>())
806        .collect::<Vec<_>>()
807        .join("/")
808}
809
810/// The checksums a multistatus says (`oc:checksums`, Nextcloud's), by the
811/// path of each response: "SHA1:… MD5:… ADLER32:…".
812fn checksums(body: &str, url: &str) -> BTreeMap<String, String> {
813    let Ok(doc) = roxmltree::Document::parse(body) else { return BTreeMap::new() };
814    doc.descendants()
815        .filter(|n| n.has_tag_name((DAV, "response")))
816        .filter_map(|response| {
817            let href = response.children().find(|c| c.has_tag_name((DAV, "href")))?.text()?.trim().to_string();
818            let sums = response.descendants().filter(|n| n.has_tag_name((OC, "checksum"))).filter_map(|n| n.text()).map(str::trim).filter(|t| !t.is_empty()).collect::<Vec<_>>().join(" ");
819            (!sums.is_empty()).then(|| (dav::path_key(&dav::absolute(url, &href)), sums))
820        })
821        .collect()
822}
823
824/// A content's SHA-1, in lower case hexadecimal, as `OC-Checksum` says it.
825fn sha1_hex(bytes: &[u8]) -> String {
826    use sha1::Digest;
827    sha1::Sha1::digest(bytes).iter().map(|b| format!("{b:02x}")).collect()
828}
829
830/// The SHA-1 among checksums as a server says them ("SHA1:ab12… MD5:…"), in lower case.
831fn sha1_of(checksums: &str) -> Option<String> {
832    checksums.split_whitespace().find_map(|sum| sum.split_once(':').filter(|(kind, _)| kind.eq_ignore_ascii_case("sha1")).map(|(_, hex)| hex.to_ascii_lowercase()))
833}
834
835/// Why, in a code the window says in words: "login:murena.io".
836fn code(error: &SyncError, host: &str) -> String {
837    let kind = match error {
838        SyncError::Login(_) | SyncError::AppPassword(_) => "login",
839        SyncError::Tls(_) => "tls",
840        SyncError::Network(_) => "network",
841        SyncError::NoPassword => "no-password",
842        SyncError::Disk(_) => "disk",
843        SyncError::Server(d) if d == "quota" => "quota",
844        _ => "server",
845    };
846    format!("{kind}:{host}")
847}
848
849/// Where an account's files are on its server, Nextcloud's way
850/// (`…/remote.php/dav/files/<user>/`, the user as the server names it in its
851/// principal), from the address its calendars were found at; none for a
852/// server of another kind. On that server only: a principal named on another
853/// is refused (`SyncError::Elsewhere`), since every request there would carry
854/// the account's login.
855pub(crate) fn files_root(server: &Server, login: &Login) -> Result<Option<String>, SyncError> {
856    let Some(at) = login.url.find("/remote.php/") else { return Ok(None) };
857    let base = format!("{}/remote.php/dav/", &login.url[..at]);
858    let mut response = server.request(false, "PROPFIND", &base, &[("Depth", "0"), ("Content-Type", XML)], PRINCIPAL.as_bytes().to_vec())?;
859    if response.status().as_u16() != 207 {
860        return Ok(None);
861    }
862    let body = response.body_mut().with_config().limit(LISTING).read_to_string().map_err(|e| failed(&e))?;
863    let (responses, _) = dav::multistatus(&body)?;
864    let principal = responses.iter().find_map(|r| r.prop(DAV, "current-user-principal").and_then(|p| p.hrefs.first().cloned()));
865    let root = principal.and_then(|href| {
866        let at = href.find("/principals/users/")?;
867        let user = href[at + "/principals/users/".len()..].trim_end_matches('/');
868        (!user.is_empty() && !user.contains('/')).then(|| dav::absolute(&base, &format!("{}/files/{user}/", &href[..at])))
869    });
870    match root {
871        Some(root) if dav::origin(&root).is_none() || dav::origin(&root) != dav::origin(&base) => Err(SyncError::Elsewhere(root)),
872        root => Ok(root),
873    }
874}
875
876// ---------------------------------------------------------------- finding it
877
878/// Where a folder may be on a server, as paths in an account's files, the
879/// likeliest first: on a phone, its path from the storage, which eDrive
880/// carries as the cloud's root (`/storage/emulated/0/Documents/Sioul` →
881/// `Documents/Sioul`), or from the Nextcloud app's own folder; under a folder
882/// the Nextcloud client carries (`nextcloud`: its `nextcloud.cfg`, as text),
883/// the same under its `targetPath`; then its last one, two and three names.
884pub fn places(folder: &Path, nextcloud: &[String]) -> Vec<String> {
885    let text = folder.to_string_lossy().replace('\\', "/");
886    let text = text.trim_end_matches('/');
887    let mut out: Vec<String> = Vec::new();
888    for storage in ["/storage/emulated/0", "/sdcard", "/storage/self/primary", "/mnt/sdcard", "/mnt/user/0/primary"] {
889        let Some(rest) = text.strip_prefix(storage).and_then(|r| r.strip_prefix('/')) else { continue };
890        // The Nextcloud app's own: Android/media/com.nextcloud.client/nextcloud/<account>/<path>.
891        match rest.strip_prefix("Android/media/com.nextcloud.client/nextcloud/").and_then(|r| r.split_once('/')) {
892            Some((_, inside)) => out.push(inside.to_string()),
893            None if !rest.starts_with("Android/") => out.push(rest.to_string()),
894            None => {}
895        }
896    }
897    for config in nextcloud {
898        out.extend(nextcloud_places(text, config));
899    }
900    let names: Vec<&str> = text.rsplit('/').filter(|n| !n.is_empty()).take(3).collect();
901    for n in 1..=names.len() {
902        out.push(names[..n].iter().rev().copied().collect::<Vec<_>>().join("/"));
903    }
904    let mut seen = BTreeSet::new();
905    out.into_iter().map(|p| p.trim_matches('/').to_string()).filter(|p| !p.is_empty() && seen.insert(p.clone())).collect()
906}
907
908/// The places a Nextcloud client's configuration gives a folder: under one of
909/// its `localPath`, the same under that folder's `targetPath`
910/// ("0\Folders\1\localPath=/home/me/Nextcloud/", "0\Folders\1\targetPath=/").
911fn nextcloud_places(folder: &str, config: &str) -> Vec<String> {
912    let mut local: BTreeMap<&str, &str> = BTreeMap::new();
913    let mut target: BTreeMap<&str, &str> = BTreeMap::new();
914    for line in config.lines() {
915        let Some((key, value)) = line.split_once('=') else { continue };
916        if let Some(prefix) = key.trim().strip_suffix("localPath") {
917            local.insert(prefix, value.trim());
918        } else if let Some(prefix) = key.trim().strip_suffix("targetPath") {
919            target.insert(prefix, value.trim());
920        }
921    }
922    local
923        .into_iter()
924        .filter_map(|(prefix, root)| {
925            let root = root.trim_end_matches('/');
926            let rest = folder.strip_prefix(root).filter(|r| !root.is_empty() && (r.is_empty() || r.starts_with('/')))?;
927            Some(format!("{}/{}", target.get(prefix)?.trim_matches('/'), rest.trim_matches('/')))
928        })
929        .collect()
930}
931
932/// Looks for the sharing folder on the accounts' servers: the place given by
933/// hand alone when there is one (`State::given`), else `places` under each
934/// account's files. Confirmed only where the server's `seal.toml` is this
935/// folder's, byte for byte; a folder sealed otherwise is never used. The
936/// state, saved, says what was found, or why nothing was.
937pub fn find(memory: &Path, folder: &Path, logins: &[Login], places: &[String], now: i64) -> State {
938    find_with(memory, folder, logins, places, now, LIMITS)
939}
940
941pub(crate) fn find_with(memory: &Path, folder: &Path, logins: &[Login], places: &[String], now: i64, limits: Limits) -> State {
942    let mut state = State::load(memory);
943    // Sioul keeping the folder itself: nothing to look for.
944    if state.mode == MIRROR {
945        return state;
946    }
947    if state.folder != shown(folder) {
948        // Another folder: what was fetched for the last one goes.
949        let _ = std::fs::remove_dir_all(cache_of(memory));
950        state = State { on: state.on, given: state.given.clone(), asked: state.asked, send: state.send, folder: shown(folder), ..State::default() };
951    }
952    state.looked = now;
953    state.confirmed = 0;
954    let Some(seal) = seal_of(folder) else {
955        state.said = "no-seal".into();
956        let _ = state.save(memory);
957        return state;
958    };
959    let given = state.given.trim().to_string();
960    let mut said: Vec<String> = Vec::new();
961    let mut looked_on: Vec<String> = Vec::new();
962    for login in logins {
963        let host = host_of(&login.url);
964        // Given as an address: on its own server only.
965        if given.contains("://") && host_of(&given) != host {
966            continue;
967        }
968        let server = match Server::new(login, limits) {
969            Ok(server) => server,
970            Err(e) => {
971                said.push(code(&e, &login.account));
972                continue;
973            }
974        };
975        let urls: Vec<String> = if given.contains("://") {
976            vec![format!("{}/", given.trim_end_matches('/'))]
977        } else {
978            let root = match files_root(&server, login) {
979                Ok(Some(root)) => root,
980                Ok(None) => continue,
981                Err(e) => {
982                    said.push(code(&e, &host));
983                    continue;
984                }
985            };
986            let wanted: Vec<String> = if given.is_empty() { places.to_vec() } else { vec![given.trim_matches('/').to_string()] };
987            wanted.iter().filter(|p| !p.is_empty()).map(|p| format!("{root}{}/", encode_path(p))).collect()
988        };
989        if !looked_on.contains(&host) {
990            looked_on.push(host.clone());
991        }
992        for url in urls {
993            match server.small(&format!("{url}seal.toml")) {
994                Ok(Some((bytes, etag))) if bytes == seal => {
995                    confirm(&mut state, memory, login, &url, &seal, etag, now);
996                    let _ = state.save(memory);
997                    return state;
998                }
999                Ok(Some(_)) => said.push(format!("seal-differs:{host}")),
1000                Ok(None) => {}
1001                Err(e) => {
1002                    said.push(code(&e, &host));
1003                    // A refused password is not tried again and again (it could lock the account); a network down neither.
1004                    if e.is_lasting() || matches!(e, SyncError::Network(_)) {
1005                        break;
1006                    }
1007                }
1008            }
1009        }
1010    }
1011    // The likeliest reason first: a folder sealed otherwise, then what you can mend.
1012    let rank = |code: &String| ["seal-differs:", "login:", "no-password:", "tls:", "network:", "server:", "disk:"].iter().position(|p| code.starts_with(p)).unwrap_or(9);
1013    said.sort_by_key(rank);
1014    state.said = match said.into_iter().next() {
1015        Some(why) => why,
1016        None if looked_on.is_empty() && given.contains("://") => format!("no-account-for:{}", host_of(&given)),
1017        None if looked_on.is_empty() => "no-account".into(),
1018        None => format!("not-found:{}", looked_on.join(", ")),
1019    };
1020    let _ = state.save(memory);
1021    state
1022}
1023
1024/// The folder found at `url`: confirmed, its seal kept in the cache, which
1025/// holds what was fetched under that seal only.
1026fn confirm(state: &mut State, memory: &Path, login: &Login, url: &str, seal: &[u8], etag: Option<String>, now: i64) {
1027    let cache = cache_of(memory);
1028    if seal_of(&cache).as_deref() != Some(seal) {
1029        let _ = std::fs::remove_dir_all(&cache);
1030        state.files.clear();
1031        state.folders.clear();
1032    }
1033    if state.url != url {
1034        state.files.clear();
1035        state.folders.clear();
1036    }
1037    let _ = crate::share::write_atomically(&cache.join("seal.toml"), seal);
1038    state.account = login.account.clone();
1039    state.url = url.to_string();
1040    state.confirmed = now;
1041    state.sealed = now;
1042    state.said.clear();
1043    // Known by its ETag there: read again only when it changes, or six hours on.
1044    if let Some(etag) = etag {
1045        state.files.insert("seal.toml".into(), Fetched { etag, size: seal.len() as u64 });
1046    }
1047}
1048
1049// ---------------------------------------------------------------- fetching
1050
1051/// How a device stands, for the pace of its pulls (`due`).
1052#[derive(Debug, Clone, Copy, Default)]
1053pub struct Pace {
1054    /// A phone; else a computer, which pulls once a minute.
1055    pub phone: bool,
1056    /// Sioul on its screen: you are there.
1057    pub shown: bool,
1058    /// Another device says it is in use, as read here (what was fetched too).
1059    pub in_use: bool,
1060    /// A dose's alarm, a waking, the background step asking for news now.
1061    pub urgent: bool,
1062    /// The phone's background step's pace now (seconds): five minutes, fifteen while you sleep.
1063    pub step: i64,
1064}
1065
1066/// Whether a pull is due now. The first after the folder is found, at once,
1067/// whatever the hour: until it comes, this device knows the others only from
1068/// the folder the sync app left stale, so whether another one is in use
1069/// cannot be told (6 October 2026, 23:30: found on the phone at night, never
1070/// pulled, the desk in use taken for closed hours before). Then a computer, a
1071/// phone with Sioul on its screen or another device in use, once a minute; a
1072/// phone in the background otherwise at its step's pace, the night's
1073/// included: a device opened at night is noticed within its fifteen minutes.
1074/// Asked for news, twenty seconds after the last.
1075pub fn due(state: &State, now: i64, pace: Pace) -> bool {
1076    let since = now - state.tried;
1077    // Tried "later" than now: the clock was set back since; never a reason to wait.
1078    if state.tried < state.confirmed || since < 0 {
1079        return true;
1080    }
1081    if pace.urgent {
1082        return since >= 20;
1083    }
1084    if !pace.phone || pace.shown || pace.in_use {
1085        return since >= 60;
1086    }
1087    since >= pace.step - 30
1088}
1089
1090/// The longest of another device's records a pull brings (`LARGEST`); a test's own, in its thread.
1091fn round_cap() -> u64 {
1092    #[cfg(test)]
1093    if let Some(cap) = tests::ROUND_CAP.with(std::cell::Cell::get) {
1094        return cap;
1095    }
1096    LARGEST
1097}
1098
1099/// How long a dose's or a waking's alarm waits for the others' news (`alarm_news`).
1100#[derive(Debug, Clone, Copy)]
1101pub struct AlarmTimes {
1102    /// Sioul's own pull alone, before the sync app is asked too.
1103    pub alone: Duration,
1104    /// In all, at most: the twenty seconds the sync app had, and the five the pull had after them.
1105    pub most: Duration,
1106}
1107
1108/// An alarm's waits, as they are on a phone.
1109pub const ALARM_TIMES: AlarmTimes = AlarmTimes { alone: Duration::from_secs(5), most: Duration::from_secs(25) };
1110
1111/// Where an alarm's news came from (`alarm_news`).
1112#[derive(Debug, Clone, PartialEq, Eq)]
1113pub enum AlarmNews {
1114    /// A pull begun since the alarm went through: what it read is trusted alone.
1115    Pulled,
1116    /// No pull since went through; the sync app asked and waited for, as
1117    /// before Sioul's own pull. `heard`: the devices whose entry or claim on
1118    /// the health part says more since the alarm began, by its own times (in
1119    /// the folder, or in what was fetched beside it). Of the others no news
1120    /// came: what this device knows
1121    /// of them is from before the alarm, and one known closed then may have
1122    /// opened since and answered (`sioul_core::health::doubts_unread`).
1123    SyncApp { heard: BTreeSet<String> },
1124}
1125
1126/// What a dose's or a waking's alarm reads before it decides, `since` its
1127/// start (Unix seconds), on a phone whose folder is found on the server
1128/// (`State::fetching`; docs/android.md, "Doses while Sioul is away"). Sioul's
1129/// own pull alone is trusted only once a pull begun since the alarm went
1130/// through (`State::went_through_since`; in this process, by its steady clock:
1131/// `PULLED`): this one, or another's that ran meanwhile in this process
1132/// (`lock`, one pull at a time: another alarm's, the window's), waited for,
1133/// never passed over. A pull that fails, or has not gone through within
1134/// `times.alone`: the sync app asked as before (`ask`, which asks it and says
1135/// how long its wait still runs: twenty seconds, less when another alarm
1136/// asked it a moment ago), and waited for, that wait ended as soon as a pull
1137/// since the alarm goes through; `times.most` in all at most. `clock` gives
1138/// the time now (Unix seconds); `key` opens the others' entries and claims.
1139#[allow(clippy::too_many_arguments)]
1140pub fn alarm_news(memory: &Path, folder: &Path, own: &str, key: [u8; 32], login: Option<Login>, lock: &'static Mutex<()>, since: i64, clock: std::sync::Arc<dyn Fn() -> i64 + Send + Sync>, ask: &mut dyn FnMut() -> Duration) -> AlarmNews {
1141    alarm_news_with(memory, folder, own, key, login, lock, since, ALARM_TIMES, clock, ask, LIMITS)
1142}
1143
1144#[allow(clippy::too_many_arguments)]
1145pub(crate) fn alarm_news_with(memory: &Path, folder: &Path, own: &str, key: [u8; 32], login: Option<Login>, lock: &'static Mutex<()>, since: i64, times: AlarmTimes, clock: std::sync::Arc<dyn Fn() -> i64 + Send + Sync>, ask: &mut dyn FnMut() -> Duration, limits: Limits) -> AlarmNews {
1146    let begun = Instant::now();
1147    let until = begun + times.most;
1148    // What each other device says of itself now, for the doses: what the sync app brings may say more.
1149    let before = said_by_each(folder, &key, own);
1150    let mut pulling = {
1151        let (memory, folder, own, clock) = (memory.to_path_buf(), folder.to_path_buf(), own.to_string(), std::sync::Arc::clone(&clock));
1152        Some(std::thread::spawn(move || pull_since_with(&memory, &folder, &own, login.as_ref(), lock, since, begun, until, &*clock, limits)))
1153    };
1154    let mut went = None;
1155    let mut asked: Option<Instant> = None;
1156    loop {
1157        if went.is_none() && pulling.as_ref().is_some_and(std::thread::JoinHandle::is_finished) {
1158            went = pulling.take().map(|pull| pull.join().unwrap_or(false));
1159        }
1160        if went == Some(true) {
1161            return AlarmNews::Pulled;
1162        }
1163        let now = Instant::now();
1164        // Failed, or not through yet: the sync app asked, as before.
1165        if asked.is_none() && (went == Some(false) || now >= begun + times.alone) {
1166            asked = Some(now + ask());
1167        }
1168        if asked.is_some_and(|end| now >= end) || now >= until {
1169            // A pull since, through at the last moment, counts all the same.
1170            if through_since(memory, folder, since, begun, clock()) {
1171                return AlarmNews::Pulled;
1172            }
1173            return AlarmNews::SyncApp { heard: heard_since(&before, &said_by_each(folder, &key, own)) };
1174        }
1175        std::thread::sleep(Duration::from_millis(50));
1176    }
1177}
1178
1179/// Whether a pull begun since the alarm (`since`, Unix seconds; `begun`, on
1180/// this process's steady clock) went through. The last one gone through in
1181/// this process, when it is the one the state says, is judged by the steady
1182/// clock (`PULLED`), which no setting of the clock moves; another process's
1183/// by its saved time, within `since` and `now`.
1184fn through_since(memory: &Path, folder: &Path, since: i64, begun: Instant, now: i64) -> bool {
1185    let state = State::load(memory);
1186    if !state.pulls_well(folder) {
1187        return false;
1188    }
1189    match PULLED.lock().ok().and_then(|all| all.get(memory).copied()) {
1190        Some((at, last)) if last == state.last => at >= begun,
1191        _ => state.went_through_since(folder, since, now),
1192    }
1193}
1194
1195/// A pull begun since the alarm (`since`, Unix seconds; `begun`, this
1196/// process's steady clock) gone through: one that ran meanwhile, waited for
1197/// (`lock`: one pull at a time in this process), else this one, begun before
1198/// `until`. False when a pull begun since failed, or none could begin in time.
1199#[allow(clippy::too_many_arguments)]
1200pub(crate) fn pull_since_with(memory: &Path, folder: &Path, own: &str, login: Option<&Login>, lock: &Mutex<()>, since: i64, begun: Instant, until: Instant, clock: &dyn Fn() -> i64, limits: Limits) -> bool {
1201    let through = || through_since(memory, folder, since, begun, clock());
1202    let _one = loop {
1203        match lock.try_lock() {
1204            Ok(held) => break held,
1205            Err(std::sync::TryLockError::Poisoned(held)) => break held.into_inner(),
1206            Err(std::sync::TryLockError::WouldBlock) if Instant::now() < until => std::thread::sleep(Duration::from_millis(50)),
1207            Err(std::sync::TryLockError::WouldBlock) => return through(),
1208        }
1209    };
1210    if through() {
1211        return true;
1212    }
1213    // One begun since that failed (another alarm's): the sync app, not another
1214    // try within this alarm. Tried "later" than now: the clock was set back,
1215    // tried again; one gone through before this alarm began: pulled again.
1216    let state = State::load(memory);
1217    if (!state.said.is_empty() && (since..=clock()).contains(&state.tried)) || Instant::now() >= until {
1218        return false;
1219    }
1220    let Some(login) = login else { return false };
1221    let _ = pull_with(memory, folder, own, login, clock(), limits);
1222    through()
1223}
1224
1225/// What each other device says of itself as the doses read it (`health::know`):
1226/// its entry (`devices`) and its claim on the health part (`lease`), the later
1227/// of the folder's copy and the one fetched beside it, each by its own times
1228/// (`devices::written`; the claim's renewal and its end).
1229type Said = BTreeMap<String, (Option<(i64, i64, i64, i64, i64)>, Option<(i64, i64)>)>;
1230
1231pub(crate) fn said_by_each(folder: &Path, key: &[u8; 32], own: &str) -> Said {
1232    let mut out: Said = BTreeMap::new();
1233    for entry in crate::devices::all(folder, key).0.into_iter().filter(|e| e.id != own) {
1234        out.entry(entry.id.clone()).or_default().0 = Some(crate::devices::written(&entry));
1235    }
1236    for claim in crate::lease::claims(folder, key, "health").into_iter().filter(|c| c.computer != own) {
1237        out.entry(claim.computer.clone()).or_default().1 = Some((claim.renewed, claim.until));
1238    }
1239    out
1240}
1241
1242/// The devices whose entry or health claim says more than before: later by
1243/// its own times. Another part's claim (notices, invoices) is no news of a
1244/// session for the doses; a file touched with the same bytes, or an older
1245/// copy put back, is none either.
1246fn heard_since(before: &Said, now: &Said) -> BTreeSet<String> {
1247    now.iter()
1248        .filter(|(id, (entry, claim))| {
1249            let (was_entry, was_claim) = before.get(*id).copied().unwrap_or_default();
1250            entry.is_some_and(|e| was_entry.is_none_or(|w| e > w)) || claim.is_some_and(|c| was_claim.is_none_or(|w| c > w))
1251        })
1252        .map(|(id, _)| id.clone())
1253        .collect()
1254}
1255
1256/// What a pull did.
1257#[derive(Debug, Default, Clone, PartialEq, Eq)]
1258pub struct Pulled {
1259    /// Files and folders the server's listings showed.
1260    pub listed: usize,
1261    /// Files that came down and are kept.
1262    pub fetched: usize,
1263    /// This device's own files that went up (`MIRROR`, or beside a sync app: `send`).
1264    pub sent: usize,
1265    /// This device's own files found there already as they are here: nothing sent (`send`).
1266    pub same: usize,
1267    /// This device's records found longer there than here (a sync app put an
1268    /// older copy back here): left as they are there (`send`).
1269    pub longer: usize,
1270    /// This device's files found otherwise there, by kind and size, for the
1271    /// log ("records 70708 here, 71234 there"; eight at most): never a name nor a content.
1272    pub differ: Vec<String>,
1273    /// Large files left for later on a metered or slow connection (`send`, `fetch_wanted`).
1274    pub paced: usize,
1275    /// Files that did not go (too slow, cut), held back until their backoff
1276    /// (`send`); sealed files fetched that did not open whole (`fetch_wanted`).
1277    pub held: usize,
1278    /// The last failure, in detail (`send`).
1279    pub failure: Option<Failure>,
1280    /// Bytes that came down.
1281    pub bytes: u64,
1282    /// The other devices' records found past what a pull brings, left to the sync app (`State::left`).
1283    pub left: usize,
1284    /// What went wrong, as `State::said` says it; none when it went through.
1285    pub problem: Option<String>,
1286}
1287
1288/// Why a pull stopped: the network or the server, or the folder no longer confirmed.
1289enum Stop {
1290    Failed(SyncError),
1291    Unconfirmed(String),
1292}
1293
1294impl From<SyncError> for Stop {
1295    fn from(e: SyncError) -> Stop {
1296        Stop::Failed(e)
1297    }
1298}
1299
1300/// The other devices' files newer on the server than here, fetched beside the
1301/// synced folder (see the module's words); `own` is this device's id, whose
1302/// files are never fetched. Nothing when the folder is not confirmed, or its
1303/// seal here is no longer the one fetched under. Never on the window's thread.
1304pub fn pull(memory: &Path, folder: &Path, own: &str, login: &Login, now: i64) -> Pulled {
1305    pull_with(memory, folder, own, login, now, LIMITS)
1306}
1307
1308pub(crate) fn pull_with(memory: &Path, folder: &Path, own: &str, login: &Login, now: i64, limits: Limits) -> Pulled {
1309    let mut state = State::load(memory);
1310    let mut pulled = Pulled::default();
1311    // Sioul keeping the folder itself: no backup beside it (`step`).
1312    if state.mode == MIRROR {
1313        pulled.problem = Some("mirror".into());
1314        return pulled;
1315    }
1316    if !state.confirmed_for(folder) || state.account != login.account {
1317        pulled.problem = Some("not-confirmed".into());
1318        return pulled;
1319    }
1320    let cache = cache_of(memory);
1321    // Switched off here: not a request; what the folder caught up with goes.
1322    if !state.fetching() {
1323        tidy_cache(&state, folder, memory, own);
1324        pulled.problem = Some("off".into());
1325        return pulled;
1326    }
1327    // This folder's seal changed here (sharing started again): what was
1328    // fetched belongs to another sharing, and the server's folder is looked at again.
1329    if !usable(folder, memory) {
1330        let _ = std::fs::remove_dir_all(&cache);
1331        state = State { on: state.on, given: state.given.clone(), asked: state.asked, send: state.send, folder: state.folder.clone(), said: "no-seal".into(), ..State::default() };
1332        let _ = state.save(memory);
1333        pulled.problem = Some(state.said);
1334        return pulled;
1335    }
1336    let loaded_left = state.left.clone();
1337    state.tried = now;
1338    let begun = Instant::now();
1339    let host = state.host();
1340    let outcome = Server::new(login, limits).map_err(Stop::Failed).and_then(|server| Puller { server: &server, folder, into: &cache, own, state: &mut state, pulled: &mut pulled, now, own_there: BTreeMap::new(), covered: BTreeSet::new(), lost: false, blobs: Vec::new(), blobs_listed: None }.run());
1341    match outcome {
1342        Ok(()) => {
1343            state.last = now;
1344            state.said.clear();
1345        }
1346        Err(Stop::Unconfirmed(why)) => {
1347            state.confirmed = 0;
1348            state.said = why;
1349        }
1350        Err(Stop::Failed(e)) => state.said = code(&e, &host),
1351    }
1352    tidy_cache(&state, folder, memory, own);
1353    if state.said.is_empty() {
1354        // Gone through, begun at this moment of this process (`went_through_since`).
1355        if let Ok(mut all) = PULLED.lock()
1356            && all.get(memory).is_none_or(|(at, _)| *at < begun)
1357        {
1358            all.insert(memory.to_path_buf(), (begun, state.last));
1359        }
1360    }
1361    let _ = state.save_pulled(memory, now + begun.elapsed().as_secs() as i64 + 1, &loaded_left);
1362    pulled.problem = (!state.said.is_empty()).then(|| state.said.clone());
1363    pulled
1364}
1365
1366/// The last pull of each memory gone through in this process: when it began
1367/// on the process's own steady clock, and its time as saved (`State::last`).
1368/// A clock set back makes a pull's saved time later than the moment; an
1369/// alarm in this process tells by this one whether that pull began before it.
1370static PULLED: Mutex<BTreeMap<PathBuf, (Instant, i64)>> = Mutex::new(BTreeMap::new());
1371
1372/// What is no longer needed in the cache, gone, without the network: this
1373/// device's own files (never fetched, never read from there), records the
1374/// synced folder caught up with, entries and notes the same as the folder's
1375/// and, the backup switched off, all of them; records longer than the
1376/// folder's stay until it catches up (read past its end here, a records'
1377/// file gone would read as cut, and the doses would doubt for hours). Sealed
1378/// files fetched for a note waiting (`fetch_wanted`) go once it is written,
1379/// once the synced folder holds them, or after a few days.
1380pub fn tidy(memory: &Path, folder: &Path, own: &str) {
1381    let state = State::load(memory);
1382    if state.folder == shown(folder) && state.mode != MIRROR {
1383        tidy_cache(&state, folder, memory, own);
1384    }
1385}
1386
1387fn tidy_cache(state: &State, folder: &Path, memory: &Path, own: &str) {
1388    let cache = &cache_of(memory);
1389    // What a crash left half written, an hour later.
1390    for dir in [cache.to_path_buf(), cache.join("devices"), cache.join("blobs")].into_iter().chain(std::fs::read_dir(cache.join("leases")).into_iter().flatten().filter_map(Result::ok).map(|e| e.path())) {
1391        crate::share::clean_leftovers(&dir);
1392    }
1393    for relative in cached_files(cache) {
1394        let (here, fetched) = (folder.join(&relative), cache.join(&relative));
1395        let name = relative.rsplit('/').next().unwrap_or_default();
1396        let gone = match kind_of(&relative) {
1397            None => continue,
1398            Some((_, id)) if id == own => true,
1399            Some((Kind::Round, _)) => std::fs::metadata(&here).is_ok_and(|h| std::fs::metadata(&fetched).is_ok_and(|f| h.len() >= f.len())),
1400            Some(_) => !state.fetching() || std::fs::read(&here).ok().is_some_and(|h| std::fs::read(&fetched).ok().is_some_and(|f| f == h)),
1401        };
1402        if gone && !name.starts_with('.') {
1403            let _ = std::fs::remove_file(&fetched);
1404        }
1405    }
1406    // Sealed files fetched because a note or a paper waited for them
1407    // (`fetch_wanted`): gone once the synced folder holds the same, once
1408    // nothing waits for them (written), after `WANTED_KEPT_DAYS` whatever,
1409    // and all of them with the backup switched off.
1410    // One aged out is no longer waited for either, until an exchange that
1411    // reads notes finds it missing again: never fetched every few days for a
1412    // note nothing here writes (Android's leave to read them taken back).
1413    let wanted = crate::blobs::Wanted::load(memory);
1414    let mut aged: Vec<String> = Vec::new();
1415    for entry in std::fs::read_dir(cache.join("blobs")).into_iter().flatten().filter_map(Result::ok) {
1416        let name = entry.file_name().to_string_lossy().to_string();
1417        let Ok(meta) = entry.metadata() else { continue };
1418        if name.starts_with('.') || !meta.is_file() {
1419            continue;
1420        }
1421        let brought = std::fs::metadata(folder.join("blobs").join(&name)).is_ok_and(|here| here.len() == meta.len());
1422        let old = meta.modified().ok().and_then(|t| t.elapsed().ok()).is_some_and(|age| age.as_secs() >= WANTED_KEPT_DAYS * 86_400);
1423        if !state.fetching() || brought || old || !wanted.blobs.contains_key(&name) {
1424            let _ = std::fs::remove_file(entry.path());
1425            if old {
1426                aged.push(name);
1427            }
1428        }
1429    }
1430    if !aged.is_empty() {
1431        let _ = crate::blobs::Wanted::change(memory, |wanted| wanted.blobs.retain(|name, _| !aged.contains(name)));
1432    }
1433}
1434
1435/// What a file of the folder is, by its path, and whose: a device's records
1436/// (`<id>-<n>.jsonl`), its notes to the others (`<id>.toml`), its entry
1437/// (`devices/<id>.device`), a claim (`leases/<part>/<id>.lease`).
1438#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1439pub(crate) enum Kind {
1440    Round,
1441    Seen,
1442    Device,
1443    Claim,
1444    /// Its texts to send (`texts/send/<id>.jsonl`): a small file of their own (`place_texts`).
1445    Texts,
1446}
1447
1448pub(crate) fn is_id(id: &str) -> bool {
1449    id.len() == 36 && uuid::Uuid::parse_str(id).is_ok()
1450}
1451
1452pub(crate) fn kind_of(relative: &str) -> Option<(Kind, &str)> {
1453    let parts: Vec<&str> = relative.split('/').collect();
1454    match parts.as_slice() {
1455        [name] => {
1456            if let Some((id, round)) = name.strip_suffix(".jsonl").and_then(|stem| stem.rsplit_once('-')) {
1457                return (is_id(id) && round.parse::<u32>().is_ok()).then_some((Kind::Round, id));
1458            }
1459            name.strip_suffix(".toml").filter(|id| is_id(id)).map(|id| (Kind::Seen, id))
1460        }
1461        ["devices", name] => name.strip_suffix(".device").filter(|id| is_id(id)).map(|id| (Kind::Device, id)),
1462        ["leases", _, name] => name.strip_suffix(".lease").filter(|id| !id.is_empty() && !id.starts_with('.')).map(|id| (Kind::Claim, id)),
1463        ["texts", "send", name] => name.strip_suffix(".jsonl").filter(|id| is_id(id)).map(|id| (Kind::Texts, id)),
1464        _ => None,
1465    }
1466}
1467
1468/// The files of the cache, by their path in the folder: the folder's own, `devices/`, `leases/<part>/`.
1469fn cached_files(cache: &Path) -> Vec<String> {
1470    let names = |dir: &Path| -> Vec<String> { std::fs::read_dir(dir).into_iter().flatten().filter_map(Result::ok).filter(|e| e.file_type().is_ok_and(|t| t.is_file())).map(|e| e.file_name().to_string_lossy().to_string()).collect() };
1471    let mut out: Vec<String> = names(cache);
1472    out.extend(names(&cache.join("devices")).into_iter().map(|n| format!("devices/{n}")));
1473    for part in std::fs::read_dir(cache.join("leases")).into_iter().flatten().filter_map(Result::ok).filter(|e| e.file_type().is_ok_and(|t| t.is_dir())) {
1474        let part = part.file_name().to_string_lossy().to_string();
1475        out.extend(names(&cache.join("leases").join(&part)).into_iter().map(|n| format!("leases/{part}/{n}")));
1476    }
1477    out.extend(names(&cache.join("texts").join("send")).into_iter().map(|n| format!("texts/send/{n}")));
1478    out
1479}
1480
1481// ---------------------------------------------------------------- sealed files waited for
1482
1483/// A step fetches at most this many sealed files waited for (`fetch_wanted`),
1484const WANTED_FILES: usize = 16;
1485/// and this many bytes of them, the first whatever its size.
1486const WANTED_BYTES: u64 = 16 << 20;
1487/// A file waited for under this size (a note, bytes) comes at once, the
1488/// connection metered or slow too; a larger one (a paper) at most every ten minutes then.
1489pub const SMALL_WANTED: u64 = 1 << 20;
1490/// A sealed file fetched for a note waiting is kept this long at most (days),
1491/// in case it was not written meanwhile (`tidy_cache`).
1492const WANTED_KEPT_DAYS: u64 = 3;
1493/// A sealed file waited for and not on the server yet (or cut on the way) is
1494/// asked again within this long (seconds): it comes as soon as it is there.
1495const WANTED_AGAIN: i64 = 5 * 60;
1496
1497/// The sealed files of the others' notes and papers this device waits for
1498/// (`blobs::Wanted`: their records came, the files they name did not),
1499/// fetched from the folder's server by their names, beside the synced folder
1500/// (`overlay`'s `blobs/`, read by `blobs::get`), when the backup is found
1501/// there under the same seal (docs/database.md, "Sealed files fetched when
1502/// waited for"). Only those neither in the folder nor fetched yet, the
1503/// smallest first, at most `WANTED_FILES` and `WANTED_BYTES` a step (the
1504/// first whatever its size). On a metered or slow connection (`frugal`),
1505/// the pace of what is sent (`send`): one of `SMALL_WANTED` or more at most
1506/// every ten minutes, a note's at once. Each kept only once it opens whole to
1507/// its content (`blobs::check`): one that does not is asked again after a
1508/// while (`backoff`, up to an hour); one not on the server yet, within five
1509/// minutes. No listing of the server's `blobs/`: the records name them. Never
1510/// on the window's thread; one fetch at a time on this device.
1511pub fn fetch_wanted(memory: &Path, folder: &Path, login: &Login, key: &[u8; 32], now: i64, frugal: bool) -> Pulled {
1512    fetch_wanted_with(memory, folder, login, key, now, frugal, LIMITS)
1513}
1514
1515/// Whether a sealed file waited for is due to be fetched (`fetch_wanted`):
1516/// cheap, nothing of the network; none while nothing waits.
1517pub fn wanted_due(memory: &Path, folder: &Path, now: i64) -> bool {
1518    crate::blobs::Wanted::path(memory).exists() && !wanted_now(&crate::blobs::Wanted::load(memory), folder, &cache_of(memory).join("blobs"), now).is_empty()
1519}
1520
1521/// The sealed files waited for that are neither in the folder nor fetched
1522/// (`into`), and not asked lately in vain.
1523fn wanted_now<'a>(wanted: &'a crate::blobs::Wanted, folder: &Path, into: &Path, now: i64) -> Vec<(&'a String, &'a crate::blobs::Want)> {
1524    let here = |dir: &Path, name: &str| std::fs::metadata(dir.join(name)).is_ok_and(|m| m.len() > 0);
1525    wanted.blobs.iter().filter(|(name, want)| want.again <= now && !here(&folder.join("blobs"), name) && !here(into, name)).collect()
1526}
1527
1528#[allow(clippy::too_many_arguments)]
1529pub(crate) fn fetch_wanted_with(memory: &Path, folder: &Path, login: &Login, key: &[u8; 32], now: i64, frugal: bool, limits: Limits) -> Pulled {
1530    let mut pulled = Pulled::default();
1531    let state = State::load(memory);
1532    if state.mode == MIRROR || !state.fetching() || !state.confirmed_for(folder) || state.account != login.account || !usable(folder, memory) {
1533        pulled.problem = Some("not-confirmed".into());
1534        return pulled;
1535    }
1536    let into = cache_of(memory).join("blobs");
1537    sioul_core::filelock::with_lock(&memory.with_file_name("wanted.step"), || {
1538        let wanted = crate::blobs::Wanted::load(memory);
1539        let mut due = wanted_now(&wanted, folder, &into, now);
1540        if due.is_empty() {
1541            return pulled;
1542        }
1543        due.sort_by_key(|(name, want)| (want.size, want.since, *name));
1544        let mut tried: BTreeMap<String, (u32, i64)> = BTreeMap::new();
1545        let mut large_at = wanted.large_at;
1546        let (mut count, mut budget) = (0, WANTED_BYTES);
1547        let outcome = Server::new(login, limits).and_then(|server| {
1548            for (name, want) in due {
1549                if count >= WANTED_FILES || (count > 0 && want.size > budget) {
1550                    break;
1551                }
1552                let large = want.size >= SMALL_WANTED;
1553                if frugal && large && now - large_at < PACE {
1554                    pulled.paced += 1;
1555                    continue;
1556                }
1557                if frugal && large {
1558                    large_at = now;
1559                }
1560                count += 1;
1561                budget = budget.saturating_sub(want.size);
1562                let path = into.join(name);
1563                let failed = |soon: bool| {
1564                    let tries = want.tries + 1;
1565                    (tries, now + if soon { backoff(tries).min(WANTED_AGAIN) } else { backoff(tries) })
1566                };
1567                match server.download(&format!("{}{}", state.url, encode_path(&format!("blobs/{name}"))), &path, BLOB_LIMIT) {
1568                    Ok(Some(size)) => match crate::blobs::check(&path, key, &want.hash) {
1569                        Ok(()) => {
1570                            pulled.fetched += 1;
1571                            pulled.bytes += size;
1572                            tried.insert(name.clone(), (0, 0));
1573                        }
1574                        // Not whole, or another content: never kept, asked again after a while.
1575                        Err(fault) => {
1576                            let _ = std::fs::remove_file(&path);
1577                            pulled.held += 1;
1578                            tried.insert(name.clone(), failed(!matches!(fault, crate::blobs::Fault::Broken)));
1579                        }
1580                    },
1581                    // Not on the server yet (its writer's sync app has not sent it), or too large.
1582                    Ok(None) => {
1583                        tried.insert(name.clone(), failed(true));
1584                    }
1585                    Err(e) => {
1586                        tried.insert(name.clone(), failed(true));
1587                        return Err(e);
1588                    }
1589                }
1590            }
1591            Ok(())
1592        });
1593        if let Err(e) = outcome {
1594            pulled.problem = Some(code(&e, &state.host()));
1595        }
1596        let _ = crate::blobs::Wanted::change(memory, |wanted| {
1597            for (name, (tries, again)) in tried {
1598                if let Some(want) = wanted.blobs.get_mut(&name) {
1599                    (want.tries, want.again) = (tries, again);
1600                }
1601            }
1602            wanted.large_at = wanted.large_at.max(large_at);
1603        });
1604        pulled
1605    })
1606}
1607
1608/// One pull: the server's folder looked through, what is newer there fetched.
1609struct Puller<'a> {
1610    server: &'a Server,
1611    /// The sharing folder as the sync app keeps it, and where what comes goes.
1612    folder: &'a Path,
1613    into: &'a Path,
1614    own: &'a str,
1615    state: &'a mut State,
1616    pulled: &'a mut Pulled,
1617    now: i64,
1618    /// This device's own files seen in the folders looked through, their ETags there (`MIRROR`).
1619    own_there: BTreeMap<String, String>,
1620    /// The folders of the server looked through in this step ("" the folder itself, "devices/").
1621    covered: BTreeSet<String>,
1622    /// Sioul keeping the folder itself, its copy lost: none of its records here.
1623    lost: bool,
1624    /// The others' sealed files to bring (`MIRROR`), listed by `run`, fetched
1625    /// last (`fetch_blobs`), once this device's own files went up.
1626    blobs: Vec<(String, Item)>,
1627    /// `blobs/`'s ETag as listed: kept once every file of it came.
1628    blobs_listed: Option<String>,
1629}
1630
1631impl Puller<'_> {
1632    fn url(&self, relative: &str) -> String {
1633        format!("{}{}", self.state.url, encode_path(relative))
1634    }
1635
1636    /// A folder of the server not changed since it was last looked through, lately.
1637    fn unchanged(&self, relative: &str, etag: &str) -> bool {
1638        !self.lost && !etag.is_empty() && self.state.folders.get(relative).is_some_and(|l| l.etag == etag && self.now - l.at < LIST_AGAIN)
1639    }
1640
1641    /// A folder looked through to its end: until it changes, not again for a while.
1642    fn looked(&mut self, relative: &str, etag: &str) {
1643        self.state.folders.insert(relative.to_string(), Looked { etag: etag.to_string(), at: self.now });
1644    }
1645
1646    fn run(&mut self) -> Result<(), Stop> {
1647        let root = self.server.list(&self.state.url).map_err(|e| match e {
1648            SyncError::NotFound(_) => Stop::Unconfirmed(format!("seal-gone:{}", self.state.host())),
1649            e => Stop::Failed(e),
1650        })?;
1651        self.pulled.listed += root.items.len();
1652        // Never fetched from a folder sealed otherwise.
1653        self.seal(&root)?;
1654        // Sioul keeping the folder itself, its copy lost (none of its records
1655        // here): everything looked through again, what it wrote brought back first.
1656        self.lost = self.mirror() && own_latest(self.folder, self.own).is_none();
1657        if self.unchanged("", &root.etag) {
1658            // What was left to the sync app and has come here since: no longer left.
1659            let (folder, into) = (self.folder, self.into);
1660            let here = |name: &str| [folder.join(name), into.join(name)].iter().filter_map(|path| std::fs::metadata(path).ok()).map(|m| m.len()).max().unwrap_or(0);
1661            self.state.left.retain(|name, size| here(name) < *size);
1662            return Ok(());
1663        }
1664        // Looked through anew: what is left to the sync app, found again.
1665        self.state.left.clear();
1666        self.own_listed("", &root);
1667        for (name, item) in root.items.iter().filter(|(_, item)| !item.dir) {
1668            match kind_of(name) {
1669                Some((Kind::Round, id)) if id != self.own => self.round(name, item)?,
1670                Some((Kind::Seen, id)) if id != self.own => self.small(name, item)?,
1671                Some(_) if self.mirror() => self.restore(name, item)?,
1672                _ => {}
1673            }
1674        }
1675        if let Some(devices) = root.items.get("devices").filter(|item| item.dir)
1676            && !self.unchanged("devices/", &devices.etag)
1677        {
1678            let listing = self.server.list(&self.url("devices/"))?;
1679            self.pulled.listed += listing.items.len();
1680            self.own_listed("devices/", &listing);
1681            // This device's own entry there, as listed: an older copy put back there is sent over (`send`).
1682            if let Some(item) = listing.items.get(&format!("{}.device", self.own)).filter(|item| !item.dir) {
1683                self.state.own_entry = Some((item.size, self.now));
1684            }
1685            for (name, item) in listing.items.iter().filter(|(_, item)| !item.dir) {
1686                let relative = format!("devices/{name}");
1687                match kind_of(&relative) {
1688                    Some((_, id)) if id != self.own => self.small(&relative, item)?,
1689                    Some(_) if self.mirror() => self.restore(&relative, item)?,
1690                    _ => {}
1691                }
1692            }
1693            self.forget_gone("devices/", &listing);
1694            self.looked("devices/", &listing.etag);
1695        }
1696        if let Some(leases) = root.items.get("leases").filter(|item| item.dir)
1697            && !self.unchanged("leases/", &leases.etag)
1698        {
1699            let parts = self.server.list(&self.url("leases/"))?;
1700            self.pulled.listed += parts.items.len();
1701            for (part, item) in parts.items.iter().filter(|(_, item)| item.dir) {
1702                let dir = format!("leases/{part}/");
1703                if self.unchanged(&dir, &item.etag) {
1704                    continue;
1705                }
1706                let listing = self.server.list(&self.url(&dir))?;
1707                self.pulled.listed += listing.items.len();
1708                self.own_listed(&dir, &listing);
1709                for (name, item) in listing.items.iter().filter(|(_, item)| !item.dir) {
1710                    let relative = format!("{dir}{name}");
1711                    match kind_of(&relative) {
1712                        Some((_, id)) if id != self.own => self.small(&relative, item)?,
1713                        Some(_) if self.mirror() => self.restore(&relative, item)?,
1714                        _ => {}
1715                    }
1716                }
1717                self.forget_gone(&dir, &listing);
1718                self.looked(&dir, &listing.etag);
1719            }
1720            self.looked("leases/", &parts.etag);
1721        }
1722        // Each device's texts to send, a small file of its own: a phone has
1723        // them within its quarter of an hour, whatever the records' size.
1724        if let Some(texts) = root.items.get("texts").filter(|item| item.dir)
1725            && !self.unchanged("texts/", &texts.etag)
1726        {
1727            match self.server.list(&self.url("texts/send/")) {
1728                Ok(listing) => {
1729                    self.pulled.listed += listing.items.len();
1730                    self.own_listed("texts/send/", &listing);
1731                    for (name, item) in listing.items.iter().filter(|(_, item)| !item.dir) {
1732                        let relative = format!("texts/send/{name}");
1733                        if let Some((Kind::Texts, id)) = kind_of(&relative)
1734                            && id != self.own
1735                        {
1736                            self.small(&relative, item)?;
1737                        }
1738                    }
1739                    self.forget_gone("texts/send/", &listing);
1740                    self.looked("texts/", &texts.etag);
1741                }
1742                Err(SyncError::NotFound(_)) => self.looked("texts/", &texts.etag),
1743                Err(e) => return Err(Stop::Failed(e)),
1744            }
1745        }
1746        // Sioul keeping the folder itself: the sealed notes and papers the others put there come too.
1747        if self.mirror()
1748            && let Some(blobs) = root.items.get("blobs").filter(|item| item.dir)
1749            && !self.unchanged("blobs/", &blobs.etag)
1750        {
1751            let listing = self.server.list(&self.url("blobs/"))?;
1752            self.pulled.listed += listing.items.len();
1753            self.own_listed("blobs/", &listing);
1754            // Brought last (`fetch_blobs`): a large one on a slow line never
1755            // holds back this device's own records going up.
1756            self.blobs = listing.items.iter().filter(|(name, item)| !item.dir && !name.starts_with('.')).map(|(name, item)| (format!("blobs/{name}"), item.clone())).collect();
1757            self.blobs_listed = Some(listing.etag.clone());
1758        }
1759        self.forget_gone("", &root);
1760        self.looked("", &root.etag);
1761        Ok(())
1762    }
1763
1764    /// Sioul keeps the folder itself (`MIRROR`): what comes goes into it, no sync app beside.
1765    fn mirror(&self) -> bool {
1766        self.into == self.folder
1767    }
1768
1769    /// A folder of the server just looked through: this device's own files
1770    /// in it (`MIRROR`), their ETags there, for what goes up next.
1771    fn own_listed(&mut self, dir: &str, listing: &Listing) {
1772        self.covered.insert(dir.to_string());
1773        for (name, item) in listing.items.iter().filter(|(_, item)| !item.dir) {
1774            let relative = format!("{dir}{name}");
1775            let own = match kind_of(&relative) {
1776                Some((_, id)) => id == self.own,
1777                None => dir == "blobs/" && self.state.sent.contains_key(&relative),
1778            };
1779            if own {
1780                self.own_there.insert(relative, item.etag.clone());
1781            }
1782        }
1783    }
1784
1785    /// The server's seal, read again when it changed there or every six
1786    /// hours: another, or none, and nothing more is fetched from there.
1787    fn seal(&mut self, root: &Listing) -> Result<(), Stop> {
1788        let host = self.state.host();
1789        let Some(item) = root.items.get("seal.toml").filter(|item| !item.dir) else { return Err(Stop::Unconfirmed(format!("seal-gone:{host}"))) };
1790        let known = Fetched { etag: item.etag.clone(), size: item.size };
1791        if !item.etag.is_empty() && self.state.files.get("seal.toml") == Some(&known) && self.now - self.state.sealed < SEAL_AGAIN {
1792            return Ok(());
1793        }
1794        let here = seal_of(self.folder).ok_or_else(|| Stop::Unconfirmed("no-seal".into()))?;
1795        match self.server.small(&self.url("seal.toml"))? {
1796            Some((bytes, _)) if bytes == here => {
1797                self.state.sealed = self.now;
1798                self.state.files.insert("seal.toml".into(), known);
1799                Ok(())
1800            }
1801            Some(_) => Err(Stop::Unconfirmed(format!("seal-differs:{host}"))),
1802            None => Err(Stop::Unconfirmed(format!("seal-gone:{host}"))),
1803        }
1804    }
1805
1806    /// Files of a folder of the server no longer there: forgotten, and their
1807    /// copies fetched gone (a records' file only once its writer removed it:
1808    /// every device had read past it).
1809    fn forget_gone(&mut self, dir: &str, listing: &Listing) {
1810        let gone: Vec<String> = self.state.files.keys().filter(|path| path.strip_prefix(dir).is_some_and(|name| !name.contains('/') && !listing.items.contains_key(name))).cloned().collect();
1811        for path in gone {
1812            self.state.files.remove(&path);
1813            // Beside a sync app, the copy fetched goes; Sioul keeping the folder
1814            // itself, another device's records its writer let go (every device read past them).
1815            let round = matches!(kind_of(&path), Some((Kind::Round, _)));
1816            if path != "seal.toml" && (self.into != self.folder || round) {
1817                let _ = std::fs::remove_file(self.into.join(&path));
1818            }
1819        }
1820    }
1821
1822    /// A small file (notes, an entry, a claim) fetched whole when it changed
1823    /// there since it was last fetched; kept only when it differs from the
1824    /// synced folder's copy, and only when all of it came.
1825    fn small(&mut self, relative: &str, item: &Item) -> Result<(), Stop> {
1826        let known = Fetched { etag: item.etag.clone(), size: item.size };
1827        // Sioul keeping the folder itself, one gone from its copy comes again.
1828        if !item.etag.is_empty() && self.state.files.get(relative) == Some(&known) && (!self.mirror() || self.folder.join(relative).exists()) {
1829            return Ok(());
1830        }
1831        let cap = if matches!(kind_of(relative), Some((Kind::Texts, _))) { TEXTS_LARGEST } else { crate::share::SMALL_FILE };
1832        if item.size > cap {
1833            return Ok(());
1834        }
1835        let Some((bytes, _)) = self.server.small_up_to(&self.url(relative), cap)? else {
1836            self.state.files.remove(relative);
1837            return Ok(());
1838        };
1839        self.pulled.bytes += bytes.len() as u64;
1840        let (here, into) = (self.folder.join(relative), self.into.join(relative));
1841        if std::fs::read(&here).ok().as_deref() == Some(&bytes[..]) {
1842            // The sync app brought it already: nothing kept twice.
1843            if into != here {
1844                let _ = std::fs::remove_file(&into);
1845            }
1846        } else {
1847            crate::share::write_atomically(&into, &bytes).map_err(|e| Stop::Failed(SyncError::Disk(e)))?;
1848            self.pulled.fetched += 1;
1849        }
1850        self.state.files.insert(relative.to_string(), known);
1851        Ok(())
1852    }
1853
1854    /// One of this device's own files there and not here (`MIRROR`), never
1855    /// sent from here, or its copy lost (`lost`: none of its records here):
1856    /// brought back, so that it goes on after its last record rather than
1857    /// numbering them again over the others' reading.
1858    fn restore(&mut self, relative: &str, item: &Item) -> Result<(), Stop> {
1859        let here = self.folder.join(relative);
1860        if here.exists() || (self.state.sent.contains_key(relative) && !self.lost) || item.size > LARGEST {
1861            return Ok(());
1862        }
1863        let Some(got) = self.server.tail(&self.url(relative), 0)? else { return Ok(()) };
1864        if let Some(e) = got.error {
1865            return Err(Stop::Failed(e));
1866        }
1867        // Records keep their whole lines only.
1868        let keep = if matches!(kind_of(relative), Some((Kind::Round, _))) { got.bytes.iter().rposition(|b| *b == b'\n').map_or(0, |at| at + 1) } else { got.bytes.len() };
1869        crate::share::write_atomically(&here, &got.bytes[..keep]).map_err(|e| Stop::Failed(SyncError::Disk(e)))?;
1870        let modified = std::fs::metadata(&here).map(|m| crate::share::modified_ns(&m)).unwrap_or(0);
1871        let etag = if keep == got.bytes.len() { item.etag.clone() } else { String::new() };
1872        self.state.sent.insert(relative.to_string(), Sent { size: keep as u64, modified, etag });
1873        self.pulled.fetched += 1;
1874        self.pulled.bytes += keep as u64;
1875        Ok(())
1876    }
1877
1878    /// The others' sealed files `run` listed, brought one after the other,
1879    /// last in the step. One that fails stops the rest until the next step,
1880    /// which looks through the folder again rather than ten minutes later.
1881    fn fetch_blobs(&mut self) -> Result<(), Stop> {
1882        for (relative, item) in std::mem::take(&mut self.blobs) {
1883            if let Err(stop) = self.blob(&relative, &item) {
1884                self.state.folders.remove("blobs/");
1885                self.state.folders.remove("");
1886                self.blobs_listed = None;
1887                return Err(stop);
1888            }
1889        }
1890        if let Some(etag) = self.blobs_listed.take() {
1891            self.looked("blobs/", &etag);
1892        }
1893        Ok(())
1894    }
1895
1896    /// A sealed file another device put there and not here (`MIRROR`):
1897    /// fetched whole, kept only whole. One sent from here never comes back.
1898    fn blob(&mut self, relative: &str, item: &Item) -> Result<(), Stop> {
1899        let here = self.folder.join(relative);
1900        if self.state.sent.contains_key(relative) || item.size > BLOB_LIMIT {
1901            return Ok(());
1902        }
1903        let known = Fetched { etag: item.etag.clone(), size: item.size };
1904        if std::fs::metadata(&here).is_ok_and(|m| m.len() == item.size) {
1905            self.state.files.insert(relative.to_string(), known);
1906            return Ok(());
1907        }
1908        if let Some(size) = self.server.download(&self.url(relative), &here, BLOB_LIMIT)? {
1909            self.pulled.fetched += 1;
1910            self.pulled.bytes += size;
1911            self.state.files.insert(relative.to_string(), known);
1912        }
1913        Ok(())
1914    }
1915
1916    /// This device's own files (`MIRROR`) changed here since they went up, or
1917    /// changed or gone there since (seen in the folders just looked through),
1918    /// sent, in `own_files`' order, each over what is there as it was seen
1919    /// (`send`). Then what it let go here goes there too: its records past
1920    /// their time (`remove_old_rounds`: every device read past them), never
1921    /// its last round, and nothing while none of its records is here; the
1922    /// sealed files it swept.
1923    fn push(&mut self) -> Result<(), Stop> {
1924        for relative in own_files(self.folder, self.own, self.state) {
1925            let Ok(meta) = std::fs::metadata(self.folder.join(&relative)) else { continue };
1926            let (size, modified) = (meta.len(), crate::share::modified_ns(&meta));
1927            let sent = self.state.sent.get(&relative).cloned();
1928            let covered = self.covered.contains(dir_of(&relative));
1929            let there = self.own_there.get(&relative).cloned();
1930            let same_here = sent.as_ref().is_some_and(|s| s.size == size && s.modified == modified);
1931            // Its ETag there, unsaid when it went up, known now.
1932            if same_here
1933                && let (Some(sent), Some(etag)) = (&sent, &there)
1934                && sent.etag.is_empty()
1935            {
1936                self.state.sent.insert(relative.clone(), Sent { etag: etag.clone(), ..sent.clone() });
1937                continue;
1938            }
1939            let same_there = !covered || there.as_deref() == sent.as_ref().map(|s| s.etag.as_str());
1940            if same_here && same_there {
1941                continue;
1942            }
1943            let seen = if covered { there } else { sent.map(|s| s.etag).filter(|e| !e.is_empty()) };
1944            self.send(&relative, size, modified, seen)?;
1945        }
1946        if let Some(latest) = own_latest(self.folder, self.own)
1947            && self.covered.contains("")
1948        {
1949            let gone: Vec<(String, String)> = self
1950                .own_there
1951                .iter()
1952                .filter(|(relative, _)| matches!(kind_of(relative), Some((Kind::Round, _))) && self.state.sent.contains_key(*relative) && !self.folder.join(relative).exists())
1953                .filter(|(relative, _)| relative.trim_end_matches(".jsonl").rsplit('-').next().and_then(|n| n.parse::<u32>().ok()).is_some_and(|round| round < latest))
1954                .map(|(relative, etag)| (relative.clone(), etag.clone()))
1955                .collect();
1956            for (relative, etag) in gone {
1957                self.take_out(&relative, Some(etag))?;
1958            }
1959        }
1960        let swept: Vec<(String, String)> = self.state.sent.iter().filter(|(relative, _)| relative.starts_with("blobs/") && !self.folder.join(relative).exists()).map(|(relative, sent)| (relative.clone(), sent.etag.clone())).collect();
1961        for (relative, etag) in swept {
1962            self.take_out(&relative, Some(etag).filter(|e| !e.is_empty()))?;
1963        }
1964        Ok(())
1965    }
1966
1967    /// One of this device's files sent whole over what is there as it was
1968    /// seen (`seen`: its ETag; none there). Refused because it changed there
1969    /// meanwhile (`412`), sent again over what is there now: it is this
1970    /// device's own. A folder missing there (`409`), made first. A sealed file
1971    /// goes only where none is: one there already is the same. Refused again
1972    /// and again: left for the next step; nothing here is lost.
1973    fn send(&mut self, relative: &str, size: u64, modified: u64, seen: Option<String>) -> Result<(), Stop> {
1974        let blob = relative.starts_with("blobs/");
1975        let bytes = std::fs::read(self.folder.join(relative)).map_err(|e| Stop::Failed(SyncError::Disk(format!("{relative}: {e}"))))?;
1976        let url = self.url(relative);
1977        let mut seen = if blob { None } else { seen };
1978        let sha1 = sha1_hex(&bytes);
1979        for attempt in 0..3 {
1980            // Given the time its size asks on a slow line (`put_file`).
1981            let (status, etag) = self.server.put_file(&url, bytes.clone(), seen.as_deref(), modified / 1_000_000_000, &sha1).map_err(|failed| Stop::Failed(failed.error))?;
1982            match status {
1983                200 | 201 | 204 => {
1984                    self.state.sent.insert(relative.to_string(), Sent { size, modified, etag: etag.unwrap_or_default() });
1985                    self.pulled.sent += 1;
1986                    self.pulled.bytes += bytes.len() as u64;
1987                    return Ok(());
1988                }
1989                412 if blob => {
1990                    self.state.sent.insert(relative.to_string(), Sent { size, modified, etag: String::new() });
1991                    return Ok(());
1992                }
1993                412 => seen = self.server.etag(&url)?,
1994                409 if attempt == 0 => self.make_folders(relative)?,
1995                507 => return Err(Stop::Failed(SyncError::Server("quota".into()))),
1996                status => return Err(Stop::Failed(SyncError::Server(format!("PUT: {status}")))),
1997            }
1998        }
1999        Err(Stop::Failed(SyncError::Server("PUT: 412".into())))
2000    }
2001
2002    /// The folders a file's path goes through, made there (`MKCOL`) when missing.
2003    fn make_folders(&mut self, relative: &str) -> Result<(), Stop> {
2004        let mut at = String::new();
2005        let names: Vec<&str> = relative.split('/').collect();
2006        for name in &names[..names.len().saturating_sub(1)] {
2007            at = format!("{at}{name}/");
2008            match self.server.mkcol(&self.url(&at))? {
2009                201 | 405 => {}
2010                status => return Err(Stop::Failed(SyncError::Server(format!("MKCOL: {status}")))),
2011            }
2012        }
2013        Ok(())
2014    }
2015
2016    /// One of this device's files taken out there, only as it was seen when
2017    /// its ETag is known; gone already, or changed there since (another device
2018    /// sealed the same content again), forgotten here all the same.
2019    fn take_out(&mut self, relative: &str, seen: Option<String>) -> Result<(), Stop> {
2020        match self.server.delete(&self.url(relative), seen.as_deref())? {
2021            200 | 204 | 404 | 412 => {
2022                self.state.sent.remove(relative);
2023                Ok(())
2024            }
2025            status => Err(Stop::Failed(SyncError::Server(format!("DELETE: {status}")))),
2026        }
2027    }
2028
2029    /// A device's records, when the server's copy is longer than both the
2030    /// synced folder's and what was fetched: what is new there, from the end
2031    /// of the longer copy here, its last whole line asked again and checked
2032    /// to be the same there (a copy that does not join comes whole). Only
2033    /// whole lines are kept: a line half arrived comes again next time.
2034    fn round(&mut self, name: &str, item: &Item) -> Result<(), Stop> {
2035        let (here, into) = (self.folder.join(name), self.into.join(name));
2036        let size = |path: &Path| std::fs::metadata(path).map_or(0, |m| m.len());
2037        let longest = size(&here).max(size(&into));
2038        let known = Fetched { etag: item.etag.clone(), size: item.size };
2039        if item.size <= longest {
2040            // Nothing more there than here.
2041            self.state.files.insert(name.to_string(), known);
2042            return Ok(());
2043        }
2044        if item.size > round_cap() {
2045            // Past what a pull brings: left to the sync app, and said (`State::left`).
2046            self.state.left.insert(name.to_string(), item.size);
2047            self.pulled.left += 1;
2048            return Ok(());
2049        }
2050        let base = if size(&into) >= size(&here) { into.clone() } else { here.clone() };
2051        let (end, last) = whole_end(&base);
2052        let from = if end - last > OVERLAP { end.saturating_sub(1) } else { last };
2053        let overlap = read_range(&base, from, end).unwrap_or_default();
2054        let url = self.url(name);
2055        let Some(mut got) = self.server.tail(&url, from)? else { return Ok(()) };
2056        let joins = got.start == from && from > 0;
2057        if joins && !got.bytes.starts_with(&overlap) {
2058            if got.error.is_some() && overlap.starts_with(&got.bytes) {
2059                // Cut within the line asked again: nothing new came.
2060                return Err(Stop::Failed(got.error.take().unwrap_or(SyncError::Network(String::new()))));
2061            }
2062            // Not the same there: the whole file.
2063            match self.server.tail(&url, 0)? {
2064                Some(whole) => got = whole,
2065                None => return Ok(()),
2066            }
2067        } else if !joins && got.start != 0 {
2068            return Err(Stop::Failed(SyncError::Server(format!("GET: from {}", got.start))));
2069        }
2070        let prefix = if got.start == from && from > 0 { from } else { 0 };
2071        self.pulled.bytes += got.bytes.len() as u64;
2072        // Whole lines only.
2073        let whole = got.bytes.iter().rposition(|b| *b == b'\n').map_or(0, |at| at + 1);
2074        if prefix + whole as u64 > longest {
2075            let temporary = crate::share::temporary(&into);
2076            let written = (|| -> std::io::Result<()> {
2077                if let Some(parent) = into.parent() {
2078                    crate::share::private_dirs(parent)?;
2079                }
2080                let mut out = crate::share::new_private(&temporary)?;
2081                if prefix > 0 {
2082                    std::io::copy(&mut std::fs::File::open(&base)?.take(prefix), &mut out)?;
2083                }
2084                out.write_all(&got.bytes[..whole])?;
2085                out.sync_all()?;
2086                std::fs::rename(&temporary, &into)
2087            })();
2088            if let Err(e) = written {
2089                let _ = std::fs::remove_file(&temporary);
2090                return Err(Stop::Failed(SyncError::Disk(format!("{name}: {e}"))));
2091            }
2092            self.pulled.fetched += 1;
2093        }
2094        match got.error {
2095            Some(e) => Err(Stop::Failed(e)),
2096            None => {
2097                self.state.files.insert(name.to_string(), known);
2098                Ok(())
2099            }
2100        }
2101    }
2102}
2103
2104/// Where a file's last whole line ends, and where it starts: (0, 0) when it holds none.
2105fn whole_end(path: &Path) -> (u64, u64) {
2106    let Ok(mut file) = std::fs::File::open(path) else { return (0, 0) };
2107    let size = file.metadata().map_or(0, |m| m.len());
2108    let Some(end) = newline_before(&mut file, size).map(|at| at + 1) else { return (0, 0) };
2109    let last = newline_before(&mut file, end - 1).map_or(0, |at| at + 1);
2110    (end, last)
2111}
2112
2113/// Where the last line break before `before` is, read back a piece at a time.
2114fn newline_before(file: &mut std::fs::File, before: u64) -> Option<u64> {
2115    let mut end = before;
2116    while end > 0 {
2117        let from = end.saturating_sub(64 << 10);
2118        let mut piece = vec![0u8; usize::try_from(end - from).ok()?];
2119        file.seek(SeekFrom::Start(from)).ok()?;
2120        file.read_exact(&mut piece).ok()?;
2121        if let Some(at) = piece.iter().rposition(|b| *b == b'\n') {
2122            return Some(from + at as u64);
2123        }
2124        end = from;
2125    }
2126    None
2127}
2128
2129fn read_range(path: &Path, from: u64, to: u64) -> Option<Vec<u8>> {
2130    let mut file = std::fs::File::open(path).ok()?;
2131    file.seek(SeekFrom::Start(from)).ok()?;
2132    let mut bytes = vec![0u8; usize::try_from(to.saturating_sub(from)).ok()?];
2133    file.read_exact(&mut bytes).ok()?;
2134    Some(bytes)
2135}
2136
2137// ---------------------------------------------------------------- kept in step by Sioul itself
2138
2139/// The mode where Sioul keeps the folder in step with the server itself, no
2140/// sync app at all (docs/database.md, "Kept in step by Sioul itself"): the
2141/// folder here is its own copy (`mirror_of`), the server's the one the
2142/// devices share, a device beside a sync app reading the same.
2143pub const MIRROR: &str = "mirror";
2144
2145/// Sioul's own copy of the folder in that mode: `<state>/share/mirror/`.
2146pub fn mirror_of(memory: &Path) -> PathBuf {
2147    memory.with_file_name("mirror")
2148}
2149
2150/// A sealed file is never fetched past this: the largest note's, sealed.
2151const BLOB_LIMIT: u64 = crate::blobs::LARGEST + (1 << 20);
2152
2153/// A folder of an account's files as Sioul keeping it itself finds it: its
2154/// address, where it is among the files, and its seal when it has one
2155/// (another device shares through it: joining), else none (a new sharing).
2156pub struct Opened {
2157    pub url: String,
2158    root: String,
2159    pub place: String,
2160    pub seal: Option<Vec<u8>>,
2161}
2162
2163/// Where `place` ("Documents/Sioul") is among `login`'s files, and what it
2164/// holds; why not, in a code as `State::said` has them.
2165pub fn open(login: &Login, place: &str) -> Result<Opened, String> {
2166    open_with(login, place, LIMITS)
2167}
2168
2169pub(crate) fn open_with(login: &Login, place: &str, limits: Limits) -> Result<Opened, String> {
2170    let host = host_of(&login.url);
2171    let place = place.trim().trim_matches('/').to_string();
2172    if place.is_empty() {
2173        return Err("no-place".into());
2174    }
2175    let server = Server::new(login, limits).map_err(|e| code(&e, &login.account))?;
2176    let root = files_root(&server, login).map_err(|e| code(&e, &host))?.ok_or_else(|| format!("no-files:{host}"))?;
2177    let url = format!("{root}{}/", encode_path(&place));
2178    let seal = server.small(&format!("{url}seal.toml")).map_err(|e| code(&e, &host))?.map(|(bytes, _)| bytes);
2179    Ok(Opened { url, root, place, seal })
2180}
2181
2182/// A new sharing there: each missing folder of its place made (`MKCOL`),
2183/// then `folder`'s seal sent only if none is there yet (`If-None-Match: *`):
2184/// two devices starting at once never seal it twice ("sealed-meanwhile").
2185pub fn create(login: &Login, opened: &Opened, folder: &Path) -> Result<(), String> {
2186    create_with(login, opened, folder, LIMITS)
2187}
2188
2189pub(crate) fn create_with(login: &Login, opened: &Opened, folder: &Path, limits: Limits) -> Result<(), String> {
2190    let host = host_of(&login.url);
2191    let failed = |e: SyncError| code(&e, &host);
2192    let server = Server::new(login, limits).map_err(failed)?;
2193    let mut at = opened.root.clone();
2194    for name in opened.place.split('/').filter(|n| !n.is_empty()) {
2195        at = format!("{at}{}/", encode_path(name));
2196        if !matches!(server.mkcol(&at).map_err(failed)?, 201 | 405) {
2197            return Err(format!("server:{host}"));
2198        }
2199    }
2200    let seal = seal_of(folder).ok_or_else(|| "no-seal".to_string())?;
2201    match server.put(&format!("{}seal.toml", opened.url), seal, None).map_err(failed)?.0 {
2202        200 | 201 | 204 => Ok(()),
2203        412 => Err("sealed-meanwhile".into()),
2204        _ => Err(format!("server:{host}")),
2205    }
2206}
2207
2208/// From now on Sioul keeps `folder` in step with `opened` itself (`MIRROR`),
2209/// with `login`'s account: what the backup knew goes.
2210pub fn begin(memory: &Path, folder: &Path, login: &Login, opened: &Opened, now: i64) -> Result<(), String> {
2211    let _ = std::fs::remove_dir_all(cache_of(memory));
2212    let state = State { mode: MIRROR.into(), place: opened.place.clone(), folder: shown(folder), account: login.account.clone(), url: opened.url.clone(), confirmed: now, sealed: now, ..State::default() };
2213    let path = state_path(memory);
2214    sioul_core::filelock::with_lock(&path, || state.write(memory))
2215}
2216
2217/// One step of Sioul keeping the folder itself (`MIRROR`): with `whole`, the
2218/// server's folder looked through first and the other devices' newer files
2219/// brought into the copy here, as `pull` brings them beside a synced folder;
2220/// then this device's own files changed since they went up, or changed there
2221/// since, sent (`Puller::push`), its records first, its sealed files last;
2222/// then the others' sealed notes, papers and spam filter's tables brought
2223/// (`Puller::fetch_blobs`). A large sealed file on a slow line, going up or
2224/// coming down, never holds back the records: the doses' answers go first.
2225/// Never another device's file written; nothing here lost when the server
2226/// refuses. One step at a time on this device, whichever process runs it.
2227pub fn step(memory: &Path, folder: &Path, own: &str, login: &Login, now: i64, whole: bool) -> Pulled {
2228    step_with(memory, folder, own, login, now, whole, LIMITS)
2229}
2230
2231pub(crate) fn step_with(memory: &Path, folder: &Path, own: &str, login: &Login, now: i64, whole: bool, limits: Limits) -> Pulled {
2232    sioul_core::filelock::with_lock(&memory.with_file_name("mirror.step"), || {
2233        let mut state = State::load(memory);
2234        let mut pulled = Pulled::default();
2235        if state.mode != MIRROR || !state.confirmed_for(folder) || state.account != login.account {
2236            pulled.problem = Some("not-confirmed".into());
2237            return pulled;
2238        }
2239        let host = state.host();
2240        if whole {
2241            state.tried = now;
2242        }
2243        let outcome = Server::new(login, limits).map_err(Stop::Failed).and_then(|server| {
2244            let mut puller = Puller { server: &server, folder, into: folder, own, state: &mut state, pulled: &mut pulled, now, own_there: BTreeMap::new(), covered: BTreeSet::new(), lost: false, blobs: Vec::new(), blobs_listed: None };
2245            if whole {
2246                puller.run()?;
2247            }
2248            // This device's own files first (its records before its sealed
2249            // files), the others' sealed files last: nothing large holds back
2250            // what the others wait for. Each said when it fails.
2251            let pushed = puller.push();
2252            let fetched = if whole { puller.fetch_blobs() } else { Ok(()) };
2253            pushed.and(fetched)
2254        });
2255        match outcome {
2256            Ok(()) => {
2257                if whole {
2258                    state.last = now;
2259                }
2260                state.pushed = now;
2261                state.said.clear();
2262            }
2263            Err(Stop::Unconfirmed(why)) => {
2264                state.confirmed = 0;
2265                state.said = why;
2266            }
2267            Err(Stop::Failed(e)) => state.said = code(&e, &host),
2268        }
2269        let _ = state.save(memory);
2270        pulled.problem = (!state.said.is_empty()).then(|| state.said.clone());
2271        pulled
2272    })
2273}
2274
2275/// The folder a file of the folder is in, as the server's listings are kept: "" the folder itself, "devices/", "leases/health/".
2276fn dir_of(relative: &str) -> &str {
2277    relative.rfind('/').map_or("", |at| &relative[..=at])
2278}
2279
2280/// This device's own files in the folder (`MIRROR`), by their path in it, in
2281/// the order they go up: its records, its notes to the others, its claims,
2282/// its entry (it says how far its records went), then the sealed files it
2283/// made (in `blobs/`, not fetched). A large sealed file on a slow line never
2284/// holds back the records, the doses' answers among them; a record naming a
2285/// sealed file not there yet is read as a sync app's late copy is: the
2286/// others wait for it (docs/database.md, "Coming in").
2287fn own_files(folder: &Path, own: &str, state: &State) -> Vec<String> {
2288    let mut out = own_plain(folder, own);
2289    out.extend(file_names(&folder.join("blobs")).into_iter().map(|n| format!("blobs/{n}")).filter(|r| !state.files.contains_key(r)));
2290    out
2291}
2292
2293/// The files of a folder, by name, hidden ones left out.
2294fn file_names(dir: &Path) -> Vec<String> {
2295    std::fs::read_dir(dir).into_iter().flatten().filter_map(Result::ok).filter(|e| e.file_type().is_ok_and(|t| t.is_file())).map(|e| e.file_name().to_string_lossy().to_string()).filter(|n| !n.starts_with('.')).collect()
2296}
2297
2298/// This device's own files in the folder but its sealed ones, in the order
2299/// they go up: its records, its notes to the others, its claims, its entry.
2300fn own_plain(folder: &Path, own: &str) -> Vec<String> {
2301    let names = file_names;
2302    let mut out: Vec<String> = Vec::new();
2303    let mut records: Vec<(u8, u32, String)> = names(folder)
2304        .into_iter()
2305        .filter_map(|name| match kind_of(&name) {
2306            Some((Kind::Round, id)) if id == own => Some((0, name.trim_end_matches(".jsonl").rsplit('-').next().and_then(|n| n.parse().ok()).unwrap_or(0), name)),
2307            Some((Kind::Seen, id)) if id == own => Some((1, 0, name)),
2308            _ => None,
2309        })
2310        .collect();
2311    records.sort();
2312    out.extend(records.into_iter().map(|(_, _, name)| name));
2313    let mut parts: Vec<String> = std::fs::read_dir(folder.join("leases")).into_iter().flatten().filter_map(Result::ok).filter(|e| e.file_type().is_ok_and(|t| t.is_dir())).map(|e| e.file_name().to_string_lossy().to_string()).collect();
2314    parts.sort();
2315    out.extend(parts.into_iter().map(|part| format!("leases/{part}/{own}.lease")).filter(|claim| folder.join(claim).is_file()));
2316    let entry = format!("devices/{own}.device");
2317    if folder.join(&entry).is_file() {
2318        out.push(entry);
2319    }
2320    out
2321}
2322
2323/// The highest round of this device's records in the folder here; none when it holds none.
2324fn own_latest(folder: &Path, own: &str) -> Option<u32> {
2325    std::fs::read_dir(folder)
2326        .into_iter()
2327        .flatten()
2328        .filter_map(Result::ok)
2329        .filter_map(|e| {
2330            let name = e.file_name().to_string_lossy().to_string();
2331            match kind_of(&name) {
2332                Some((Kind::Round, id)) if id == own => name.trim_end_matches(".jsonl").rsplit('-').next().and_then(|n| n.parse().ok()),
2333                _ => None,
2334            }
2335        })
2336        .max()
2337}
2338
2339// ---------------------------------------------------------------- sent beside the sync app
2340
2341/// A full check of what the server holds of this device's files, beside the
2342/// quick sends (each a few requests): every six hours, and when asked
2343/// ("Send everything again").
2344const CHECK_AGAIN: i64 = 6 * 3600;
2345/// A file past this size (bytes) is a large one: on a metered or slow
2346/// connection (`Sending::frugal`), sent at most every `PACE` seconds.
2347pub const LARGE_FILE: u64 = 256 << 10;
2348const PACE: i64 = 10 * 60;
2349/// Under this rate (bytes a second), measured on a file sent of 64 KiB at
2350/// least, the connection counts as slow: paced as a metered one.
2351pub const SLOW: u64 = 64 << 10;
2352
2353/// How long a failed send waits before it is tried again, by how many failed
2354/// in a row: a minute, two, five, a quarter of an hour, half an hour, an hour.
2355pub fn backoff(tries: u32) -> i64 {
2356    [60, 120, 300, 900, 1800, 3600][tries.saturating_sub(1).min(5) as usize]
2357}
2358
2359/// The two ways this device's files go (`send`, `send_urgent`), each with its
2360/// own lock and its own memory, so that a large file on a slow line, going
2361/// up for minutes, never holds back a text to send.
2362#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2363enum Lane {
2364    /// Its records, notes, claims, entry and sealed files.
2365    Main,
2366    /// What must reach the others within minutes, in a small file of its own:
2367    /// its texts to send (`texts/send/<id>.jsonl`, `place_texts`).
2368    Urgent,
2369}
2370
2371impl Lane {
2372    fn memory(self, memory: &Path) -> PathBuf {
2373        memory.with_file_name(match self {
2374            Lane::Main => "sending.toml",
2375            Lane::Urgent => "sending-urgent.toml",
2376        })
2377    }
2378
2379    fn lock(self, memory: &Path) -> PathBuf {
2380        memory.with_file_name(match self {
2381            Lane::Main => "send.step",
2382            Lane::Urgent => "send.urgent",
2383        })
2384    }
2385}
2386
2387/// What sending this device's own files beside the sync app knows (`send`):
2388/// `share/sending.toml` (`share/sending-urgent.toml` for the urgent lane),
2389/// never shared, apart from the backup's state, so that a pull and a send
2390/// never write over each other's.
2391#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
2392pub struct Sending {
2393    /// The folder there the files went to: another, and each is looked at again.
2394    #[serde(default, skip_serializing_if = "String::is_empty")]
2395    pub url: String,
2396    /// When a send last went through whole, and was last tried (Unix seconds).
2397    #[serde(default)]
2398    pub last: i64,
2399    #[serde(default)]
2400    pub tried: i64,
2401    /// When the server's copies were last listed and compared with the files here (Unix seconds).
2402    #[serde(default)]
2403    pub checked: i64,
2404    /// Why the last send stopped, a code as `State::said` has them; "" when it went through.
2405    #[serde(default, skip_serializing_if = "String::is_empty")]
2406    pub said: String,
2407    /// The last failure, in detail: which file, how, how long, how many of its bytes went.
2408    #[serde(default, skip_serializing_if = "Option::is_none")]
2409    pub failure: Option<Failure>,
2410    /// The server unreachable (a connection, a certificate, a password
2411    /// refused): how many sends failed in a row, and none tried before
2412    /// `retry_at` (Unix seconds; "Send everything again" apart).
2413    #[serde(default)]
2414    pub tries: u32,
2415    #[serde(default)]
2416    pub retry_at: i64,
2417    /// Files that did not go (too slow, cut): how many tries in a row, and not
2418    /// tried again before when (`backoff`).
2419    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
2420    pub held: BTreeMap<String, Held>,
2421    /// The rate of the last file of 64 KiB or more that went, or of what went
2422    /// of one that failed (bytes a second); 0 unknown.
2423    #[serde(default)]
2424    pub rate: u64,
2425    /// When a large file was last sent or tried (Unix seconds), for the pace.
2426    #[serde(default)]
2427    pub large_at: i64,
2428    /// What "Send everything again" last did, in either mode.
2429    #[serde(default)]
2430    pub again: Again,
2431    /// This device's own files as they last went up, or were found there the same, by their path in the folder.
2432    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
2433    pub sent: BTreeMap<String, Sent>,
2434}
2435
2436/// A failed send, in detail: what the panel and the log say of it.
2437#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
2438pub struct Failure {
2439    /// "timeout" (slower than the floor), "stalled" (nothing moved for half
2440    /// a minute), "unreachable", "tls", "login", "server", "quota", "network".
2441    #[serde(default)]
2442    pub kind: String,
2443    /// The file, by kind: "records", "notes", "claim", "entry", "texts", "sealed".
2444    #[serde(default)]
2445    pub file: String,
2446    #[serde(default)]
2447    pub seconds: u64,
2448    #[serde(default)]
2449    pub sent: u64,
2450    #[serde(default)]
2451    pub total: u64,
2452    /// The error's own words, for the log.
2453    #[serde(default, skip_serializing_if = "String::is_empty")]
2454    pub words: String,
2455    /// When (Unix seconds).
2456    #[serde(default)]
2457    pub at: i64,
2458}
2459
2460/// A file held back after it failed: tries in a row, and when to try again (Unix seconds).
2461#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
2462pub struct Held {
2463    #[serde(default)]
2464    pub tries: u32,
2465    #[serde(default)]
2466    pub until: i64,
2467}
2468
2469/// What "Send everything again" did: when (Unix seconds), how many files
2470/// went, how many were there already as they are here, and why it stopped
2471/// ("" when it went through).
2472#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
2473pub struct Again {
2474    #[serde(default)]
2475    pub at: i64,
2476    #[serde(default)]
2477    pub sent: usize,
2478    #[serde(default)]
2479    pub same: usize,
2480    #[serde(default, skip_serializing_if = "String::is_empty")]
2481    pub said: String,
2482}
2483
2484#[cfg(test)]
2485fn sending_path(memory: &Path) -> PathBuf {
2486    Lane::Main.memory(memory)
2487}
2488
2489impl Sending {
2490    pub fn load(memory: &Path) -> Sending {
2491        Sending::load_lane(memory, Lane::Main)
2492    }
2493
2494    /// The urgent lane's (`send_urgent`).
2495    pub fn load_urgent(memory: &Path) -> Sending {
2496        Sending::load_lane(memory, Lane::Urgent)
2497    }
2498
2499    fn load_lane(memory: &Path, lane: Lane) -> Sending {
2500        std::fs::read_to_string(lane.memory(memory)).ok().and_then(|text| toml::from_str(&text).ok()).unwrap_or_default()
2501    }
2502
2503    fn save(&self, memory: &Path, lane: Lane) -> Result<(), String> {
2504        let text = toml::to_string(self).map_err(|e| e.to_string())?;
2505        crate::share::write_atomically(&lane.memory(memory), format!("# Sioul: this device's files sent to the sharing folder's server beside the sync app (docs/database.md). Never shared.\n{text}").as_bytes())
2506    }
2507
2508    /// "Send everything again" could not start (no account for the server,
2509    /// the folder not found, sending switched off): why, said as a code.
2510    pub fn note_again(memory: &Path, now: i64, why: &str) {
2511        let mut sending = Sending::load(memory);
2512        sending.again = Again { at: now, sent: 0, same: 0, said: why.to_string() };
2513        let _ = sending.save(memory, Lane::Main);
2514    }
2515
2516    /// The connection measured slow: under `SLOW` on the last large file.
2517    pub fn slow(&self) -> bool {
2518        self.rate > 0 && self.rate < SLOW
2519    }
2520
2521    /// When the next try is due, when a send waits (Unix seconds): the server
2522    /// unreachable, or a file held back; 0 when nothing waits.
2523    pub fn next_try(&self) -> i64 {
2524        self.held.values().map(|h| h.until).chain([self.retry_at]).filter(|at| *at > 0).min().unwrap_or(0)
2525    }
2526}
2527
2528/// This device's own files sent to the server beside the sync app that
2529/// carries the folder (docs/database.md, "Sent to the server too"), once the
2530/// backup found the folder there under the same seal. Only its own files
2531/// (`own_plain`, and the sealed files it made, `sealed`: their name and the
2532/// size it sealed, sent whole only), each over what is there as it was seen;
2533/// a file found there as it is here is not sent again (the sync app sent it,
2534/// or will find it there as its own); records only grow (never one shorter
2535/// than what went, or than the server's), and go in order; never another
2536/// device's file. Each file is given the time its size asks (`FLOOR`); one
2537/// that does not go is held back (`backoff`), never tried sooner, the next
2538/// records with it, the other files still going. The server unreachable,
2539/// nothing is tried before its backoff. With `frugal` (a metered connection),
2540/// or the connection measured slow, a large file goes at most every ten
2541/// minutes. With `again`, or every six hours, or the first time, the server's
2542/// folders are listed and every file compared: one lost there is sent again.
2543/// This device's records past their time and the sealed files it swept are
2544/// taken out there. One send at a time on this device, whichever process runs it.
2545#[allow(clippy::too_many_arguments)]
2546pub fn send(memory: &Path, folder: &Path, own: &str, login: &Login, sealed: &BTreeMap<String, u64>, now: i64, again: bool, frugal: bool) -> Pulled {
2547    send_lane(Lane::Main, memory, folder, own, login, sealed, now, again, frugal, SEND_LIMITS)
2548}
2549
2550/// This device's texts to send (`texts/send/<id>.jsonl`, `place_texts`), on a
2551/// lane of their own: a small file, sent first and alone, never waiting
2552/// behind a large one going up on a slow line; Sioul keeping the folder
2553/// itself too (`MIRROR`).
2554pub fn send_urgent(memory: &Path, folder: &Path, own: &str, login: &Login, now: i64) -> Pulled {
2555    send_lane(Lane::Urgent, memory, folder, own, login, &BTreeMap::new(), now, false, false, SEND_LIMITS)
2556}
2557
2558#[cfg(test)]
2559pub(crate) fn send_urgent_with(memory: &Path, folder: &Path, own: &str, login: &Login, now: i64, limits: Limits) -> Pulled {
2560    send_lane(Lane::Urgent, memory, folder, own, login, &BTreeMap::new(), now, false, false, limits)
2561}
2562
2563#[cfg(test)]
2564#[allow(clippy::too_many_arguments)]
2565pub(crate) fn send_with(memory: &Path, folder: &Path, own: &str, login: &Login, sealed: &BTreeMap<String, u64>, now: i64, again: bool, limits: Limits) -> Pulled {
2566    send_lane(Lane::Main, memory, folder, own, login, sealed, now, again, false, limits)
2567}
2568
2569#[allow(clippy::too_many_arguments)]
2570fn send_lane(lane: Lane, memory: &Path, folder: &Path, own: &str, login: &Login, sealed: &BTreeMap<String, u64>, now: i64, again: bool, frugal: bool, limits: Limits) -> Pulled {
2571    sioul_core::filelock::with_lock(&lane.lock(memory), || {
2572        let state = State::load(memory);
2573        let mut pulled = Pulled::default();
2574        let refused = if state.mode == MIRROR && lane == Lane::Main {
2575            Some("mirror")
2576        } else if !state.confirmed_for(folder) || state.account != login.account {
2577            Some("not-confirmed")
2578        } else if state.mode != MIRROR && !state.sending() {
2579            Some("off")
2580        } else {
2581            None
2582        };
2583        if let Some(why) = refused {
2584            if again {
2585                Sending::note_again(memory, now, why);
2586            }
2587            pulled.problem = Some(why.into());
2588            return pulled;
2589        }
2590        let mut sending = Sending::load_lane(memory, lane);
2591        if sending.url != state.url {
2592            sending = Sending { url: state.url.clone(), again: sending.again.clone(), ..Sending::default() };
2593        }
2594        // This device's entry listed there by a pull after this send last went
2595        // through, of another size than it was sent: an older copy put back
2596        // there (by another device's sync app, the server's own versions).
2597        // Looked at again now, and sent over it, rather than at the check of
2598        // every six hours.
2599        let entry = format!("devices/{own}.device");
2600        if lane == Lane::Main
2601            && let Some((size, at)) = state.own_entry
2602            && at > sending.last
2603            && sending.sent.get(&entry).is_some_and(|sent| sent.size != size)
2604        {
2605            sending.sent.remove(&entry);
2606        }
2607        let host = state.host();
2608        // The server unreachable a moment ago: not tried before its backoff.
2609        if !again && now < sending.retry_at {
2610            pulled.problem = Some(format!("waiting:{host}"));
2611            return pulled;
2612        }
2613        sending.tried = now;
2614        let check = again || sending.checked == 0 || now - sending.checked >= CHECK_AGAIN;
2615        let frugal = frugal || sending.slow();
2616        let files: Vec<String> = match lane {
2617            Lane::Urgent => own_urgent(folder, own),
2618            Lane::Main => {
2619                let mut files = own_plain(folder, own);
2620                let mut names: Vec<&String> = sealed.keys().collect();
2621                names.sort();
2622                files.extend(names.into_iter().map(|name| format!("blobs/{name}")).filter(|relative| folder.join(relative).is_file()));
2623                files
2624            }
2625        };
2626        let outcome = Server::new(login, limits).map_err(Stop::Failed).and_then(|server| Sender { server: &server, folder, own, url: &state.url, sealed, sending: &mut sending, pulled: &mut pulled, now, frugal, lane }.run(&files, check, again));
2627        match outcome {
2628            Ok(()) => {
2629                sending.tries = 0;
2630                sending.retry_at = 0;
2631                if sending.held.is_empty() {
2632                    sending.last = now;
2633                    sending.said.clear();
2634                    sending.failure = None;
2635                } else if let Some(failure) = &sending.failure {
2636                    sending.said = format!("network:{host}");
2637                    pulled.failure = Some(failure.clone());
2638                }
2639                if check {
2640                    sending.checked = now;
2641                }
2642            }
2643            Err(Stop::Unconfirmed(why)) => sending.said = why,
2644            Err(Stop::Failed(e)) => {
2645                // Unreachable, refused, out of time: nothing more before the backoff.
2646                sending.tries += 1;
2647                sending.retry_at = now + backoff(sending.tries);
2648                sending.said = code(&e, &host);
2649                if sending.failure.as_ref().is_none_or(|f| f.at != now) {
2650                    sending.failure = Some(Failure { kind: kind_of_error(&e).into(), file: String::new(), words: e.to_string(), at: now, ..Failure::default() });
2651                }
2652                pulled.failure = sending.failure.clone();
2653            }
2654        }
2655        if again {
2656            sending.again = Again { at: now, sent: pulled.sent, same: pulled.same, said: sending.said.clone() };
2657        }
2658        let _ = sending.save(memory, lane);
2659        pulled.problem = (!sending.said.is_empty()).then(|| sending.said.clone());
2660        pulled
2661    })
2662}
2663
2664/// A failure's kind, from its error alone (no file under way).
2665fn kind_of_error(e: &SyncError) -> &'static str {
2666    match e {
2667        SyncError::Login(_) | SyncError::AppPassword(_) => "login",
2668        SyncError::Tls(_) => "tls",
2669        SyncError::Server(d) if d == "quota" => "quota",
2670        SyncError::Server(_) => "server",
2671        SyncError::Disk(_) => "disk",
2672        SyncError::Network(d) if d == "out of time" => "out-of-time",
2673        _ => "unreachable",
2674    }
2675}
2676
2677/// Sioul keeping the folder itself (`MIRROR`): "Send everything again". What
2678/// it remembers of sending this device's own files is forgotten (its sealed
2679/// files apart, which tell its own from the others'), and the server's
2680/// folders are looked through again: each own file goes over what is there
2681/// now, the others' newer files come. Said as `Sending::again` says it.
2682pub fn again_mirror(memory: &Path, folder: &Path, own: &str, login: &Login, now: i64) -> Pulled {
2683    again_mirror_with(memory, folder, own, login, now, LIMITS)
2684}
2685
2686pub(crate) fn again_mirror_with(memory: &Path, folder: &Path, own: &str, login: &Login, now: i64, limits: Limits) -> Pulled {
2687    let forgot = sioul_core::filelock::with_lock(&memory.with_file_name("mirror.step"), || {
2688        let mut state = State::load(memory);
2689        if state.mode != MIRROR {
2690            return false;
2691        }
2692        state.sent.retain(|relative, _| relative.starts_with("blobs/"));
2693        state.folders.clear();
2694        state.save(memory).is_ok()
2695    });
2696    let pulled = if forgot { step_with(memory, folder, own, login, now, true, limits) } else { Pulled { problem: Some("not-confirmed".into()), ..Pulled::default() } };
2697    let mut sending = Sending::load(memory);
2698    sending.again = Again { at: now, sent: pulled.sent, same: 0, said: pulled.problem.clone().unwrap_or_default() };
2699    let _ = sending.save(memory, Lane::Main);
2700    pulled
2701}
2702
2703/// This device's texts to send copied into the sharing folder as a small
2704/// file of its own, `texts/send/<id>.jsonl` (`source`: its own file of the
2705/// part "texts", every line sealed already): the sync app carries it, Sioul
2706/// sends it first (`send_urgent`), the others read it beside the records'
2707/// copy, so that a request reaches the phone within its quarter of an hour
2708/// whatever the records' size. None here, none there. Whether it changed.
2709pub fn place_texts(folder: &Path, own: &str, source: &Path) -> Result<bool, String> {
2710    let target = texts_file(folder, own);
2711    let Ok(bytes) = std::fs::read(source) else {
2712        return Ok(std::fs::remove_file(&target).is_ok());
2713    };
2714    if std::fs::read(&target).ok().as_deref() == Some(&bytes[..]) {
2715        return Ok(false);
2716    }
2717    if let Some(parent) = target.parent() {
2718        crate::share::private_dirs(parent).map_err(|e| format!("{}: {e}", parent.display()))?;
2719    }
2720    crate::share::write_atomically(&target, &bytes).map(|()| true)
2721}
2722
2723/// Where a device's texts to send are, in the sharing folder (or a copy fetched from its server).
2724pub fn texts_file(folder: &Path, device: &str) -> PathBuf {
2725    folder.join("texts").join("send").join(format!("{device}.jsonl"))
2726}
2727
2728/// This device's urgent files in the folder: its texts to send.
2729fn own_urgent(folder: &Path, own: &str) -> Vec<String> {
2730    let relative = format!("texts/send/{own}.jsonl");
2731    if folder.join(&relative).is_file() { vec![relative] } else { Vec::new() }
2732}
2733
2734/// A file of the folder, by its kind, for what is said: never its name.
2735fn file_kind(relative: &str) -> &'static str {
2736    match kind_of(relative) {
2737        Some((Kind::Round, _)) => "records",
2738        Some((Kind::Seen, _)) => "notes",
2739        Some((Kind::Device, _)) => "entry",
2740        Some((Kind::Claim, _)) => "claim",
2741        Some((Kind::Texts, _)) => "texts",
2742        None => "sealed",
2743    }
2744}
2745
2746/// What came of one file: gone or the same there, left (unchanged, paced,
2747/// longer there), or held back after a failure.
2748#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2749enum Went {
2750    Done,
2751    Left,
2752    Held,
2753}
2754
2755/// One send beside the sync app.
2756struct Sender<'a> {
2757    server: &'a Server,
2758    folder: &'a Path,
2759    own: &'a str,
2760    /// The folder there, an address ending with "/".
2761    url: &'a str,
2762    sealed: &'a BTreeMap<String, u64>,
2763    sending: &'a mut Sending,
2764    pulled: &'a mut Pulled,
2765    now: i64,
2766    frugal: bool,
2767    lane: Lane,
2768}
2769
2770impl Sender<'_> {
2771    fn url(&self, relative: &str) -> String {
2772        format!("{}{}", self.url, encode_path(relative))
2773    }
2774
2775    /// `files` in their order (records first, by round, then the notes to
2776    /// the others, the claims, the entry, the sealed files last): nothing
2777    /// large holds back what the others wait for. A records file held back
2778    /// holds the next ones (records go in order); the other files still go.
2779    fn run(&mut self, files: &[String], check: bool, again: bool) -> Result<(), Stop> {
2780        let listed = if check { Some(self.list_own()?) } else { None };
2781        let mut records_held = false;
2782        for relative in files {
2783            let records = matches!(kind_of(relative), Some((Kind::Round, _)));
2784            if records && records_held {
2785                continue;
2786            }
2787            let there = listed.as_ref().map(|listed| listed.get(relative).cloned());
2788            let went = self.one(relative, there, again)?;
2789            records_held |= records && went == Went::Held;
2790        }
2791        if self.lane == Lane::Main {
2792            self.take_out_gone()?;
2793        }
2794        Ok(())
2795    }
2796
2797    /// The server's copies of this device's files, by their path in the
2798    /// folder: the folder itself, `devices/`, each part's claims, `blobs/`
2799    /// (when this device sealed any), `texts/send/`. A folder not there holds none.
2800    fn list_own(&mut self) -> Result<BTreeMap<String, Item>, Stop> {
2801        let mut out = BTreeMap::new();
2802        let root = self.server.list(self.url).map_err(|e| match e {
2803            SyncError::NotFound(_) => Stop::Unconfirmed(format!("seal-gone:{}", host_of(self.url))),
2804            e => Stop::Failed(e),
2805        })?;
2806        self.pulled.listed += root.items.len();
2807        let mut dirs: Vec<String> = Vec::new();
2808        for (name, item) in &root.items {
2809            if item.dir {
2810                if name == "devices" || (name == "blobs" && !self.sealed.is_empty()) {
2811                    dirs.push(format!("{name}/"));
2812                }
2813                if name == "texts" {
2814                    dirs.push("texts/send/".into());
2815                }
2816                if name == "leases" {
2817                    match self.server.list(&self.url("leases/")) {
2818                        Ok(parts) => dirs.extend(parts.items.iter().filter(|(_, item)| item.dir).map(|(part, _)| format!("leases/{part}/"))),
2819                        Err(SyncError::NotFound(_)) => {}
2820                        Err(e) => return Err(Stop::Failed(e)),
2821                    }
2822                }
2823            } else {
2824                out.insert(name.clone(), item.clone());
2825            }
2826        }
2827        for dir in dirs {
2828            match self.server.list(&self.url(&dir)) {
2829                Ok(listing) => {
2830                    self.pulled.listed += listing.items.len();
2831                    out.extend(listing.items.into_iter().filter(|(_, item)| !item.dir).map(|(name, item)| (format!("{dir}{name}"), item)));
2832                }
2833                Err(SyncError::NotFound(_)) => {}
2834                Err(e) => return Err(Stop::Failed(e)),
2835            }
2836        }
2837        Ok(out)
2838    }
2839
2840    /// Whether the server's copy is this file as it is here: a sealed file
2841    /// by its size (its name is its content's); records by their size (they
2842    /// only grow, one writer) and their checksum when the server keeps one;
2843    /// the rest by their checksum, else read there and compared.
2844    fn same(&self, relative: &str, there: &Item, bytes: &[u8], sha1: &str) -> Result<bool, Stop> {
2845        if there.dir || there.size != bytes.len() as u64 {
2846            return Ok(false);
2847        }
2848        if relative.starts_with("blobs/") {
2849            return Ok(true);
2850        }
2851        if let Some(theirs) = sha1_of(&there.checksum) {
2852            return Ok(theirs == sha1);
2853        }
2854        if matches!(kind_of(relative), Some((Kind::Round, _))) {
2855            return Ok(true);
2856        }
2857        Ok(self.server.small_up_to(&self.url(relative), TEXTS_LARGEST)?.is_some_and(|(theirs, _)| theirs == bytes))
2858    }
2859
2860    /// One of this device's files, sent when it changed here since it went
2861    /// (or, `there` listed, when the server's copy is not this one), over
2862    /// what is there as it was seen. Refused because it changed there
2863    /// meanwhile (`412`): the server's copy looked at; the same as here (the
2864    /// sync app sent it), nothing more; else sent over it. A folder missing
2865    /// there (`409`), made first. Too slow or cut: held back (`backoff`).
2866    fn one(&mut self, relative: &str, listed: Option<Option<Item>>, again: bool) -> Result<Went, Stop> {
2867        let path = self.folder.join(relative);
2868        let Ok(meta) = std::fs::metadata(&path) else { return Ok(Went::Left) };
2869        let (size, modified) = (meta.len(), crate::share::modified_ns(&meta));
2870        let blob = relative.strip_prefix("blobs/");
2871        // A sealed file goes whole, as this device sealed it, or not at all.
2872        if blob.is_some_and(|name| self.sealed.get(name) != Some(&size)) {
2873            return Ok(Went::Left);
2874        }
2875        let records = matches!(kind_of(relative), Some((Kind::Round, _)));
2876        let sent = self.sending.sent.get(relative).cloned();
2877        let same_here = sent.as_ref().is_some_and(|s| s.size == size && s.modified == modified);
2878        match &listed {
2879            None if same_here => return Ok(Went::Left),
2880            Some(there) if same_here && !again && there.as_ref().is_some_and(|t| t.etag == sent.as_ref().map_or("", |s| s.etag.as_str())) => return Ok(Went::Left),
2881            _ => {}
2882        }
2883        // Held back after a failure: never tried before its backoff.
2884        if !again && self.sending.held.get(relative).is_some_and(|held| self.now < held.until) {
2885            return Ok(Went::Held);
2886        }
2887        // Records only grow: one cut here (a sync app put an older copy back)
2888        // is never sent over a longer one; the exchange opens a new round.
2889        if records && sent.as_ref().is_some_and(|s| size < s.size) {
2890            self.pulled.longer += 1;
2891            return Ok(Went::Left);
2892        }
2893        // A large file on a metered or slow connection: at most every ten minutes.
2894        let large = size > LARGE_FILE;
2895        if large && self.frugal && !again && self.now - self.sending.large_at < PACE {
2896            self.pulled.paced += 1;
2897            return Ok(if records { Went::Held } else { Went::Left });
2898        }
2899        let bytes = std::fs::read(&path).map_err(|e| Stop::Failed(SyncError::Disk(format!("{relative}: {e}"))))?;
2900        // Read while it grew: what was read is what goes, said as it is.
2901        let size = bytes.len() as u64;
2902        let sha1 = sha1_hex(&bytes);
2903        let url = self.url(relative);
2904        // What is there: as listed; else, never sent or its ETag unsaid, asked.
2905        let mut there: Option<Option<Item>> = listed;
2906        if there.is_none() && sent.as_ref().is_none_or(|s| s.etag.is_empty()) {
2907            there = Some(self.server.stat(&url)?);
2908        }
2909        let modified_s = modified / 1_000_000_000;
2910        for attempt in 0..3 {
2911            if let Some(Some(item)) = &there {
2912                if self.same(relative, item, &bytes, &sha1)? {
2913                    self.sending.sent.insert(relative.to_string(), Sent { size, modified, etag: item.etag.clone() });
2914                    self.sending.held.remove(relative);
2915                    self.pulled.same += 1;
2916                    return Ok(Went::Done);
2917                }
2918                // Not the same: said by kind and size, for the log (nothing of its name or content).
2919                if self.pulled.differ.len() < 8 {
2920                    self.pulled.differ.push(format!("{} {size} here, {} there", file_kind(relative), item.size));
2921                }
2922                // Longer there than here: never cut. Remembered at the server's
2923                // size, so that what is appended here later, while still
2924                // shorter, is never sent over it either.
2925                if records && item.size > size {
2926                    self.sending.sent.insert(relative.to_string(), Sent { size: item.size, modified, etag: item.etag.clone() });
2927                    self.pulled.longer += 1;
2928                    return Ok(Went::Left);
2929                }
2930            }
2931            let seen: Option<String> = match &there {
2932                Some(Some(item)) => Some(item.etag.clone()),
2933                Some(None) => None,
2934                None => sent.as_ref().map(|s| s.etag.clone()),
2935            };
2936            if large {
2937                self.sending.large_at = self.now;
2938            }
2939            let started = Instant::now();
2940            let (status, etag) = match self.server.put_file(&url, bytes.clone(), seen.as_deref(), modified_s, &sha1) {
2941                Ok(answer) => answer,
2942                Err(failed) => return self.failed(relative, failed),
2943            };
2944            match status {
2945                200 | 201 | 204 => {
2946                    self.sending.sent.insert(relative.to_string(), Sent { size, modified, etag: etag.unwrap_or_default() });
2947                    self.sending.held.remove(relative);
2948                    self.pulled.sent += 1;
2949                    self.pulled.bytes += size;
2950                    if size >= 64 << 10 {
2951                        self.sending.rate = size * 1000 / (started.elapsed().as_millis() as u64).max(1);
2952                    }
2953                    return Ok(Went::Done);
2954                }
2955                // Changed there meanwhile, or gone: looked at again.
2956                412 | 404 => there = Some(self.server.stat(&url)?),
2957                409 if attempt == 0 => self.make_folders(relative)?,
2958                507 => return Err(Stop::Failed(SyncError::Server("quota".into()))),
2959                status => return Err(Stop::Failed(SyncError::Server(format!("PUT: {status}")))),
2960            }
2961        }
2962        Err(Stop::Failed(SyncError::Server("PUT: 412".into())))
2963    }
2964
2965    /// A file that did not go, said in detail (`Sending::failure`). Too slow
2966    /// or cut on its way (the connection made, its bytes moving): that file
2967    /// held back (`backoff`), the others still going. Else (the server
2968    /// unreachable, refused, out of time): the send stops.
2969    fn failed(&mut self, relative: &str, failed: PutFailed) -> Result<Went, Stop> {
2970        self.sending.failure = Some(Failure { kind: failed.kind.into(), file: file_kind(relative).into(), seconds: failed.seconds, sent: failed.sent, total: failed.total, words: failed.words.clone(), at: self.now });
2971        if failed.sent > 0 && failed.seconds > 0 {
2972            self.sending.rate = failed.sent / failed.seconds;
2973        }
2974        if matches!(failed.kind, "timeout" | "stalled" | "network") && failed.sent > 0 {
2975            let held = self.sending.held.entry(relative.to_string()).or_default();
2976            held.tries += 1;
2977            held.until = self.now + backoff(held.tries);
2978            self.pulled.held += 1;
2979            return Ok(Went::Held);
2980        }
2981        Err(Stop::Failed(failed.error))
2982    }
2983
2984    /// The folders a file's path goes through, made there (`MKCOL`) when missing.
2985    fn make_folders(&mut self, relative: &str) -> Result<(), Stop> {
2986        let mut at = String::new();
2987        let names: Vec<&str> = relative.split('/').collect();
2988        for name in &names[..names.len().saturating_sub(1)] {
2989            at = format!("{at}{name}/");
2990            match self.server.mkcol(&self.url(&at))? {
2991                201 | 405 => {}
2992                status => return Err(Stop::Failed(SyncError::Server(format!("MKCOL: {status}")))),
2993            }
2994        }
2995        Ok(())
2996    }
2997
2998    /// What this device sent and let go here, taken out there as it was
2999    /// sent (`If-Match`): its records past their time (removed here once
3000    /// every device read past them), never its last round, and nothing while
3001    /// none of its records is here; the sealed files it swept. Gone there
3002    /// already, or changed there since: forgotten all the same. Another file
3003    /// gone here (a claim of a part no longer kept) is forgotten, left there.
3004    fn take_out_gone(&mut self) -> Result<(), Stop> {
3005        let latest = own_latest(self.folder, self.own);
3006        let gone: Vec<(String, String)> = self.sending.sent.iter().filter(|(relative, _)| !self.folder.join(relative).exists()).map(|(relative, sent)| (relative.clone(), sent.etag.clone())).collect();
3007        for (relative, etag) in gone {
3008            let round = relative.trim_end_matches(".jsonl").rsplit('-').next().and_then(|n| n.parse::<u32>().ok());
3009            let past = matches!(kind_of(&relative), Some((Kind::Round, _))) && round.zip(latest).is_some_and(|(round, latest)| round < latest);
3010            let swept = relative.strip_prefix("blobs/").is_some_and(|name| !self.sealed.contains_key(name));
3011            if past || swept {
3012                match self.server.delete(&self.url(&relative), Some(&etag).filter(|e| !e.is_empty()).map(String::as_str))? {
3013                    200 | 204 | 404 | 412 => {}
3014                    status => return Err(Stop::Failed(SyncError::Server(format!("DELETE: {status}")))),
3015                }
3016            }
3017            if past || swept || !matches!(kind_of(&relative), Some((Kind::Round, _))) {
3018                self.sending.sent.remove(&relative);
3019                self.sending.held.remove(&relative);
3020            }
3021        }
3022        Ok(())
3023    }
3024}
3025
3026#[cfg(test)]
3027pub(crate) mod fake {
3028    //! A Nextcloud in a test: an account's files, a folder on the disk,
3029    //! served over WebDAV on 127.0.0.1 (plain HTTP, which only a test build
3030    //! reaches: `dav::allowed`), a request and a thread per connection; faults
3031    //! to inject, and every request seen.
3032
3033    use std::collections::hash_map::DefaultHasher;
3034    use std::hash::{Hash, Hasher};
3035    use std::io::{BufRead, BufReader, Read, Write};
3036    use std::net::{TcpListener, TcpStream};
3037    use std::path::{Path, PathBuf};
3038    use std::sync::{Arc, Mutex};
3039    use std::time::Duration;
3040
3041    pub(crate) const USER: &str = "jane";
3042    pub(crate) const PASSWORD: &str = "correct horse battery staple";
3043    const FILES: &str = "/remote.php/dav/files/jane";
3044
3045    /// How the server misbehaves, for one request.
3046    #[derive(Debug, Clone, Copy)]
3047    pub(crate) enum Fault {
3048        /// Answers this status, doing nothing.
3049        Status(u16),
3050        /// Waits this long, then answers.
3051        Late(Duration),
3052        /// Says the whole answer's length, sends this many bytes of it, and closes.
3053        Cut(usize),
3054        /// Closes without a word.
3055        Drop,
3056    }
3057
3058    /// A request, as read: its path decoded.
3059    pub(crate) struct Request {
3060        pub method: String,
3061        pub path: String,
3062        headers: Vec<(String, String)>,
3063        pub body: Vec<u8>,
3064    }
3065
3066    impl Request {
3067        pub fn header(&self, name: &str) -> Option<&str> {
3068            self.headers.iter().find(|(n, _)| n.eq_ignore_ascii_case(name)).map(|(_, v)| v.as_str())
3069        }
3070    }
3071
3072    type Faults = Vec<Box<dyn FnMut(&Request) -> Option<Fault> + Send>>;
3073    pub(crate) type Reply = (u16, Vec<(&'static str, String)>, Vec<u8>);
3074
3075    pub(crate) struct Fake {
3076        /// The account's files: `<files>/<path>` is `/remote.php/dav/files/jane/<path>`.
3077        pub(crate) files: PathBuf,
3078        pub(crate) base: String,
3079        /// The checksums files were sent with (`OC-Checksum`), as Nextcloud
3080        /// keeps them: by file, with its ETag then; a file changed since by
3081        /// other hands (a test's own copy) has none.
3082        sums: Mutex<std::collections::BTreeMap<PathBuf, (String, String)>>,
3083        /// An uplink this slow (bytes a second) for what is sent to it; none: as fast as it comes.
3084        pub(crate) throttle: Mutex<Option<u64>>,
3085        /// The bytes of every `PUT` it took, whole or not: what a sending costs.
3086        pub(crate) received: std::sync::atomic::AtomicU64,
3087        faults: Mutex<Faults>,
3088        seen: Mutex<Vec<String>>,
3089        /// Serving one request at a time: a listing never sees a file half written by a test's own copy.
3090        pub(crate) turn: Mutex<()>,
3091    }
3092
3093    impl Fake {
3094        pub(crate) fn start(files: &Path) -> Arc<Fake> {
3095            let listener = TcpListener::bind("127.0.0.1:0").unwrap();
3096            let base = format!("http://127.0.0.1:{}", listener.local_addr().unwrap().port());
3097            let fake = Arc::new(Fake {
3098                files: files.to_path_buf(),
3099                base,
3100                sums: Mutex::new(std::collections::BTreeMap::new()),
3101                throttle: Mutex::new(None),
3102                received: std::sync::atomic::AtomicU64::new(0),
3103                faults: Mutex::new(Vec::new()),
3104                seen: Mutex::new(Vec::new()),
3105                turn: Mutex::new(()),
3106            });
3107            let serving = Arc::clone(&fake);
3108            std::thread::spawn(move || {
3109                for stream in listener.incoming().filter_map(Result::ok) {
3110                    let fake = Arc::clone(&serving);
3111                    std::thread::spawn(move || fake.exchange(stream));
3112                }
3113            });
3114            fake
3115        }
3116
3117        /// The account, as a contacts-and-calendars account found it: at its calendars' address.
3118        pub(crate) fn login(&self) -> super::Login {
3119            super::Login { account: "cloud".into(), url: format!("{}/remote.php/dav/calendars/{USER}/", self.base), user: USER.into(), password: Some(PASSWORD.into()) }
3120        }
3121
3122        /// A folder of the account's files, as an address ending with "/".
3123        pub(crate) fn folder_url(&self, path: &str) -> String {
3124            format!("{}{FILES}/{}/", self.base, path.trim_matches('/'))
3125        }
3126
3127        /// A fault, asked of each request until one says one.
3128        pub(crate) fn fault(&self, fault: impl FnMut(&Request) -> Option<Fault> + Send + 'static) {
3129            self.faults.lock().unwrap().push(Box::new(fault));
3130        }
3131
3132        pub(crate) fn clear_faults(&self) {
3133            self.faults.lock().unwrap().clear();
3134        }
3135
3136        /// The requests seen, in order: "GET /remote.php/dav/files/jane/Documents/Sioul/seal.toml 200".
3137        pub(crate) fn seen(&self) -> Vec<String> {
3138            self.seen.lock().unwrap().clone()
3139        }
3140
3141        pub(crate) fn forget_seen(&self) {
3142            self.seen.lock().unwrap().clear();
3143        }
3144
3145        fn exchange(&self, mut stream: TcpStream) {
3146            let throttle = *self.throttle.lock().unwrap();
3147            let Some(request) = read(&stream, throttle, &self.received) else { return };
3148            let fault = self.faults.lock().unwrap().iter_mut().find_map(|f| f(&request));
3149            let (status, headers, body) = match fault {
3150                Some(Fault::Drop) => {
3151                    self.note(&request, "dropped");
3152                    return;
3153                }
3154                Some(Fault::Status(status)) => (status, Vec::new(), Vec::new()),
3155                Some(Fault::Late(wait)) => {
3156                    std::thread::sleep(wait);
3157                    self.answer(&request)
3158                }
3159                _ => self.answer(&request),
3160            };
3161            self.note(&request, &status.to_string());
3162            let mut head = format!("HTTP/1.1 {status} Fake\r\nContent-Length: {}\r\nConnection: close\r\n", body.len());
3163            for (name, value) in &headers {
3164                head.push_str(&format!("{name}: {value}\r\n"));
3165            }
3166            head.push_str("\r\n");
3167            if stream.write_all(head.as_bytes()).is_err() {
3168                return;
3169            }
3170            let sent = match fault {
3171                Some(Fault::Cut(n)) => &body[..n.min(body.len())],
3172                _ => &body[..],
3173            };
3174            let _ = stream.write_all(sent);
3175            let _ = stream.flush();
3176        }
3177
3178        /// A file's checksums as kept, while it is as it was sent.
3179        fn sum_of(&self, path: &Path) -> Option<String> {
3180            let sums = self.sums.lock().unwrap();
3181            sums.get(path).filter(|(etag, _)| *etag == etag_of(path)).map(|(_, sum)| sum.clone())
3182        }
3183
3184        fn propfind(&self, request: &Request, relative: &str, path: &Path) -> Reply {
3185            let Ok(meta) = std::fs::metadata(path) else { return (404, Vec::new(), Vec::new()) };
3186            let mut out = entry(&href(relative, meta.is_dir()), meta.is_dir(), &etag_of(path), meta.len(), self.sum_of(path));
3187            if meta.is_dir() && request.header("Depth") != Some("0") {
3188                let mut children: Vec<_> = std::fs::read_dir(path).into_iter().flatten().filter_map(Result::ok).collect();
3189                children.sort_by_key(|e| e.file_name());
3190                for child in children {
3191                    let name = child.file_name().to_string_lossy().to_string();
3192                    let inside = if relative.is_empty() { name } else { format!("{relative}/{name}") };
3193                    let dir = child.file_type().is_ok_and(|t| t.is_dir());
3194                    out += &entry(&href(&inside, dir), dir, &etag_of(&child.path()), child.metadata().map_or(0, |m| m.len()), self.sum_of(&child.path()));
3195                }
3196            }
3197            (207, vec![("Content-Type", "application/xml; charset=utf-8".into())], multistatus(&out))
3198        }
3199
3200        /// A file written whole (`PUT`), as Nextcloud does: its folder there,
3201        /// the precondition met (`If-None-Match: *` none there, `If-Match` its
3202        /// ETag); dated as `X-OC-Mtime` says, its `OC-Checksum` kept.
3203        fn put(&self, request: &Request, path: &Path) -> Reply {
3204            let reply = put(request, path);
3205            if matches!(reply.0, 201 | 204) {
3206                if let Some(seconds) = request.header("X-OC-Mtime").and_then(|t| t.parse::<u64>().ok())
3207                    && let Ok(file) = std::fs::File::options().write(true).open(path)
3208                {
3209                    let _ = file.set_modified(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(seconds));
3210                }
3211                let mut sums = self.sums.lock().unwrap();
3212                match request.header("OC-Checksum") {
3213                    Some(sum) => sums.insert(path.to_path_buf(), (etag_of(path), sum.to_string())),
3214                    None => sums.remove(path),
3215                };
3216            }
3217            reply
3218        }
3219
3220        fn note(&self, request: &Request, what: &str) {
3221            self.seen.lock().unwrap().push(format!("{} {} {what}", request.method, request.path));
3222        }
3223
3224        pub(crate) fn answer(&self, request: &Request) -> Reply {
3225            let _turn = self.turn.lock().unwrap();
3226            let expected = format!("Basic {}", sioul_core::lines::base64_encode(format!("{USER}:{PASSWORD}").as_bytes()));
3227            if request.header("Authorization") != Some(expected.as_str()) {
3228                return (401, Vec::new(), Vec::new());
3229            }
3230            if request.method == "PROPFIND" && request.path == "/remote.php/dav" {
3231                let principal = format!("<d:response><d:href>/remote.php/dav/</d:href><d:propstat><d:prop><d:current-user-principal><d:href>/remote.php/dav/principals/users/{USER}/</d:href></d:current-user-principal></d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response>");
3232                return (207, vec![("Content-Type", "application/xml".into())], multistatus(&principal));
3233            }
3234            let Some(relative) = request.path.strip_prefix(FILES).map(|r| r.trim_start_matches('/').to_string()) else { return (404, Vec::new(), Vec::new()) };
3235            let path = if relative.is_empty() { self.files.clone() } else { self.files.join(&relative) };
3236            match request.method.as_str() {
3237                "PROPFIND" => self.propfind(request, &relative, &path),
3238                "GET" => get(request, &path),
3239                "PUT" => self.put(request, &path),
3240                "MKCOL" => mkcol(&path),
3241                "DELETE" => delete(request, &path),
3242                _ => (405, Vec::new(), Vec::new()),
3243            }
3244        }
3245    }
3246
3247    /// A request as it comes, its body at `throttle` bytes a second at most (a slow uplink).
3248    fn read(stream: &TcpStream, throttle: Option<u64>, received: &std::sync::atomic::AtomicU64) -> Option<Request> {
3249        let mut reader = BufReader::new(stream);
3250        let mut line = String::new();
3251        reader.read_line(&mut line).ok()?;
3252        let mut words = line.split_whitespace();
3253        let (method, raw) = (words.next()?.to_string(), words.next()?.to_string());
3254        let mut headers: Vec<(String, String)> = Vec::new();
3255        loop {
3256            let mut line = String::new();
3257            reader.read_line(&mut line).ok()?;
3258            let Some((name, value)) = line.trim_end().split_once(':') else { break };
3259            headers.push((name.trim().to_string(), value.trim().to_string()));
3260        }
3261        let length = headers.iter().find(|(n, _)| n.eq_ignore_ascii_case("content-length")).and_then(|(_, v)| v.parse().ok()).unwrap_or(0);
3262        let mut body = vec![0; length];
3263        let piece = throttle.map_or(length.max(1), |rate| usize::try_from(rate / 20).unwrap_or(usize::MAX).max(1));
3264        let mut at = 0;
3265        while at < length {
3266            let end = (at + piece).min(length);
3267            let read = reader.read_exact(&mut body[at..end]);
3268            if method == "PUT" {
3269                received.fetch_add((end - at) as u64, std::sync::atomic::Ordering::Relaxed);
3270            }
3271            read.ok()?;
3272            at = end;
3273            if throttle.is_some() && at < length {
3274                std::thread::sleep(Duration::from_millis(50));
3275            }
3276        }
3277        Some(Request { method, path: crate::dav::path_key(&raw), headers, body })
3278    }
3279
3280    pub(crate) fn multistatus(responses: &str) -> Vec<u8> {
3281        format!(r#"<?xml version="1.0" encoding="utf-8"?><d:multistatus xmlns:d="DAV:" xmlns:oc="http://owncloud.org/ns">{responses}</d:multistatus>"#).into_bytes()
3282    }
3283
3284    /// A file's ETag, from its content; a folder's, from all it holds, as Nextcloud's change with anything below them.
3285    pub(crate) fn etag_of(path: &Path) -> String {
3286        let mut hasher = DefaultHasher::new();
3287        if path.is_dir() {
3288            let mut names: Vec<_> = std::fs::read_dir(path).into_iter().flatten().filter_map(Result::ok).map(|e| e.file_name()).collect();
3289            names.sort();
3290            for name in names {
3291                name.hash(&mut hasher);
3292                etag_of(&path.join(&name)).hash(&mut hasher);
3293            }
3294        } else {
3295            std::fs::read(path).unwrap_or_default().hash(&mut hasher);
3296            GENERATIONS.lock().unwrap().get(path).copied().unwrap_or(0).hash(&mut hasher);
3297        }
3298        format!("{:016x}", hasher.finish())
3299    }
3300
3301    /// How many times each file was written whole there: Nextcloud gives a
3302    /// file a new ETag at each `PUT`, the same bytes sent again included.
3303    static GENERATIONS: Mutex<std::collections::BTreeMap<PathBuf, u64>> = Mutex::new(std::collections::BTreeMap::new());
3304
3305    /// A file written there anew (a `PUT`, or a test's sync app sending it): a new ETag, whatever it holds.
3306    pub(crate) fn bump(path: &Path) {
3307        *GENERATIONS.lock().unwrap().entry(path.to_path_buf()).or_insert(0) += 1;
3308    }
3309
3310    fn entry(href: &str, dir: bool, etag: &str, size: u64, sum: Option<String>) -> String {
3311        let kind = if dir { "<d:resourcetype><d:collection/></d:resourcetype>" } else { "<d:resourcetype/>" };
3312        let length = if dir { String::new() } else { format!("<d:getcontentlength>{size}</d:getcontentlength>") };
3313        let sums = sum.map(|sum| format!("<oc:checksums><oc:checksum>{sum}</oc:checksum></oc:checksums>")).unwrap_or_default();
3314        format!("<d:response><d:href>{href}</d:href><d:propstat><d:prop>{kind}<d:getetag>&quot;{etag}&quot;</d:getetag>{length}{sums}</d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response>")
3315    }
3316
3317    pub(crate) fn href(relative: &str, dir: bool) -> String {
3318        let path = super::encode_path(relative);
3319        match (relative.is_empty(), dir) {
3320            (true, _) => format!("{FILES}/"),
3321            (false, true) => format!("{FILES}/{path}/"),
3322            (false, false) => format!("{FILES}/{path}"),
3323        }
3324    }
3325
3326    /// A file written whole (`PUT`), as Nextcloud does: its folder there, the
3327    /// precondition met (`If-None-Match: *` none there, `If-Match` its ETag).
3328    fn put(request: &Request, path: &Path) -> Reply {
3329        if !path.parent().is_some_and(Path::is_dir) {
3330            return (409, Vec::new(), Vec::new());
3331        }
3332        let reply = put_once(request, path);
3333        if matches!(reply.0, 201 | 204) {
3334            bump(path);
3335            return (reply.0, vec![("ETag", format!("\"{}\"", etag_of(path)))], Vec::new());
3336        }
3337        reply
3338    }
3339
3340    fn put_once(request: &Request, path: &Path) -> Reply {
3341        let current = path.is_file().then(|| format!("\"{}\"", etag_of(path)));
3342        let refused = match (request.header("If-None-Match"), request.header("If-Match")) {
3343            (Some("*"), _) => current.is_some(),
3344            (_, Some(wanted)) => current.as_deref() != Some(wanted),
3345            _ => false,
3346        };
3347        if refused {
3348            return (412, Vec::new(), Vec::new());
3349        }
3350        std::fs::write(path, &request.body).unwrap();
3351        (if current.is_some() { 204 } else { 201 }, vec![("ETag", format!("\"{}\"", etag_of(path)))], Vec::new())
3352    }
3353
3354    fn mkcol(path: &Path) -> Reply {
3355        if path.exists() {
3356            return (405, Vec::new(), Vec::new());
3357        }
3358        if !path.parent().is_some_and(Path::is_dir) {
3359            return (409, Vec::new(), Vec::new());
3360        }
3361        std::fs::create_dir(path).unwrap();
3362        (201, Vec::new(), Vec::new())
3363    }
3364
3365    fn delete(request: &Request, path: &Path) -> Reply {
3366        if !path.exists() {
3367            return (404, Vec::new(), Vec::new());
3368        }
3369        if request.header("If-Match").is_some_and(|wanted| format!("\"{}\"", etag_of(path)) != wanted) {
3370            return (412, Vec::new(), Vec::new());
3371        }
3372        if path.is_dir() {
3373            std::fs::remove_dir_all(path).unwrap();
3374        } else {
3375            std::fs::remove_file(path).unwrap();
3376        }
3377        (204, Vec::new(), Vec::new())
3378    }
3379
3380    fn get(request: &Request, path: &Path) -> Reply {
3381        let Ok(bytes) = std::fs::read(path) else { return (404, Vec::new(), Vec::new()) };
3382        let etag = format!("\"{}\"", etag_of(path));
3383        let from = request.header("Range").and_then(|r| r.strip_prefix("bytes=")).and_then(|r| r.strip_suffix('-')).and_then(|s| s.parse::<usize>().ok());
3384        match from {
3385            Some(from) if from >= bytes.len() => (416, vec![("Content-Range", format!("bytes */{}", bytes.len()))], Vec::new()),
3386            Some(from) => (206, vec![("ETag", etag), ("Content-Range", format!("bytes {from}-{}/{}", bytes.len() - 1, bytes.len()))], bytes[from..].to_vec()),
3387            None => (200, vec![("ETag", etag)], bytes),
3388        }
3389    }
3390}
3391
3392#[cfg(test)]
3393mod tests {
3394    use super::fake::{Fake, Fault, PASSWORD};
3395    use super::*;
3396    use crate::devices::{COMPUTER, Entry, PHONE};
3397    use crate::share::{Outcome, Roots, Sharing, Store};
3398    use sioul_core::config::Config;
3399    use sioul_core::everywhere::{Press, Switch, change_own};
3400    use sioul_core::health::{Doubt, Peer, doubts_now};
3401    use std::sync::Arc;
3402
3403    const KEY: [u8; 32] = [11u8; 32];
3404    const SEAL: &str = "# Sioul: how your devices' shared records are sealed (docs/database.md).\nversion = 1\nsalt = \"AAAAAAAAAAAAAAAAAAAAAA==\"\nmemory_kib = 65536\npasses = 3\ncheck = \"one\"\n";
3405    const OTHER_SEAL: &str = "# Sioul: how your devices' shared records are sealed (docs/database.md).\nversion = 1\nsalt = \"BBBBBBBBBBBBBBBBBBBBBB==\"\nmemory_kib = 65536\npasses = 3\ncheck = \"two\"\n";
3406    /// The network's limits, short: a test does not wait ten seconds.
3407    const TEST: Limits = Limits { wait: Duration::from_millis(700), small: Duration::from_secs(3), large: Duration::from_secs(5), pull: Duration::from_secs(30), blob: Duration::from_secs(5), floor: 1 << 30, stall: Duration::from_millis(700) };
3408    thread_local! {
3409        /// The longest round a pull brings, in this test's thread (`round_cap`).
3410        pub(super) static ROUND_CAP: std::cell::Cell<Option<u64>> = const { std::cell::Cell::new(None) };
3411    }
3412    const DUE: i64 = 1_800_000_000;
3413
3414    fn scratch(name: &str) -> PathBuf {
3415        let dir = std::env::temp_dir().join(format!("sioul-remote-{name}-{}", std::process::id()));
3416        let _ = std::fs::remove_dir_all(&dir);
3417        std::fs::create_dir_all(&dir).unwrap();
3418        dir
3419    }
3420
3421    /// A server whose principal it names on another server: the files are
3422    /// not looked for there, and nothing is asked there with the login.
3423    #[test]
3424    fn the_files_stay_on_the_account_s_server() {
3425        use crate::dav::stand_in::{Reply, Request, Server as StandIn, serve};
3426        /// Notes what it is asked, and with what login.
3427        struct Noted(std::sync::Mutex<Vec<String>>);
3428        impl StandIn for Noted {
3429            fn misbehaves(&self, _: &Request) -> Option<crate::dav::stand_in::Fault> {
3430                None
3431            }
3432            fn answer(&self, request: &Request) -> Reply {
3433                self.0.lock().unwrap().push(format!("{} {} {}", request.method, request.path, request.header("Authorization").unwrap_or("")));
3434                Reply::new(404, "")
3435            }
3436        }
3437        /// Names its principal at the address it is given.
3438        struct Pointing(String);
3439        impl StandIn for Pointing {
3440            fn misbehaves(&self, _: &Request) -> Option<crate::dav::stand_in::Fault> {
3441                None
3442            }
3443            fn answer(&self, _: &Request) -> Reply {
3444                Reply::new(207, format!(r#"<?xml version="1.0" encoding="utf-8"?><d:multistatus xmlns:d="DAV:"><d:response><d:href>/remote.php/dav/</d:href><d:propstat><d:prop><d:current-user-principal><d:href>{}</d:href></d:current-user-principal></d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response></d:multistatus>"#, self.0))
3445            }
3446        }
3447        let noted = Arc::new(Noted(std::sync::Mutex::new(Vec::new())));
3448        let there = serve(noted.clone());
3449        let here = serve(Arc::new(Pointing(format!("{there}/remote.php/dav/principals/users/jane/"))));
3450        let login = Login { account: "cloud".into(), url: format!("{here}/remote.php/dav/calendars/jane/"), user: "jane".into(), password: Some("secret".into()) };
3451        let found = files_root(&Server::new(&login, TEST).unwrap(), &login);
3452        assert!(matches!(&found, Err(SyncError::Elsewhere(root)) if root.starts_with(&there)), "{found:?}");
3453        assert!(noted.0.lock().unwrap().is_empty());
3454        // Named on its own server, as Nextcloud does: the files are there.
3455        let own = serve(Arc::new(Pointing("/remote.php/dav/principals/users/jane/".into())));
3456        let login = Login { url: format!("{own}/remote.php/dav/calendars/jane/"), ..login };
3457        assert_eq!(files_root(&Server::new(&login, TEST).unwrap(), &login).unwrap(), Some(format!("{own}/remote.php/dav/files/jane/")));
3458    }
3459
3460    fn put(dir: &Path, relative: &str, text: &str) {
3461        let path = dir.join(relative);
3462        std::fs::create_dir_all(path.parent().unwrap()).unwrap();
3463        std::fs::write(path, text).unwrap();
3464    }
3465
3466    /// The files under a folder, hidden ones left out, by their path in it.
3467    fn walk(dir: &Path) -> BTreeMap<String, Vec<u8>> {
3468        let mut out = BTreeMap::new();
3469        let mut stack = vec![dir.to_path_buf()];
3470        while let Some(at) = stack.pop() {
3471            for entry in std::fs::read_dir(&at).into_iter().flatten().filter_map(Result::ok) {
3472                let path = entry.path();
3473                if entry.file_name().to_string_lossy().starts_with('.') {
3474                    continue;
3475                }
3476                if path.is_dir() {
3477                    stack.push(path);
3478                } else {
3479                    let relative = path.strip_prefix(dir).unwrap().to_string_lossy().replace('\\', "/");
3480                    out.insert(relative, std::fs::read(&path).unwrap());
3481                }
3482            }
3483        }
3484        out
3485    }
3486
3487    fn copy_dir(from: &Path, to: &Path) {
3488        for (relative, bytes) in walk(from) {
3489            put_bytes(to, &relative, &bytes);
3490        }
3491    }
3492
3493    fn put_bytes(dir: &Path, relative: &str, bytes: &[u8]) {
3494        let path = dir.join(relative);
3495        std::fs::create_dir_all(path.parent().unwrap()).unwrap();
3496        std::fs::write(path, bytes).unwrap();
3497    }
3498
3499    /// A device of its own: its folders, its id, its memory.
3500    struct Device {
3501        roots: Roots,
3502        id: String,
3503        memory: PathBuf,
3504        kind: &'static str,
3505    }
3506
3507    impl Device {
3508        fn new(base: &Path, name: &str, kind: &'static str) -> Device {
3509            let root = base.join(name);
3510            let roots = Roots { config: root.join("config"), data: root.join("data"), state: root.join("state") };
3511            for dir in [&roots.config, &roots.data, &roots.state] {
3512                std::fs::create_dir_all(dir).unwrap();
3513            }
3514            let memory = roots.state.join("share").join("memory.json");
3515            Device { roots, id: uuid::Uuid::new_v4().to_string(), memory, kind }
3516        }
3517
3518        fn stores(&self) -> Vec<Store> {
3519            crate::share::stores(&Config::default(), &self.roots)
3520        }
3521
3522        /// An exchange (seconds), then its entry says what it read and wrote.
3523        fn exchange(&self, folder: &Path, now: i64) -> Outcome {
3524            let outcome = crate::share::exchange(&Sharing { folder, computer: &self.id, key: &KEY, memory: &self.memory, files: false, hurry: None }, &self.stores(), now * 1000).unwrap();
3525            crate::devices::change(&crate::devices::own_path(&self.roots.state), Some((folder, &KEY)), |e| e.exported(now * 1000, outcome.wrote, now)).unwrap();
3526            outcome
3527        }
3528
3529        fn session(&self, folder: &Path, f: impl FnOnce(&mut Entry)) {
3530            let (kind, id) = (self.kind, self.id.clone());
3531            crate::devices::change(&crate::devices::own_path(&self.roots.state), Some((folder, &KEY)), |e| {
3532                e.id = id;
3533                e.kind = kind.into();
3534                e.doses = true;
3535                f(e);
3536            })
3537            .unwrap();
3538        }
3539
3540        fn switch(&self) -> PathBuf {
3541            self.roots.state.join(sioul_core::everywhere::SWITCH_FILE)
3542        }
3543
3544        /// Do-not-disturb pressed here (seconds).
3545        fn press(&self, on: bool, at: i64) {
3546            change_own(&self.switch(), &self.id, |s| {
3547                s.press(&self.id, on, 0, at * 1000);
3548                true
3549            })
3550            .unwrap();
3551        }
3552
3553        fn dnd(&self) -> Option<Press> {
3554            Switch::read(&self.switch()).unwrap().latest()
3555        }
3556
3557        fn write(&self, relative: &str, text: &str) {
3558            put(&self.roots.state, relative, text);
3559        }
3560
3561        fn read(&self, relative: &str) -> String {
3562            std::fs::read_to_string(self.roots.state.join(relative)).unwrap_or_default()
3563        }
3564
3565        /// What this device knows of `other`, as the window's `know` builds it.
3566        fn sees(&self, folder: &Path, other: &Device, now: i64) -> Peer {
3567            let entry = crate::devices::all(folder, &KEY).0.into_iter().find(|e| e.id == other.id).expect("its entry");
3568            let heard = crate::share::heard(&self.memory, &self.id);
3569            Peer { id: other.id.clone(), name: entry.name.clone(), said: Some(entry.said()), complete: entry.wrote.is_none_or(|wrote| heard.complete(&other.id, wrote)), seen: now, heard: entry.exported, ..Peer::default() }
3570        }
3571
3572        /// What it keeps of its own, as files: the shared ones, written by the exchanges (not its memory).
3573        fn kept(&self) -> BTreeMap<String, Vec<u8>> {
3574            let mut out = BTreeMap::new();
3575            for (root, dir) in [("config", &self.roots.config), ("data", &self.roots.data), ("state", &self.roots.state)] {
3576                for (relative, bytes) in walk(dir) {
3577                    if !relative.starts_with("share/") {
3578                        out.insert(format!("{root}/{relative}"), bytes);
3579                    }
3580                }
3581            }
3582            out
3583        }
3584    }
3585
3586    /// A desk and a phone sharing through a Nextcloud: the desk's folder is the
3587    /// server's (its client keeps the two equal); the phone's sync app carried
3588    /// the folder down once, then only carries the phone's own files up (eDrive
3589    /// tonight, 6 October 2026).
3590    struct World {
3591        base: PathBuf,
3592        fake: Arc<Fake>,
3593        server: PathBuf,
3594        phone_folder: PathBuf,
3595        desk: Device,
3596        phone: Device,
3597    }
3598
3599    impl World {
3600        fn new(name: &str) -> World {
3601            let base = scratch(name);
3602            let files = base.join("cloud");
3603            let server = files.join("Documents").join("Sioul");
3604            let phone_folder = base.join("phone").join("storage").join("Documents").join("Sioul");
3605            put(&server, "seal.toml", SEAL);
3606            put(&phone_folder, "seal.toml", SEAL);
3607            let fake = Fake::start(&files);
3608            let desk = Device::new(&base, "desk", COMPUTER);
3609            let phone = Device::new(&base, "phone-state", PHONE);
3610            let world = World { base, fake, server, phone_folder, desk, phone };
3611            world.desk.session(&world.server, |e| e.start(DUE - 7_200));
3612            world.desk.exchange(&world.server, DUE - 7_100);
3613            world.down();
3614            world.phone.session(&world.phone_folder, |e| e.start(DUE - 7_050));
3615            world.phone.exchange(&world.phone_folder, DUE - 7_000);
3616            world.up();
3617            world.desk.exchange(&world.server, DUE - 6_900);
3618            world.down();
3619            world
3620        }
3621
3622        /// The phone's sync app carries its own files up.
3623        fn up(&self) {
3624            let _turn = self.fake.turn.lock().unwrap();
3625            for (relative, bytes) in walk(&self.phone_folder).into_iter().filter(|(r, _)| r.contains(&self.phone.id)) {
3626                put_bytes(&self.server, &relative, &bytes);
3627            }
3628        }
3629
3630        /// The phone's sync app brings the others' files down, as it should.
3631        fn down(&self) {
3632            for (relative, bytes) in walk(&self.server).into_iter().filter(|(r, _)| !r.contains(&self.phone.id)) {
3633                put_bytes(&self.phone_folder, &relative, &bytes);
3634            }
3635        }
3636
3637        fn find(&self, now: i64) -> State {
3638            find_with(&self.phone.memory, &self.phone_folder, &[self.fake.login()], &places(&self.phone_folder, &[]), now, TEST)
3639        }
3640
3641        fn pull(&self, now: i64) -> Pulled {
3642            pull_with(&self.phone.memory, &self.phone_folder, &self.phone.id, &self.fake.login(), now, TEST)
3643        }
3644
3645        fn cache(&self) -> PathBuf {
3646            cache_of(&self.phone.memory)
3647        }
3648
3649        /// The desk's last records' file, by its name in the folder.
3650        fn desk_round(&self) -> String {
3651            let mut names: Vec<String> = walk(&self.server).into_keys().filter(|n| n.starts_with(&self.desk.id) && n.ends_with(".jsonl")).collect();
3652            names.sort_by_key(|n| n.trim_end_matches(".jsonl").rsplit('-').next().unwrap().parse::<u32>().unwrap());
3653            names.pop().unwrap()
3654        }
3655
3656        /// The phone as it is now, aside, to put back (`restore`): the same device run twice.
3657        fn snapshot(&self) -> PathBuf {
3658            let aside = self.base.join("aside");
3659            let _ = std::fs::remove_dir_all(&aside);
3660            copy_dir(&self.base.join("phone-state"), &aside.join("phone-state"));
3661            copy_dir(&self.phone_folder, &aside.join("folder"));
3662            aside
3663        }
3664
3665        fn restore(&self, aside: &Path) {
3666            let _ = std::fs::remove_dir_all(self.base.join("phone-state"));
3667            let _ = std::fs::remove_dir_all(&self.phone_folder);
3668            copy_dir(&aside.join("phone-state"), &self.base.join("phone-state"));
3669            copy_dir(&aside.join("folder"), &self.phone_folder);
3670        }
3671
3672        /// The other devices' files in the phone's synced folder.
3673        fn others_here(&self) -> BTreeMap<String, Vec<u8>> {
3674            walk(&self.phone_folder).into_iter().filter(|(r, _)| !r.contains(&self.phone.id)).collect()
3675        }
3676    }
3677
3678    impl Drop for World {
3679        fn drop(&mut self) {
3680            let _ = std::fs::remove_dir_all(&self.base);
3681        }
3682    }
3683
3684    /// Only once, then never again.
3685    fn once(matches: impl Fn(&super::fake::Request) -> bool + Send + 'static, fault: Fault) -> impl FnMut(&super::fake::Request) -> Option<Fault> + Send + 'static {
3686        let mut done = false;
3687        move |request| {
3688            if done || !matches(request) {
3689                return None;
3690            }
3691            done = true;
3692            Some(fault)
3693        }
3694    }
3695
3696    #[test]
3697    fn a_stale_folder_is_caught_up_from_the_server() {
3698        let w = World::new("stale");
3699        // The dose falls due; the desk, in use, turns do-not-disturb on and
3700        // shares after it. Its client carries it to the server at once; the
3701        // phone's sync app carries the phone's files up, and brings nothing down.
3702        w.desk.press(true, DUE + 30);
3703        w.desk.exchange(&w.server, DUE + 150);
3704        w.up();
3705        let alone = w.phone.exchange(&w.phone_folder, DUE + 170);
3706        assert!(alone.problems.is_empty(), "{:?}", alone.problems);
3707        assert!(w.phone.dnd().is_none_or(|p| !p.on), "nothing came yet");
3708        let stale = w.phone.sees(&w.phone_folder, &w.desk, DUE + 180);
3709        assert!(matches!(doubts_now(DUE, DUE + 180, None, &[stale]).as_slice(), [Doubt::Working { .. }]), "the desk's news has not come: a doubt");
3710        // The backup: the folder found on the server under the same seal, the desk's news fetched.
3711        let state = w.find(DUE + 180);
3712        assert!(state.confirmed_for(&w.phone_folder), "{state:?}");
3713        assert_eq!(state.url, w.fake.folder_url("Documents/Sioul"));
3714        let before = w.others_here();
3715        let pulled = w.pull(DUE + 181);
3716        assert!(pulled.problem.is_none() && pulled.fetched >= 2, "{pulled:?}");
3717        assert_eq!(w.others_here(), before, "nothing written into the synced folder");
3718        let outcome = w.phone.exchange(&w.phone_folder, DUE + 182);
3719        assert!(outcome.problems.is_empty(), "{:?}", outcome.problems);
3720        let press = w.phone.dnd().expect("the desk's press");
3721        assert!(press.on && press.from == w.desk.id, "{press:?}");
3722        // Its entry the fresh one: the dose known not taken, as with the folder fresh.
3723        let fresh = w.phone.sees(&w.phone_folder, &w.desk, DUE + 183);
3724        assert!(doubts_now(DUE, DUE + 183, None, std::slice::from_ref(&fresh)).is_empty(), "{fresh:?}");
3725        let direct = crate::devices::all(&w.server, &KEY).0.into_iter().find(|e| e.id == w.desk.id).unwrap();
3726        assert_eq!(fresh.said, Some(direct.said()), "the same as the server's folder says");
3727        assert!(crate::share::others(&w.phone_folder, &w.phone.id).iter().any(|o| o.id == w.desk.id && o.heard >= DUE + 150));
3728        // Read only, never one of the phone's own files; the password nowhere in what it keeps.
3729        for line in w.fake.seen() {
3730            assert!(line.starts_with("PROPFIND ") || line.starts_with("GET "), "{line}");
3731            assert!(!(line.starts_with("GET ") && line.contains(&w.phone.id)), "{line}");
3732        }
3733        let kept = std::fs::read_to_string(state_path(&w.phone.memory)).unwrap();
3734        assert!(!kept.contains(PASSWORD) && !kept.contains("Basic"), "{kept}");
3735        assert!(walk(&w.cache()).keys().all(|name| !name.contains(&w.phone.id)), "never the phone's own");
3736    }
3737
3738    #[test]
3739    fn with_the_folder_fresh_the_backup_changes_nothing() {
3740        let w = World::new("fresh");
3741        w.desk.press(true, DUE + 30);
3742        w.desk.exchange(&w.server, DUE + 150);
3743        crate::lease::renew(&w.server, &KEY, "health", &w.desk.id, DUE + 150, DUE + 150, false, crate::lease::Rule::FollowsYou, crate::share::written(&w.desk.memory, &w.desk.id)).unwrap();
3744        w.up();
3745        w.down();
3746        // The phone exchanges without the backup...
3747        let aside = w.snapshot();
3748        let without = w.phone.exchange(&w.phone_folder, DUE + 200);
3749        let kept_without = w.phone.kept();
3750        // ...and, put back as it was, with it: the same.
3751        w.restore(&aside);
3752        assert!(w.find(DUE + 190).confirmed_for(&w.phone_folder));
3753        w.fake.forget_seen();
3754        let pulled = w.pull(DUE + 195);
3755        assert!(pulled.problem.is_none(), "{pulled:?}");
3756        assert_eq!(pulled.fetched, 0, "nothing the folder holds is kept twice");
3757        assert!(w.fake.seen().iter().all(|line| !(line.starts_with("GET ") && line.contains(".jsonl"))), "no records fetched: {:?}", w.fake.seen());
3758        assert_eq!(walk(&w.cache()).into_keys().collect::<Vec<_>>(), ["seal.toml"]);
3759        let with = w.phone.exchange(&w.phone_folder, DUE + 200);
3760        assert_eq!((with.received, with.sent, &with.problems), (without.received, without.sent, &without.problems));
3761        assert_eq!(w.phone.kept(), kept_without, "the same files, byte for byte");
3762        // Fetched again: nothing, the server's files known by their ETag.
3763        w.fake.forget_seen();
3764        w.pull(DUE + 260);
3765        assert!(w.fake.seen().iter().all(|line| line.starts_with("PROPFIND ")), "{:?}", w.fake.seen());
3766    }
3767
3768    #[test]
3769    fn a_half_downloaded_file_is_never_read_as_whole() {
3770        let w = World::new("half");
3771        // Two doses marked on the desk, carried to the phone as they should be.
3772        let first = "\"first@1800000000\" = 1800000000\n\"second@1800000000\" = 1800000001\n";
3773        w.desk.write("health-state.toml", &format!("[taken]\n{first}"));
3774        w.desk.exchange(&w.server, DUE + 50);
3775        w.down();
3776        // Forty more: forty lines of records, carried to the server only.
3777        let marks: String = (0..40).map(|n| format!("\"dose-{n}@1800000000\" = {}\n", DUE + n)).collect();
3778        w.desk.write("health-state.toml", &format!("[taken]\n{first}{marks}"));
3779        w.desk.exchange(&w.server, DUE + 100);
3780        assert!(w.find(DUE + 110).confirmed_for(&w.phone_folder));
3781        let round = w.desk_round();
3782        let whole = std::fs::read(w.server.join(&round)).unwrap();
3783        let (end, last) = whole_end(&w.phone_folder.join(&round));
3784        assert!(end > 0 && whole.len() as u64 > end, "the phone's copy is behind");
3785        // The records come cut in the middle of a line.
3786        let cut = (whole.len() - last as usize) / 2;
3787        let name = round.clone();
3788        w.fake.fault(once(move |r| r.method == "GET" && r.path.ends_with(&name), Fault::Cut(cut)));
3789        let pulled = w.pull(DUE + 120);
3790        assert!(pulled.problem.as_deref().is_some_and(|p| p.starts_with("network:")), "{pulled:?}");
3791        assert!(w.fake.seen().iter().any(|l| l.starts_with("GET ") && l.contains(&round) && l.ends_with(" 206")), "asked from where the phone's copy ends: {:?}", w.fake.seen());
3792        let cached = w.cache().join(&round);
3793        let arrived = std::fs::read(&cached).unwrap_or_default();
3794        assert!(arrived.is_empty() || (arrived.ends_with(b"\n") && whole.starts_with(&arrived) && arrived.len() < whole.len()), "whole lines only: {} of {}", arrived.len(), whole.len());
3795        let first = w.phone.exchange(&w.phone_folder, DUE + 130);
3796        assert!(first.problems.iter().all(|p| !p.starts_with("share-other")), "{:?}", first.problems);
3797        // Then the desk's entry comes cut: never kept, never read.
3798        let entry = format!("devices/{}.device", w.desk.id);
3799        let name = entry.clone();
3800        w.fake.fault(once(move |r| r.method == "GET" && r.path.ends_with(&name), Fault::Cut(40)));
3801        let pulled = w.pull(DUE + 140);
3802        assert!(pulled.problem.as_deref().is_some_and(|p| p.starts_with("network:")), "{pulled:?}");
3803        assert_eq!(std::fs::read(&cached).unwrap(), whole, "the records whole now");
3804        assert!(!w.cache().join(&entry).exists(), "an entry cut is not kept");
3805        let (entries, unread) = crate::devices::all(&w.phone_folder, &KEY);
3806        assert!(entries.iter().any(|e| e.id == w.desk.id) && !unread.contains(&w.desk.id), "the folder's copy read meanwhile");
3807        let second = w.phone.exchange(&w.phone_folder, DUE + 150);
3808        assert!(second.problems.iter().all(|p| !p.starts_with("share-other")), "{:?}", second.problems);
3809        assert!(w.pull(DUE + 160).problem.is_none());
3810        assert!(w.cache().join(&entry).exists());
3811        // Every dose came, each line read once, as far as the desk wrote.
3812        let record = w.phone.read("health-state.toml");
3813        for n in 0..40 {
3814            assert!(record.contains(&format!("dose-{n}@")), "dose-{n}:\n{record}");
3815        }
3816        let heard = crate::share::heard(&w.phone.memory, &w.phone.id);
3817        assert!(heard.complete(&w.desk.id, crate::share::written(&w.desk.memory, &w.desk.id).unwrap()), "{heard:?}");
3818        assert!(!heard.broken.contains_key(&w.desk.id), "no line lost nor read twice: {heard:?}");
3819    }
3820
3821    #[test]
3822    fn a_seal_that_differs_is_never_used() {
3823        let w = World::new("seal");
3824        // A folder of the same name on the server, sealed otherwise: refused, and nothing fetched from it.
3825        put(&w.server, "seal.toml", OTHER_SEAL);
3826        let state = w.find(DUE);
3827        assert!(!state.confirmed_for(&w.phone_folder) && state.said.starts_with("seal-differs:"), "{state:?}");
3828        assert!(attach(&w.phone_folder, &w.phone.memory).is_none());
3829        assert_eq!(w.pull(DUE + 1).problem.as_deref(), Some("not-confirmed"));
3830        assert!(w.fake.seen().iter().filter(|l| l.starts_with("GET ")).all(|l| l.ends_with("seal.toml 200") || l.ends_with("seal.toml 404")), "{:?}", w.fake.seen());
3831        // Sealed the same: confirmed. Then sealed otherwise there: nothing more fetched.
3832        put(&w.server, "seal.toml", SEAL);
3833        State::choose(&w.phone.memory, |s| s.asked = DUE + 2).unwrap();
3834        assert!(w.find(DUE + 3).confirmed_for(&w.phone_folder));
3835        assert!(w.pull(DUE + 4).problem.is_none());
3836        put(&w.server, "seal.toml", OTHER_SEAL);
3837        w.desk.press(true, DUE + 5);
3838        w.desk.exchange(&w.server, DUE + 6);
3839        w.fake.forget_seen();
3840        let pulled = w.pull(DUE + 7);
3841        assert!(pulled.problem.as_deref().is_some_and(|p| p.starts_with("seal-differs:")), "{pulled:?}");
3842        assert!(w.fake.seen().iter().filter(|l| l.starts_with("GET ")).all(|l| l.contains("seal.toml")), "{:?}", w.fake.seen());
3843        assert!(!State::load(&w.phone.memory).confirmed_for(&w.phone_folder));
3844        // Sharing started again here, under another seal: what was fetched is not read.
3845        put(&w.server, "seal.toml", SEAL);
3846        State::choose(&w.phone.memory, |s| s.asked = DUE + 8).unwrap();
3847        assert!(w.find(DUE + 9).confirmed_for(&w.phone_folder));
3848        assert!(w.pull(DUE + 10).problem.is_none());
3849        assert!(attach(&w.phone_folder, &w.phone.memory).is_some());
3850        put(&w.phone_folder, "seal.toml", OTHER_SEAL);
3851        assert!(attach(&w.phone_folder, &w.phone.memory).is_none(), "another sharing's");
3852        assert_eq!(w.pull(DUE + 11).problem.as_deref(), Some("no-seal"));
3853        assert!(!w.cache().exists());
3854    }
3855
3856    #[test]
3857    fn server_errors_and_timeouts_leave_the_exchange_as_before() {
3858        let w = World::new("errors");
3859        assert!(w.find(DUE).confirmed_for(&w.phone_folder));
3860        w.desk.press(true, DUE + 30);
3861        w.desk.exchange(&w.server, DUE + 60);
3862        w.up();
3863        for (n, fault) in [Fault::Status(500), Fault::Status(503), Fault::Late(Duration::from_secs(2)), Fault::Drop, Fault::Cut(20)].into_iter().enumerate() {
3864            w.fake.clear_faults();
3865            w.fake.fault(move |_| Some(fault));
3866            let pulled = w.pull(DUE + 100 + n as i64);
3867            let said = pulled.problem.clone().unwrap_or_default();
3868            assert!(said.starts_with("server:") || said.starts_with("network:"), "{fault:?}: {pulled:?}");
3869            assert_eq!(walk(&w.cache()).into_keys().collect::<Vec<_>>(), ["seal.toml"], "{fault:?}: nothing kept");
3870            assert_eq!(State::load(&w.phone.memory).said, said);
3871        }
3872        w.fake.clear_faults();
3873        // The exchange then is the one without the backup, byte for byte.
3874        let aside = w.snapshot();
3875        let with = w.phone.exchange(&w.phone_folder, DUE + 200);
3876        let kept_with = w.phone.kept();
3877        w.restore(&aside);
3878        forget(&w.phone.memory, &w.phone_folder);
3879        let without = w.phone.exchange(&w.phone_folder, DUE + 200);
3880        assert_eq!((with.received, &with.problems), (without.received, &without.problems));
3881        assert_eq!(w.phone.kept(), kept_with);
3882    }
3883
3884    #[test]
3885    fn what_was_fetched_never_wins_over_a_newer_synced_copy() {
3886        let w = World::new("older");
3887        assert!(w.find(DUE).confirmed_for(&w.phone_folder));
3888        let renew = |now: i64| crate::lease::renew(&w.server, &KEY, "health", &w.desk.id, now, now, false, crate::lease::Rule::FollowsYou, crate::share::written(&w.desk.memory, &w.desk.id)).unwrap();
3889        w.desk.press(true, DUE + 30);
3890        w.desk.exchange(&w.server, DUE + 60);
3891        renew(DUE + 60);
3892        assert!(w.pull(DUE + 70).problem.is_none());
3893        let round = w.desk_round();
3894        assert!(w.cache().join(&round).exists() && w.cache().join(format!("devices/{}.device", w.desk.id)).exists());
3895        // The desk goes on, a quarter of an hour later; the sync app brings it all at last: the folder's copies newer.
3896        w.desk.press(false, DUE + 1_000);
3897        w.desk.exchange(&w.server, DUE + 1_100);
3898        renew(DUE + 1_100);
3899        w.up();
3900        w.down();
3901        let entry = crate::devices::all(&w.phone_folder, &KEY).0.into_iter().find(|e| e.id == w.desk.id).unwrap();
3902        assert_eq!(entry.exported, DUE + 1_100, "the folder's entry");
3903        let claim = crate::lease::claims(&w.phone_folder, &KEY, "health").into_iter().find(|c| c.computer == w.desk.id).unwrap();
3904        assert_eq!(claim.renewed, DUE + 1_100, "the folder's claim");
3905        let heard = crate::share::others(&w.phone_folder, &w.phone.id).into_iter().find(|o| o.id == w.desk.id).unwrap().heard;
3906        assert_eq!(heard, DUE + 1_100, "the folder's notes");
3907        w.phone.exchange(&w.phone_folder, DUE + 1_200);
3908        assert!(!w.phone.dnd().unwrap().on, "the later press, from the folder's longer records");
3909        // What the folder caught up with goes from the cache.
3910        tidy(&w.phone.memory, &w.phone_folder, &w.phone.id);
3911        assert!(!w.cache().join(&round).exists());
3912        // The backup switched off: the entries and notes fetched go too; records longer than the folder's would stay.
3913        State::choose(&w.phone.memory, |s| s.on = Some(false)).unwrap();
3914        tidy(&w.phone.memory, &w.phone_folder, &w.phone.id);
3915        assert_eq!(walk(&w.cache()).into_keys().collect::<Vec<_>>(), ["seal.toml"]);
3916        w.fake.forget_seen();
3917        assert_eq!(w.pull(DUE + 1_300).problem.as_deref(), Some("off"));
3918        assert!(w.fake.seen().is_empty(), "switched off: not a request");
3919    }
3920
3921    /// The doses' rule (docs/health.md, "Knowing") with the phone's sync app
3922    /// stuck, bringing nothing down, and the backup fetching each minute:
3923    /// what the phone takes as known is there, and every dose comes.
3924    #[test]
3925    fn doses_are_never_known_wrongly_with_a_stuck_sync_app_and_the_backup() {
3926        let w = World::new("stuck");
3927        assert!(w.find(DUE).confirmed_for(&w.phone_folder));
3928        let mut marked: Vec<(String, i64)> = Vec::new();
3929        for minute in 0..90 {
3930            let now = DUE + minute * 60;
3931            if minute % 10 == 0 {
3932                marked.push((format!("dose@{minute}"), minute));
3933                let record: String = marked.iter().map(|(d, m)| format!("\"{d}\" = {m}\n")).collect();
3934                w.desk.write("health-state.toml", &format!("[taken]\n{record}"));
3935            }
3936            w.desk.exchange(&w.server, now);
3937            crate::lease::renew(&w.server, &KEY, "health", &w.desk.id, now, now, false, crate::lease::Rule::FollowsYou, crate::share::written(&w.desk.memory, &w.desk.id)).unwrap();
3938            w.up();
3939            let _ = w.pull(now + 20);
3940            w.phone.exchange(&w.phone_folder, now + 30);
3941            if let Some(claim) = crate::lease::claims(&w.phone_folder, &KEY, "health").into_iter().find(|c| c.computer == w.desk.id)
3942                && let Some(wrote) = claim.wrote
3943                && crate::share::heard(&w.phone.memory, &w.phone.id).complete(&w.desk.id, wrote)
3944            {
3945                let record = w.phone.read("health-state.toml");
3946                let claimed = (claim.renewed - DUE) / 60;
3947                for (dose, at) in marked.iter().filter(|(_, m)| *m < claimed) {
3948                    assert!(record.contains(dose.as_str()), "minute {minute}: {dose} ({at}) taken as known, not there:\n{record}");
3949                }
3950            }
3951        }
3952        let record = w.phone.read("health-state.toml");
3953        for (dose, _) in &marked {
3954            assert!(record.contains(dose.as_str()), "{dose} never came:\n{record}");
3955        }
3956        // The synced folder never had them: the backup brought them.
3957        let synced = std::fs::read(w.phone_folder.join(w.desk_round())).map(|b| b.len()).unwrap_or(0);
3958        assert!(synced < std::fs::read(w.server.join(w.desk_round())).unwrap().len());
3959    }
3960
3961    /// An alarm's waits, short: a test does not wait twenty-five seconds.
3962    const QUICK: AlarmTimes = AlarmTimes { alone: Duration::from_secs(2), most: Duration::from_secs(5) };
3963
3964    /// A dose's alarm on the phone (`health::alarm_decide`, through the app's
3965    /// `exchange_here`) on its own pull while it works (`alarm_news`): a pull
3966    /// begun since the alarm, gone through, and the sync app neither asked to
3967    /// look nor waited for. Minute after minute, the desk in use marking
3968    /// doses, the phone's sync app bringing nothing down: from the same
3969    /// moment, the alarm on its pull knows exactly what it knows with the sync
3970    /// app asked as before (it brought everything down), its doubt never
3971    /// weaker, and a dose marked elsewhere is never known not taken.
3972    #[test]
3973    fn a_doses_alarm_on_its_pull_knows_what_it_knew_with_the_sync_app_asked() {
3974        static ONE_PULL: Mutex<()> = Mutex::new(());
3975        let w = World::new("alarm-alone");
3976        assert!(w.find(DUE).confirmed_for(&w.phone_folder));
3977        assert!(w.pull(DUE).problem.is_none());
3978        let mut marked: Vec<(String, i64)> = Vec::new();
3979        for minute in 0..16 {
3980            let now = DUE + minute * 60;
3981            if minute % 3 == 0 {
3982                marked.push((format!("dose@{minute}"), now));
3983                let record: String = marked.iter().map(|(d, at)| format!("\"{d}\" = {at}\n")).collect();
3984                w.desk.write("health-state.toml", &format!("[taken]\n{record}"));
3985            }
3986            w.desk.exchange(&w.server, now);
3987            w.up();
3988            // As before: the sync app asked brings the server's files down, the pull meanwhile.
3989            let aside = w.snapshot();
3990            w.down();
3991            let _ = w.pull(now + 20);
3992            w.phone.exchange(&w.phone_folder, now + 25);
3993            let asked = (w.phone.read("health-state.toml"), w.phone.sees(&w.phone_folder, &w.desk, now + 30));
3994            // The alarm on its pull, from the same moment: the phone's folder as stale as the sync app left it.
3995            w.restore(&aside);
3996            assert!(State::load(&w.phone.memory).pulls_well(&w.phone_folder), "minute {minute}");
3997            let mut ask = || -> Duration { panic!("minute {minute}: the pull went through, the sync app is not asked") };
3998            let came = alarm_news_with(&w.phone.memory, &w.phone_folder, &w.phone.id, KEY, Some(w.fake.login()), &ONE_PULL, now + 20, QUICK, Arc::new(move || now + 20), &mut ask, TEST);
3999            assert_eq!(came, AlarmNews::Pulled, "minute {minute}");
4000            w.phone.exchange(&w.phone_folder, now + 25);
4001            let alone = (w.phone.read("health-state.toml"), w.phone.sees(&w.phone_folder, &w.desk, now + 30));
4002            assert_eq!(alone.0, asked.0, "minute {minute}: the same answers here");
4003            for (dose, due) in &marked {
4004                let doubted = |peer: &Peer| !doubts_now(*due, now + 30, None, std::slice::from_ref(peer)).is_empty();
4005                assert_eq!(doubted(&alone.1), doubted(&asked.1), "minute {minute}, {dose}: the same doubt");
4006                assert!(alone.0.contains(dose.as_str()) || doubted(&alone.1), "minute {minute}: {dose}, marked on the desk, known not taken");
4007            }
4008        }
4009        // The synced folder never had them: the pull brought them.
4010        assert!(std::fs::read(w.phone_folder.join(w.desk_round())).map_or(0, |b| b.len()) < std::fs::read(w.server.join(w.desk_round())).unwrap().len());
4011    }
4012
4013    /// The desk closed an hour before the dose, the phone heard it through a
4014    /// good pull; a minute before the dose the desk opens, the dose is marked
4015    /// taken there and shared. The phone's alarm: its own pull fails. It never
4016    /// decides on what the last good pull said (the desk closed: the dose
4017    /// "known" not taken); it asks the sync app, as before Sioul's own pull,
4018    /// and waits for it: the answer comes, or the doubt is said. The next
4019    /// alarm asks the sync app first; the server back, the pull is trusted
4020    /// again (review of battery part B, F2).
4021    #[test]
4022    fn a_doses_alarm_whose_pull_fails_asks_the_sync_app_and_never_knows_wrongly() {
4023        static ONE_PULL: Mutex<()> = Mutex::new(());
4024        let w = World::new("alarm-fails");
4025        assert!(w.find(DUE - 3_000).confirmed_for(&w.phone_folder));
4026        w.desk.exchange(&w.server, DUE - 3_600);
4027        w.desk.session(&w.server, |e| e.close(DUE - 3_590));
4028        assert!(w.pull(DUE - 3_000).problem.is_none());
4029        w.phone.exchange(&w.phone_folder, DUE - 2_990);
4030        assert!(State::load(&w.phone.memory).pulls_well(&w.phone_folder), "the pull works");
4031        assert!(doubts_now(DUE, DUE + 10, None, &[w.phone.sees(&w.phone_folder, &w.desk, DUE + 10)]).is_empty(), "the desk closed before the dose: known");
4032        w.desk.session(&w.server, |e| e.start(DUE - 60));
4033        w.desk.write("health-state.toml", "[taken]\n\"dose@0\" = 1\n");
4034        w.desk.exchange(&w.server, DUE + 30);
4035        // The alarm at DUE + 60: its pull refused (503); the sync app asked brings the server's files down.
4036        w.fake.fault(|_| Some(Fault::Status(503)));
4037        let mut asked = 0;
4038        let mut ask = || {
4039            asked += 1;
4040            w.down();
4041            Duration::from_millis(200)
4042        };
4043        let came = alarm_news_with(&w.phone.memory, &w.phone_folder, &w.phone.id, KEY, Some(w.fake.login()), &ONE_PULL, DUE + 60, QUICK, Arc::new(|| DUE + 60), &mut ask, TEST);
4044        assert!(matches!(came, AlarmNews::SyncApp { ref heard } if heard.contains(&w.desk.id)) && asked == 1, "the pull failed: the sync app asked, once, the desk's entry brought: {came:?}");
4045        w.phone.exchange(&w.phone_folder, DUE + 65);
4046        let taken = w.phone.read("health-state.toml").contains("dose@0");
4047        let doubts = doubts_now(DUE, DUE + 70, None, &[w.phone.sees(&w.phone_folder, &w.desk, DUE + 70)]);
4048        assert!(taken || !doubts.is_empty(), "the dose marked on the desk is never known not taken: {doubts:?}");
4049        assert!(taken, "the sync app brought it");
4050        assert!(!State::load(&w.phone.memory).pulls_well(&w.phone_folder), "the next alarm asks the sync app first, as before");
4051        // The server back: the pull is trusted again.
4052        w.fake.clear_faults();
4053        assert!(w.pull(DUE + 180).problem.is_none());
4054        assert!(State::load(&w.phone.memory).pulls_well(&w.phone_folder));
4055    }
4056
4057    /// Two doses due at the same minute: two alarms at once, each on its own
4058    /// thread (review of battery part B, F1). The second finds the first one
4059    /// pulling: it waits for that pull, never decides on what came before it,
4060    /// and knows what it brought; the sync app is asked by neither.
4061    #[test]
4062    fn two_doses_at_once_the_second_alarm_waits_for_the_first_ones_pull() {
4063        static ONE_PULL: Mutex<()> = Mutex::new(());
4064        let w = World::new("alarm-twice");
4065        assert!(w.find(DUE - 3_000).confirmed_for(&w.phone_folder));
4066        w.desk.session(&w.server, |e| e.close(DUE - 3_590));
4067        assert!(w.pull(DUE - 3_000).problem.is_none());
4068        w.phone.exchange(&w.phone_folder, DUE - 2_990);
4069        // The desk opens a minute before, the doses are marked taken there.
4070        w.desk.session(&w.server, |e| e.start(DUE - 60));
4071        w.desk.write("health-state.toml", "[taken]\n\"a@0\" = 1\n\"b@0\" = 2\n");
4072        w.desk.exchange(&w.server, DUE + 10);
4073        // The first alarm's pull is slow: its first listing takes half a second (within the test's wait for an answer).
4074        w.fake.fault(once(|r| r.method == "PROPFIND", Fault::Late(Duration::from_millis(450))));
4075        let outcomes: Vec<(AlarmNews, Duration)> = std::thread::scope(|scope| {
4076            let alarm = |pause: u64| {
4077                let w = &w;
4078                scope.spawn(move || {
4079                    std::thread::sleep(Duration::from_millis(pause));
4080                    let begun = Instant::now();
4081                    let mut ask = || -> Duration { panic!("the pull went through: the sync app is not asked") };
4082                    (alarm_news_with(&w.phone.memory, &w.phone_folder, &w.phone.id, KEY, Some(w.fake.login()), &ONE_PULL, DUE + 60, QUICK, Arc::new(|| DUE + 60), &mut ask, TEST), begun.elapsed())
4083                })
4084            };
4085            let (first, second) = (alarm(0), alarm(150));
4086            vec![first.join().unwrap(), second.join().unwrap()]
4087        });
4088        assert!(outcomes.iter().all(|(came, _)| *came == AlarmNews::Pulled), "{outcomes:?}");
4089        assert!(outcomes[1].1 >= Duration::from_millis(250), "the second waited for the first one's pull: {outcomes:?}");
4090        w.phone.exchange(&w.phone_folder, DUE + 70);
4091        let record = w.phone.read("health-state.toml");
4092        assert!(record.contains("a@0") && record.contains("b@0"), "{record}");
4093        let seen = w.phone.sees(&w.phone_folder, &w.desk, DUE + 70);
4094        assert!(seen.said.as_ref().is_some_and(|s| s.working), "the desk known in use, from the pull: {seen:?}");
4095    }
4096
4097    /// An alarm while the window's pull runs (Sioul on the screen, its minute's
4098    /// pull begun before the alarm): the alarm waits for it, then, that pull
4099    /// begun before it, pulls again itself, and knows what came since (review
4100    /// of battery part B, F1).
4101    #[test]
4102    fn an_alarm_while_the_windows_pull_runs_pulls_again_after_it() {
4103        static ONE_PULL: Mutex<()> = Mutex::new(());
4104        let w = World::new("alarm-window");
4105        assert!(w.find(DUE - 3_000).confirmed_for(&w.phone_folder));
4106        w.desk.session(&w.server, |e| e.close(DUE - 3_590));
4107        assert!(w.pull(DUE - 3_000).problem.is_none());
4108        w.phone.exchange(&w.phone_folder, DUE - 2_990);
4109        w.desk.session(&w.server, |e| e.start(DUE - 60));
4110        w.desk.write("health-state.toml", "[taken]\n\"dose@0\" = 1\n");
4111        w.desk.exchange(&w.server, DUE + 10);
4112        w.fake.fault(once(|r| r.method == "PROPFIND", Fault::Late(Duration::from_millis(450))));
4113        let came = std::thread::scope(|scope| {
4114            // The window's pull, begun at DUE + 50, holding the one pull at a time.
4115            let window = scope.spawn(|| {
4116                let _one = ONE_PULL.lock().unwrap();
4117                pull_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), DUE + 50, TEST)
4118            });
4119            std::thread::sleep(Duration::from_millis(200));
4120            let mut ask = || -> Duration { panic!("a pull went through: the sync app is not asked") };
4121            let came = alarm_news_with(&w.phone.memory, &w.phone_folder, &w.phone.id, KEY, Some(w.fake.login()), &ONE_PULL, DUE + 60, QUICK, Arc::new(|| DUE + 61), &mut ask, TEST);
4122            assert!(window.join().unwrap().problem.is_none());
4123            came
4124        });
4125        assert_eq!(came, AlarmNews::Pulled);
4126        assert_eq!(State::load(&w.phone.memory).last, DUE + 61, "pulled again, after the window's");
4127        w.phone.exchange(&w.phone_folder, DUE + 70);
4128        assert!(w.phone.read("health-state.toml").contains("dose@0"));
4129    }
4130
4131    /// An alarm whose pull is slower than its wait (a large round first, a
4132    /// slow network): the sync app is asked after the pull's few seconds, and
4133    /// waited for; the alarm decides on what it brought, never on what came
4134    /// before it (review of battery part B, F2 and F7).
4135    #[test]
4136    fn an_alarm_whose_pull_does_not_go_through_in_time_asks_the_sync_app() {
4137        static ONE_PULL: Mutex<()> = Mutex::new(());
4138        let w = World::new("alarm-slow");
4139        assert!(w.find(DUE - 3_000).confirmed_for(&w.phone_folder));
4140        w.desk.session(&w.server, |e| e.close(DUE - 3_590));
4141        assert!(w.pull(DUE - 3_000).problem.is_none());
4142        w.phone.exchange(&w.phone_folder, DUE - 2_990);
4143        w.desk.session(&w.server, |e| e.start(DUE - 60));
4144        w.desk.write("health-state.toml", "[taken]\n\"dose@0\" = 1\n");
4145        w.desk.exchange(&w.server, DUE + 10);
4146        // Every answer of the server comes after six seconds: past the alarm's wait.
4147        w.fake.fault(|_| Some(Fault::Late(Duration::from_secs(6))));
4148        let mut asked = 0;
4149        let mut ask = || {
4150            asked += 1;
4151            w.down();
4152            Duration::from_millis(300)
4153        };
4154        let begun = Instant::now();
4155        let came = alarm_news_with(&w.phone.memory, &w.phone_folder, &w.phone.id, KEY, Some(w.fake.login()), &ONE_PULL, DUE + 60, QUICK, Arc::new(|| DUE + 60), &mut ask, TEST);
4156        let waited = begun.elapsed();
4157        assert!(matches!(came, AlarmNews::SyncApp { .. }) && asked == 1, "not through in time: the sync app asked, once: {came:?}");
4158        assert!(waited < QUICK.most, "never past the alarm's wait: {waited:?}");
4159        w.phone.exchange(&w.phone_folder, DUE + 65);
4160        assert!(w.phone.read("health-state.toml").contains("dose@0"), "what the sync app brought is read");
4161        w.fake.clear_faults();
4162    }
4163
4164    /// The scene of the review's F2: the desk closed an hour before the dose,
4165    /// heard through a good pull (`pulled_at`, by the phone's clock then);
4166    /// then, if `opens`, it opens a minute before the dose, the dose is marked
4167    /// taken there and shared.
4168    fn alarm_scene(name: &str, pulled_at: i64, opens: bool) -> World {
4169        let w = World::new(name);
4170        assert!(w.find(DUE - 3_000).confirmed_for(&w.phone_folder));
4171        w.desk.exchange(&w.server, DUE - 3_600);
4172        w.desk.session(&w.server, |e| e.close(DUE - 3_590));
4173        assert!(w.pull(pulled_at).problem.is_none());
4174        w.phone.exchange(&w.phone_folder, DUE - 2_990);
4175        assert!(State::load(&w.phone.memory).pulls_well(&w.phone_folder));
4176        if opens {
4177            w.desk.session(&w.server, |e| e.start(DUE - 60));
4178            w.desk.write("health-state.toml", "[taken]\n\"dose@0\" = 1\n");
4179            w.desk.exchange(&w.server, DUE + 30);
4180        }
4181        w
4182    }
4183
4184    /// The phone's clock an hour ahead when it last pulled, then set right
4185    /// (review of battery part B, S1). That pull, said begun after the alarm,
4186    /// is never taken for one since it: pulled again (refused here), the sync
4187    /// app asked; nothing came, so the doubt is said (S2). The clock set back
4188    /// never stops the background pulls either.
4189    #[test]
4190    fn an_alarm_never_takes_a_pull_from_before_the_clock_was_set_back() {
4191        static ONE_PULL: Mutex<()> = Mutex::new(());
4192        let w = alarm_scene("alarm-clock", DUE + 600, true);
4193        w.fake.fault(|_| Some(Fault::Status(503)));
4194        let mut asked = 0;
4195        let mut ask = || {
4196            asked += 1;
4197            Duration::from_millis(200)
4198        };
4199        let came = alarm_news_with(&w.phone.memory, &w.phone_folder, &w.phone.id, KEY, Some(w.fake.login()), &ONE_PULL, DUE + 60, QUICK, Arc::new(|| DUE + 60), &mut ask, TEST);
4200        assert_eq!((&came, asked), (&AlarmNews::SyncApp { heard: BTreeSet::new() }, 1), "the old pull not taken: pulled again, refused; the sync app asked, nothing came");
4201        w.phone.exchange(&w.phone_folder, DUE + 65);
4202        assert!(!w.phone.read("health-state.toml").contains("dose@0"));
4203        let peer = w.phone.sees(&w.phone_folder, &w.desk, DUE + 70);
4204        assert!(doubts_now(DUE, DUE + 70, None, std::slice::from_ref(&peer)).is_empty(), "what was read before the alarm says the desk closed");
4205        assert!(!sioul_core::health::doubts_unread(DUE, DUE + 70, None, std::slice::from_ref(&peer), DUE + 60, &BTreeSet::new()).is_empty(), "nothing read since the alarm: the doubt said, never known");
4206        let state = State::load(&w.phone.memory);
4207        assert!(!state.pulls_well(&w.phone_folder), "the refused pull is the later one, whatever the clock said: {state:?}");
4208        let pace = Pace { phone: true, shown: false, in_use: false, urgent: false, step: 5 * 60 };
4209        assert!(due(&State { tried: DUE + 600, ..state }, DUE + 120, pace), "tried \"later\" than now: due at once");
4210        w.fake.clear_faults();
4211    }
4212
4213    /// Neither route brings anything at an alarm (the server out of reach, the
4214    /// sync app bringing nothing): the desk, known closed before the dose by
4215    /// what was read before, may have opened since and answered; the doubt is
4216    /// said, never "not taken" (review of battery part B, S2). An alarm whose
4217    /// pull went through is not doubted by that rule: the desk still closed,
4218    /// the dose known.
4219    #[test]
4220    fn an_alarm_that_reads_no_news_says_the_doubt_and_one_that_pulls_does_not() {
4221        static ONE_PULL: Mutex<()> = Mutex::new(());
4222        let w = alarm_scene("alarm-nothing", DUE - 3_000, true);
4223        w.fake.fault(|_| Some(Fault::Status(503)));
4224        let mut ask = || Duration::from_millis(200);
4225        let came = alarm_news_with(&w.phone.memory, &w.phone_folder, &w.phone.id, KEY, Some(w.fake.login()), &ONE_PULL, DUE + 60, QUICK, Arc::new(|| DUE + 60), &mut ask, TEST);
4226        assert_eq!(came, AlarmNews::SyncApp { heard: BTreeSet::new() });
4227        w.phone.exchange(&w.phone_folder, DUE + 65);
4228        let peer = w.phone.sees(&w.phone_folder, &w.desk, DUE + 70);
4229        assert!(!w.phone.read("health-state.toml").contains("dose@0"));
4230        let doubts = sioul_core::health::doubts_unread(DUE, DUE + 70, None, std::slice::from_ref(&peer), DUE + 60, &BTreeSet::new());
4231        assert!(matches!(doubts.as_slice(), [Doubt::Unheard { closed: true, until, .. }] if *until == DUE - 3_590), "the desk as last heard, closed: {doubts:?}");
4232        w.fake.clear_faults();
4233        // The desk stays closed; the alarm's pull goes through: known, no doubt.
4234        static ANOTHER: Mutex<()> = Mutex::new(());
4235        let quiet = alarm_scene("alarm-pulled", DUE - 3_000, false);
4236        let mut never = || -> Duration { panic!("the pull went through: the sync app is not asked") };
4237        let came = alarm_news_with(&quiet.phone.memory, &quiet.phone_folder, &quiet.phone.id, KEY, Some(quiet.fake.login()), &ANOTHER, DUE + 60, QUICK, Arc::new(|| DUE + 60), &mut never, TEST);
4238        assert_eq!(came, AlarmNews::Pulled);
4239        quiet.phone.exchange(&quiet.phone_folder, DUE + 65);
4240        assert!(doubts_now(DUE, DUE + 70, None, &[quiet.phone.sees(&quiet.phone_folder, &quiet.desk, DUE + 70)]).is_empty(), "pulled since the alarm, the desk closed: known");
4241    }
4242
4243    /// What the phone's alarm decides of the desk once its news came (as
4244    /// `alarm_decide` does: `doubts_unread` when no pull went through since
4245    /// it began): whether the dose is known wrongly, read taken, the doubts.
4246    fn alarm_verdict(w: &World, came: &AlarmNews, since: i64, at: i64) -> (bool, bool, Vec<Doubt>) {
4247        w.phone.exchange(&w.phone_folder, at);
4248        let taken = w.phone.read("health-state.toml").contains("dose@0");
4249        let peer = w.phone.sees(&w.phone_folder, &w.desk, at + 5);
4250        let doubts = match came {
4251            AlarmNews::SyncApp { heard } => sioul_core::health::doubts_unread(DUE, at + 5, None, std::slice::from_ref(&peer), since, heard),
4252            AlarmNews::Pulled => doubts_now(DUE, at + 5, None, std::slice::from_ref(&peer)),
4253        };
4254        (!taken && doubts.is_empty(), taken, doubts)
4255    }
4256
4257    /// The pull refused; the sync app brings one file of the desk's that says
4258    /// nothing of its session (its notes), not its entry: the desk, opened
4259    /// since and marked the dose there, is never known closed by what was read
4260    /// before; the doubt is said (review of battery part B, T2). Its entry
4261    /// brought, it is as its entry says.
4262    #[test]
4263    fn an_alarm_doubts_each_device_whose_own_entry_did_not_come() {
4264        static ONE_PULL: Mutex<()> = Mutex::new(());
4265        let w = alarm_scene("alarm-unrelated", DUE - 3_000, true);
4266        w.fake.fault(|_| Some(Fault::Status(503)));
4267        let notes = format!("{}.toml", w.desk.id);
4268        let mut ask = || {
4269            // The sync app brings only the desk's notes (how far it read): not its entry, not its records.
4270            std::fs::copy(w.server.join(&notes), w.phone_folder.join(&notes)).unwrap();
4271            std::fs::File::options().write(true).open(w.phone_folder.join(&notes)).unwrap().set_modified(std::time::SystemTime::now()).unwrap();
4272            Duration::from_millis(200)
4273        };
4274        let came = alarm_news_with(&w.phone.memory, &w.phone_folder, &w.phone.id, KEY, Some(w.fake.login()), &ONE_PULL, DUE + 60, QUICK, Arc::new(|| DUE + 60), &mut ask, TEST);
4275        w.fake.clear_faults();
4276        assert_eq!(came, AlarmNews::SyncApp { heard: BTreeSet::new() }, "the notes are no news of the desk's session");
4277        let (wrong, taken, doubts) = alarm_verdict(&w, &came, DUE + 60, DUE + 65);
4278        assert!(!wrong && !taken && matches!(doubts.as_slice(), [Doubt::Unheard { closed: true, .. }]), "{doubts:?}");
4279        // The sync app brings the desk's entry too: news of it came, as it says (in use, not after the dose: a doubt of its own).
4280        static AGAIN: Mutex<()> = Mutex::new(());
4281        let other = alarm_scene("alarm-entry-came", DUE - 3_000, true);
4282        other.fake.fault(|_| Some(Fault::Status(503)));
4283        let entry = format!("devices/{}.device", other.desk.id);
4284        let mut ask = || {
4285            std::fs::copy(other.server.join(&entry), other.phone_folder.join(&entry)).unwrap();
4286            Duration::from_millis(200)
4287        };
4288        let came = alarm_news_with(&other.phone.memory, &other.phone_folder, &other.phone.id, KEY, Some(other.fake.login()), &AGAIN, DUE + 60, QUICK, Arc::new(|| DUE + 60), &mut ask, TEST);
4289        other.fake.clear_faults();
4290        assert!(matches!(came, AlarmNews::SyncApp { ref heard } if heard.contains(&other.desk.id)), "{came:?}");
4291        let (wrong, _, doubts) = alarm_verdict(&other, &came, DUE + 60, DUE + 65);
4292        assert!(!wrong && matches!(doubts.as_slice(), [Doubt::Working { .. }] | [Doubt::Coming { .. }]), "the desk in use, as its entry says: {doubts:?}");
4293    }
4294
4295    /// The clock set forward, a pull in this process, then set back: the
4296    /// alarm's own pull goes through and the desk is truly closed: counted,
4297    /// known, no needless doubt (review of battery part B, T1).
4298    #[test]
4299    fn a_clock_set_back_never_hides_the_alarms_own_pull() {
4300        static ONE_PULL: Mutex<()> = Mutex::new(());
4301        let w = alarm_scene("alarm-forward-back", DUE - 3_000, false);
4302        assert!(w.pull(DUE + 600).problem.is_none());
4303        let mut never = || -> Duration { panic!("the alarm's own pull went through: the sync app is not asked") };
4304        let came = alarm_news_with(&w.phone.memory, &w.phone_folder, &w.phone.id, KEY, Some(w.fake.login()), &ONE_PULL, DUE + 60, QUICK, Arc::new(|| DUE + 60), &mut never, TEST);
4305        assert_eq!(came, AlarmNews::Pulled);
4306        assert_eq!(State::load(&w.phone.memory).last, DUE + 60, "the time from before the set-back gives way");
4307        let (_, _, doubts) = alarm_verdict(&w, &came, DUE + 60, DUE + 65);
4308        assert!(doubts.is_empty(), "{doubts:?}");
4309    }
4310
4311    /// A pull that began with a round left to the sync app and found it again,
4312    /// grown, while the other process's save found it brought: still left, its
4313    /// own finding (review of battery part B, T3).
4314    #[test]
4315    fn a_round_found_again_grown_stays_left_in_the_merge() {
4316        let w = World::new("left-regrown");
4317        assert!(w.find(DUE).confirmed_for(&w.phone_folder));
4318        assert!(w.pull(DUE + 10).problem.is_none());
4319        let disk = State::load(&w.phone.memory);
4320        // The other process saved first: R brought, nothing left.
4321        State { left: BTreeMap::new(), tried: DUE + 20, last: DUE + 20, ..disk.clone() }.write(&w.phone.memory).unwrap();
4322        // This pull began with R left at 100 bytes; its look found it at 200.
4323        let loaded: BTreeMap<String, u64> = [("r-1.jsonl".to_string(), 100)].into();
4324        let ours = State { tried: DUE + 21, last: DUE + 21, left: [("r-1.jsonl".to_string(), 200)].into(), ..disk };
4325        ours.save_pulled(&w.phone.memory, DUE + 25, &loaded).unwrap();
4326        let merged = State::load(&w.phone.memory);
4327        assert_eq!(merged.left.get("r-1.jsonl"), Some(&200), "{:?}", merged.left);
4328        assert!(!merged.pulls_well(&w.phone_folder));
4329    }
4330
4331    /// Only what the doses read of a device is news of its session (review of
4332    /// battery part B, fourth pass): its claim on another part (notices),
4333    /// brought alone, is none (U1); its stale entry merely touched, same bytes
4334    /// at a new time, is none either (U2). Either way the desk, opened since and
4335    /// the dose marked there, is said in doubt, never known closed.
4336    #[test]
4337    fn only_a_newer_entry_or_health_claim_is_news_of_a_device() {
4338        static ONE_PULL: Mutex<()> = Mutex::new(());
4339        let w = alarm_scene("alarm-notices", DUE - 3_000, true);
4340        crate::lease::renew(&w.server, &KEY, "notices", &w.desk.id, DUE + 20, DUE + 20, false, crate::lease::Rule::FollowsYou, crate::share::written(&w.desk.memory, &w.desk.id)).unwrap();
4341        w.fake.fault(|_| Some(Fault::Status(503)));
4342        let lease = format!("leases/notices/{}.lease", w.desk.id);
4343        let mut ask = || {
4344            let to = w.phone_folder.join(&lease);
4345            std::fs::create_dir_all(to.parent().unwrap()).unwrap();
4346            std::fs::copy(w.server.join(&lease), &to).unwrap();
4347            Duration::from_millis(200)
4348        };
4349        let came = alarm_news_with(&w.phone.memory, &w.phone_folder, &w.phone.id, KEY, Some(w.fake.login()), &ONE_PULL, DUE + 60, QUICK, Arc::new(|| DUE + 60), &mut ask, TEST);
4350        w.fake.clear_faults();
4351        assert_eq!(came, AlarmNews::SyncApp { heard: BTreeSet::new() }, "a notices claim is no news of the session");
4352        let (wrong, _, doubts) = alarm_verdict(&w, &came, DUE + 60, DUE + 65);
4353        assert!(!wrong && !doubts.is_empty(), "{doubts:?}");
4354        // The stale entry touched: same bytes, a new time.
4355        static AGAIN: Mutex<()> = Mutex::new(());
4356        let touched = alarm_scene("alarm-touched", DUE - 3_000, true);
4357        touched.fake.fault(|_| Some(Fault::Status(503)));
4358        let entry = touched.phone_folder.join(format!("devices/{}.device", touched.desk.id));
4359        let mut ask = || {
4360            std::fs::File::options().write(true).open(&entry).unwrap().set_modified(std::time::SystemTime::now()).unwrap();
4361            Duration::from_millis(200)
4362        };
4363        let came = alarm_news_with(&touched.phone.memory, &touched.phone_folder, &touched.phone.id, KEY, Some(touched.fake.login()), &AGAIN, DUE + 60, QUICK, Arc::new(|| DUE + 60), &mut ask, TEST);
4364        touched.fake.clear_faults();
4365        assert_eq!(came, AlarmNews::SyncApp { heard: BTreeSet::new() }, "the same entry at a new time is no news");
4366        let (wrong, _, doubts) = alarm_verdict(&touched, &came, DUE + 60, DUE + 65);
4367        assert!(!wrong && !doubts.is_empty(), "{doubts:?}");
4368    }
4369
4370    /// Two pulls at once in two processes (the window's or an alarm's, and the
4371    /// background step's), here two threads without a common lock: neither
4372    /// undoes the other (review of battery part B, S3). The one begun later says
4373    /// when a pull was tried and went through; a round left to the sync app,
4374    /// found by the slower one, stays left after the other's save.
4375    #[test]
4376    fn two_pulls_at_once_keep_what_each_found() {
4377        let w = World::new("two-pulls");
4378        assert!(w.find(DUE).confirmed_for(&w.phone_folder));
4379        w.desk.write("health-state.toml", "[taken]\n\"dose@0\" = 1\n");
4380        w.desk.exchange(&w.server, DUE + 10);
4381        w.fake.fault(once(|r| r.method == "PROPFIND", Fault::Late(Duration::from_millis(450))));
4382        std::thread::scope(|scope| {
4383            let slow = scope.spawn(|| pull_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), DUE + 50, TEST));
4384            std::thread::sleep(Duration::from_millis(100));
4385            // This one's look (begun later, done first): rounds past 64 bytes are left to the sync app.
4386            ROUND_CAP.with(|cap| cap.set(Some(64)));
4387            // Begun a moment later by the same clock (within the slower one's run, as a real one would).
4388            let quick = pull_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), DUE + 51, TEST);
4389            ROUND_CAP.with(|cap| cap.set(None));
4390            assert!(quick.problem.is_none() && quick.left >= 1, "{quick:?}");
4391            let slow = slow.join().unwrap();
4392            assert!(slow.problem.is_none(), "{slow:?}");
4393        });
4394        let state = State::load(&w.phone.memory);
4395        assert_eq!((state.tried, state.last), (DUE + 51, DUE + 51), "the later begun says when");
4396        assert!(!state.left.is_empty() && !state.pulls_well(&w.phone_folder), "found left by the other, never erased by the slower one's save: {:?}", state.left);
4397        // A look that finds it brought (the slower one fetched it) clears it.
4398        assert!(w.pull(DUE + 120).problem.is_none());
4399        assert!(State::load(&w.phone.memory).pulls_well(&w.phone_folder));
4400    }
4401
4402    /// A round on the server past what a pull brings (`LARGEST`; a small cap
4403    /// in this test) is left to the sync app, and said: the pull alone is no
4404    /// longer trusted while it lasts; once the sync app brought it, the pull is
4405    /// again (review of battery part B, F4).
4406    #[test]
4407    fn a_round_too_large_for_the_pull_is_said_and_left_to_the_sync_app() {
4408        let w = World::new("too-large");
4409        assert!(w.find(DUE).confirmed_for(&w.phone_folder));
4410        w.desk.write("health-state.toml", "[taken]\n\"dose@0\" = 1\n");
4411        w.desk.exchange(&w.server, DUE + 10);
4412        ROUND_CAP.with(|cap| cap.set(Some(64)));
4413        let pulled = w.pull(DUE + 20);
4414        assert!(pulled.problem.is_none() && pulled.left >= 1, "{pulled:?}");
4415        let state = State::load(&w.phone.memory);
4416        assert!(!state.left.is_empty() && !state.pulls_well(&w.phone_folder) && !state.went_through_since(&w.phone_folder, DUE + 20, DUE + 20), "{:?}", state.left);
4417        // The sync app brings it: nothing left there that is not here.
4418        w.down();
4419        let pulled = w.pull(DUE + 100);
4420        ROUND_CAP.with(|cap| cap.set(None));
4421        assert_eq!(pulled.left, 0, "{pulled:?}");
4422        assert!(State::load(&w.phone.memory).pulls_well(&w.phone_folder));
4423    }
4424
4425    /// The phone's entry on the server put back to an older copy (another
4426    /// device's sync app, the server's versions): the next pull lists it of
4427    /// another size than the phone sent, and the next send sends it over that
4428    /// copy, rather than at the check of every six hours (review of battery
4429    /// part B, F6).
4430    #[test]
4431    fn the_phones_entry_put_back_older_on_the_server_is_sent_again() {
4432        let w = World::new("entry-back");
4433        assert!(w.find(DUE).confirmed_for(&w.phone_folder));
4434        let entry = format!("devices/{}.device", w.phone.id);
4435        let older = std::fs::read(w.phone_folder.join(&entry)).unwrap();
4436        w.phone.session(&w.phone_folder, |e| e.close(DUE + 10));
4437        assert!(phone_sends(&w, DUE + 20, false).problem.is_none());
4438        let current = std::fs::read(w.phone_folder.join(&entry)).unwrap();
4439        assert_eq!(std::fs::read(w.server.join(&entry)).unwrap(), current);
4440        assert_ne!(older.len(), current.len());
4441        // Put back there; the desk changes too, so that the folder's listing is looked at again.
4442        std::fs::write(w.server.join(&entry), &older).unwrap();
4443        w.desk.exchange(&w.server, DUE + 30);
4444        assert!(w.pull(DUE + 40).problem.is_none());
4445        assert_eq!(State::load(&w.phone.memory).own_entry.map(|(size, _)| size), Some(older.len() as u64));
4446        assert!(phone_sends(&w, DUE + 50, false).problem.is_none());
4447        assert_eq!(std::fs::read(w.server.join(&entry)).unwrap(), current, "sent over the copy put back");
4448    }
4449
4450    /// The owner's phone, 6 October 2026, 23:30: the folder found on the
4451    /// server at night, the phone in the background, its folder telling the
4452    /// desk closed hours ago (eDrive left it so) while the desk is in use.
4453    /// The first pull comes at once; then the desk is known in use, and
4454    /// followed once a minute, night or not; closed, at the night's pace.
4455    #[test]
4456    fn found_at_night_the_first_pull_comes_at_once() {
4457        let w = World::new("night");
4458        w.desk.press(true, DUE + 30);
4459        w.desk.exchange(&w.server, DUE + 60);
4460        w.up();
4461        let in_use = |now: i64| crate::devices::all(&w.phone_folder, &KEY).0.iter().any(|e| e.id == w.desk.id && e.working && !e.left && now - e.exported <= sioul_core::health::FRESH + sioul_core::health::SKEW);
4462        let night = |now: i64| Pace { phone: true, shown: false, in_use: in_use(now), urgent: false, step: 15 * 60 };
4463        assert!(!in_use(DUE + 70), "the stale folder tells the desk not in use");
4464        let state = w.find(DUE + 70);
4465        assert!(state.confirmed_for(&w.phone_folder));
4466        attach(&w.phone_folder, &w.phone.memory);
4467        assert!(due(&State::load(&w.phone.memory), DUE + 80, night(DUE + 80)), "found: pulled at once, whatever the hour");
4468        let pulled = w.pull(DUE + 80);
4469        assert!(pulled.problem.is_none() && pulled.listed > 0 && pulled.fetched > 0, "{pulled:?}");
4470        assert!(in_use(DUE + 90), "the desk known in use, from the server");
4471        w.phone.exchange(&w.phone_folder, DUE + 90);
4472        assert!(w.phone.dnd().is_some_and(|p| p.on), "its do-not-disturb too");
4473        let state = State::load(&w.phone.memory);
4474        assert!(!due(&state, DUE + 100, night(DUE + 100)), "not twice in a minute");
4475        w.desk.exchange(&w.server, DUE + 130);
4476        assert!(w.pull(DUE + 140).problem.is_none(), "once a minute while the desk is in use, at night too");
4477        // The desk closes: at the night's pace again, fifteen minutes.
4478        w.desk.session(&w.server, |e| e.close(DUE + 150));
4479        assert!(w.pull(DUE + 200).problem.is_none());
4480        let state = State::load(&w.phone.memory);
4481        assert!(!in_use(DUE + 260) && !due(&state, DUE + 260, night(DUE + 260)));
4482        assert!(due(&state, DUE + 200 + 15 * 60, night(DUE + 200 + 15 * 60)), "a device opened at night is noticed within the step's fifteen minutes");
4483    }
4484
4485    #[test]
4486    fn the_pace_of_pulls() {
4487        let state = State { confirmed: 1_000, tried: 1_000, ..State::default() };
4488        let pace = |phone, shown, in_use, urgent| Pace { phone, shown, in_use, urgent, step: 5 * 60 };
4489        // A computer, a phone on its screen or another device in use: once a minute.
4490        assert!(due(&state, 1_060, pace(false, false, false, false)) && !due(&state, 1_059, pace(false, false, false, false)));
4491        assert!(due(&state, 1_060, pace(true, true, false, false)));
4492        assert!(due(&state, 1_060, pace(true, false, true, false)));
4493        // A phone in the background, nobody at another device: at its step's pace.
4494        assert!(!due(&state, 1_120, pace(true, false, false, false)) && due(&state, 1_270, pace(true, false, false, false)));
4495        // Asked for news: twenty seconds after the last.
4496        assert!(due(&state, 1_020, pace(true, false, false, true)) && !due(&state, 1_010, pace(true, false, false, true)));
4497        // Found again since the last pull: at once.
4498        assert!(due(&State { confirmed: 1_005, ..state }, 1_006, pace(true, false, false, false)));
4499    }
4500
4501    /// A phone's background step leaves the sync app alone only while its own
4502    /// pull works: on, confirmed for this folder, gone through, and not failing.
4503    #[test]
4504    fn the_pull_alone_is_trusted_only_while_it_goes_through() {
4505        let folder = Path::new("/storage/emulated/0/Documents/Sioul");
4506        let well = State { folder: shown(folder), url: "https://cloud.example.org/remote.php/dav/files/jane/Documents/Sioul/".into(), confirmed: 1_000, last: 1_200, tried: 1_200, ..State::default() };
4507        assert!(well.pulls_well(folder));
4508        assert!(!State { on: Some(false), ..well.clone() }.pulls_well(folder), "switched off here");
4509        assert!(State { on: Some(true), ..well.clone() }.pulls_well(folder));
4510        assert!(!State { confirmed: 0, ..well.clone() }.pulls_well(folder), "not confirmed");
4511        assert!(!State { last: 0, ..well.clone() }.pulls_well(folder), "never pulled yet");
4512        assert!(!State { said: "network:cloud.example.org".into(), ..well.clone() }.pulls_well(folder), "the last pull failed");
4513        assert!(!State { said: "not-found:cloud.example.org".into(), ..well.clone() }.pulls_well(folder), "looked for again, not found");
4514        assert!(!State { mode: MIRROR.into(), ..well.clone() }.pulls_well(folder), "Sioul keeping the folder: no sync app to ask anyway");
4515        assert!(!well.pulls_well(Path::new("/storage/emulated/0/Documents/Other")), "another folder");
4516    }
4517
4518    /// A step's requests, a pull's then a send's (each a `Server` of its own,
4519    /// as `pull` and `send` make them, with their own limits), go through one
4520    /// connection while it stays open: the process's one agent. On mobile
4521    /// data, one TLS handshake a step instead of one per pull, per send and
4522    /// per file sent.
4523    #[test]
4524    fn a_steps_requests_go_through_one_connection() {
4525        use std::io::{BufRead, BufReader, Write};
4526        use std::sync::atomic::{AtomicUsize, Ordering};
4527        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
4528        let base = format!("http://127.0.0.1:{}", listener.local_addr().unwrap().port());
4529        let (connections, requests) = (Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0)));
4530        let (counted, served) = (Arc::clone(&connections), Arc::clone(&requests));
4531        // A server that keeps its connections open (HTTP/1.1's default), as Nextcloud's does.
4532        std::thread::spawn(move || {
4533            for stream in listener.incoming().filter_map(Result::ok) {
4534                counted.fetch_add(1, Ordering::SeqCst);
4535                let served = Arc::clone(&served);
4536                std::thread::spawn(move || {
4537                    let mut reader = BufReader::new(stream.try_clone().unwrap());
4538                    let mut stream = stream;
4539                    loop {
4540                        let mut line = String::new();
4541                        if reader.read_line(&mut line).unwrap_or(0) == 0 {
4542                            return;
4543                        }
4544                        let mut length = 0;
4545                        loop {
4546                            let mut header = String::new();
4547                            reader.read_line(&mut header).unwrap();
4548                            if header.trim().is_empty() {
4549                                break;
4550                            }
4551                            if let Some((name, value)) = header.split_once(':')
4552                                && name.eq_ignore_ascii_case("content-length")
4553                            {
4554                                length = value.trim().parse().unwrap();
4555                            }
4556                        }
4557                        let mut body = vec![0; length];
4558                        reader.read_exact(&mut body).unwrap();
4559                        served.fetch_add(1, Ordering::SeqCst);
4560                        let (status, answer) = match line.split_whitespace().next() {
4561                            Some("PROPFIND") => ("207 Multi-Status", super::fake::multistatus("")),
4562                            Some("GET") => ("200 OK", b"version = 1\n".to_vec()),
4563                            _ => ("201 Created", Vec::new()),
4564                        };
4565                        let head = format!("HTTP/1.1 {status}\r\nContent-Length: {}\r\nETag: \"e\"\r\n\r\n", answer.len());
4566                        if stream.write_all(head.as_bytes()).and_then(|()| stream.write_all(&answer)).is_err() {
4567                            return;
4568                        }
4569                    }
4570                });
4571            }
4572        });
4573        let login = Login { account: "cloud".into(), url: format!("{base}/remote.php/dav/"), user: "jane".into(), password: Some(PASSWORD.into()) };
4574        let folder = format!("{base}/remote.php/dav/files/jane/Documents/Sioul/");
4575        // The pull: a listing and a small file.
4576        let pull = Server::new(&login, TEST).unwrap();
4577        pull.list(&folder).unwrap();
4578        assert!(pull.small(&format!("{folder}seal.toml")).unwrap().is_some());
4579        // The send after the exchange, with a send's limits: a look and a file sent whole.
4580        let send = Server::new(&login, Limits { wait: Duration::from_secs(2), stall: Duration::from_secs(3), ..TEST }).unwrap();
4581        assert!(send.stat(&format!("{folder}devices/")).unwrap().is_none_or(|item| item.etag.is_empty()));
4582        let (status, _) = send.put_file(&format!("{folder}phone.toml"), b"read = 1\n".to_vec(), None, 1, &sha1_hex(b"read = 1\n")).unwrap();
4583        assert_eq!(status, 201);
4584        // The second send of a step.
4585        let again = Server::new(&login, TEST).unwrap();
4586        again.list(&folder).unwrap();
4587        assert_eq!(requests.load(Ordering::SeqCst), 5);
4588        assert_eq!(connections.load(Ordering::SeqCst), 1, "one connection for the whole step");
4589    }
4590
4591    #[test]
4592    fn where_the_folder_may_be_on_the_server() {
4593        let found = |folder: &str, configs: &[&str]| places(Path::new(folder), &configs.iter().map(|c| c.to_string()).collect::<Vec<_>>());
4594        // A phone: eDrive carries its storage as the cloud's root, whatever name the storage goes by.
4595        assert_eq!(found("/storage/emulated/0/Documents/Sioul", &[]), ["Documents/Sioul", "Sioul", "0/Documents/Sioul"]);
4596        assert_eq!(found("/sdcard/Documents/Sioul/", &[])[0], "Documents/Sioul");
4597        // The Nextcloud app's own folder: after its account's.
4598        assert_eq!(found("/storage/emulated/0/Android/media/com.nextcloud.client/nextcloud/jane@murena.io/Shared/Sioul", &[])[0], "Shared/Sioul");
4599        // A computer: the Nextcloud client's folders, by their target there.
4600        let cfg = "[Accounts]\n0\\Folders\\1\\localPath=/home/me/Nextcloud/\n0\\Folders\\1\\targetPath=/\n0\\Folders\\2\\localPath=/home/me/Work\n0\\Folders\\2\\targetPath=/Projects/Work\n0\\url=https://cloud.example.org\n";
4601        assert_eq!(found("/home/me/Nextcloud/Documents/Sioul", &[cfg])[0], "Documents/Sioul");
4602        assert_eq!(found("/home/me/Work/Sioul", &[cfg])[0], "Projects/Work/Sioul");
4603        assert_eq!(found("/home/me/Workshop/Sioul", &[cfg]), ["Sioul", "Workshop/Sioul", "me/Workshop/Sioul"], "not inside /home/me/Work");
4604        // Addresses and names.
4605        assert_eq!(encode_path("Mes documents/Sioul"), "Mes%20documents/Sioul");
4606        assert_eq!(host_of("https://jane@murena.io/remote.php/dav/"), "murena.io");
4607        assert_eq!(kind_of("devices/0f8fad5b-d9cb-469f-a165-70867728950e.device").map(|k| k.0), Some(Kind::Device));
4608        assert_eq!(kind_of("0f8fad5b-d9cb-469f-a165-70867728950e-12.jsonl").map(|k| k.0), Some(Kind::Round));
4609        assert_eq!(kind_of("0f8fad5b-d9cb-469f-a165-70867728950e-12 (conflicted copy).jsonl"), None);
4610        assert_eq!(kind_of("leases/health/desktop.lease").map(|k| k.0), Some(Kind::Claim));
4611    }
4612
4613    #[test]
4614    fn found_on_the_account_given_by_hand_or_not_at_all() {
4615        let w = World::new("given");
4616        // Given by hand, a path in the account's files.
4617        State::choose(&w.phone.memory, |s| s.given = "Documents/Sioul".into()).unwrap();
4618        let state = find_with(&w.phone.memory, &w.phone_folder, &[w.fake.login()], &[], DUE, TEST);
4619        assert!(state.confirmed_for(&w.phone_folder), "{state:?}");
4620        // Given as an address.
4621        State::choose(&w.phone.memory, |s| s.given = w.fake.folder_url("Documents/Sioul")).unwrap();
4622        assert!(find_with(&w.phone.memory, &w.phone_folder, &[w.fake.login()], &[], DUE + 1, TEST).confirmed_for(&w.phone_folder));
4623        // Given wrong: not found, said, and nothing looked for by itself meanwhile.
4624        State::choose(&w.phone.memory, |s| s.given = "Elsewhere".into()).unwrap();
4625        let state = find_with(&w.phone.memory, &w.phone_folder, &[w.fake.login()], &places(&w.phone_folder, &[]), DUE + 2, TEST);
4626        assert!(state.said.starts_with("not-found:") && !state.confirmed_for(&w.phone_folder), "{state:?}");
4627        State::choose(&w.phone.memory, |s| s.given.clear()).unwrap();
4628        // A refused password, said; no password kept, said; no account, said.
4629        let wrong = Login { password: Some("wrong".into()), ..w.fake.login() };
4630        assert!(find_with(&w.phone.memory, &w.phone_folder, &[wrong], &places(&w.phone_folder, &[]), DUE + 3, TEST).said.starts_with("login:"));
4631        let none = Login { password: None, ..w.fake.login() };
4632        assert_eq!(find_with(&w.phone.memory, &w.phone_folder, &[none], &[], DUE + 4, TEST).said, "no-password:cloud");
4633        assert_eq!(find_with(&w.phone.memory, &w.phone_folder, &[], &[], DUE + 5, TEST).said, "no-account");
4634        // A server of another kind (no Nextcloud files): not looked on.
4635        let other = Login { url: format!("{}/dav/calendars/jane/", w.fake.base), ..w.fake.login() };
4636        assert_eq!(find_with(&w.phone.memory, &w.phone_folder, &[other], &places(&w.phone_folder, &[]), DUE + 6, TEST).said, "no-account");
4637        // When to look again: at once when asked, else six hours on.
4638        let state = State::load(&w.phone.memory);
4639        assert!(!state.look_due(&w.phone_folder, DUE + 7) && state.look_due(&w.phone_folder, DUE + 6 + LOOK_AGAIN));
4640        State::choose(&w.phone.memory, |s| s.asked = DUE + 8).unwrap();
4641        assert!(State::load(&w.phone.memory).look_due(&w.phone_folder, DUE + 9));
4642    }
4643
4644    #[test]
4645    fn your_choices_are_never_written_over_by_a_pull() {
4646        let w = World::new("choices");
4647        assert!(w.find(DUE).confirmed_for(&w.phone_folder));
4648        // A pull holds the state while it works; you switch it off meanwhile: off stays.
4649        let held = State::load(&w.phone.memory);
4650        State::choose(&w.phone.memory, |s| s.on = Some(false)).unwrap();
4651        held.save(&w.phone.memory).unwrap();
4652        assert_eq!(State::load(&w.phone.memory).on, Some(false));
4653        State::choose(&w.phone.memory, |s| s.on = None).unwrap();
4654        assert!(State::load(&w.phone.memory).fetching());
4655    }
4656
4657    // ------------------------------------------------ kept in step by Sioul itself
4658
4659    /// Two devices with no sync app at all, each keeping its own copy of the
4660    /// folder in step with the server through Sioul alone (`MIRROR`).
4661    struct Pair {
4662        base: PathBuf,
4663        fake: Arc<Fake>,
4664        desk: Device,
4665        desk_folder: PathBuf,
4666        phone: Device,
4667        phone_folder: PathBuf,
4668    }
4669
4670    impl Pair {
4671        fn new(name: &str) -> Pair {
4672            let base = scratch(name);
4673            let files = base.join("cloud");
4674            std::fs::create_dir_all(&files).unwrap();
4675            let fake = Fake::start(&files);
4676            let desk = Device::new(&base, "desk", COMPUTER);
4677            let phone = Device::new(&base, "phone", PHONE);
4678            let (desk_folder, phone_folder) = (mirror_of(&desk.memory), mirror_of(&phone.memory));
4679            let login = fake.login();
4680            // The desk starts sharing there: the folder made, its seal sent.
4681            let opened = open_with(&login, "Documents/Sioul", TEST).unwrap();
4682            assert!(opened.seal.is_none(), "nothing there yet");
4683            put(&desk_folder, "seal.toml", SEAL);
4684            create_with(&login, &opened, &desk_folder, TEST).unwrap();
4685            begin(&desk.memory, &desk_folder, &login, &opened, DUE - 7_300).unwrap();
4686            // A second start there at once finds it sealed: never sealed twice.
4687            assert_eq!(create_with(&login, &opened, &desk_folder, TEST), Err("sealed-meanwhile".to_string()));
4688            // The phone joins it: the seal brought into its own copy.
4689            let joined = open_with(&login, "/Documents/Sioul/", TEST).unwrap();
4690            assert_eq!(joined.seal.as_deref(), Some(SEAL.as_bytes()));
4691            put_bytes(&phone_folder, "seal.toml", joined.seal.as_deref().unwrap());
4692            begin(&phone.memory, &phone_folder, &login, &joined, DUE - 7_250).unwrap();
4693            let pair = Pair { base, fake, desk, desk_folder, phone, phone_folder };
4694            pair.desk.session(&pair.desk_folder, |e| e.start(DUE - 7_200));
4695            pair.phone.session(&pair.phone_folder, |e| e.start(DUE - 7_200));
4696            pair.turn(true, DUE - 7_100);
4697            pair.turn(false, DUE - 7_090);
4698            pair.turn(true, DUE - 7_080);
4699            pair
4700        }
4701
4702        fn device(&self, desk: bool) -> (&Device, &Path) {
4703            if desk { (&self.desk, &self.desk_folder) } else { (&self.phone, &self.phone_folder) }
4704        }
4705
4706        fn step(&self, desk: bool, now: i64, whole: bool) -> Pulled {
4707            let (device, folder) = self.device(desk);
4708            step_with(&device.memory, folder, &device.id, &self.fake.login(), now, whole, TEST)
4709        }
4710
4711        /// As the window does each minute: the server looked through, an exchange, what it wrote sent.
4712        fn turn(&self, desk: bool, now: i64) -> Outcome {
4713            let before = self.step(desk, now, true);
4714            assert!(before.problem.is_none(), "{before:?}");
4715            let outcome = self.device(desk).0.exchange(self.device(desk).1, now);
4716            let after = self.step(desk, now, false);
4717            assert!(after.problem.is_none(), "{after:?}");
4718            outcome
4719        }
4720    }
4721
4722    impl Drop for Pair {
4723        fn drop(&mut self) {
4724            let _ = std::fs::remove_dir_all(&self.base);
4725        }
4726    }
4727
4728    #[test]
4729    fn two_devices_without_a_sync_app_share_through_the_server_alone() {
4730        let p = Pair::new("mirror");
4731        // Do-not-disturb turned on at the desk: on the phone at its next turn.
4732        p.desk.press(true, DUE + 10);
4733        p.turn(true, DUE + 20);
4734        p.turn(false, DUE + 30);
4735        let press = p.phone.dnd().expect("the desk's press");
4736        assert!(press.on && press.from == p.desk.id, "{press:?}");
4737        // A dose marked taken on the phone: on the desk at its next turn.
4738        p.phone.write("health-state.toml", "[taken]\n\"levo@1800000000\" = 1800000040\n");
4739        p.turn(false, DUE + 40);
4740        p.turn(true, DUE + 50);
4741        assert!(p.desk.read("health-state.toml").contains("levo@1800000000"));
4742        // Turned off on the phone: off on the desk.
4743        p.phone.press(false, DUE + 60);
4744        p.turn(false, DUE + 70);
4745        p.turn(true, DUE + 80);
4746        assert!(!p.desk.dnd().unwrap().on);
4747        // Each knows how the other is, from the server alone.
4748        let phone = p.desk.sees(&p.desk_folder, &p.phone, DUE + 90);
4749        assert!(phone.complete && phone.said.as_ref().is_some_and(|s| s.working && s.exported >= DUE + 70), "{phone:?}");
4750        assert!(doubts_now(DUE - 100, DUE + 90, None, &[phone]).is_empty(), "a dose due before its last export is known");
4751        // Neither the backup nor its reading beside the folder ever run on a folder Sioul keeps itself.
4752        assert_eq!(pull_with(&p.phone.memory, &p.phone_folder, &p.phone.id, &p.fake.login(), DUE + 95, TEST).problem.as_deref(), Some("mirror"));
4753        assert!(attach(&p.phone_folder, &p.phone.memory).is_none());
4754    }
4755
4756    #[test]
4757    fn a_device_never_writes_another_devices_file() {
4758        let p = Pair::new("own");
4759        for minute in 0..6 {
4760            for desk in [true, false] {
4761                let device = p.device(desk).0;
4762                if minute % 2 == 0 {
4763                    device.press(minute % 4 == 0, DUE + minute * 60 + 1);
4764                }
4765                p.fake.forget_seen();
4766                p.turn(desk, DUE + minute * 60 + if desk { 0 } else { 30 });
4767                for line in p.fake.seen() {
4768                    let path = line.split(' ').nth(1).unwrap_or_default();
4769                    if line.starts_with("PUT ") || line.starts_with("DELETE ") {
4770                        assert!(path.contains(&device.id) || path.contains("/blobs/"), "{line}");
4771                    } else {
4772                        assert!(line.starts_with("GET ") || line.starts_with("PROPFIND ") || line.starts_with("MKCOL "), "{line}");
4773                    }
4774                }
4775            }
4776        }
4777    }
4778
4779    #[test]
4780    fn a_refused_or_failed_upload_is_never_lost() {
4781        let p = Pair::new("refused");
4782        p.phone.write("health-state.toml", "[taken]\n\"levo@1800000000\" = 1800000040\n");
4783        for (n, fault) in [Fault::Status(412), Fault::Status(500), Fault::Drop].into_iter().enumerate() {
4784            p.fake.clear_faults();
4785            let id = p.phone.id.clone();
4786            p.fake.fault(move |r| (r.method == "PUT" && r.path.contains(&id) && r.path.ends_with(".jsonl")).then_some(fault));
4787            let now = DUE + 40 + n as i64 * 10;
4788            // Its step looks through the server, and sends what waits: refused again.
4789            let _ = p.step(false, now, true);
4790            p.phone.exchange(&p.phone_folder, now);
4791            let pushed = p.step(false, now, false);
4792            assert!(pushed.problem.is_some(), "{fault:?}: {pushed:?}");
4793            assert!(p.phone.read("health-state.toml").contains("levo@"), "nothing lost here");
4794            p.turn(true, now + 5);
4795            assert!(!p.desk.read("health-state.toml").contains("levo@"), "{fault:?}: not there yet");
4796        }
4797        // The server takes it again: it goes, and the desk has it.
4798        p.fake.clear_faults();
4799        p.turn(false, DUE + 100);
4800        p.turn(true, DUE + 110);
4801        assert!(p.desk.read("health-state.toml").contains("levo@1800000000"));
4802        let heard = crate::share::heard(&p.desk.memory, &p.desk.id);
4803        assert!(!heard.broken.contains_key(&p.phone.id), "{heard:?}");
4804    }
4805
4806    #[test]
4807    fn its_copy_lost_a_device_goes_on_after_its_last_record() {
4808        let p = Pair::new("lost");
4809        p.phone.write("health-state.toml", "[taken]\n\"one@1800000000\" = 1800000010\n");
4810        p.turn(false, DUE + 20);
4811        p.turn(true, DUE + 25);
4812        // The phone's copy of the folder lost, its memory kept: its records come back from the server, numbering goes on.
4813        for entry in std::fs::read_dir(&p.phone_folder).unwrap().filter_map(Result::ok) {
4814            if entry.file_name() != "seal.toml" {
4815                let _ = std::fs::remove_dir_all(entry.path());
4816                let _ = std::fs::remove_file(entry.path());
4817            }
4818        }
4819        p.phone.write("health-state.toml", "[taken]\n\"one@1800000000\" = 1800000010\n\"two@1800000000\" = 1800000030\n");
4820        let outcome = p.turn(false, DUE + 40);
4821        assert!(outcome.problems.iter().all(|p| !p.starts_with("share-own")), "{:?}", outcome.problems);
4822        p.turn(true, DUE + 50);
4823        let record = p.desk.read("health-state.toml");
4824        assert!(record.contains("one@") && record.contains("two@"), "{record}");
4825        let heard = crate::share::heard(&p.desk.memory, &p.desk.id);
4826        assert!(!heard.broken.contains_key(&p.phone.id), "no gap nor cut on the desk: {heard:?}");
4827        // The others' files came back too.
4828        assert!(p.phone_folder.join(format!("devices/{}.device", p.desk.id)).exists());
4829    }
4830
4831    #[test]
4832    fn sealed_files_travel_whole_and_go_when_swept() {
4833        let p = Pair::new("blobs");
4834        let source = p.base.join("note.md");
4835        std::fs::write(&source, "a note\n".repeat(1000)).unwrap();
4836        let (hash, _) = crate::blobs::hash_file(&source).unwrap();
4837        crate::blobs::put(&p.desk_folder, &KEY, &source, &hash).unwrap();
4838        let name = crate::blobs::name(&KEY, &hash);
4839        p.turn(true, DUE + 10);
4840        p.turn(false, DUE + 20);
4841        let sealed = std::fs::read(p.desk_folder.join("blobs").join(&name)).unwrap();
4842        assert_eq!(std::fs::read(p.phone_folder.join("blobs").join(&name)).unwrap(), sealed);
4843        // Never sent back by the phone, which did not make it.
4844        p.fake.forget_seen();
4845        p.turn(false, DUE + 30);
4846        assert!(p.fake.seen().iter().all(|l| !(l.starts_with("PUT ") && l.contains("/blobs/"))), "{:?}", p.fake.seen());
4847        // Swept on the desk: taken out there.
4848        std::fs::remove_file(p.desk_folder.join("blobs").join(&name)).unwrap();
4849        p.turn(true, DUE + 40);
4850        assert!(!p.fake.files.join("Documents/Sioul/blobs").join(&name).exists());
4851    }
4852
4853    /// A sealed file going up slowly or not at all, or coming down slowly,
4854    /// never holds back a device's records: its own records go up first, the
4855    /// doses' answers among them, sealed files last (the spam filter's table
4856    /// here, made by a training on the desk).
4857    #[test]
4858    fn a_slow_or_failing_sealed_file_never_holds_back_the_records() {
4859        let p = Pair::new("sealed-last");
4860        let made = |seed: u32| {
4861            let mut x = seed.wrapping_mul(2_654_435_761) | 1;
4862            (0..600_000)
4863                .map(|_| {
4864                    x ^= x << 13;
4865                    x ^= x >> 17;
4866                    x ^= x << 5;
4867                    x as u8
4868                })
4869                .collect::<Vec<u8>>()
4870        };
4871        let table = p.desk.roots.data.join("spam").join("table.bin");
4872        std::fs::create_dir_all(table.parent().unwrap()).unwrap();
4873        std::fs::write(&table, made(1)).unwrap();
4874        let mut taken = String::from("[taken]\n");
4875        // Late past the time a file's size asks (an upload's answer is awaited that long, `put_file`).
4876        for (n, fault) in [Fault::Status(500), Fault::Drop, Fault::Late(Duration::from_secs(6))].into_iter().enumerate() {
4877            p.fake.clear_faults();
4878            p.fake.fault(move |r| (r.method == "PUT" && r.path.contains("/blobs/")).then_some(fault));
4879            let now = DUE + 100 + n as i64 * 60;
4880            // A dose marked on the desk, beside its new table that cannot go up.
4881            taken.push_str(&format!("\"levo@{}\" = {}\n", 1_800_000_000 + n, 1_800_000_040 + n));
4882            p.desk.write("health-state.toml", &taken);
4883            let _ = p.step(true, now, true);
4884            p.desk.exchange(&p.desk_folder, now);
4885            let pushed = p.step(true, now, false);
4886            assert!(pushed.problem.is_some(), "{fault:?}: {pushed:?}");
4887            // The phone has the dose all the same; the table waits for its sealed file.
4888            p.turn(false, now + 30);
4889            assert!(p.phone.read("health-state.toml").contains(&format!("levo@{}", 1_800_000_000 + n)), "{fault:?}");
4890        }
4891        // The server takes it: it goes, and comes.
4892        p.fake.clear_faults();
4893        p.turn(true, DUE + 400);
4894        p.turn(false, DUE + 410);
4895        let phone_table = p.phone.roots.data.join("spam").join("table.bin");
4896        assert_eq!(std::fs::read(&phone_table).unwrap(), std::fs::read(&table).unwrap());
4897        // A new table coming down slowly to the phone: the phone's own answer went up before it.
4898        std::fs::write(&table, made(2)).unwrap();
4899        p.turn(true, DUE + 500);
4900        p.phone.write("health-state.toml", &format!("{taken}\"two@1800000000\" = 1800000090\n"));
4901        p.phone.exchange(&p.phone_folder, DUE + 505);
4902        p.fake.fault(|r| (r.method == "GET" && r.path.contains("/blobs/")).then_some(Fault::Late(Duration::from_secs(2))));
4903        let stepped = p.step(false, DUE + 510, true);
4904        assert!(stepped.problem.is_some(), "{stepped:?}");
4905        p.fake.clear_faults();
4906        p.turn(true, DUE + 520);
4907        assert!(p.desk.read("health-state.toml").contains("two@1800000000"));
4908        // The table comes at the phone's next step: looked for again at once.
4909        p.turn(false, DUE + 530);
4910        assert_eq!(std::fs::read(&phone_table).unwrap(), std::fs::read(&table).unwrap());
4911    }
4912
4913    /// A device beside its sync app (the desk: its folder is the server's) and
4914    /// a phone with none, Sioul keeping its copy: one folder, both ways.
4915    #[test]
4916    fn a_device_with_a_sync_app_and_one_without_share_one_folder() {
4917        let w = World::new("mixed");
4918        let lone = Device::new(&w.base, "lone", PHONE);
4919        let folder = mirror_of(&lone.memory);
4920        let login = w.fake.login();
4921        let opened = open_with(&login, "Documents/Sioul", TEST).unwrap();
4922        put_bytes(&folder, "seal.toml", opened.seal.as_deref().expect("sealed by the desk"));
4923        begin(&lone.memory, &folder, &login, &opened, DUE).unwrap();
4924        lone.session(&folder, |e| e.start(DUE));
4925        let turn = |now: i64| {
4926            assert!(step_with(&lone.memory, &folder, &lone.id, &login, now, true, TEST).problem.is_none());
4927            lone.exchange(&folder, now);
4928            assert!(step_with(&lone.memory, &folder, &lone.id, &login, now, false, TEST).problem.is_none());
4929        };
4930        turn(DUE + 10);
4931        // Do-not-disturb turned on at the desk: the phone without a sync app has it.
4932        w.desk.press(true, DUE + 20);
4933        w.desk.exchange(&w.server, DUE + 30);
4934        turn(DUE + 40);
4935        assert!(lone.dnd().is_some_and(|p| p.on && p.from == w.desk.id));
4936        // A dose answered on that phone: the desk reads it in its own folder.
4937        lone.write("health-state.toml", "[taken]\n\"levo@1800000000\" = 1800000050\n");
4938        turn(DUE + 50);
4939        w.desk.exchange(&w.server, DUE + 60);
4940        assert!(w.desk.read("health-state.toml").contains("levo@1800000000"));
4941        let seen = w.desk.sees(&w.server, &lone, DUE + 70);
4942        assert!(seen.complete && seen.said.is_some_and(|s| s.working), "the desk knows how the phone is");
4943    }
4944
4945    #[test]
4946    fn keeping_the_folder_is_inert_until_set_up() {
4947        let w = World::new("inert");
4948        w.fake.forget_seen();
4949        let stepped = step_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), DUE, true, TEST);
4950        assert_eq!(stepped.problem.as_deref(), Some("not-confirmed"));
4951        assert!(w.fake.seen().is_empty(), "not a request");
4952        // The backup confirmed is not Sioul keeping the folder: nothing sent either.
4953        assert!(w.find(DUE).confirmed_for(&w.phone_folder));
4954        w.fake.forget_seen();
4955        assert_eq!(step_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), DUE + 1, false, TEST).problem.as_deref(), Some("not-confirmed"));
4956        assert!(w.fake.seen().is_empty());
4957    }
4958
4959    // ------------------------------------------------ sent beside the sync app
4960
4961    fn mtime(path: &Path) -> Option<SystemTime> {
4962        std::fs::metadata(path).and_then(|m| m.modified()).ok()
4963    }
4964
4965    fn set_mtime(path: &Path, at: SystemTime) {
4966        if let Ok(file) = std::fs::File::options().write(true).open(path) {
4967            let _ = file.set_modified(at);
4968        }
4969    }
4970
4971    /// A file written there by a sync app, as a `PUT` writes it: a new ETag, dated as the file here.
4972    fn sync_up(local: &Path, server: &Path) {
4973        std::fs::create_dir_all(server.parent().unwrap()).unwrap();
4974        std::fs::write(server, std::fs::read(local).unwrap()).unwrap();
4975        fake::bump(server);
4976        if let Some(at) = mtime(local) {
4977            set_mtime(server, at);
4978        }
4979    }
4980
4981    /// A file brought down by a sync app: a new file put in its place, dated as the server's.
4982    fn sync_down(server: &Path, local: &Path) {
4983        std::fs::create_dir_all(local.parent().unwrap()).unwrap();
4984        let temporary = local.with_file_name(format!(".{}.download", local.file_name().unwrap().to_string_lossy()));
4985        std::fs::write(&temporary, std::fs::read(server).unwrap()).unwrap();
4986        if let Some(at) = mtime(server) {
4987            set_mtime(&temporary, at);
4988        }
4989        std::fs::rename(&temporary, local).unwrap();
4990    }
4991
4992    fn folder_of(relative: &str) -> &str {
4993        relative.rfind('/').map_or("", |at| &relative[..at])
4994    }
4995
4996    /// Murena's eDrive as its source reads (eDrive `main`, 31 August 2026;
4997    /// docs/database.md, "Sent to the server too"): the server's files first,
4998    /// an ETag changed there brought down when the size differs, else taken
4999    /// as known; then the phone's, a time later than the one known sent up,
5000    /// `If-Match` the ETag it knows (the Documents folder is a "media"
5001    /// folder), replacing the download planned. A folder whose time did not
5002    /// change is not looked at here (a file appended in place changes none):
5003    /// the download planned then goes ahead over the phone's newer file. A
5004    /// refused upload holds the file back, and after four, for good. Never a
5005    /// conflicted copy.
5006    #[derive(Default)]
5007    struct EDrive {
5008        known: BTreeMap<String, (String, Option<SystemTime>)>,
5009        dirs: BTreeMap<String, Option<SystemTime>>,
5010        refused: BTreeMap<String, u32>,
5011        uploads: usize,
5012        refusals: usize,
5013        downloads_over_newer: usize,
5014    }
5015
5016    impl EDrive {
5017        fn scan(&mut self, fake: &Fake, server: &Path, phone: &Path) {
5018            let _turn = fake.turn.lock().unwrap();
5019            let (there, here) = (walk(server), walk(phone));
5020            let mut downloads: BTreeSet<String> = BTreeSet::new();
5021            for (relative, bytes) in there.iter().filter(|(r, _)| *r != "seal.toml") {
5022                let etag = fake::etag_of(&server.join(relative));
5023                if self.known.get(relative).is_some_and(|(known, _)| *known == etag) {
5024                    continue;
5025                }
5026                if here.get(relative).map(Vec::len) == Some(bytes.len()) {
5027                    let time = self.known.get(relative).and_then(|(_, t)| *t);
5028                    self.known.insert(relative.clone(), (etag, time));
5029                } else {
5030                    downloads.insert(relative.clone());
5031                }
5032            }
5033            let mut uploads: Vec<String> = Vec::new();
5034            for relative in here.keys().filter(|r| *r != "seal.toml") {
5035                let dir = folder_of(relative).to_string();
5036                let looked = self.dirs.get(&dir).copied().flatten() != mtime(&phone.join(&dir));
5037                let later = self.known.get(relative).is_none_or(|(_, t)| mtime(&phone.join(relative)) > *t);
5038                if looked && later {
5039                    downloads.remove(relative);
5040                    uploads.push(relative.clone());
5041                }
5042            }
5043            for relative in downloads {
5044                if here.get(&relative).is_some_and(|mine| there[&relative].len() < mine.len()) {
5045                    self.downloads_over_newer += 1;
5046                }
5047                sync_down(&server.join(&relative), &phone.join(&relative));
5048                self.known.insert(relative.clone(), (fake::etag_of(&server.join(&relative)), mtime(&phone.join(&relative))));
5049            }
5050            for relative in uploads {
5051                self.upload(server, phone, &relative);
5052            }
5053            self.dirs = here.keys().map(|r| folder_of(r).to_string()).chain([String::new()]).map(|d| (d.clone(), mtime(&phone.join(&d)))).collect();
5054        }
5055
5056        /// One file sent up, as eDrive sends it: `If-Match` the ETag it knows.
5057        fn upload(&mut self, server: &Path, phone: &Path, relative: &str) {
5058            if self.refused.get(relative).is_some_and(|n| *n >= 4) {
5059                return;
5060            }
5061            let there = server.join(relative);
5062            if let Some((known, _)) = self.known.get(relative)
5063                && there.is_file()
5064                && fake::etag_of(&there) != *known
5065            {
5066                self.refusals += 1;
5067                *self.refused.entry(relative.to_string()).or_insert(0) += 1;
5068                return;
5069            }
5070            sync_up(&phone.join(relative), &there);
5071            self.uploads += 1;
5072            self.known.insert(relative.to_string(), (fake::etag_of(&there), mtime(&phone.join(relative))));
5073        }
5074
5075        /// A file closed after writing (`CLOSE_WRITE`), sent at once.
5076        fn closed(&mut self, fake: &Fake, server: &Path, phone: &Path, relative: &str) {
5077            let _turn = fake.turn.lock().unwrap();
5078            if phone.join(relative).is_file() {
5079                self.upload(server, phone, relative);
5080            }
5081        }
5082    }
5083
5084    /// The Nextcloud desktop client as its source reads (v34; docs/database.md,
5085    /// "Sent to the server too"): a file changed on one side only goes the
5086    /// other way (`If-Match`); changed on both, the contents compared: the
5087    /// same, it takes the server's as its own; else the file here is renamed
5088    /// "(conflicted copy …)", which never goes up, and the server's takes its
5089    /// name. Hidden files left out.
5090    #[derive(Default)]
5091    struct Client {
5092        journal: BTreeMap<String, (String, u64, Option<SystemTime>)>,
5093        conflicts: usize,
5094        uploads: usize,
5095    }
5096
5097    impl Client {
5098        fn stamp(path: &Path) -> Option<(u64, Option<SystemTime>)> {
5099            std::fs::metadata(path).ok().filter(|m| m.is_file()).map(|m| (m.len(), m.modified().ok()))
5100        }
5101
5102        fn sync(&mut self, fake: &Fake, server: &Path, local: &Path) {
5103            let _turn = fake.turn.lock().unwrap();
5104            let names: BTreeSet<String> = walk(server).into_keys().chain(walk(local).into_keys()).filter(|r| r != "seal.toml" && !r.contains("(conflicted copy")).collect();
5105            for relative in names {
5106                let (there, here) = (server.join(&relative), local.join(&relative));
5107                let etag = there.is_file().then(|| fake::etag_of(&there));
5108                let stamp = Self::stamp(&here);
5109                let known = self.journal.get(&relative).cloned();
5110                let local_changed = known.as_ref().map(|(_, size, at)| Some((*size, *at))) != Some(stamp);
5111                let remote_changed = known.as_ref().map(|(e, ..)| e.clone()) != etag;
5112                let remember = |journal: &mut BTreeMap<String, (String, u64, Option<SystemTime>)>| {
5113                    let (size, at) = Self::stamp(&here).unwrap_or((0, None));
5114                    journal.insert(relative.clone(), (fake::etag_of(&there), size, at));
5115                };
5116                match (stamp.is_some(), etag.is_some()) {
5117                    (true, true) if !local_changed && !remote_changed => {}
5118                    (true, true) if !remote_changed => {
5119                        sync_up(&here, &there);
5120                        self.uploads += 1;
5121                        remember(&mut self.journal);
5122                    }
5123                    (true, true) if !local_changed => {
5124                        sync_down(&there, &here);
5125                        remember(&mut self.journal);
5126                    }
5127                    (true, true) => {
5128                        if std::fs::read(&here).ok() != std::fs::read(&there).ok() {
5129                            let name = here.file_name().unwrap().to_string_lossy().to_string();
5130                            let (stem, extension) = name.rsplit_once('.').unwrap_or((&name, ""));
5131                            std::fs::rename(&here, here.with_file_name(format!("{stem} (conflicted copy 20261008 104200).{extension}"))).unwrap();
5132                            sync_down(&there, &here);
5133                            self.conflicts += 1;
5134                        }
5135                        remember(&mut self.journal);
5136                    }
5137                    (true, false) if known.is_some() && !local_changed => {
5138                        let _ = std::fs::remove_file(&here);
5139                        self.journal.remove(&relative);
5140                    }
5141                    (true, false) => {
5142                        sync_up(&here, &there);
5143                        self.uploads += 1;
5144                        remember(&mut self.journal);
5145                    }
5146                    (false, true) if known.is_some() && !remote_changed => {
5147                        let _ = std::fs::remove_file(&there);
5148                        self.journal.remove(&relative);
5149                    }
5150                    (false, true) => {
5151                        sync_down(&there, &here);
5152                        remember(&mut self.journal);
5153                    }
5154                    (false, false) => {
5155                        self.journal.remove(&relative);
5156                    }
5157                }
5158            }
5159        }
5160    }
5161
5162    /// The sizes of a device's records on the server, which must only grow.
5163    fn records_there(server: &Path, id: &str) -> BTreeMap<String, usize> {
5164        walk(server).into_iter().filter(|(r, _)| r.starts_with(id) && r.ends_with(".jsonl")).map(|(r, b)| (r, b.len())).collect()
5165    }
5166
5167    fn never_shorter(before: &mut BTreeMap<String, usize>, now: BTreeMap<String, usize>, at: &str) {
5168        for (name, size) in &now {
5169            assert!(before.get(name).is_none_or(|was| size >= was), "{at}: {name} shorter there: {size} after {:?}", before.get(name));
5170        }
5171        before.extend(now);
5172    }
5173
5174    /// The phone's own files sent beside its sync app.
5175    fn phone_sends(w: &World, now: i64, again: bool) -> Pulled {
5176        send_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), &BTreeMap::new(), now, again, TEST)
5177    }
5178
5179    /// This device's own files go up at once, beside the sync app; the sync
5180    /// app sending the same file after is a no-op here; a file it sent before
5181    /// Sioul is taken as sent (`412`, then the same there); records never
5182    /// sent shorter; switched off, not a request; never another device's file.
5183    #[test]
5184    fn beside_a_sync_app_only_this_devices_files_go_up_and_never_twice() {
5185        let w = World::new("beside");
5186        assert!(w.find(DUE).confirmed_for(&w.phone_folder));
5187        // A dose answered on the phone: its records and entry on the server within the send.
5188        w.phone.write("health-state.toml", "[taken]\n\"levo@1800000000\" = 1800000040\n");
5189        w.phone.exchange(&w.phone_folder, DUE + 40);
5190        w.fake.forget_seen();
5191        let sent = phone_sends(&w, DUE + 41, false);
5192        assert!(sent.problem.is_none() && sent.sent >= 2, "{sent:?}");
5193        let mine = |line: &String| line.split(' ').nth(1).is_some_and(|path| path.contains(&w.phone.id));
5194        for line in w.fake.seen() {
5195            if line.starts_with("PUT ") || line.starts_with("DELETE ") {
5196                assert!(mine(&line), "never another device's file: {line}");
5197            }
5198        }
5199        for (relative, bytes) in walk(&w.phone_folder).into_iter().filter(|(r, _)| r.contains(&w.phone.id)) {
5200            assert_eq!(std::fs::read(w.server.join(&relative)).ok(), Some(bytes), "{relative} there as here");
5201        }
5202        let kept = std::fs::read_to_string(sending_path(&w.phone.memory)).unwrap();
5203        assert!(!kept.contains(PASSWORD) && !kept.contains("Basic"), "{kept}");
5204        w.desk.exchange(&w.server, DUE + 45);
5205        assert!(w.desk.read("health-state.toml").contains("levo@1800000000"), "the desk has it, no sync app between");
5206        // The sync app sends them again, the same: nothing more from Sioul, not a request.
5207        w.up();
5208        for relative in walk(&w.phone_folder).into_keys().filter(|r| r.contains(&w.phone.id)) {
5209            fake::bump(&w.server.join(relative));
5210        }
5211        w.fake.forget_seen();
5212        let again = phone_sends(&w, DUE + 50, false);
5213        assert_eq!((again.sent, &again.problem), (0, &None));
5214        assert!(w.fake.seen().is_empty(), "{:?}", w.fake.seen());
5215        // Another answer: the sync app sends it first; Sioul finds it there (412, the same) and sends nothing twice.
5216        w.phone.write("health-state.toml", "[taken]\n\"levo@1800000000\" = 1800000040\n\"iron@1800000000\" = 1800000060\n");
5217        w.phone.exchange(&w.phone_folder, DUE + 60);
5218        {
5219            let _turn = w.fake.turn.lock().unwrap();
5220            for relative in walk(&w.phone_folder).into_keys().filter(|r| r.contains(&w.phone.id)) {
5221                sync_up(&w.phone_folder.join(&relative), &w.server.join(&relative));
5222            }
5223        }
5224        w.fake.forget_seen();
5225        let raced = phone_sends(&w, DUE + 61, false);
5226        assert!(raced.problem.is_none() && raced.sent == 0 && raced.same >= 2, "{raced:?} {:?}", w.fake.seen());
5227        assert!(w.fake.seen().iter().all(|l| !(l.starts_with("PUT ") && !l.ends_with(" 412"))), "{:?}", w.fake.seen());
5228        // The phone's records put back shorter here (a sync app's older copy): never sent over the longer one there.
5229        let round = walk(&w.phone_folder).into_keys().find(|r| r.starts_with(&w.phone.id) && r.ends_with(".jsonl")).unwrap();
5230        let longer = std::fs::read(w.server.join(&round)).unwrap();
5231        let first = longer.iter().position(|b| *b == b'\n').unwrap() + 1;
5232        std::fs::write(w.phone_folder.join(&round), &longer[..first]).unwrap();
5233        w.fake.forget_seen();
5234        let cut = phone_sends(&w, DUE + 70, false);
5235        assert!(cut.problem.is_none(), "{cut:?}");
5236        assert_eq!(std::fs::read(w.server.join(&round)).unwrap(), longer, "never shorter there");
5237        assert!(w.fake.seen().iter().all(|l| !(l.starts_with("PUT ") && l.contains(&round))), "{:?}", w.fake.seen());
5238        // Listed (every six hours), found longer there; then a line appended here, still shorter: never sent either.
5239        assert_eq!(phone_sends(&w, DUE + 72, true).longer, 1);
5240        let mut appended = longer[..first].to_vec();
5241        appended.extend_from_slice(&longer[first..first + (longer.len() - first) / 2]);
5242        appended.push(b'\n');
5243        std::fs::write(w.phone_folder.join(&round), &appended).unwrap();
5244        w.fake.forget_seen();
5245        assert!(phone_sends(&w, DUE + 75, false).problem.is_none());
5246        assert_eq!(std::fs::read(w.server.join(&round)).unwrap(), longer, "an append here, still shorter, never sent over the longer copy");
5247        assert!(w.fake.seen().iter().all(|l| !(l.starts_with("PUT ") && l.contains(&round))), "{:?}", w.fake.seen());
5248        // Switched off: not a request; the backup's fetching goes on.
5249        State::choose(&w.phone.memory, |s| s.send = Some(false)).unwrap();
5250        w.fake.forget_seen();
5251        assert_eq!(phone_sends(&w, DUE + 80, true).problem.as_deref(), Some("off"));
5252        assert!(w.fake.seen().is_empty());
5253        assert!(State::load(&w.phone.memory).fetching());
5254    }
5255
5256    /// Sealed files: only those this device sealed, whole, and taken out there once swept.
5257    #[test]
5258    fn sealed_files_go_up_whole_and_only_this_devices() {
5259        let w = World::new("beside-blobs");
5260        assert!(w.find(DUE).confirmed_for(&w.phone_folder));
5261        let source = w.base.join("note.md");
5262        std::fs::write(&source, "a note\n".repeat(500)).unwrap();
5263        let (hash, _) = crate::blobs::hash_file(&source).unwrap();
5264        crate::blobs::put(&w.phone_folder, &KEY, &source, &hash).unwrap();
5265        let name = crate::blobs::name(&KEY, &hash);
5266        let size = std::fs::metadata(w.phone_folder.join("blobs").join(&name)).unwrap().len();
5267        // Another device's sealed file, brought down by the sync app: never sent.
5268        put(&w.phone_folder, "blobs/0123456789abcdef", "another device's");
5269        // Not whole yet (the size sealed differs): not sent.
5270        let half: BTreeMap<String, u64> = [(name.clone(), size + 1)].into();
5271        send_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), &half, DUE + 10, false, TEST);
5272        assert!(!w.server.join("blobs").join(&name).exists());
5273        let whole: BTreeMap<String, u64> = [(name.clone(), size)].into();
5274        let sent = send_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), &whole, DUE + 20, false, TEST);
5275        assert!(sent.problem.is_none(), "{sent:?}");
5276        assert_eq!(std::fs::read(w.server.join("blobs").join(&name)).unwrap(), std::fs::read(w.phone_folder.join("blobs").join(&name)).unwrap());
5277        assert!(!w.server.join("blobs/0123456789abcdef").exists(), "never another device's");
5278        // Swept here: taken out there.
5279        std::fs::remove_file(w.phone_folder.join("blobs").join(&name)).unwrap();
5280        send_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), &BTreeMap::new(), DUE + 30, false, TEST);
5281        assert!(!w.server.join("blobs").join(&name).exists());
5282    }
5283
5284    /// "Send everything again" after the server lost this device's files:
5285    /// each one goes back, said; a second time, nothing goes, all found there.
5286    #[test]
5287    fn send_everything_again_after_the_server_lost_this_devices_files() {
5288        let w = World::new("again");
5289        assert!(w.find(DUE).confirmed_for(&w.phone_folder));
5290        w.phone.write("health-state.toml", "[taken]\n\"levo@1800000000\" = 1800000040\n");
5291        w.phone.exchange(&w.phone_folder, DUE + 40);
5292        assert!(phone_sends(&w, DUE + 41, false).problem.is_none());
5293        let own: Vec<String> = walk(&w.phone_folder).into_keys().filter(|r| r.contains(&w.phone.id)).collect();
5294        assert!(own.len() >= 2, "{own:?}");
5295        // The server loses them (restored from a backup, a folder emptied by hand).
5296        for relative in &own {
5297            std::fs::remove_file(w.server.join(relative)).unwrap();
5298        }
5299        assert!(crate::devices::all(&w.server, &KEY).0.iter().all(|e| e.id != w.phone.id));
5300        // Nothing changed here: a quick send asks nothing, sends nothing.
5301        w.fake.forget_seen();
5302        assert_eq!(phone_sends(&w, DUE + 50, false).sent, 0);
5303        assert!(w.fake.seen().is_empty());
5304        // Sent again, whatever was sent before.
5305        let again = phone_sends(&w, DUE + 60, true);
5306        assert!(again.problem.is_none() && again.sent == own.len(), "{again:?}");
5307        for relative in &own {
5308            assert_eq!(std::fs::read(w.server.join(relative)).ok(), std::fs::read(w.phone_folder.join(relative)).ok(), "{relative}");
5309        }
5310        assert_eq!(Sending::load(&w.phone.memory).again, Again { at: DUE + 60, sent: own.len(), same: 0, said: String::new() });
5311        w.desk.exchange(&w.server, DUE + 70);
5312        assert!(w.desk.read("health-state.toml").contains("levo@1800000000"));
5313        assert!(crate::devices::all(&w.server, &KEY).0.iter().any(|e| e.id == w.phone.id), "its entry back");
5314        // Again: all there, nothing sent.
5315        w.fake.forget_seen();
5316        let twice = phone_sends(&w, DUE + 80, true);
5317        assert!(twice.problem.is_none() && twice.sent == 0 && twice.same == own.len(), "{twice:?}");
5318        assert!(w.fake.seen().iter().all(|l| !l.starts_with("PUT ")), "{:?}", w.fake.seen());
5319        // The server unreachable: said, nothing lost here.
5320        w.fake.fault(|_| Some(Fault::Status(503)));
5321        let failed = phone_sends(&w, DUE + 90, true);
5322        assert!(failed.problem.as_deref().is_some_and(|p| p.starts_with("server:")), "{failed:?}");
5323        assert!(Sending::load(&w.phone.memory).again.said.starts_with("server:"));
5324    }
5325
5326    /// The doses' rule (docs/health.md, "Knowing") with the phone's own files
5327    /// sent by Sioul right after each exchange and by eDrive too, late, and
5328    /// now and then in the moment between a write and Sioul's send (eDrive
5329    /// bringing the older copy down over the phone's newer one, or refusing
5330    /// its own upload): what the desk takes as known is there, every dose
5331    /// comes, the phone's records there never shorter, no conflicted copy.
5332    #[test]
5333    fn doses_are_never_known_wrongly_with_edrive_sending_too() {
5334        let w = World::new("edrive-sends");
5335        assert!(w.find(DUE).confirmed_for(&w.phone_folder));
5336        let mut edrive = EDrive::default();
5337        edrive.scan(&w.fake, &w.server, &w.phone_folder);
5338        let mut marked: Vec<(String, i64)> = Vec::new();
5339        let mut sizes = BTreeMap::new();
5340        for minute in 0..150 {
5341            let now = DUE + minute * 60;
5342            // Every ten minutes; and twice close together, the second where eDrive looks before Sioul sends.
5343            if (minute % 10 == 0 || [17, 19, 77, 79].contains(&minute)) && minute < 130 {
5344                marked.push((format!("dose@{minute}"), now));
5345                let record: String = marked.iter().map(|(d, at)| format!("\"{d}\" = {at}\n")).collect();
5346                w.phone.write("health-state.toml", &format!("[taken]\n{record}"));
5347            }
5348            w.phone.exchange(&w.phone_folder, now);
5349            let round = walk(&w.phone_folder).into_keys().filter(|r| r.starts_with(&w.phone.id) && r.ends_with(".jsonl")).max_by_key(|r| r.trim_end_matches(".jsonl").rsplit('-').next().unwrap().parse::<u32>().unwrap()).unwrap();
5350            match minute % 6 {
5351                // Between the write and Sioul's send: eDrive looks (asked by another app, its half hour come).
5352                1 => edrive.scan(&w.fake, &w.server, &w.phone_folder),
5353                // Its upload of the appended file at once (CLOSE_WRITE), before Sioul's.
5354                3 => edrive.closed(&w.fake, &w.server, &w.phone_folder, &round),
5355                _ => {}
5356            }
5357            let sent = phone_sends(&w, now + 1, false);
5358            assert!(sent.problem.is_none(), "minute {minute}: {sent:?}");
5359            // Then eDrive, asked to look after Sioul's send, or not at all for a while.
5360            if minute % 4 == 0 {
5361                edrive.scan(&w.fake, &w.server, &w.phone_folder);
5362            }
5363            never_shorter(&mut sizes, records_there(&w.server, &w.phone.id), &format!("minute {minute}"));
5364            let outcome = w.desk.exchange(&w.server, now + 20);
5365            assert!(outcome.problems.iter().all(|p| !p.starts_with("share-other-cut") && !p.starts_with("share-other-gap")), "minute {minute}: {:?}", outcome.problems);
5366            if let Some(entry) = crate::devices::all(&w.server, &KEY).0.into_iter().find(|e| e.id == w.phone.id)
5367                && let Some(wrote) = entry.wrote
5368                && crate::share::heard(&w.desk.memory, &w.desk.id).complete(&w.phone.id, wrote)
5369            {
5370                let record = w.desk.read("health-state.toml");
5371                for (dose, at) in marked.iter().filter(|(_, at)| *at <= entry.exported) {
5372                    assert!(record.contains(dose.as_str()), "minute {minute}: the desk takes {dose} ({at}) as known, its record lacks it:\n{record}");
5373                }
5374            }
5375        }
5376        let record = w.desk.read("health-state.toml");
5377        for (dose, _) in &marked {
5378            assert!(record.contains(dose.as_str()), "{dose} never came:\n{record}");
5379        }
5380        assert!(walk(&w.server).keys().all(|r| !r.contains("conflicted")));
5381        eprintln!("eDrive beside Sioul: {} uploads, {} refused, {} brought down over a newer copy", edrive.uploads, edrive.refusals, edrive.downloads_over_newer);
5382        assert!(edrive.downloads_over_newer > 0 && edrive.refusals > 0, "the races were run");
5383    }
5384
5385    /// The same with the desk's folder carried by the Nextcloud client and
5386    /// the desk sending its own files too: the client looking between a write
5387    /// and Sioul's send makes a conflicted copy here and puts the older copy
5388    /// back; the desk starts a new round with all it holds; the phone never
5389    /// takes a dose as known wrongly, never reads a copy, gets every dose.
5390    #[test]
5391    fn doses_are_never_known_wrongly_with_the_nextcloud_client_sending_too() {
5392        let w = World::new("client-sends");
5393        let desk_folder = w.base.join("desk-folder").join("Sioul");
5394        copy_dir(&w.server, &desk_folder);
5395        State::choose(&w.desk.memory, |s| s.given = "Documents/Sioul".into()).unwrap();
5396        assert!(find_with(&w.desk.memory, &desk_folder, &[w.fake.login()], &[], DUE, TEST).confirmed_for(&desk_folder));
5397        let mut client = Client::default();
5398        client.sync(&w.fake, &w.server, &desk_folder);
5399        let phone_down = || {
5400            let _turn = w.fake.turn.lock().unwrap();
5401            for (relative, bytes) in walk(&w.server).into_iter().filter(|(r, _)| !r.contains(&w.phone.id)) {
5402                put_bytes(&w.phone_folder, &relative, &bytes);
5403            }
5404        };
5405        let phone_up = || {
5406            let _turn = w.fake.turn.lock().unwrap();
5407            for relative in walk(&w.phone_folder).into_keys().filter(|r| r.contains(&w.phone.id)) {
5408                sync_up(&w.phone_folder.join(&relative), &w.server.join(&relative));
5409            }
5410        };
5411        let mut marked: Vec<(String, i64)> = Vec::new();
5412        let mut sizes = BTreeMap::new();
5413        let mut cuts = 0;
5414        for minute in 0..150 {
5415            let now = DUE + minute * 60;
5416            // Every ten minutes; and right after, where the client looks before Sioul sends.
5417            if (minute % 10 == 0 || [21, 71].contains(&minute)) && minute < 130 {
5418                marked.push((format!("dose@{minute}"), now));
5419                let record: String = marked.iter().map(|(d, at)| format!("\"{d}\" = {at}\n")).collect();
5420                w.desk.write("health-state.toml", &format!("[taken]\n{record}"));
5421            }
5422            let outcome = w.desk.exchange(&desk_folder, now);
5423            cuts += outcome.problems.iter().filter(|p| *p == "share-own-cut").count();
5424            // Now and then the client looks before Sioul's send.
5425            if minute % 5 == 1 {
5426                client.sync(&w.fake, &w.server, &desk_folder);
5427            }
5428            let sent = send_with(&w.desk.memory, &desk_folder, &w.desk.id, &w.fake.login(), &BTreeMap::new(), now + 1, false, TEST);
5429            assert!(sent.problem.is_none(), "minute {minute}: {sent:?}");
5430            if minute % 3 == 0 {
5431                client.sync(&w.fake, &w.server, &desk_folder);
5432            }
5433            never_shorter(&mut sizes, records_there(&w.server, &w.desk.id), &format!("minute {minute}"));
5434            phone_down();
5435            let read = w.phone.exchange(&w.phone_folder, now + 30);
5436            assert!(read.problems.iter().all(|p| !p.starts_with("share-other-cut") && !p.starts_with("share-other-gap")), "minute {minute}: {:?}", read.problems);
5437            phone_up();
5438            if let Some(entry) = crate::devices::all(&w.phone_folder, &KEY).0.into_iter().find(|e| e.id == w.desk.id)
5439                && let Some(wrote) = entry.wrote
5440                && crate::share::heard(&w.phone.memory, &w.phone.id).complete(&w.desk.id, wrote)
5441            {
5442                let record = w.phone.read("health-state.toml");
5443                for (dose, at) in marked.iter().filter(|(_, at)| *at <= entry.exported) {
5444                    assert!(record.contains(dose.as_str()), "minute {minute}: the phone takes {dose} ({at}) as known, its record lacks it:\n{record}");
5445                }
5446            }
5447        }
5448        let record = w.phone.read("health-state.toml");
5449        for (dose, _) in &marked {
5450            assert!(record.contains(dose.as_str()), "{dose} never came:\n{record}");
5451        }
5452        assert!(walk(&w.server).keys().all(|r| !r.contains("conflicted")), "a conflicted copy never goes up");
5453        eprintln!("Nextcloud client beside Sioul: {} uploads, {} conflicted copies here, {} rounds started again", client.uploads, client.conflicts, cuts);
5454        assert!(client.conflicts > 0 && cuts > 0, "the races were run");
5455    }
5456
5457    // ------------------------------------------------ a slow uplink, a text to send, the cost
5458
5459    /// The desk beside its own sync app, sending its files: its folder apart
5460    /// from the server's, found there under the same seal.
5461    fn desk_beside(w: &World) -> PathBuf {
5462        let desk_folder = w.base.join("desk-folder").join("Sioul");
5463        copy_dir(&w.server, &desk_folder);
5464        State::choose(&w.desk.memory, |s| s.given = "Documents/Sioul".into()).unwrap();
5465        assert!(find_with(&w.desk.memory, &desk_folder, &[w.fake.login()], &[], DUE, TEST).confirmed_for(&desk_folder));
5466        desk_folder
5467    }
5468
5469    /// A desk holding much (its records' opening far past a megabyte).
5470    fn a_desk_holding_much(w: &World, folder: &Path, at: i64) {
5471        let many: String = (0..8000).map(|n| format!("sender{n}@example.org\n")).collect();
5472        put(&w.desk.roots.config, "known-senders.txt", &many);
5473        w.desk.exchange(folder, at);
5474    }
5475
5476    fn desk_records(folder: &Path, id: &str) -> Vec<(String, u64)> {
5477        let mut out: Vec<(String, u64)> = walk(folder).into_iter().filter(|(r, _)| r.starts_with(id) && r.ends_with(".jsonl")).map(|(r, b)| (r, b.len() as u64)).collect();
5478        out.sort_by_key(|(r, _)| r.trim_end_matches(".jsonl").rsplit('-').next().unwrap().parse::<u32>().unwrap());
5479        out
5480    }
5481
5482    /// The owner's desk, 8 October 2026: a records file of 2.5 MB on a
5483    /// phone's hotspot (3G) never went, cut at a flat minute and tried again
5484    /// from zero every minute. Now a file is given the time its size asks
5485    /// (`FLOOR`), still going is not cut; a file too slow even so is said in
5486    /// detail and held back, never tried before its backoff.
5487    #[test]
5488    fn a_large_file_goes_through_a_slow_uplink() {
5489        let w = World::new("slow-uplink");
5490        let desk_folder = desk_beside(&w);
5491        a_desk_holding_much(&w, &desk_folder, DUE);
5492        // The first large one in order: records go in order, the next ones wait behind it.
5493        let (round, size) = desk_records(&desk_folder, &w.desk.id).into_iter().find(|(_, s)| *s > 1 << 20).unwrap();
5494        assert!(size > 1 << 20, "a large records file: {size}");
5495        // An uplink of 512 KiB a second; a flat second for a file, as a flat minute was on 3G.
5496        *w.fake.throttle.lock().unwrap() = Some(512 << 10);
5497        let flat = Limits { large: Duration::from_secs(1), floor: 1 << 30, ..TEST };
5498        let tried = send_with(&w.desk.memory, &desk_folder, &w.desk.id, &w.fake.login(), &BTreeMap::new(), DUE + 10, false, flat);
5499        let failure = tried.failure.clone().expect("said in detail");
5500        assert!(failure.kind == "timeout" && failure.file == "records" && failure.sent > 0 && failure.total == size && failure.seconds >= 1, "{failure:?}");
5501        assert!(tried.held == 1, "{tried:?}");
5502        let sending = Sending::load(&w.desk.memory);
5503        assert_eq!(sending.failure.as_ref().map(|f| f.kind.as_str()), Some("timeout"));
5504        assert!(sending.held.get(&round).is_some_and(|h| h.tries == 1 && h.until == DUE + 10 + backoff(1)), "{:?}", sending.held);
5505        assert!(sending.rate > 0, "its rate measured from what went");
5506        // Before its backoff: not tried again, not a byte spent.
5507        let spent = w.fake.received.load(std::sync::atomic::Ordering::Relaxed);
5508        let early = send_with(&w.desk.memory, &desk_folder, &w.desk.id, &w.fake.login(), &BTreeMap::new(), DUE + 40, false, flat);
5509        assert!(early.held == 0 && w.fake.received.load(std::sync::atomic::Ordering::Relaxed) - spent < size / 2, "{early:?}");
5510        assert!(std::fs::metadata(w.server.join(&round)).map_or(0, |m| m.len()) < size);
5511        // Given the time its size asks (a floor of 128 KiB a second here): it goes, slowly, whole.
5512        // A stall of seconds: a busy machine (macOS on CI) may pause the fake server longer than TEST's.
5513        let scaled = Limits { large: Duration::from_secs(1), floor: 128 << 10, stall: Duration::from_secs(3), ..TEST };
5514        let went = send_with(&w.desk.memory, &desk_folder, &w.desk.id, &w.fake.login(), &BTreeMap::new(), DUE + 10 + backoff(1), false, scaled);
5515        assert!(went.problem.is_none() && went.held == 0, "{went:?}");
5516        assert_eq!(std::fs::read(w.server.join(&round)).unwrap(), std::fs::read(desk_folder.join(&round)).unwrap());
5517        let sending = Sending::load(&w.desk.memory);
5518        assert!(sending.held.is_empty() && sending.failure.is_none() && sending.last == DUE + 10 + backoff(1), "{sending:?}");
5519    }
5520
5521    /// The server unreachable, or refusing: nothing tried again before the backoff.
5522    #[test]
5523    fn a_failed_send_waits_for_its_backoff() {
5524        let w = World::new("backoff");
5525        let desk_folder = desk_beside(&w);
5526        w.desk.write("health-state.toml", "[taken]\n\"levo@1800000000\" = 1800000040\n");
5527        w.desk.exchange(&desk_folder, DUE + 40);
5528        w.fake.fault(|r| (r.method == "PUT").then_some(Fault::Status(503)));
5529        let failed = send_with(&w.desk.memory, &desk_folder, &w.desk.id, &w.fake.login(), &BTreeMap::new(), DUE + 41, false, TEST);
5530        assert!(failed.problem.as_deref().is_some_and(|p| p.starts_with("server:")), "{failed:?}");
5531        assert_eq!(Sending::load(&w.desk.memory).retry_at, DUE + 41 + backoff(1));
5532        w.fake.forget_seen();
5533        let early = send_with(&w.desk.memory, &desk_folder, &w.desk.id, &w.fake.login(), &BTreeMap::new(), DUE + 41 + backoff(1) - 1, false, TEST);
5534        assert!(early.problem.as_deref().is_some_and(|p| p.starts_with("waiting:")) && w.fake.seen().is_empty(), "{early:?} {:?}", w.fake.seen());
5535        // Failing again: two minutes, then five.
5536        let again = send_with(&w.desk.memory, &desk_folder, &w.desk.id, &w.fake.login(), &BTreeMap::new(), DUE + 41 + backoff(1), false, TEST);
5537        assert!(again.problem.is_some());
5538        assert_eq!(Sending::load(&w.desk.memory).retry_at, DUE + 41 + backoff(1) + backoff(2));
5539        assert_eq!((backoff(1), backoff(2), backoff(3), backoff(9)), (60, 120, 300, 3600));
5540        // The server back: it goes, and the count starts again.
5541        w.fake.clear_faults();
5542        let at = DUE + 41 + backoff(1) + backoff(2);
5543        assert!(send_with(&w.desk.memory, &desk_folder, &w.desk.id, &w.fake.login(), &BTreeMap::new(), at, false, TEST).problem.is_none());
5544        let sending = Sending::load(&w.desk.memory);
5545        assert!(sending.tries == 0 && sending.retry_at == 0 && sending.failure.is_none());
5546    }
5547
5548    /// A text written on the desk reaches the phone within a minute while a
5549    /// large records file is still crawling up: its small file of its own
5550    /// goes first and alone (`send_urgent`), the phone's pull brings it.
5551    #[test]
5552    fn a_text_to_send_reaches_the_phone_while_the_records_crawl() {
5553        let w = World::new("texts-lane");
5554        let desk_folder = desk_beside(&w);
5555        assert!(w.find(DUE).confirmed_for(&w.phone_folder));
5556        a_desk_holding_much(&w, &desk_folder, DUE);
5557        // An uplink of 256 KiB a second: the records take seconds to go, given the time.
5558        *w.fake.throttle.lock().unwrap() = Some(256 << 10);
5559        let slow = Limits { large: Duration::from_secs(2), floor: 64 << 10, stall: Duration::from_secs(5), pull: Duration::from_secs(120), ..TEST };
5560        let crawling = {
5561            let (memory, folder, id, login) = (w.desk.memory.clone(), desk_folder.clone(), w.desk.id.clone(), w.fake.login());
5562            std::thread::spawn(move || send_with(&memory, &folder, &id, &login, &BTreeMap::new(), DUE + 10, false, slow))
5563        };
5564        std::thread::sleep(Duration::from_millis(500));
5565        // A text written now: its lines sealed already, in the desk's own file of the part "texts".
5566        let source = w.base.join("desk-texts-send.jsonl");
5567        std::fs::write(&source, "a sealed request line\n").unwrap();
5568        let written = Instant::now();
5569        assert!(place_texts(&desk_folder, &w.desk.id, &source).unwrap());
5570        let urgent = send_urgent_with(&w.desk.memory, &desk_folder, &w.desk.id, &w.fake.login(), DUE + 11, slow);
5571        assert!(urgent.problem.is_none() && urgent.sent == 1, "{urgent:?}");
5572        // The phone's next pull brings it; the records still on their way.
5573        let pulled = pull_with(&w.phone.memory, &w.phone_folder, &w.phone.id, &w.fake.login(), DUE + 20, TEST);
5574        assert!(pulled.problem.is_none(), "{pulled:?}");
5575        assert_eq!(std::fs::read(texts_file(&cache_of(&w.phone.memory), &w.desk.id)).unwrap(), b"a sealed request line\n");
5576        assert!(!crawling.is_finished(), "the records still going up");
5577        assert!(written.elapsed() < Duration::from_secs(10), "{:?}", written.elapsed());
5578        let crawled = crawling.join().unwrap();
5579        assert!(crawled.problem.is_none(), "{crawled:?}");
5580        // Never sent twice by the main lane; a change goes again; none here, none there.
5581        assert!(!own_plain(&desk_folder, &w.desk.id).iter().any(|r| r.starts_with("texts/")));
5582        assert!(!place_texts(&desk_folder, &w.desk.id, &source).unwrap(), "unchanged: nothing to place");
5583        std::fs::remove_file(&source).unwrap();
5584        assert!(place_texts(&desk_folder, &w.desk.id, &source).unwrap() && !texts_file(&desk_folder, &w.desk.id).exists());
5585    }
5586
5587    /// What a computer writing each minute costs, sent beside its sync app:
5588    /// a change a minute for an hour, its entry and its claim renewed each
5589    /// minute. Its records' file stays small (`share::SEGMENT`): measured here.
5590    #[test]
5591    fn bytes_an_hour_of_a_computer_writing_each_minute() {
5592        let w = World::new("an-hour");
5593        let desk_folder = desk_beside(&w);
5594        a_desk_holding_much(&w, &desk_folder, DUE);
5595        assert!(send_with(&w.desk.memory, &desk_folder, &w.desk.id, &w.fake.login(), &BTreeMap::new(), DUE + 1, false, TEST).problem.is_none());
5596        let start = w.fake.received.load(std::sync::atomic::Ordering::Relaxed);
5597        let mut minutes: String = String::new();
5598        for minute in 1..=60 {
5599            let now = DUE + minute * 60;
5600            minutes.push_str(&format!("\"dose@{minute}\" = {now}\n"));
5601            w.desk.write("health-state.toml", &format!("[taken]\n{minutes}"));
5602            w.desk.exchange(&desk_folder, now);
5603            crate::lease::renew(&desk_folder, &KEY, "health", &w.desk.id, now, now, false, crate::lease::Rule::FollowsYou, crate::share::written(&w.desk.memory, &w.desk.id)).unwrap();
5604            let sent = send_with(&w.desk.memory, &desk_folder, &w.desk.id, &w.fake.login(), &BTreeMap::new(), now + 1, false, TEST);
5605            assert!(sent.problem.is_none(), "minute {minute}: {sent:?}");
5606        }
5607        let hour = w.fake.received.load(std::sync::atomic::Ordering::Relaxed) - start;
5608        let largest = desk_records(&desk_folder, &w.desk.id).last().map_or(0, |(_, s)| *s);
5609        eprintln!("A computer writing each minute, sent beside its sync app: {hour} bytes in an hour; its current records' file {largest} bytes");
5610        assert!(hour < 3 << 20, "{hour} bytes an hour");
5611        assert!(largest < 64 << 10, "{largest}");
5612        // Every dose there for the others.
5613        w.phone.exchange(&w.server, DUE + 3_700);
5614        assert!(w.phone.read("health-state.toml").contains("dose@60"));
5615    }
5616
5617    // ------------------------------------------------ sealed files waited for
5618
5619    impl Device {
5620        /// An exchange of its notes too (`files`), kept in `notes`.
5621        fn exchange_notes(&self, folder: &Path, notes: &Path, now: i64) -> Outcome {
5622            let config = Config { notes_root: Some(notes.display().to_string()), ..Config::default() };
5623            let stores = crate::share::stores_of(&config, &self.roots, &|part| part == "notes" || crate::share::shared_by_default(part, &config));
5624            crate::share::exchange(&Sharing { folder, computer: &self.id, key: &KEY, memory: &self.memory, files: true, hurry: None }, &stores, now * 1000).unwrap()
5625        }
5626    }
5627
5628    /// The desk and the phone sharing their notes, the backup found for the
5629    /// phone; the phone's sync app brings nothing down from now on (eDrive's
5630    /// half-hour, 10 October 2026). Their notes' folders.
5631    fn notes_world(name: &str) -> (World, PathBuf, PathBuf) {
5632        let w = World::new(name);
5633        let (desk_notes, phone_notes) = (w.base.join("desk-notes"), w.base.join("phone-notes"));
5634        for dir in [&desk_notes, &phone_notes] {
5635            std::fs::create_dir_all(dir).unwrap();
5636        }
5637        w.desk.exchange_notes(&w.server, &desk_notes, DUE + 10);
5638        w.phone.exchange_notes(&w.phone_folder, &phone_notes, DUE + 20);
5639        w.up();
5640        assert!(w.find(DUE + 30).confirmed_for(&w.phone_folder));
5641        (w, desk_notes, phone_notes)
5642    }
5643
5644    /// A note written on the desk, its client carrying it to the server at once: its sealed file's name there.
5645    fn desk_writes(w: &World, desk_notes: &Path, file: &str, text: &str, now: i64) -> String {
5646        put(desk_notes, file, text);
5647        w.desk.exchange_notes(&w.server, desk_notes, now);
5648        let name = crate::blobs::name(&KEY, &crate::blobs::hash_file(&desk_notes.join(file)).unwrap().0);
5649        assert!(w.server.join("blobs").join(&name).exists());
5650        name
5651    }
5652
5653    fn fetch(w: &World, now: i64, frugal: bool) -> Pulled {
5654        fetch_wanted_with(&w.phone.memory, &w.phone_folder, &w.fake.login(), &KEY, now, frugal, TEST)
5655    }
5656
5657    fn blob_gets(w: &World) -> usize {
5658        w.fake.seen().iter().filter(|l| l.starts_with("GET ") && l.contains("/blobs/")).count()
5659    }
5660
5661    /// A note written on the desk reaches a phone whose sync app brings no
5662    /// sealed file within one step: the pull brings the record, the exchange
5663    /// says the sealed file it waits for, the fetch brings that file alone,
5664    /// by its name, beside the synced folder, and the next exchange writes
5665    /// the note. Nothing listed in the server's `blobs/`, nothing sent; the
5666    /// copy fetched goes once the note is written.
5667    #[test]
5668    fn a_note_waited_for_is_fetched_by_its_name_and_written_within_one_step() {
5669        let (w, desk_notes, phone_notes) = notes_world("wanted");
5670        let name = desk_writes(&w, &desk_notes, "lease.md", "The lease, signed.\n", DUE + 60);
5671        w.fake.forget_seen();
5672        // The step: the pull, then the exchange.
5673        let pulled = w.pull(DUE + 70);
5674        assert!(pulled.problem.is_none(), "{pulled:?}");
5675        let first = w.phone.exchange_notes(&w.phone_folder, &phone_notes, DUE + 71);
5676        assert_eq!(first.waits.get("no-blob"), Some(&1), "{first:?}");
5677        assert_eq!((first.received, first.pending), (0, 1));
5678        assert!(!phone_notes.join("lease.md").exists());
5679        let wanted = crate::blobs::Wanted::load(&w.phone.memory);
5680        assert_eq!(wanted.blobs.keys().cloned().collect::<Vec<_>>(), [name.clone()]);
5681        assert!(wanted.blobs[&name].size == 19 && wanted.blobs[&name].since == DUE + 71, "{wanted:?}");
5682        assert!(wanted_due(&w.phone.memory, &w.phone_folder, DUE + 71));
5683        // Its sealed file fetched, then the exchange again.
5684        let fetched = fetch(&w, DUE + 72, false);
5685        assert_eq!((fetched.fetched, fetched.problem.as_deref()), (1, None), "{fetched:?}");
5686        assert!(w.cache().join("blobs").join(&name).exists());
5687        assert!(!w.phone_folder.join("blobs").join(&name).exists(), "never written into the synced folder");
5688        assert!(!wanted_due(&w.phone.memory, &w.phone_folder, DUE + 72), "fetched: no longer due");
5689        let second = w.phone.exchange_notes(&w.phone_folder, &phone_notes, DUE + 73);
5690        assert_eq!((second.received, second.pending), (1, 0), "{second:?}");
5691        assert!(second.waits.is_empty() && second.problems.is_empty(), "{second:?}");
5692        assert_eq!(std::fs::read_to_string(phone_notes.join("lease.md")).unwrap(), "The lease, signed.\n");
5693        assert!(!crate::blobs::Wanted::path(&w.phone.memory).exists(), "nothing waits");
5694        // By its name: one GET, no listing of `blobs/`, nothing sent.
5695        assert_eq!(blob_gets(&w), 1, "{:?}", w.fake.seen());
5696        for line in w.fake.seen() {
5697            assert!(!(line.starts_with("PROPFIND") && line.contains("/blobs")), "{line}");
5698            assert!(line.starts_with("PROPFIND ") || line.starts_with("GET "), "{line}");
5699        }
5700        // Nothing waits: no request.
5701        w.fake.forget_seen();
5702        assert_eq!(fetch(&w, DUE + 74, false), Pulled::default());
5703        assert!(w.fake.seen().is_empty());
5704        // Written: the copy fetched goes at the next tidying.
5705        tidy(&w.phone.memory, &w.phone_folder, &w.phone.id);
5706        assert!(!w.cache().join("blobs").join(&name).exists());
5707    }
5708
5709    /// A copy fetched that does not open whole is never read: one damaged on
5710    /// this phone is taken out and fetched again; one damaged on the server
5711    /// is never kept, and asked again after a while; whole again there, it
5712    /// comes, and the note is written.
5713    #[test]
5714    fn a_damaged_copy_fetched_is_refused_and_fetched_again() {
5715        let (w, desk_notes, phone_notes) = notes_world("wanted-damaged");
5716        let name = desk_writes(&w, &desk_notes, "lease.md", "The lease, signed.\n", DUE + 60);
5717        assert!(w.pull(DUE + 70).problem.is_none());
5718        assert_eq!(w.phone.exchange_notes(&w.phone_folder, &phone_notes, DUE + 71).waits.get("no-blob"), Some(&1));
5719        // Damaged here, after it came: refused, taken out, waited for again.
5720        let whole = std::fs::read(w.server.join("blobs").join(&name)).unwrap();
5721        let mut damaged = whole.clone();
5722        let middle = damaged.len() / 2;
5723        damaged[middle] ^= 0x10;
5724        put_bytes(&w.cache(), &format!("blobs/{name}"), &damaged);
5725        let outcome = w.phone.exchange_notes(&w.phone_folder, &phone_notes, DUE + 72);
5726        assert_eq!(outcome.waits.get("no-blob"), Some(&1), "{outcome:?}");
5727        assert!(outcome.problems.is_empty(), "the copy fetched is not the folder's: nothing said damaged, {outcome:?}");
5728        assert!(!w.cache().join("blobs").join(&name).exists());
5729        assert!(!phone_notes.join("lease.md").exists());
5730        // Damaged on the server: fetched, refused, never kept; not asked again at once.
5731        std::fs::write(w.server.join("blobs").join(&name), &damaged).unwrap();
5732        let fetched = fetch(&w, DUE + 73, false);
5733        assert_eq!((fetched.fetched, fetched.held), (0, 1), "{fetched:?}");
5734        assert!(!w.cache().join("blobs").join(&name).exists());
5735        assert_eq!(crate::blobs::Wanted::load(&w.phone.memory).blobs[&name].again, DUE + 73 + 60);
5736        w.fake.forget_seen();
5737        assert_eq!(fetch(&w, DUE + 100, false).fetched, 0);
5738        assert_eq!(blob_gets(&w), 0, "within its wait: not asked");
5739        // Whole there again: it comes, the note is written.
5740        std::fs::write(w.server.join("blobs").join(&name), &whole).unwrap();
5741        assert_eq!(fetch(&w, DUE + 140, false).fetched, 1);
5742        let outcome = w.phone.exchange_notes(&w.phone_folder, &phone_notes, DUE + 141);
5743        assert_eq!(outcome.received, 1, "{outcome:?}");
5744        assert_eq!(std::fs::read_to_string(phone_notes.join("lease.md")).unwrap(), "The lease, signed.\n");
5745    }
5746
5747    /// A sealed file the sync app is late with is not on the server yet
5748    /// either: asked again within minutes, never backed off for long; there,
5749    /// it comes at the next ask.
5750    #[test]
5751    fn a_sealed_file_not_on_the_server_yet_is_asked_again_within_minutes() {
5752        let (w, desk_notes, phone_notes) = notes_world("wanted-late");
5753        let name = desk_writes(&w, &desk_notes, "lease.md", "The lease, signed.\n", DUE + 60);
5754        let aside = w.base.join("aside-blob");
5755        std::fs::rename(w.server.join("blobs").join(&name), &aside).unwrap();
5756        assert!(w.pull(DUE + 70).problem.is_none());
5757        assert_eq!(w.phone.exchange_notes(&w.phone_folder, &phone_notes, DUE + 71).waits.get("no-blob"), Some(&1));
5758        let mut at = DUE + 72;
5759        for _ in 0..8 {
5760            let fetched = fetch(&w, at, false);
5761            assert!(fetched.fetched == 0 && fetched.problem.is_none(), "{fetched:?}");
5762            let again = crate::blobs::Wanted::load(&w.phone.memory).blobs[&name].again;
5763            assert!(again > at && again <= at + WANTED_AGAIN, "{again} after {at}");
5764            at = again;
5765        }
5766        std::fs::rename(&aside, w.server.join("blobs").join(&name)).unwrap();
5767        assert_eq!(fetch(&w, at, false).fetched, 1);
5768        assert_eq!(w.phone.exchange_notes(&w.phone_folder, &phone_notes, at + 1).received, 1);
5769    }
5770
5771    /// A copy fetched, then the same brought by the sync app into the synced
5772    /// folder: the folder's is read, identical, and the copy fetched goes,
5773    /// even while the note still waits.
5774    #[test]
5775    fn a_sealed_file_fetched_then_brought_by_the_sync_app_is_kept_once() {
5776        let (w, desk_notes, phone_notes) = notes_world("wanted-brought");
5777        let name = desk_writes(&w, &desk_notes, "lease.md", "The lease, signed.\n", DUE + 60);
5778        assert!(w.pull(DUE + 70).problem.is_none());
5779        assert_eq!(w.phone.exchange_notes(&w.phone_folder, &phone_notes, DUE + 71).waits.get("no-blob"), Some(&1));
5780        assert_eq!(fetch(&w, DUE + 72, false).fetched, 1);
5781        // The sync app catches up before the next exchange.
5782        w.down();
5783        let (fetched, brought) = (std::fs::read(w.cache().join("blobs").join(&name)).unwrap(), std::fs::read(w.phone_folder.join("blobs").join(&name)).unwrap());
5784        assert_eq!(fetched, brought, "the same file");
5785        tidy(&w.phone.memory, &w.phone_folder, &w.phone.id);
5786        assert!(!w.cache().join("blobs").join(&name).exists(), "the synced folder holds it: the copy fetched goes");
5787        assert!(crate::blobs::Wanted::path(&w.phone.memory).exists(), "the note still waits");
5788        let outcome = w.phone.exchange_notes(&w.phone_folder, &phone_notes, DUE + 73);
5789        assert_eq!(outcome.received, 1, "{outcome:?}");
5790        assert!(!crate::blobs::Wanted::path(&w.phone.memory).exists());
5791    }
5792
5793    /// What was fetched beside the folder never piles up: a copy no note
5794    /// waits for goes, one kept past a few days goes and is no longer waited
5795    /// for (an exchange that reads notes says it again if it still waits),
5796    /// all of them with the backup switched off.
5797    #[test]
5798    fn the_sealed_files_fetched_are_pruned() {
5799        let (w, desk_notes, phone_notes) = notes_world("wanted-pruned");
5800        let name = desk_writes(&w, &desk_notes, "lease.md", "The lease, signed.\n", DUE + 60);
5801        assert!(w.pull(DUE + 70).problem.is_none());
5802        assert_eq!(w.phone.exchange_notes(&w.phone_folder, &phone_notes, DUE + 71).waits.get("no-blob"), Some(&1));
5803        assert_eq!(fetch(&w, DUE + 72, false).fetched, 1);
5804        let blobs = w.cache().join("blobs");
5805        put(&blobs, "0123abcd", "a copy no note waits for");
5806        tidy(&w.phone.memory, &w.phone_folder, &w.phone.id);
5807        assert!(blobs.join(&name).exists(), "waited for: kept");
5808        assert!(!blobs.join("0123abcd").exists());
5809        set_mtime(&blobs.join(&name), SystemTime::now() - Duration::from_secs(4 * 86_400));
5810        tidy(&w.phone.memory, &w.phone_folder, &w.phone.id);
5811        assert!(!blobs.join(&name).exists(), "past a few days");
5812        assert!(!crate::blobs::Wanted::path(&w.phone.memory).exists(), "no longer waited for");
5813        w.fake.forget_seen();
5814        assert_eq!(fetch(&w, DUE + 80, false).fetched, 0);
5815        assert_eq!(blob_gets(&w), 0);
5816        // Still waiting, said again by the next exchange that reads notes: fetched again.
5817        assert_eq!(w.phone.exchange_notes(&w.phone_folder, &phone_notes, DUE + 81).waits.get("no-blob"), Some(&1));
5818        assert_eq!(fetch(&w, DUE + 82, false).fetched, 1);
5819        // The backup switched off: what was fetched goes.
5820        State::choose(&w.phone.memory, |s| s.on = Some(false)).unwrap();
5821        tidy(&w.phone.memory, &w.phone_folder, &w.phone.id);
5822        assert!(!blobs.join(&name).exists());
5823        assert_eq!(fetch(&w, DUE + 83, false).problem.as_deref(), Some("not-confirmed"));
5824    }
5825
5826    /// Waited for: a step brings at most `WANTED_FILES` and `WANTED_BYTES`,
5827    /// the smallest first; on a metered or slow connection a note's sealed
5828    /// file comes at once, a large paper's at most every ten minutes.
5829    #[test]
5830    fn sealed_files_waited_for_come_bounded_and_paced() {
5831        let (w, _, _) = notes_world("wanted-paced");
5832        let made = |seed: u32, size: usize| {
5833            let mut x = seed.wrapping_mul(2_654_435_761) | 1;
5834            (0..size)
5835                .map(|_| {
5836                    x ^= x << 13;
5837                    x ^= x >> 17;
5838                    x ^= x << 5;
5839                    x as u8
5840                })
5841                .collect::<Vec<u8>>()
5842        };
5843        // Sealed on the server by hand, said waited for as an exchange says it.
5844        let sealed = |seed: u32, size: usize| {
5845            let source = w.base.join(format!("source-{seed}"));
5846            std::fs::write(&source, made(seed, size)).unwrap();
5847            let (hash, size) = crate::blobs::hash_file(&source).unwrap();
5848            crate::blobs::put(&w.server, &KEY, &source, &hash).unwrap();
5849            let name = crate::blobs::name(&KEY, &hash);
5850            crate::blobs::Wanted::change(&w.phone.memory, |wanted| {
5851                wanted.blobs.insert(name.clone(), crate::blobs::Want { hash, size, since: DUE, ..Default::default() });
5852            })
5853            .unwrap();
5854            name
5855        };
5856        let papers = [sealed(1, 1_500_000), sealed(2, 1_200_000)];
5857        let note = sealed(3, 3_000);
5858        // Metered: the note and one paper; the other paper waits ten minutes.
5859        let first = fetch(&w, DUE + 100, true);
5860        assert_eq!((first.fetched, first.paced), (2, 1), "{first:?}");
5861        let blobs = w.cache().join("blobs");
5862        assert!(blobs.join(&note).exists() && blobs.join(&papers[1]).exists() && !blobs.join(&papers[0]).exists());
5863        w.fake.forget_seen();
5864        assert_eq!((fetch(&w, DUE + 400, true).fetched, blob_gets(&w)), (0, 0), "paced: not asked");
5865        assert_eq!(fetch(&w, DUE + 100 + PACE, true).fetched, 1);
5866        assert!(blobs.join(&papers[0]).exists());
5867        // Many notes at once: `WANTED_FILES` a step, the rest at the next.
5868        let notes: Vec<String> = (10..10 + WANTED_FILES as u32 + 2).map(|seed| sealed(seed, 500)).collect();
5869        assert_eq!(fetch(&w, DUE + 2_000, true).fetched, WANTED_FILES);
5870        assert_eq!(fetch(&w, DUE + 2_001, true).fetched, 2);
5871        assert!(notes.iter().all(|n| blobs.join(n).exists()));
5872        // `WANTED_BYTES` a step, the first whatever its size: on an unmetered connection, all at once otherwise.
5873        let large: Vec<String> = (40..43).map(|seed| sealed(seed, 7 << 20)).collect();
5874        assert_eq!(fetch(&w, DUE + 3_000, false).fetched, 2, "14 MiB, then the third past 16");
5875        assert_eq!(fetch(&w, DUE + 3_001, false).fetched, 1);
5876        assert!(large.iter().all(|n| blobs.join(n).exists()));
5877    }
5878}