1use chacha20poly1305::aead::{Aead, AeadCore, KeyInit, OsRng, Payload};
21use chacha20poly1305::{XChaCha20Poly1305, XNonce};
22use hmac::{Hmac, Mac};
23use sha2::{Digest, Sha256};
24use std::collections::{BTreeMap, BTreeSet};
25use std::io::{BufReader, BufWriter, Read, Write};
26use std::path::{Path, PathBuf};
27
28pub const LARGEST: u64 = 64 << 20;
30pub const UNUSED_DAYS: i64 = 90;
32const PIECE: usize = 1 << 20;
34const INFLATED: usize = 64 << 10;
36const MAGIC: &[u8] = b"sioul-blob 1\n";
38const END: u32 = 1 << 31;
40const AROUND: usize = 24 + 16;
42
43#[derive(Debug, PartialEq, Eq)]
45pub enum Fault {
46 Missing,
48 Broken,
50 Local(String),
52 Room,
54}
55
56fn keys(key: &[u8; 32]) -> ([u8; 32], [u8; 32]) {
59 let made = hkdf::Hkdf::<Sha256>::new(None, key);
60 let (mut sealing, mut naming) = ([0u8; 32], [0u8; 32]);
61 made.expand(b"sioul blobs 1: sealing", &mut sealing).expect("32 bytes is a length HKDF gives");
62 made.expand(b"sioul blobs 1: naming", &mut naming).expect("32 bytes is a length HKDF gives");
63 (sealing, naming)
64}
65
66pub fn name(key: &[u8; 32], hash: &str) -> String {
69 let mut mac = <Hmac<Sha256> as Mac>::new_from_slice(&keys(key).1).expect("HMAC takes any key");
70 mac.update(hash.as_bytes());
71 hex(&mac.finalize().into_bytes())
72}
73
74pub fn path(folder: &Path, key: &[u8; 32], hash: &str) -> PathBuf {
75 folder.join("blobs").join(name(key, hash))
76}
77
78fn hex(bytes: &[u8]) -> String {
79 bytes.iter().map(|b| format!("{b:02x}")).collect()
80}
81
82#[cfg(test)]
83thread_local! {
84 pub(crate) static HASHED: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
86}
87
88pub fn hash_file(path: &Path) -> std::io::Result<(String, u64)> {
90 #[cfg(test)]
91 HASHED.with(|n| n.set(n.get() + 1));
92 let mut reading = Hashing { inner: std::fs::File::open(path)?, hasher: Sha256::new(), size: 0 };
93 std::io::copy(&mut reading, &mut std::io::sink())?;
94 Ok((hex(&reading.hasher.finalize()), reading.size))
95}
96
97pub fn fingerprint(meta: &std::fs::Metadata) -> (u64, u64, u64) {
101 let modified = meta.modified().ok().and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()).map_or(0, |d| d.as_nanos() as u64);
102 #[cfg(unix)]
103 let other = {
104 use std::os::unix::fs::MetadataExt;
105 meta.ino().wrapping_mul(0x9e37_79b9_7f4a_7c15) ^ (meta.ctime() as u64).wrapping_mul(1_000_000_000).wrapping_add(meta.ctime_nsec() as u64)
106 };
107 #[cfg(not(unix))]
108 let other = meta.created().ok().and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()).map_or(0, |d| d.as_nanos() as u64);
109 (meta.len(), modified, other)
110}
111
112struct Hashing<T> {
114 inner: T,
115 hasher: Sha256,
116 size: u64,
117}
118
119impl<R: Read> Read for Hashing<R> {
120 fn read(&mut self, buffer: &mut [u8]) -> std::io::Result<usize> {
121 let n = self.inner.read(buffer)?;
122 self.hasher.update(&buffer[..n]);
123 self.size += n as u64;
124 Ok(n)
125 }
126}
127
128impl<W: Write> Write for Hashing<W> {
129 fn write(&mut self, buffer: &[u8]) -> std::io::Result<usize> {
130 let n = self.inner.write(buffer)?;
131 self.hasher.update(&buffer[..n]);
132 self.size += n as u64;
133 Ok(n)
134 }
135
136 fn flush(&mut self) -> std::io::Result<()> {
137 self.inner.flush()
138 }
139}
140
141fn bound(name: &str, index: u64, end: bool) -> String {
143 format!("blob\u{1f}{name}\u{1f}{index}{}", if end { "\u{1f}end" } else { "" })
144}
145
146fn fill(reader: &mut impl Read, buffer: &mut [u8]) -> std::io::Result<usize> {
148 let mut filled = 0;
149 while filled < buffer.len() {
150 match reader.read(&mut buffer[filled..]) {
151 Ok(0) => break,
152 Ok(n) => filled += n,
153 Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {}
154 Err(e) => return Err(e),
155 }
156 }
157 Ok(filled)
158}
159
160pub fn put(folder: &Path, key: &[u8; 32], source: &Path, hash: &str) -> Result<Option<u64>, String> {
166 let target = path(folder, key, hash);
167 if let Ok(meta) = std::fs::metadata(&target)
168 && meta.len() > 0
169 {
170 let old = meta.modified().ok().and_then(|t| t.elapsed().ok()).is_some_and(|age| age.as_secs() > 30 * 86_400);
171 if old {
172 let _ = std::fs::File::options().write(true).open(&target).and_then(|f| f.set_modified(std::time::SystemTime::now()));
173 }
174 return Ok(None);
175 }
176 seal_into(folder, key, source, hash).map(Some)
177}
178
179pub fn put_again(folder: &Path, key: &[u8; 32], source: &Path, hash: &str) -> Result<u64, String> {
182 seal_into(folder, key, source, hash)
183}
184
185fn seal_into(folder: &Path, key: &[u8; 32], source: &Path, hash: &str) -> Result<u64, String> {
186 let target = path(folder, key, hash);
187 let name = name(key, hash);
188 let dir = folder.join("blobs");
189 crate::share::private_dirs(&dir).map_err(|e| format!("{}: {e}", dir.display()))?;
190 let temporary = crate::share::temporary(&target);
193 let sealed = (|| {
194 let fail = |e: std::io::Error| format!("{}: {e}", temporary.display());
195 let mut reading = Hashing { inner: std::fs::File::open(source).map_err(|e| format!("{}: {e}", source.display()))?, hasher: Sha256::new(), size: 0 };
196 let mut packed = flate2::read::GzEncoder::new(&mut reading, flate2::Compression::default());
197 let mut out = BufWriter::new(crate::share::new_private(&temporary).map_err(fail)?);
198 out.write_all(MAGIC).map_err(fail)?;
199 let cipher = XChaCha20Poly1305::new((&keys(key).0).into());
200 let mut piece = vec![0u8; PIECE];
201 let mut index = 0;
202 loop {
203 let n = fill(&mut packed, &mut piece).map_err(|e| format!("{}: {e}", source.display()))?;
204 if n == 0 {
205 break;
206 }
207 if packed.get_ref().size > LARGEST {
208 return Err(format!("{}: changed while it was sealed", source.display()));
209 }
210 write_piece(&mut out, &cipher, &name, index, false, &piece[..n]).map_err(fail)?;
211 index += 1;
212 }
213 write_piece(&mut out, &cipher, &name, index, true, &[]).map_err(fail)?;
214 drop(packed);
215 if hex(&reading.hasher.finalize()) != hash {
216 return Err(format!("{}: changed while it was sealed", source.display()));
217 }
218 let file = out.into_inner().map_err(|e| fail(e.into_error()))?;
219 file.sync_all().map_err(fail)?;
220 let size = file.metadata().map_err(fail)?.len();
221 std::fs::rename(&temporary, &target).map_err(|e| format!("{}: {e}", target.display()))?;
222 Ok(size)
223 })();
224 if sealed.is_err() {
225 let _ = std::fs::remove_file(&temporary);
226 }
227 sealed
228}
229
230fn write_piece(out: &mut impl Write, cipher: &XChaCha20Poly1305, name: &str, index: u64, end: bool, plain: &[u8]) -> std::io::Result<()> {
232 let nonce = XChaCha20Poly1305::generate_nonce(&mut OsRng);
233 let sealed = cipher.encrypt(&nonce, Payload { msg: plain, aad: bound(name, index, end).as_bytes() }).map_err(|_| std::io::Error::other("sealing"))?;
234 let length = (nonce.len() + sealed.len()) as u32 | if end { END } else { 0 };
235 out.write_all(&length.to_be_bytes())?;
236 out.write_all(&nonce)?;
237 out.write_all(&sealed)
238}
239
240pub fn get(folder: &Path, key: &[u8; 32], hash: &str, target: &Path) -> Result<(), Fault> {
248 get_unless(folder, key, hash, target, false)
249}
250
251pub(crate) fn get_unless(folder: &Path, key: &[u8; 32], hash: &str, target: &Path, damaged: bool) -> Result<(), Fault> {
254 let here = if damaged { Err(Fault::Broken) } else { get_from(folder, key, hash, target) };
255 let Err(fault @ (Fault::Missing | Fault::Broken)) = here else { return here };
256 let Some(fetched) = crate::remote::overlay(folder) else { return Err(fault) };
257 match get_from(&fetched, key, hash, target) {
258 Ok(()) => Ok(()),
259 Err(Fault::Broken) => {
260 let _ = std::fs::remove_file(path(&fetched, key, hash));
261 Err(fault)
262 }
263 Err(Fault::Missing) => Err(fault),
264 Err(other) => Err(other),
265 }
266}
267
268fn get_from(folder: &Path, key: &[u8; 32], hash: &str, target: &Path) -> Result<(), Fault> {
269 let opened = open_into(folder, key, hash, target);
270 if opened.is_err() {
271 let _ = std::fs::remove_file(target);
272 }
273 opened
274}
275
276pub fn check(path: &Path, key: &[u8; 32], hash: &str) -> Result<(), Fault> {
280 let Some(file) = present(path) else { return Err(Fault::Missing) };
281 let mut reader = BufReader::new(file);
282 begins(&mut reader)?;
283 unseal(reader, key, &name(key, hash), hash, std::io::sink(), &|e| Fault::Local(format!("{}: {e}", path.display()))).map(|_| ())
284}
285
286fn present(path: &Path) -> Option<std::fs::File> {
289 let file = std::fs::File::open(path).ok()?;
290 (file.metadata().map_or(0, |m| m.len()) > 0).then_some(file)
291}
292
293fn begins(reader: &mut impl Read) -> Result<(), Fault> {
295 let mut magic = [0u8; MAGIC.len()];
296 if reader.read_exact(&mut magic).is_err() || magic != MAGIC {
297 return Err(Fault::Broken);
298 }
299 Ok(())
300}
301
302fn open_into(folder: &Path, key: &[u8; 32], hash: &str, target: &Path) -> Result<(), Fault> {
303 let name = name(key, hash);
304 let Some(file) = present(&folder.join("blobs").join(&name)) else { return Err(Fault::Missing) };
305 let mut reader = BufReader::new(file);
306 begins(&mut reader)?;
307 let local = |e: std::io::Error| Fault::Local(format!("{}: {e}", target.display()));
308 let out = BufWriter::new(crate::share::new_private(target).map_err(local)?);
310 let out = unseal(reader, key, &name, hash, out, &local)?;
311 out.into_inner().map_err(|e| local(e.into_error()))?.sync_all().map_err(local)
312}
313
314fn unseal<W: Write>(mut reader: impl Read, key: &[u8; 32], name: &str, hash: &str, out: W, local: &dyn Fn(std::io::Error) -> Fault) -> Result<W, Fault> {
317 let mut unpacked = flate2::write::GzDecoder::new(Hashing { inner: out, hasher: Sha256::new(), size: 0 });
318 let cipher = XChaCha20Poly1305::new((&keys(key).0).into());
319 let fault = |e: std::io::Error| if matches!(e.kind(), std::io::ErrorKind::InvalidInput | std::io::ErrorKind::InvalidData | std::io::ErrorKind::WriteZero) { Fault::Broken } else { local(e) };
322 let mut index = 0;
323 loop {
324 let mut length = [0u8; 4];
325 reader.read_exact(&mut length).map_err(|_| Fault::Broken)?;
327 let length = u32::from_be_bytes(length);
328 let (end, length) = (length & END != 0, (length & !END) as usize);
329 if !(AROUND..=PIECE + AROUND).contains(&length) {
330 return Err(Fault::Broken);
331 }
332 let mut sealed = vec![0u8; length];
333 reader.read_exact(&mut sealed).map_err(|_| Fault::Broken)?;
334 let (nonce, body) = sealed.split_at(24);
335 let plain = cipher.decrypt(XNonce::from_slice(nonce), Payload { msg: body, aad: bound(name, index, end).as_bytes() }).map_err(|_| Fault::Broken)?;
336 if end {
337 break;
338 }
339 for part in plain.chunks(INFLATED) {
341 unpacked.write_all(part).map_err(fault)?;
342 if unpacked.get_ref().size > LARGEST {
343 return Err(Fault::Broken);
344 }
345 }
346 index += 1;
347 }
348 if reader.read(&mut [0u8; 1]).map_or(true, |n| n > 0) {
350 return Err(Fault::Broken);
351 }
352 let Hashing { inner, hasher, .. } = unpacked.finish().map_err(fault)?;
353 if hex(&hasher.finalize()) != hash {
354 return Err(Fault::Broken);
355 }
356 Ok(inner)
357}
358
359pub fn sweep(folder: &Path, mine: &mut BTreeMap<String, i64>, used: &BTreeSet<String>, now_ms: i64) {
365 let unused = UNUSED_DAYS * 86_400_000;
366 let day = 86_400_000;
370 mine.retain(|name, last| {
371 if used.contains(name) {
372 if now_ms - *last >= day {
373 *last = now_ms;
374 }
375 return true;
376 }
377 if now_ms - *last < unused {
378 return true;
379 }
380 let path = folder.join("blobs").join(name);
381 let fresh = std::fs::metadata(&path).ok().and_then(|m| m.modified().ok()).and_then(|t| t.elapsed().ok()).is_some_and(|age| (age.as_millis() as i64) < unused);
382 if fresh {
383 *last = now_ms;
384 return true;
385 }
386 let _ = std::fs::remove_file(path);
387 false
388 });
389}
390
391#[derive(Debug, Default, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
399pub struct Wanted {
400 #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
402 pub blobs: BTreeMap<String, Want>,
403 #[serde(default)]
405 pub large_at: i64,
406}
407
408#[derive(Debug, Default, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
410pub struct Want {
411 #[serde(default)]
413 pub hash: String,
414 #[serde(default)]
416 pub size: u64,
417 #[serde(default)]
419 pub since: i64,
420 #[serde(default)]
422 pub tries: u32,
423 #[serde(default)]
424 pub again: i64,
425}
426
427impl Wanted {
428 pub fn path(memory: &Path) -> PathBuf {
430 memory.with_file_name("wanted.toml")
431 }
432
433 pub fn load(memory: &Path) -> Wanted {
434 std::fs::read_to_string(Wanted::path(memory)).ok().and_then(|text| toml::from_str(&text).ok()).unwrap_or_default()
435 }
436
437 pub fn change(memory: &Path, change: impl FnOnce(&mut Wanted)) -> Result<(), String> {
440 let path = Wanted::path(memory);
441 sioul_core::filelock::with_lock(&path, || {
442 let before = Wanted::load(memory);
443 let mut wanted = before.clone();
444 change(&mut wanted);
445 if wanted == before {
446 return Ok(());
447 }
448 if wanted.blobs.is_empty() {
449 return match std::fs::remove_file(&path) {
450 Err(e) if e.kind() != std::io::ErrorKind::NotFound => Err(format!("{}: {e}", path.display())),
451 _ => Ok(()),
452 };
453 }
454 let text = toml::to_string(&wanted).map_err(|e| e.to_string())?;
455 crate::share::write_atomically(&path, format!("# Sioul: the others' sealed notes and papers this device waits for (docs/database.md). Never shared.\n{text}").as_bytes())
456 })
457 }
458}
459
460#[cfg(test)]
461mod tests {
462 use super::*;
463
464 fn scratch(name: &str) -> PathBuf {
465 let dir = std::env::temp_dir().join(format!("sioul-blobs-{name}-{}", std::process::id()));
466 let _ = std::fs::remove_dir_all(&dir);
467 std::fs::create_dir_all(&dir).unwrap();
468 dir
469 }
470
471 fn temporary(base: &Path) -> PathBuf {
472 crate::share::temporary(&base.join("scan.pdf"))
473 }
474
475 #[test]
476 fn sealed_whole_and_opened_whole() {
477 let base = scratch("whole");
478 let folder = base.join("vault");
479 let key = [3u8; 32];
480 let mut bytes = Vec::new();
482 let mut x = 7u32;
483 while bytes.len() < 2 * PIECE + 1000 {
484 x = x.wrapping_mul(1_103_515_245).wrapping_add(12_345);
485 bytes.extend_from_slice(&x.to_le_bytes());
486 }
487 let source = base.join("scan.pdf");
488 std::fs::write(&source, &bytes).unwrap();
489 let (hash, size) = hash_file(&source).unwrap();
490 assert_eq!(size, bytes.len() as u64);
491 let sealed_size = put(&folder, &key, &source, &hash).unwrap().unwrap();
492 assert_eq!(std::fs::metadata(path(&folder, &key, &hash)).unwrap().len(), sealed_size);
493 assert_eq!(put(&folder, &key, &source, &hash).unwrap(), None, "a content already there is not sealed again");
494 let target = temporary(&base);
495 get(&folder, &key, &hash, &target).unwrap();
496 assert_eq!(std::fs::read(&target).unwrap(), bytes);
497 let other = temporary(&base);
499 assert_eq!(get(&folder, &[4u8; 32], &hash, &other), Err(Fault::Missing), "another key names it otherwise");
500 let sealed = path(&folder, &key, &hash);
501 std::fs::copy(&sealed, path(&folder, &[4u8; 32], &hash)).unwrap();
502 assert_eq!(get(&folder, &[4u8; 32], &hash, &other), Err(Fault::Broken));
503 assert!(!other.exists(), "nothing half written is left");
504 let whole = std::fs::read(&sealed).unwrap();
506 let mut damaged = whole.clone();
507 damaged[whole.len() / 2] ^= 0x40;
508 for (bytes, fault) in [(damaged, Fault::Broken), (whole[..whole.len() - 50].to_vec(), Fault::Broken), ([whole.clone(), b"more".to_vec()].concat(), Fault::Broken), (Vec::new(), Fault::Missing)] {
509 std::fs::write(&sealed, &bytes).unwrap();
510 assert_eq!(get(&folder, &key, &hash, &temporary(&base)), Err(fault));
511 }
512 put_again(&folder, &key, &source, &hash).unwrap();
514 get(&folder, &key, &hash, &temporary(&base)).unwrap();
515 assert!(!name(&key, &hash).contains(&hash[..16]));
517 let mut plain = <Hmac<Sha256> as Mac>::new_from_slice(&key).unwrap();
518 plain.update(hash.as_bytes());
519 assert_ne!(name(&key, &hash), hex(&plain.finalize().into_bytes()));
520 let _ = std::fs::remove_dir_all(&base);
521 }
522
523 #[test]
526 fn a_copy_is_checked_whole_without_being_written() {
527 let base = scratch("check");
528 let folder = base.join("vault");
529 let key = [6u8; 32];
530 let source = base.join("note.md");
531 std::fs::write(&source, "a note\n".repeat(300)).unwrap();
532 let (hash, _) = hash_file(&source).unwrap();
533 put(&folder, &key, &source, &hash).unwrap();
534 let sealed = path(&folder, &key, &hash);
535 assert_eq!(check(&sealed, &key, &hash), Ok(()));
536 assert_eq!(check(&sealed, &key, &"0".repeat(64)), Err(Fault::Broken), "another content");
537 let whole = std::fs::read(&sealed).unwrap();
538 let mut damaged = whole.clone();
539 damaged[whole.len() - 30] ^= 0x01;
540 std::fs::write(&sealed, &damaged).unwrap();
541 assert_eq!(check(&sealed, &key, &hash), Err(Fault::Broken));
542 std::fs::write(&sealed, b"").unwrap();
543 assert_eq!(check(&sealed, &key, &hash), Err(Fault::Missing));
544 assert_eq!(check(&base.join("none"), &key, &hash), Err(Fault::Missing));
545 assert_eq!(std::fs::read_dir(folder.join("blobs")).unwrap().count(), 1, "nothing written beside");
546 let _ = std::fs::remove_dir_all(&base);
547 }
548
549 #[test]
550 fn a_bomb_stops_at_the_largest_file() {
551 let base = scratch("bomb");
552 let folder = base.join("vault");
553 let key = [5u8; 32];
554 let zeros = vec![0u8; 3 * PIECE];
556 let mut packed = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::best());
557 for _ in 0..30 {
558 packed.write_all(&zeros).unwrap();
559 }
560 let packed = packed.finish().unwrap();
561 let hash = "0".repeat(64);
562 let name = name(&key, &hash);
563 let cipher = XChaCha20Poly1305::new((&keys(&key).0).into());
564 let mut out = MAGIC.to_vec();
565 for (index, piece) in packed.chunks(PIECE).enumerate() {
566 write_piece(&mut out, &cipher, &name, index as u64, false, piece).unwrap();
567 }
568 write_piece(&mut out, &cipher, &name, packed.chunks(PIECE).count() as u64, true, &[]).unwrap();
569 std::fs::create_dir_all(folder.join("blobs")).unwrap();
570 std::fs::write(folder.join("blobs").join(&name), &out).unwrap();
571 let target = temporary(&base);
572 assert_eq!(get(&folder, &key, &hash, &target), Err(Fault::Broken));
573 assert!(!target.exists());
574 let _ = std::fs::remove_dir_all(&base);
575 }
576
577 #[test]
578 fn a_file_changed_while_sealed_is_not_sent() {
579 let base = scratch("changed");
580 let folder = base.join("vault");
581 let source = base.join("note.md");
582 std::fs::write(&source, "before").unwrap();
583 let (hash, _) = hash_file(&source).unwrap();
584 std::fs::write(&source, "after, longer").unwrap();
585 assert!(put(&folder, &[1u8; 32], &source, &hash).is_err());
586 assert!(!path(&folder, &[1u8; 32], &hash).exists());
587 assert_eq!(std::fs::read_dir(folder.join("blobs")).unwrap().count(), 0, "no hidden file left behind");
588 let _ = std::fs::remove_dir_all(&base);
589 }
590
591 #[test]
592 fn unused_ones_go_after_ninety_days() {
593 let base = scratch("sweep");
594 let folder = base.join("vault");
595 std::fs::create_dir_all(folder.join("blobs")).unwrap();
596 let old = std::time::SystemTime::now() - std::time::Duration::from_secs(100 * 86_400);
597 for name in ["used", "unused", "recent", "fresh"] {
598 std::fs::write(folder.join("blobs").join(name), "x").unwrap();
599 if name != "fresh" {
600 std::fs::File::options().write(true).open(folder.join("blobs").join(name)).unwrap().set_modified(old).unwrap();
601 }
602 }
603 let day = 86_400_000;
604 let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_millis() as i64;
605 let mut mine: BTreeMap<String, i64> = [("used".to_string(), now - 100 * day), ("unused".to_string(), now - 100 * day), ("recent".to_string(), now - 20 * day), ("fresh".to_string(), now - 100 * day)].into();
606 sweep(&folder, &mut mine, &["used".to_string()].into(), now);
607 assert!(folder.join("blobs/used").exists() && folder.join("blobs/recent").exists());
608 assert!(folder.join("blobs/fresh").exists(), "sealed or named again lately: kept");
609 assert!(!folder.join("blobs/unused").exists());
610 assert_eq!(mine.get("used"), Some(&now));
611 let _ = std::fs::remove_dir_all(&base);
612 }
613}