sioul_learn/
detail.rs

1// SPDX-License-Identifier: GPL-3.0-or-later
2// Copyright © 2026 Aurélien Pierre
3
4//! What a test said message by message (`sioul spam eval --errors`, `sioul
5//! spam train --errors`, and the MCP's spam_eval and spam_train): the test
6//! messages counted by account and by folder, their label (spam or ham, as
7//! decided) against the filter's class (probably spam, maybe spam, probably
8//! not spam); the numbers at a grid of thresholds, so that one can be
9//! chosen; and the worst errors: ham the filter called spam or maybe spam,
10//! the surest first, then spam it did not call spam, the least sure first.
11//!
12//! An error is said by where it is and by its card, as the Porch reads the
13//! message (its sender, its subject, the start of its text): the command
14//! line prints its sender and its subject, masked as the MCP masks them
15//! (codes, sign-in links, account numbers), never its text. Nothing here is
16//! written to a file: `trained.toml` keeps numbers only.
17
18use crate::corpus::{self, Place};
19use crate::eval::{self, AtThreshold};
20use crate::external;
21use crate::labels::{Evidence, Label};
22use crate::{Dirs, LearnError};
23use serde::Serialize;
24use sioul_core::card::Card;
25use sioul_core::spam::Class;
26use std::collections::{BTreeMap, HashMap, HashSet};
27
28/// The thresholds the grid gives numbers at.
29pub const GRID: [f64; 10] = [0.5, 0.6, 0.7, 0.8, 0.85, 0.9, 0.95, 0.97, 0.98, 0.99];
30
31/// Errors listed at most, of each kind.
32pub const MOST_ERRORS: usize = 500;
33
34/// One test message, as the filter judged it.
35#[derive(Debug, Clone, PartialEq)]
36pub struct Tested {
37    /// How likely spam, as the filter says.
38    pub p: f64,
39    /// What it is, as its label says.
40    pub label: Label,
41    /// When it arrived (Unix seconds).
42    pub date: i64,
43    pub origin: Origin,
44    /// Ham in an inbox, come less than `UNSETTLED_DAYS` ago: its label may
45    /// not be settled yet (spam nobody has looked at stays in an inbox as ham).
46    pub unsettled: bool,
47}
48
49/// Days after which a message left in an inbox counts as said ham: before,
50/// it may be spam nobody has looked at yet.
51pub const UNSETTLED_DAYS: i64 = 30;
52
53/// Whether a message's label may not be settled yet: ham in an inbox, come
54/// less than `UNSETTLED_DAYS` before `now`.
55pub fn unsettled(label: Label, role: sioul_core::folders::Role, date: i64, now: i64) -> bool {
56    label == Label::Ham && role == sioul_core::folders::Role::Inbox && date > now - UNSETTLED_DAYS * 86_400
57}
58
59/// Where a test message comes from.
60#[derive(Debug, Clone, PartialEq, Eq)]
61pub enum Origin {
62    /// Your own mail: the copy learned from, and what decided its label.
63    Own { place: Place, evidence: Evidence },
64    /// Outside material: its source, and its rank among that source's
65    /// messages as `external::read_all` reads them, from 0.
66    Outside { source: String, ordinal: u64 },
67}
68
69/// How many messages of one label the filter judged probably spam, maybe
70/// spam, probably not spam.
71#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
72pub struct Row {
73    pub spam: u64,
74    pub unsure: u64,
75    pub ham: u64,
76}
77
78impl Row {
79    pub fn total(&self) -> u64 {
80        self.spam + self.unsure + self.ham
81    }
82}
83
84/// Messages by their label, then by the filter's class.
85#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
86pub struct Confusion {
87    pub ham: Row,
88    pub spam: Row,
89}
90
91impl Confusion {
92    fn add(&mut self, label: Label, class: Class) {
93        let row = match label {
94            Label::Ham => &mut self.ham,
95            Label::Spam => &mut self.spam,
96        };
97        match class {
98            Class::Spam => row.spam += 1,
99            Class::Unsure => row.unsure += 1,
100            Class::Ham => row.ham += 1,
101        }
102    }
103}
104
105/// One error: a ham the filter called spam or maybe spam, or a spam it did
106/// not call spam.
107#[derive(Debug, Clone)]
108pub struct Wrong {
109    pub p: f64,
110    pub label: Label,
111    pub class: Class,
112    /// When it arrived (Unix seconds).
113    pub date: i64,
114    /// What decided its label: "folder", "junk-folder", "keyword", "log";
115    /// "outside" for outside material.
116    pub evidence: &'static str,
117    /// Its account; the source's name for outside material.
118    pub account: String,
119    /// Its folder on the server (IMAP's modified UTF-7); empty for outside material.
120    pub folder: String,
121    pub outside: bool,
122    /// The message as the Porch reads it; none when it could not be read again.
123    pub card: Option<Card>,
124}
125
126/// The worst errors of a test set.
127#[derive(Debug, Clone, Default)]
128pub struct Errors {
129    /// Ham judged spam or maybe spam, the most likely spam first.
130    pub ham_called_spam: Vec<Wrong>,
131    /// Spam judged maybe spam or not spam, the least likely spam first.
132    pub spam_missed: Vec<Wrong>,
133}
134
135/// An outside source's held-out part, message by message.
136#[derive(Debug, Clone, Default)]
137pub struct OutsideDetail {
138    pub confusion: Confusion,
139    /// Its held-out part at each threshold of `GRID`.
140    pub grid: Vec<AtThreshold>,
141    /// Each held-out message's probability, and whether it is spam.
142    pub scores: Vec<(f64, bool)>,
143    pub errors: Errors,
144}
145
146/// What a test said message by message.
147#[derive(Debug, Clone, Default)]
148pub struct Detail {
149    /// Your own mail's test messages, by account.
150    pub by_account: BTreeMap<String, Confusion>,
151    /// The same, by account, then by folder (its server name).
152    pub by_folder: BTreeMap<String, BTreeMap<String, Confusion>>,
153    /// Your own mail's test messages at each threshold of `GRID`.
154    pub grid: Vec<AtThreshold>,
155    /// The same, unsettled ham left out (`Tested::unsettled`).
156    pub grid_settled: Vec<AtThreshold>,
157    /// Each test message's probability, whether it is spam, whether its
158    /// label is unsettled: numbers only, for a curve or another threshold.
159    pub scores: Vec<(f64, bool, bool)>,
160    pub errors: Errors,
161    /// Each outside source's held-out part.
162    pub outside: BTreeMap<String, OutsideDetail>,
163}
164
165/// What a label's evidence is called in the errors listed.
166pub fn evidence_name(evidence: Evidence) -> &'static str {
167    match evidence {
168        Evidence::Folder => "folder",
169        Evidence::Filter => "filter",
170        Evidence::JunkFolder => "junk-folder",
171        Evidence::Keyword => "keyword",
172        Evidence::Log => "log",
173    }
174}
175
176/// The class a probability falls in, between the two thresholds.
177pub fn class_of(p: f64, threshold_spam: f64, threshold_unsure: f64) -> Class {
178    if p >= threshold_spam {
179        Class::Spam
180    } else if p >= threshold_unsure {
181        Class::Unsure
182    } else {
183        Class::Ham
184    }
185}
186
187/// The detail of a test: counts, the grid, and up to `errors` errors of
188/// each kind for your own mail and for each outside source. The errors'
189/// cards are read again from the corpus and the outside material, once.
190pub fn of(dirs: &Dirs, tested: &[Tested], threshold_spam: f64, threshold_unsure: f64, errors: usize) -> Result<Detail, LearnError> {
191    let class = |p: f64| class_of(p, threshold_spam, threshold_unsure);
192    let mut detail = Detail::default();
193    let mut own = Vec::new();
194    let mut settled = Vec::new();
195    let mut outside: BTreeMap<&str, Vec<(f64, bool)>> = BTreeMap::new();
196    for t in tested {
197        match &t.origin {
198            Origin::Own { place, .. } => {
199                detail.by_account.entry(place.account.clone()).or_default().add(t.label, class(t.p));
200                detail.by_folder.entry(place.account.clone()).or_default().entry(place.folder.clone()).or_default().add(t.label, class(t.p));
201                own.push((t.p, t.label == Label::Spam));
202                detail.scores.push((t.p, t.label == Label::Spam, t.unsettled));
203                if !t.unsettled {
204                    settled.push((t.p, t.label == Label::Spam));
205                }
206            }
207            Origin::Outside { source, .. } => {
208                detail.outside.entry(source.clone()).or_default().confusion.add(t.label, class(t.p));
209                outside.entry(source.as_str()).or_default().push((t.p, t.label == Label::Spam));
210            }
211        }
212    }
213    detail.grid = eval::grid(&own, &GRID);
214    detail.grid_settled = eval::grid(&settled, &GRID);
215    for (source, scored) in outside {
216        let part = detail.outside.entry(source.to_string()).or_default();
217        part.grid = eval::grid(&scored, &GRID);
218        part.scores = scored;
219    }
220    let errors = errors.min(MOST_ERRORS);
221    if errors == 0 {
222        return Ok(detail);
223    }
224    // The worst of each kind, in each set: your own mail (None), each outside source.
225    let mut sets: Vec<Option<&str>> = vec![None];
226    sets.extend(detail.outside.keys().map(|s| Some(s.as_str())));
227    let in_set = |t: &&Tested, set: Option<&str>| match (&t.origin, set) {
228        (Origin::Own { .. }, None) => true,
229        (Origin::Outside { source, .. }, Some(set)) => source == set,
230        _ => false,
231    };
232    let mut chosen: Vec<(Option<String>, Vec<&Tested>, Vec<&Tested>)> = Vec::new();
233    for set in sets {
234        let mut ham: Vec<&Tested> = tested.iter().filter(|t| in_set(t, set) && t.label == Label::Ham && class(t.p) != Class::Ham).collect();
235        ham.sort_by(|a, b| b.p.total_cmp(&a.p).then(b.date.cmp(&a.date)));
236        ham.truncate(errors);
237        let mut spam: Vec<&Tested> = tested.iter().filter(|t| in_set(t, set) && t.label == Label::Spam && class(t.p) != Class::Spam).collect();
238        spam.sort_by(|a, b| a.p.total_cmp(&b.p).then(b.date.cmp(&a.date)));
239        spam.truncate(errors);
240        chosen.push((set.map(str::to_string), ham, spam));
241    }
242    // Their cards, read again: your own mail's from the corpus, outside material's from its files.
243    let places: HashSet<&Place> = chosen.iter().flat_map(|(_, h, s)| h.iter().chain(s)).filter_map(|t| if let Origin::Own { place, .. } = &t.origin { Some(place) } else { None }).collect();
244    let ranks: HashSet<(&str, u64)> =
245        chosen.iter().flat_map(|(_, h, s)| h.iter().chain(s)).filter_map(|t| if let Origin::Outside { source, ordinal } = &t.origin { Some((source.as_str(), *ordinal)) } else { None }).collect();
246    let mut cards: HashMap<Place, Card> = HashMap::new();
247    if !places.is_empty() {
248        corpus::read_all(dirs, |record| {
249            let place = record.place();
250            if places.contains(&place)
251                && !cards.contains_key(&place)
252                && let Some(card) = crate::spamcore::card_of(&record)
253            {
254                cards.insert(place, card);
255            }
256        })?;
257    }
258    let mut outside_cards: HashMap<(String, u64), Card> = HashMap::new();
259    if !ranks.is_empty() {
260        let mut counted: HashMap<String, u64> = HashMap::new();
261        external::read_all(dirs, |source, message| {
262            let rank = counted.entry(source.to_string()).or_default();
263            let ordinal = *rank;
264            *rank += 1;
265            if ranks.contains(&(source, ordinal))
266                && let Some(card) = message.card()
267            {
268                outside_cards.insert((source.to_string(), ordinal), card);
269            }
270        })?;
271    }
272    let wrong = |t: &Tested| -> Wrong {
273        let (evidence, account, folder, outside, card) = match &t.origin {
274            Origin::Own { place, evidence } => (evidence_name(*evidence), place.account.clone(), place.folder.clone(), false, cards.get(place).cloned()),
275            Origin::Outside { source, ordinal } => ("outside", source.clone(), String::new(), true, outside_cards.get(&(source.clone(), *ordinal)).cloned()),
276        };
277        Wrong { p: t.p, label: t.label, class: class(t.p), date: t.date, evidence, account, folder, outside, card }
278    };
279    for (set, ham, spam) in chosen {
280        let listed = Errors { ham_called_spam: ham.into_iter().map(wrong).collect(), spam_missed: spam.into_iter().map(wrong).collect() };
281        match set {
282            None => detail.errors = listed,
283            Some(source) => detail.outside.entry(source).or_default().errors = listed,
284        }
285    }
286    Ok(detail)
287}
288
289#[cfg(test)]
290mod tests {
291    use super::*;
292    use crate::synthetic;
293
294    fn scratch(name: &str) -> std::path::PathBuf {
295        let dir = std::env::temp_dir().join(format!("sioul-learn-detail-{name}-{}", std::process::id()));
296        let _ = std::fs::remove_dir_all(&dir);
297        std::fs::create_dir_all(&dir).unwrap();
298        dir
299    }
300
301    /// Counted by account and folder, the grid, and the worst errors of each
302    /// kind with their cards read again; outside material apart.
303    #[test]
304    fn the_worst_errors_with_their_cards() {
305        let root = scratch("errors");
306        let dirs = Dirs::under(&root);
307        let mail = synthetic::mailbox(5, 6, 4);
308        let records: Vec<_> = mail.iter().enumerate().map(|(i, m)| synthetic::record_of(m, 1, i as u32 + 1)).collect();
309        corpus::store(&dirs, &records).unwrap();
310        let file = root.join("old.jsonl");
311        std::fs::write(&file, "{\"label\":\"ham\",\"date\":1600000000,\"subject\":\"Lunch tomorrow\",\"text\":\"See you\",\"from\":\"Jane <jane@example.org>\"}\n{\"label\":\"spam\",\"date\":1600000100,\"subject\":\"You won\",\"text\":\"Claim it\"}\n").unwrap();
312        external::import(&dirs, &file, None).unwrap();
313        // Each message judged: ham at 0.1 but two, spam at 0.99 but one.
314        let mut tested: Vec<Tested> = records
315            .iter()
316            .map(|r| {
317                let label = if r.role == sioul_core::folders::Role::Junk || r.has_flag("$Junk") { Label::Spam } else { Label::Ham };
318                Tested { p: if label == Label::Spam { 0.99 } else { 0.1 }, label, date: r.date, origin: Origin::Own { place: r.place(), evidence: Evidence::Folder }, unsettled: false }
319            })
320            .collect();
321        let hams: Vec<usize> = (0..tested.len()).filter(|&i| tested[i].label == Label::Ham).collect();
322        let spams: Vec<usize> = (0..tested.len()).filter(|&i| tested[i].label == Label::Spam).collect();
323        tested[hams[0]].p = 0.97;
324        tested[hams[1]].p = 0.6;
325        tested[spams[0]].p = 0.2;
326        tested.push(Tested { p: 0.8, label: Label::Ham, date: 1_600_000_000, origin: Origin::Outside { source: "old".into(), ordinal: 0 }, unsettled: false });
327        tested.push(Tested { p: 0.9, label: Label::Spam, date: 1_600_000_100, origin: Origin::Outside { source: "old".into(), ordinal: 1 }, unsettled: false });
328        let detail = of(&dirs, &tested, 0.95, 0.5, 5).unwrap();
329        // Counted: every own message once by account and once by folder.
330        let counted: u64 = detail.by_account.values().map(|c| c.ham.total() + c.spam.total()).sum();
331        let by_folder: u64 = detail.by_folder.values().flat_map(|f| f.values()).map(|c| c.ham.total() + c.spam.total()).sum();
332        assert_eq!((counted, by_folder), (records.len() as u64, records.len() as u64));
333        assert_eq!(detail.grid.len(), GRID.len());
334        assert_eq!(detail.grid[0].ham_called_spam.count, 2, "two ham from 0.5 on");
335        assert_eq!(detail.grid[6].ham_called_spam.count, 1, "one from 0.95 on");
336        // The worst first, each with its card: the ham at 0.97, then at 0.6; the spam at 0.2.
337        let ham: Vec<f64> = detail.errors.ham_called_spam.iter().map(|w| w.p).collect();
338        assert_eq!(ham, vec![0.97, 0.6]);
339        assert_eq!(detail.errors.ham_called_spam[0].class, Class::Spam);
340        assert_eq!(detail.errors.spam_missed.iter().map(|w| (w.p, w.class)).collect::<Vec<_>>(), vec![(0.2, Class::Ham)]);
341        let card = detail.errors.ham_called_spam[0].card.as_ref().expect("read again");
342        let record = &records[hams[0]];
343        assert_eq!(card.message_id.as_deref().map(sioul_core::mailindex::bare_id), crate::labels::message_id(&record.header_bytes()));
344        assert_eq!(detail.errors.ham_called_spam[0].evidence, "folder");
345        // Outside material apart: its own counts and errors, its card made of its fields.
346        let old = &detail.outside["old"];
347        assert_eq!((old.confusion.ham.unsure, old.confusion.spam.unsure), (1, 1));
348        let outside = &old.errors.ham_called_spam[0];
349        assert!(outside.outside && outside.evidence == "outside" && outside.account == "old");
350        let card = outside.card.as_ref().expect("made of its fields");
351        assert_eq!((card.subject.as_str(), card.from_address.as_deref()), ("Lunch tomorrow", Some("jane@example.org")));
352        assert_eq!(old.errors.spam_missed[0].card.as_ref().map(|c| c.subject.clone()), Some("You won".to_string()));
353        // No errors asked: counts and the grid alone, no card read.
354        let plain = of(&dirs, &tested, 0.95, 0.5, 0).unwrap();
355        assert!(plain.errors.ham_called_spam.is_empty() && plain.outside["old"].errors.spam_missed.is_empty() && !plain.grid.is_empty());
356        let _ = std::fs::remove_dir_all(root);
357    }
358}
sioul_learn/detail.rs

sioul_learn/
detail.rs
