sioul_learn/
labels.rs

1// SPDX-License-Identifier: GPL-3.0-or-later
2// Copyright © 2026 Aurélien Pierre
3
4//! Spam or ham for each message of the corpus, from what is already there
5//! (docs/spam-filter.md, "The labels"): what you said, your mail apps'
6//! keywords, the Junk folders, your own filter's flags, the other folders.
7//!
8//! - **Spam**: in a Junk-role folder, whoever put it there (your provider,
9//!   your own filter's "Move to spam", a mail filter's "spam", you, another
10//!   mail app); carrying `$Junk` (or Thunderbird's older `Junk`); flagged by
11//!   your own filter as probably spam (every device's log of what it
12//!   flagged), wherever it is now.
13//! - **Ham**: in any other folder kept (the corpus leaves out Trash, Drafts
14//!   and Sent; of Gmail's All Mail it keeps what no other folder has, your own
15//!   messages aside), or carrying `$NotJunk` (or `NonJunk`, `NotJunk`,
16//!   `$NonJunk`, other mail apps' words for it).
17//! - **Left out**: flagged by your own filter as maybe spam, until you say:
18//!   a doubt is no label.
19//! - **Your actions** (every device's label log: junk, not junk, "Spam",
20//!   "Not spam", block) beat everything, the newest one winning: a message
21//!   you said is not spam is never learned as spam, whatever its folder,
22//!   its keywords or your filter say of it.
23//! - **One message, one label**: copies (the same Message-ID in several
24//!   folders or accounts) become one; when they disagree, the log beats the
25//!   keywords, which beat the Junk folder, which beats your filter's flag,
26//!   which beats the other folders. Copies that disagree at the same level
27//!   are left out.
28//!
29//! Your filter's verdicts count as they are until you correct them: you are
30//! never asked to confirm one, only to say "Not spam" of one that is wrong.
31//! Trained again and again, the growing mail and your corrections outweigh
32//! its mistakes.
33
34use crate::corpus::{Place, Record};
35use serde::{Deserialize, Serialize};
36use sioul_core::folders::Role;
37/// The label logs are phase 0's (`sioul_core::spam::labels`): one reader for their lines.
38pub use sioul_core::spam::labels::{Entry, Flagged, Label, Moved};
39use sioul_core::spam::Class;
40use std::collections::{BTreeMap, HashMap, HashSet};
41
42/// What decided a label, weakest first.
43#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
44#[serde(rename_all = "kebab-case")]
45pub enum Evidence {
46    /// A folder of yours other than Junk.
47    Folder,
48    /// Your own filter's flag: probably spam, which you did not correct.
49    Filter,
50    /// The Junk folder, whoever put the message there: mostly your
51    /// provider's filing; your own filter's move; a mail filter's; yours.
52    JunkFolder,
53    /// A junk or not-junk keyword, set by you, your other mail apps or a mail filter.
54    Keyword,
55    /// The label log: what you did in Sioul.
56    Log,
57}
58
59/// What training needs of one record to decide its label (read in a first
60/// pass, so the corpus is never held in memory whole).
61#[derive(Debug, Clone, PartialEq, Eq)]
62pub struct Copy {
63    pub place: Place,
64    pub role: Role,
65    pub date: i64,
66    /// The bare Message-ID, else `header:<FNV-1a of the header block>`.
67    pub key: String,
68    pub junk: bool,
69    pub not_junk: bool,
70}
71
72/// The keywords that say junk: `$Junk` (RFC 5788's registry), and `Junk`,
73/// Thunderbird's older word, which other apps read too.
74pub const JUNK: &[&str] = &["$Junk", "Junk"];
75/// The keywords that say not junk: `$NotJunk`, Thunderbird's older
76/// `NonJunk`, and `NotJunk` and `$NonJunk`, which other apps write.
77pub const NOT_JUNK: &[&str] = &["$NotJunk", "NonJunk", "NotJunk", "$NonJunk"];
78
79impl Copy {
80    pub fn of(record: &Record) -> Copy {
81        let header = record.header_bytes();
82        let key = message_id(&header).unwrap_or_else(|| format!("header:{:016x}", fnv64(&header)));
83        let junk = JUNK.iter().any(|k| record.has_flag(k));
84        let not_junk = NOT_JUNK.iter().any(|k| record.has_flag(k));
85        Copy { place: record.place(), role: record.role, date: record.date, key, junk, not_junk }
86    }
87
88    /// What this copy says by itself: a not-junk keyword (your correction)
89    /// before a junk one, then the folder.
90    fn says(&self) -> (Label, Evidence) {
91        if self.not_junk {
92            (Label::Ham, Evidence::Keyword)
93        } else if self.junk {
94            (Label::Spam, Evidence::Keyword)
95        } else if self.role == Role::Junk {
96            (Label::Spam, Evidence::JunkFolder)
97        } else {
98            (Label::Ham, Evidence::Folder)
99        }
100    }
101}
102
103/// A header block's Message-ID, bare, read as the label log reads it
104/// (`spam::labels`), so that your actions find their message.
105pub(crate) fn message_id(header: &[u8]) -> Option<String> {
106    let headers = mail_parser::MessageParser::default().parse_headers(header)?;
107    let id = sioul_core::mailindex::bare_id(headers.message_id()?);
108    (!id.trim().is_empty()).then_some(id)
109}
110
111/// A header block's From address, lowercase.
112pub(crate) fn from_address(header: &[u8]) -> Option<String> {
113    let headers = mail_parser::MessageParser::default().parse_headers(header)?;
114    let address = headers.from()?.first()?.address()?.trim().to_lowercase();
115    (!address.is_empty()).then_some(address)
116}
117
118/// Your actions, as every device's label log keeps them (`$XDG_STATE_HOME/sioul/spam/labels/`), oldest first.
119pub fn read_log(dirs: &crate::Dirs) -> Vec<Entry> {
120    sioul_core::spam::labels::read_all_in(&dirs.state)
121}
122
123/// What your own filter moved, every device's log, oldest first.
124pub fn read_moved(dirs: &crate::Dirs) -> Vec<Moved> {
125    sioul_core::spam::labels::read_moved_in(&dirs.state)
126}
127
128/// What your own filter flagged where it is, every device's log, oldest first.
129pub fn read_flagged(dirs: &crate::Dirs) -> Vec<Flagged> {
130    sioul_core::spam::labels::read_flagged_in(&dirs.state)
131}
132
133/// One message to learn from.
134#[derive(Debug, Clone, PartialEq, Eq)]
135pub struct Labeled {
136    pub key: String,
137    /// The copy whose words and headers are learned from.
138    pub place: Place,
139    pub label: Label,
140    pub evidence: Evidence,
141    /// When it first arrived: the earliest INTERNALDATE of its copies.
142    pub date: i64,
143    /// The role of the folder the copy learned from is in (an inbox, a Junk folder…).
144    pub role: Role,
145}
146
147/// What deciding the labels found, in numbers only.
148#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
149pub struct Summary {
150    /// Records read, copies included.
151    pub records: u64,
152    pub ham: u64,
153    pub spam: u64,
154    /// Messages whose copies disagreed at the same level: left out.
155    pub ambiguous: u64,
156    /// Messages your own filter flagged as maybe spam, about which you said
157    /// nothing: left out until you say (a doubt is no label).
158    #[serde(default)]
159    pub unsure: u64,
160    /// Spam in a Junk folder because your own filter moved it there, about
161    /// which you said nothing: learned as all Junk mail is (among `by_junk_folder`).
162    #[serde(default)]
163    pub filter_moved: u64,
164    /// Labels decided by each kind of evidence.
165    pub by_log: u64,
166    pub by_keyword: u64,
167    pub by_junk_folder: u64,
168    /// By your own filter's flag: probably spam, uncorrected.
169    #[serde(default)]
170    pub by_filter: u64,
171    pub by_folder: u64,
172    /// Ham and spam per account (the copy learned from).
173    pub accounts: BTreeMap<String, (u64, u64)>,
174}
175
176/// One label per message, from every copy read and the logs (see the
177/// module): `log`, what you said; `moved` and `flagged`, what your own
178/// filter moved into a Junk folder and what it flagged where it is.
179pub fn decide(copies: Vec<Copy>, log: &[Entry], moved: &[Moved], flagged: &[Flagged]) -> (Vec<Labeled>, Summary) {
180    let mut summary = Summary { records: copies.len() as u64, ..Summary::default() };
181    // What your filter moved, by account and Message-ID: learned as all Junk mail is, counted apart.
182    let moved: HashSet<(&str, &str)> = moved.iter().filter_map(|m| Some((m.account.as_str(), m.message_id.as_deref()?))).collect();
183    // What your filter flagged, the newest flag of each message: by account
184    // and Message-ID, wherever its copies are now; else by its very place.
185    let mut flags_by_id: HashMap<(&str, &str), Class> = HashMap::new();
186    let mut flags_by_place: HashMap<(&str, &str, u32, u32), Class> = HashMap::new();
187    for flag in flagged {
188        match flag.message_id.as_deref().filter(|id| !id.is_empty()) {
189            Some(id) => flags_by_id.insert((&flag.account, id), flag.class),
190            None => flags_by_place.insert((&flag.account, &flag.folder, flag.uidvalidity, flag.uid), flag.class),
191        };
192    }
193    let flag_of = |c: &Copy| -> Option<Class> {
194        let account = c.place.account.as_str();
195        flags_by_id.get(&(account, c.key.as_str())).or_else(|| flags_by_place.get(&(account, c.place.folder.as_str(), c.place.uidvalidity, c.place.uid))).copied()
196    };
197    // What a copy says, your filter's flag included: flagged as probably
198    // spam, a message in a folder of yours is spam (a keyword or a Junk
199    // folder speaks louder).
200    let word = |c: &Copy| -> (Label, Evidence) {
201        match (c.says(), flag_of(c)) {
202            ((_, Evidence::Folder), Some(Class::Spam)) => (Label::Spam, Evidence::Filter),
203            (own, _) => own,
204        }
205    };
206    // Flagged as maybe spam, in a folder of yours: no say, until you say.
207    let doubted = |c: &Copy| c.says().1 == Evidence::Folder && flag_of(c) == Some(Class::Unsure);
208    // The newest action for each Message-ID and each place.
209    let mut by_id: HashMap<&str, &Entry> = HashMap::new();
210    let mut by_place: HashMap<(&str, &str, u32, u32), &Entry> = HashMap::new();
211    for action in log {
212        if let Some(id) = action.message_id.as_deref().filter(|id| !id.is_empty()) {
213            let newest = by_id.entry(id).or_insert(action);
214            if action.at >= newest.at {
215                *newest = action;
216            }
217        }
218        let newest = by_place.entry((&action.account, &action.folder, action.uidvalidity, action.uid)).or_insert(action);
219        if action.at >= newest.at {
220            *newest = action;
221        }
222    }
223    let mut groups: BTreeMap<String, Vec<Copy>> = BTreeMap::new();
224    for copy in copies {
225        groups.entry(copy.key.clone()).or_default().push(copy);
226    }
227    let mut labeled = Vec::with_capacity(groups.len());
228    for (key, mut copies) in groups {
229        let date = copies.iter().map(|c| c.date).filter(|d| *d > 0).min().unwrap_or(0);
230        let action = std::iter::once(by_id.get(key.as_str()).copied())
231            .chain(copies.iter().map(|c| by_place.get(&(c.place.account.as_str(), c.place.folder.as_str(), c.place.uidvalidity, c.place.uid)).copied()))
232            .flatten()
233            .max_by_key(|a| a.at);
234        // Your filter's doubt, about which you said nothing: that copy has no say.
235        if action.is_none() && copies.iter().any(|c| doubted(c)) {
236            copies.retain(|c| !doubted(c));
237            if copies.is_empty() {
238                summary.unsure += 1;
239                continue;
240            }
241        }
242        let decided = match action {
243            Some(action) => {
244                // The copy where you acted, else the one whose own word agrees, else the first.
245                let at = copies.iter().find(|c| c.place.account == action.account && c.place.folder == action.folder && c.place.uidvalidity == action.uidvalidity && c.place.uid == action.uid);
246                let agreeing = copies.iter().filter(|c| word(c).0 == action.label).max_by_key(|c| (word(c).1, std::cmp::Reverse((c.date, c.place.clone()))));
247                let copy = at.or(agreeing).unwrap_or(&copies[0]);
248                Some((copy, action.label, Evidence::Log))
249            }
250            None => {
251                let strongest = copies.iter().map(|c| word(c).1).max().expect("a group has copies");
252                let top: Vec<&Copy> = copies.iter().filter(|c| word(c).1 == strongest).collect();
253                if top.iter().any(|c| word(c).0 != word(top[0]).0) {
254                    summary.ambiguous += 1;
255                    None
256                } else {
257                    let copy = top.iter().min_by_key(|c| (c.date, c.place.clone())).expect("not empty");
258                    Some((*copy, word(copy).0, strongest))
259                }
260            }
261        };
262        let Some((copy, label, evidence)) = decided else { continue };
263        match label {
264            Label::Ham => summary.ham += 1,
265            Label::Spam => summary.spam += 1,
266        }
267        match evidence {
268            Evidence::Log => summary.by_log += 1,
269            Evidence::Keyword => summary.by_keyword += 1,
270            Evidence::JunkFolder => summary.by_junk_folder += 1,
271            Evidence::Filter => summary.by_filter += 1,
272            Evidence::Folder => summary.by_folder += 1,
273        }
274        // In a Junk folder because your own filter moved it there.
275        if evidence == Evidence::JunkFolder && copies.iter().any(|c| c.says().1 == Evidence::JunkFolder && moved.contains(&(c.place.account.as_str(), key.as_str()))) {
276            summary.filter_moved += 1;
277        }
278        let counts = summary.accounts.entry(copy.place.account.clone()).or_default();
279        match label {
280            Label::Ham => counts.0 += 1,
281            Label::Spam => counts.1 += 1,
282        }
283        labeled.push(Labeled { key, place: copy.place.clone(), label, evidence, date, role: copy.role });
284    }
285    (labeled, summary)
286}
287
288/// 64-bit FNV-1a.
289pub(crate) fn fnv64(bytes: &[u8]) -> u64 {
290    bytes.iter().fold(0xcbf2_9ce4_8422_2325u64, |h, &b| (h ^ u64::from(b)).wrapping_mul(0x0000_0100_0000_01b3))
291}
292
293#[cfg(test)]
294mod tests {
295    use super::*;
296
297    fn copy(account: &str, folder: &str, role: Role, uid: u32, key: &str) -> Copy {
298        Copy { place: Place { account: account.into(), folder: folder.into(), uidvalidity: 1, uid }, role, date: i64::from(uid), key: key.into(), junk: false, not_junk: false }
299    }
300
301    fn action(at: i64, folder: &str, uid: u32, id: Option<&str>, label: Label) -> Entry {
302        use sioul_core::spam::labels::Source;
303        let source = if label == Label::Spam { Source::Junk } else { Source::NotSpam };
304        Entry { at, account: "home".into(), folder: folder.into(), uidvalidity: 1, uid, message_id: id.map(str::to_string), label, source }
305    }
306
307    fn label_of<'a>(labeled: &'a [Labeled], key: &str) -> Option<&'a Labeled> {
308        labeled.iter().find(|l| l.key == key)
309    }
310
311    #[test]
312    fn folders_and_keywords() {
313        let copies = vec![
314            copy("home", "INBOX", Role::Inbox, 1, "a@example.org"),
315            copy("home", "Junk", Role::Junk, 2, "b@example.org"),
316            Copy { junk: true, ..copy("home", "INBOX", Role::Inbox, 3, "c@example.org") },
317            Copy { not_junk: true, ..copy("home", "Junk", Role::Junk, 4, "d@example.org") },
318            Copy { junk: true, not_junk: true, ..copy("home", "INBOX", Role::Inbox, 5, "e@example.org") },
319        ];
320        let (labeled, summary) = decide(copies, &[], &[], &[]);
321        let says = |key| label_of(&labeled, key).map(|l| (l.label, l.evidence));
322        assert_eq!(says("a@example.org"), Some((Label::Ham, Evidence::Folder)));
323        assert_eq!(says("b@example.org"), Some((Label::Spam, Evidence::JunkFolder)));
324        assert_eq!(says("c@example.org"), Some((Label::Spam, Evidence::Keyword)));
325        assert_eq!(says("d@example.org"), Some((Label::Ham, Evidence::Keyword)), "$NotJunk beats the Junk folder");
326        assert_eq!(says("e@example.org"), Some((Label::Ham, Evidence::Keyword)), "your correction beats $Junk");
327        assert_eq!((summary.ham, summary.spam, summary.records), (3, 2, 5));
328        assert_eq!(summary.accounts["home"], (3, 2));
329    }
330
331    /// Thunderbird's older keywords, and other apps' words, count as `$Junk` and `$NotJunk` do.
332    #[test]
333    fn older_keywords_are_labels_too() {
334        let record = |uid: u32, folder: &str, role: Role, flags: &[&str]| Record {
335            account: "home".into(),
336            folder: folder.into(),
337            role,
338            uidvalidity: 1,
339            uid,
340            date: i64::from(uid),
341            flags: flags.iter().map(|f| f.to_string()).collect(),
342            size: 0,
343            header: format!("Message-ID: <{uid}@example.org>\r\n"),
344            header_latin1: false,
345            structure: None,
346            plain: None,
347            html: None,
348            checked: None,
349            fetched: 0,
350        };
351        let copies = vec![
352            Copy::of(&record(1, "INBOX", Role::Inbox, &["Junk"])),
353            Copy::of(&record(2, "Junk", Role::Junk, &["NonJunk"])),
354            Copy::of(&record(3, "Junk", Role::Junk, &["notjunk"])),
355            Copy::of(&record(4, "Junk", Role::Junk, &["$NonJunk", "\\Seen"])),
356            Copy::of(&record(5, "INBOX", Role::Inbox, &["Junk", "NonJunk"])),
357            Copy::of(&record(6, "INBOX", Role::Inbox, &["JunkMail"])),
358        ];
359        assert_eq!(copies[0].key, "1@example.org");
360        let (labeled, summary) = decide(copies, &[], &[], &[]);
361        let says = |n: u32| labeled.iter().find(|l| l.key == format!("{n}@example.org")).map(|l| (l.label, l.evidence));
362        assert_eq!(says(1), Some((Label::Spam, Evidence::Keyword)));
363        assert_eq!(says(2), Some((Label::Ham, Evidence::Keyword)), "NonJunk beats the Junk folder");
364        assert_eq!(says(3), Some((Label::Ham, Evidence::Keyword)), "keywords' case aside");
365        assert_eq!(says(4), Some((Label::Ham, Evidence::Keyword)));
366        assert_eq!(says(5), Some((Label::Ham, Evidence::Keyword)), "Thunderbird left both: the correction wins");
367        assert_eq!(says(6), Some((Label::Ham, Evidence::Folder)), "another keyword is no label");
368        assert_eq!(summary.by_keyword, 5);
369    }
370
371    #[test]
372    fn headers_read_as_the_porch_reads_them() {
373        let header = b"From: =?utf-8?Q?Aur=C3=A9lien?= <Owner@Example.ORG>\r\nMessage-ID: <abc@example.org>\r\n";
374        assert_eq!(from_address(header).as_deref(), Some("owner@example.org"));
375        assert_eq!(message_id(header).as_deref(), Some("abc@example.org"));
376        assert_eq!(message_id(b"Subject: none\r\n"), None);
377    }
378
379    /// Copies become one message; the stronger evidence wins, the same level disagreeing leaves it out.
380    #[test]
381    fn copies_and_their_disagreements() {
382        let copies = vec![
383            // In the inbox of one account, in Junk of another: Junk wins.
384            copy("home", "INBOX", Role::Inbox, 10, "x@example.org"),
385            copy("work", "Junk", Role::Junk, 11, "x@example.org"),
386            // $Junk on one copy, $NotJunk on another: no way to tell.
387            Copy { junk: true, ..copy("home", "INBOX", Role::Inbox, 12, "y@example.org") },
388            Copy { not_junk: true, ..copy("work", "INBOX", Role::Inbox, 13, "y@example.org") },
389            // The same message twice in ham folders: one message, its first arrival.
390            copy("home", "Archive", Role::Archive, 21, "z@example.org"),
391            copy("home", "INBOX", Role::Inbox, 20, "z@example.org"),
392        ];
393        let (labeled, summary) = decide(copies, &[], &[], &[]);
394        let x = label_of(&labeled, "x@example.org").unwrap();
395        assert_eq!((x.label, x.evidence, x.place.account.as_str(), x.date), (Label::Spam, Evidence::JunkFolder, "work", 10));
396        assert!(label_of(&labeled, "y@example.org").is_none());
397        assert_eq!(summary.ambiguous, 1);
398        let z = label_of(&labeled, "z@example.org").unwrap();
399        assert_eq!((z.place.uid, z.date), (20, 20));
400        assert_eq!(labeled.len(), 2);
401    }
402
403    /// Your actions beat everything, the newest first; by Message-ID, or by
404    /// place for a message that has none.
405    #[test]
406    fn the_log_beats_folders_and_keywords() {
407        let copies = vec![
408            Copy { not_junk: true, ..copy("home", "Junk", Role::Junk, 1, "p@example.org") },
409            copy("home", "INBOX", Role::Inbox, 2, "header:0000000000000001"),
410            copy("home", "Junk", Role::Junk, 3, "q@example.org"),
411        ];
412        let log = vec![
413            action(100, "INBOX", 9, Some("p@example.org"), Label::Ham),
414            action(200, "INBOX", 9, Some("p@example.org"), Label::Spam),
415            action(150, "INBOX", 2, None, Label::Spam),
416            action(300, "Junk", 3, Some("q@example.org"), Label::Ham),
417        ];
418        let (labeled, summary) = decide(copies, &log, &[], &[]);
419        let says = |key| label_of(&labeled, key).map(|l| (l.label, l.evidence));
420        assert_eq!(says("p@example.org"), Some((Label::Spam, Evidence::Log)), "the newest action");
421        assert_eq!(says("header:0000000000000001"), Some((Label::Spam, Evidence::Log)), "by place");
422        assert_eq!(says("q@example.org"), Some((Label::Ham, Evidence::Log)), "Not spam on a junk message");
423        assert_eq!(summary.by_log, 3);
424    }
425
426    #[test]
427    fn the_log_reads_phase_zeros_lines() {
428        let dir = std::env::temp_dir().join(format!("sioul-learn-labels-{}", std::process::id()));
429        let dirs = crate::Dirs::under(&dir);
430        std::fs::create_dir_all(&dirs.state).unwrap();
431        // The older single log, before the logs were shared; then another device's own.
432        std::fs::write(
433            dirs.state.join(sioul_core::spam::labels::FILE),
434            "{\"at\":1791360000,\"account\":\"home\",\"folder\":\"INBOX\",\"uidvalidity\":1700000000,\"uid\":4521,\"message_id\":\"abc@example.org\",\"label\":\"ham\",\"source\":\"not-spam\"}\nnot json\n{\"at\":1,\"account\":\"home\",\"folder\":\"INBOX\",\"uidvalidity\":1,\"uid\":2,\"message_id\":null,\"label\":\"spam\",\"source\":\"block\"}\n",
435        )
436        .unwrap();
437        std::fs::create_dir_all(dirs.labels()).unwrap();
438        std::fs::write(dirs.labels().join("phone.jsonl"), "{\"at\":1791360500,\"account\":\"home\",\"folder\":\"Junk\",\"uidvalidity\":3,\"uid\":7,\"message_id\":\"def@example.org\",\"label\":\"spam\",\"source\":\"junk\"}\n").unwrap();
439        let log = read_log(&dirs);
440        assert_eq!(log.len(), 3, "every device's, oldest first");
441        assert_eq!((log[0].label, log[0].message_id.as_deref()), (Label::Spam, None));
442        assert_eq!((log[1].label, log[1].message_id.as_deref(), log[1].source), (Label::Ham, Some("abc@example.org"), sioul_core::spam::labels::Source::NotSpam));
443        assert_eq!(log[2].message_id.as_deref(), Some("def@example.org"));
444        let _ = std::fs::remove_dir_all(dir);
445    }
446
447    /// Your own filter's word counts until you say otherwise: what it moved
448    /// into a Junk folder is spam at once, as all Junk mail is; what it
449    /// flagged as probably spam is spam at once, wherever the message is now;
450    /// what it flagged as maybe spam is left out, unless another copy is in
451    /// a Junk folder. What you said wins over all of it, "Not spam" above
452    /// all; another mail app's `$NotJunk` speaks louder than a flag; a mail
453    /// filter's "spam" (`$Junk`, in the Junk folder) is spam.
454    #[test]
455    fn the_filters_word_counts_until_you_say() {
456        let judged = |folder: &str, uid: u32, id: &str, class: Class| Moved { at: 50, account: "home".into(), folder: folder.into(), uidvalidity: 1, uid, message_id: Some(id.into()), class };
457        let copies = vec![
458            copy("home", "Junk", Role::Junk, 1, "moved@example.org"),
459            copy("home", "INBOX", Role::Inbox, 2, "flagged@example.org"),
460            copy("home", "INBOX", Role::Inbox, 3, "doubt@example.org"),
461            copy("home", "INBOX", Role::Inbox, 4, "doubt-elsewhere@example.org"),
462            copy("home", "Junk", Role::Junk, 5, "doubt-elsewhere@example.org"),
463            copy("home", "Junk", Role::Junk, 6, "not-spam@example.org"),
464            copy("home", "INBOX", Role::Inbox, 7, "said-after-flag@example.org"),
465            Copy { not_junk: true, ..copy("home", "INBOX", Role::Inbox, 8, "kept@example.org") },
466            Copy { junk: true, ..copy("home", "Junk", Role::Junk, 9, "rule@example.org") },
467            copy("home", "Archive", Role::Archive, 10, "archived@example.org"),
468            copy("home", "INBOX", Role::Inbox, 11, "unjudged@example.org"),
469            copy("work", "INBOX", Role::Inbox, 12, "other-account@example.org"),
470        ];
471        let moved = [judged("INBOX", 91, "moved@example.org", Class::Spam), judged("INBOX", 96, "not-spam@example.org", Class::Unsure)];
472        let flagged = [
473            judged("INBOX", 2, "flagged@example.org", Class::Spam),
474            judged("INBOX", 3, "doubt@example.org", Class::Unsure),
475            judged("INBOX", 4, "doubt-elsewhere@example.org", Class::Unsure),
476            judged("INBOX", 7, "said-after-flag@example.org", Class::Spam),
477            judged("INBOX", 8, "kept@example.org", Class::Spam),
478            // Flagged in the inbox, then moved into the archive: still the filter's word, until you say.
479            judged("INBOX", 99, "archived@example.org", Class::Spam),
480            // Flagged in another account than the one holding the copy.
481            judged("INBOX", 12, "other-account@example.org", Class::Spam),
482        ];
483        let log = vec![action(100, "Junk", 6, Some("not-spam@example.org"), Label::Ham), action(100, "INBOX", 7, Some("said-after-flag@example.org"), Label::Ham)];
484        let (labeled, summary) = decide(copies, &log, &moved, &flagged);
485        let says = |key| label_of(&labeled, key).map(|l| (l.label, l.evidence));
486        assert_eq!(says("moved@example.org"), Some((Label::Spam, Evidence::JunkFolder)), "moved: spam at once, as all Junk mail");
487        assert_eq!(says("flagged@example.org"), Some((Label::Spam, Evidence::Filter)), "flagged as probably spam: spam at once");
488        assert_eq!(says("doubt@example.org"), None, "maybe spam: left out until you say");
489        assert_eq!(says("doubt-elsewhere@example.org"), Some((Label::Spam, Evidence::JunkFolder)), "its copy in a Junk folder speaks");
490        assert_eq!(says("not-spam@example.org"), Some((Label::Ham, Evidence::Log)), "Not spam wins over the filter's move");
491        assert_eq!(says("said-after-flag@example.org"), Some((Label::Ham, Evidence::Log)), "and over its flag");
492        assert_eq!(says("kept@example.org"), Some((Label::Ham, Evidence::Keyword)), "$NotJunk from another mail app speaks louder than a flag");
493        assert_eq!(says("rule@example.org"), Some((Label::Spam, Evidence::Keyword)), "a mail filter's spam: $Junk, in Junk");
494        assert_eq!(says("archived@example.org"), Some((Label::Spam, Evidence::Filter)), "moving it is no correction: only Spam or Not spam are");
495        assert_eq!(says("unjudged@example.org"), Some((Label::Ham, Evidence::Folder)));
496        assert_eq!(says("other-account@example.org"), Some((Label::Ham, Evidence::Folder)), "a flag is its account's");
497        assert_eq!((summary.spam, summary.ham, summary.unsure, summary.filter_moved), (5, 5, 1, 1));
498        assert_eq!((summary.by_log, summary.by_keyword, summary.by_junk_folder, summary.by_filter, summary.by_folder), (2, 2, 2, 2, 2));
499        // The newest flag wins: flagged maybe, then judged again probably spam.
500        let again = [judged("INBOX", 3, "doubt@example.org", Class::Unsure), Moved { at: 60, ..judged("INBOX", 3, "doubt@example.org", Class::Spam) }];
501        let (labeled, _) = decide(vec![copy("home", "INBOX", Role::Inbox, 3, "doubt@example.org")], &[], &[], &again);
502        assert_eq!(labeled.first().map(|l| (l.label, l.evidence)), Some((Label::Spam, Evidence::Filter)));
503        // A message without a Message-ID: by its very place.
504        let nameless = Moved { message_id: None, ..judged("INBOX", 2, "", Class::Unsure) };
505        let (labeled, summary) = decide(vec![copy("home", "INBOX", Role::Inbox, 2, "header:0000000000000002")], &[], &[], &[nameless]);
506        assert!(labeled.is_empty() && summary.unsure == 1);
507    }
508}
sioul_learn/labels.rs

sioul_learn/
labels.rs
