sioul_core/spam/
mod.rs

1// SPDX-License-Identifier: GPL-3.0-or-later
2// Copyright © 2026 Aurélien Pierre
3
4//! Sioul's own spam filter, the part every device runs
5//! (docs/spam-filter.md): the tokenizer (`tokenize`), the header
6//! features (`features`), the reduced table that scores both (`table`), the
7//! label log and the log of what the filter moved (`labels`), and the filter
8//! as the Porch asks it (`Filter`). The training, on one computer, by hand,
9//! is the `sioul-learn` crate's; it writes the table with
10//! `table::Table::write`, and shares these functions with the phone, so that
11//! both read a message the same way.
12//!
13//! Where its verdict goes, and what it never touches (people you know, codes,
14//! projects, your own mail), is the Porch's: `porch::triage`. What it does
15//! with each verdict is yours to choose, class by class (`Actions`).
16
17pub mod features;
18pub mod labels;
19pub mod table;
20pub mod tokenize;
21
22use crate::card::Card;
23use serde::{Deserialize, Serialize};
24use std::collections::HashMap;
25use std::path::PathBuf;
26use std::sync::{Arc, Mutex};
27use table::Table;
28
29/// From this probability on, a message is spam, unless said otherwise (`[spam] threshold_spam`).
30pub const THRESHOLD_SPAM: f32 = 0.95;
31/// From this one on, it is unsure (`[spam] threshold_unsure`).
32pub const THRESHOLD_UNSURE: f32 = 0.5;
33
34/// Where a probability falls between the two thresholds: probably spam,
35/// maybe spam, probably not spam.
36#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
37#[serde(rename_all = "lowercase")]
38pub enum Class {
39    Spam,
40    Unsure,
41    Ham,
42}
43
44impl Class {
45    /// In the order the matrix shows them: the surest spam first.
46    pub const ALL: [Class; 3] = [Class::Spam, Class::Unsure, Class::Ham];
47
48    /// "spam", "unsure", "ham".
49    pub fn as_str(self) -> &'static str {
50        match self {
51            Class::Spam => "spam",
52            Class::Unsure => "unsure",
53            Class::Ham => "ham",
54        }
55    }
56
57    pub fn read(text: &str) -> Option<Class> {
58        Class::ALL.into_iter().find(|c| c.as_str() == text.trim())
59    }
60}
61
62/// What the filter does with a message of a class, as you chose it (the
63/// matrix in Settings ▸ Mail ▸ Your own spam filter).
64#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
65pub enum Action {
66    /// Into the account's Junk folder on the server, as the Junk button does;
67    /// it waits in the Porch's review queue, where "Not spam" brings it back
68    /// to the inbox. Only mail as it arrives: what was there before stays put.
69    Move,
70    /// Where it is, marked, in the Porch's review queue; never notified.
71    Flag,
72    /// Nothing shows.
73    Nothing,
74}
75
76impl Action {
77    pub const ALL: [Action; 3] = [Action::Move, Action::Flag, Action::Nothing];
78
79    /// "move", "flag", "nothing".
80    pub fn as_str(self) -> &'static str {
81        match self {
82            Action::Move => "move",
83            Action::Flag => "flag",
84            Action::Nothing => "nothing",
85        }
86    }
87
88    pub fn read(text: &str) -> Option<Action> {
89        Action::ALL.into_iter().find(|a| a.as_str() == text.trim())
90    }
91}
92
93/// What the filter does with each class: the matrix {probably spam, maybe
94/// spam, probably not spam} × {move to spam, flag only, do nothing}.
95#[derive(Debug, Clone, Copy, PartialEq, Eq)]
96pub struct Actions {
97    pub spam: Action,
98    pub unsure: Action,
99    pub ham: Action,
100}
101
102impl Default for Actions {
103    /// Until you choose: spam and doubts flagged, waiting for your review; nothing moves.
104    fn default() -> Actions {
105        Actions { spam: Action::Flag, unsure: Action::Flag, ham: Action::Nothing }
106    }
107}
108
109impl Actions {
110    /// What an older Sioul's `mode` meant: "off", nothing at all; "say", spam
111    /// and doubts flagged; "act", spam moved, doubts flagged.
112    pub fn of_mode(mode: &str) -> Option<Actions> {
113        let (spam, unsure) = match mode.trim() {
114            "off" => (Action::Nothing, Action::Nothing),
115            "say" => (Action::Flag, Action::Flag),
116            "act" => (Action::Move, Action::Flag),
117            _ => return None,
118        };
119        Some(Actions { spam, unsure, ham: Action::Nothing })
120    }
121
122    /// The action for a class.
123    pub fn of(&self, class: Class) -> Action {
124        match class {
125            Class::Spam => self.spam,
126            Class::Unsure => self.unsure,
127            Class::Ham => self.ham,
128        }
129    }
130
131    /// The same, one class's action changed.
132    pub fn with(mut self, class: Class, action: Action) -> Actions {
133        match class {
134            Class::Spam => self.spam = action,
135            Class::Unsure => self.unsure = action,
136            Class::Ham => self.ham = action,
137        }
138        self
139    }
140
141    /// Nothing done with any class: no message is judged at all.
142    pub fn idle(&self) -> bool {
143        Class::ALL.iter().all(|c| self.of(*c) == Action::Nothing)
144    }
145}
146
147/// The filter's verdict on one message: how likely it is spam, from 0 to 1.
148/// What weighed stays in the table's score (`table::Score`), for `sioul spam
149/// why` in your terminal; the Porch never says it.
150#[derive(Debug, Clone, Copy, PartialEq)]
151pub struct Verdict {
152    pub p: f32,
153}
154
155/// The table's place on this device: `$XDG_DATA_HOME/sioul/spam/table.bin`.
156pub fn table_path() -> PathBuf {
157    crate::config::data_dir().join("spam").join("table.bin")
158}
159
160/// The filter as the Porch asks it: your choices, and where its table is.
161#[derive(Debug, Clone, PartialEq)]
162pub struct Filter {
163    /// What is done with each class.
164    pub actions: Actions,
165    /// From this probability on, spam; from `threshold_unsure` on, unsure.
166    pub threshold_spam: f32,
167    pub threshold_unsure: f32,
168    /// The table's file; with none there, the filter says nothing.
169    pub table: PathBuf,
170}
171
172impl Filter {
173    /// The filter as the configuration sets it (`[spam]`), its table where this device keeps it.
174    pub fn of(config: &crate::config::Config) -> Filter {
175        let (threshold_spam, threshold_unsure) = config.spam.thresholds();
176        Filter { actions: config.spam.actions(), threshold_spam, threshold_unsure, table: table_path() }
177    }
178
179    /// Where a probability falls.
180    pub fn class(&self, p: f32) -> Class {
181        if p >= self.threshold_spam {
182            Class::Spam
183        } else if p >= self.threshold_unsure {
184            Class::Unsure
185        } else {
186            Class::Ham
187        }
188    }
189
190    /// What is done with a verdict: its class's action.
191    pub fn action(&self, verdict: Verdict) -> Action {
192        self.actions.of(self.class(verdict.p))
193    }
194
195    /// How likely a message is spam; none when nothing is done with any
196    /// class, or with no table yet. `trusted_ids`: its account's
197    /// authserv-ids (`score`). Each stored message is read once per table:
198    /// the Porch asks again at each look.
199    pub fn judge(&self, card: &Card, trusted_ids: &[String]) -> Option<Verdict> {
200        if self.actions.idle() {
201            return None;
202        }
203        let table = Table::cached(&self.table)?;
204        let key = card.path.as_ref().map(|p| p.to_string_lossy().into_owned());
205        if let Some(verdict) = key.as_deref().and_then(|key| remembered(&table, key)) {
206            return Some(verdict);
207        }
208        let verdict = score(&table, card, trusted_ids);
209        if let Some(key) = key {
210            remember(&table, key, verdict);
211        }
212        Some(verdict)
213    }
214}
215
216/// A message's verdict by a table: its words, its header features (its
217/// file's time standing for when its provider received it, as Sioul stores
218/// mail), these reading Sioul's own checks first (its stamp, made as the
219/// message was stored), your provider's second, as the training reads the
220/// corpus's own checks (`features::auth_results`; `trusted_ids`: the
221/// account's authserv-ids, Sioul's own first). The words are read with the
222/// table's own (`Table::lexicon`).
223pub fn score(table: &Table, card: &Card, trusted_ids: &[String]) -> Verdict {
224    let tokens = tokenize::tokens_with(&table.lexicon(), &card.subject, &card.excerpt);
225    let received = card
226        .path
227        .as_deref()
228        .and_then(|p| std::fs::metadata(p).ok()?.modified().ok()?.duration_since(std::time::UNIX_EPOCH).ok())
229        .and_then(|d| i64::try_from(d.as_secs()).ok());
230    let auth = features::auth_results(&card.headers, trusted_ids);
231    let x = features::features(card, auth.as_ref(), received, &tokens.links);
232    Verdict { p: table.score(&tokens.words, &x).p }
233}
234
235/// The verdicts given with the table in use, by message file: forgotten when the table changes.
236type Remembered = Option<(Arc<Table>, HashMap<String, Verdict>)>;
237static REMEMBERED: Mutex<Remembered> = Mutex::new(None);
238
239/// At most this many verdicts kept, then forgotten at once.
240const REMEMBERED_MOST: usize = 20_000;
241
242fn remembered(table: &Arc<Table>, key: &str) -> Option<Verdict> {
243    let kept = REMEMBERED.lock().unwrap_or_else(|e| e.into_inner());
244    kept.as_ref().filter(|(t, _)| Arc::ptr_eq(t, table)).and_then(|(_, verdicts)| verdicts.get(key).copied())
245}
246
247fn remember(table: &Arc<Table>, key: String, verdict: Verdict) {
248    let mut kept = REMEMBERED.lock().unwrap_or_else(|e| e.into_inner());
249    if !kept.as_ref().is_some_and(|(t, verdicts)| Arc::ptr_eq(t, table) && verdicts.len() < REMEMBERED_MOST) {
250        *kept = Some((Arc::clone(table), HashMap::new()));
251    }
252    if let Some((_, verdicts)) = kept.as_mut() {
253        verdicts.insert(key, verdict);
254    }
255}
sioul_core/spam/mod.rs

sioul_core/spam/
mod.rs
