sioul_learn/
spamcore.rs

1// SPDX-License-Identifier: GPL-3.0-or-later
2// Copyright © 2026 Aurélien Pierre
3
4//! The seam to sioul-core's spam module: the tokenizer, the header features
5//! and the reduced table, the same code the phone runs.
6//!
7//! Training reads a corpus record as the Porch reads the message itself, by
8//! the same code: `card_of` makes the message again from what the record
9//! keeps (its header block; its MIME structure from BODYSTRUCTURE, part by
10//! part; the text and the HTML the Porch reads, in their places; every other
11//! part empty) and hands it to `Card::from_bytes`. So the card's excerpt
12//! (the text part, the HTML read as text after a stand-in) and its shape (its
13//! text and HTML versions, its other parts' names and types) are made by
14//! sioul-core's own code, mail-parser sorting the parts. `bodies` asks the
15//! same parser, before the texts are downloaded, which parts they are.
16
17use crate::corpus::{Leaf, Node, Record, Text};
18use mail_parser::{MessageParser, PartType};
19pub(crate) use sioul_core::spam::features::{FEATURES, N, NAMES, all_header_words, features, header_words};
20#[cfg(test)]
21pub(crate) use sioul_core::spam::table::ngram_buckets;
22pub(crate) use sioul_core::spam::table::{Meta, Table, word_hash};
23pub(crate) use sioul_core::spam::tokenize::{Lexicon, TOKENIZER, tokens_with};
24use sioul_core::card::Card;
25
26/// The Porch's card for a corpus record, made by `Card::from_bytes` from the
27/// message made again (see the module); its headers are the record's own,
28/// MIME fields included.
29pub(crate) fn card_of(record: &Record) -> Option<Card> {
30    let header = record.header_bytes();
31    let raw = rebuild(&header, record.structure.as_ref(), record.plain.as_ref(), record.html.as_ref());
32    let mut card = Card::from_bytes(&raw)?;
33    card.headers = sioul_core::headers::RawHeaders::parse(&[header.as_slice(), b"\r\n\r\n"].concat());
34    Some(card)
35}
36
37/// Which parts of a structure are its text and HTML bodies, as mail-parser
38/// sorts them for the Porch: its first text body that is text, its first
39/// HTML body that is HTML (mail-parser lends one as the other when a message
40/// has only one).
41pub(crate) fn bodies(structure: &Node) -> (Option<Vec<u32>>, Option<Vec<u32>>) {
42    let raw = rebuild(b"", Some(structure), None, None);
43    let Some(message) = MessageParser::default().parse(&raw) else { return (None, None) };
44    let numbers = numbered(structure);
45    let find = |ids: &[u32], html: bool| {
46        ids.iter()
47            .find(|&&id| match message.parts.get(id as usize).map(|p| &p.body) {
48                Some(PartType::Text(_)) => !html,
49                Some(PartType::Html(_)) => html,
50                _ => false,
51            })
52            .and_then(|&id| numbers.get(id as usize).cloned().flatten())
53    };
54    (find(&message.text_body, false), find(&message.html_body, true))
55}
56
57/// Every part's number in mail-parser's order (each multipart before its
58/// parts): the leaves' part numbers, none for the multiparts.
59fn numbered(structure: &Node) -> Vec<Option<Vec<u32>>> {
60    fn walk(node: &Node, path: &mut Vec<u32>, out: &mut Vec<Option<Vec<u32>>>) {
61        match node {
62            Node::Multipart { parts, .. } => {
63                out.push(None);
64                for (i, part) in parts.iter().enumerate() {
65                    path.push(i as u32 + 1);
66                    walk(part, path, out);
67                    path.pop();
68                }
69            }
70            Node::Leaf(_) => out.push(Some(if path.is_empty() { vec![1] } else { path.clone() })),
71        }
72    }
73    let mut out = Vec::new();
74    walk(structure, &mut Vec::new(), &mut out);
75    out
76}
77
78/// A message made again from a record (see the module). Without a structure
79/// (a server that gave none), one part: the HTML if that is all there is,
80/// else the text.
81pub(crate) fn rebuild(header: &[u8], structure: Option<&Node>, plain: Option<&Text>, html: Option<&Text>) -> Vec<u8> {
82    let texts: Vec<(&[u32], &str)> = plain.iter().chain(html.iter()).map(|t| (t.at.as_slice(), t.text.as_str())).collect();
83    let single;
84    let structure = match structure {
85        Some(structure) => structure,
86        None => {
87            let mime = if plain.is_none() && html.is_some() { "text/html" } else { "text/plain" };
88            single = Node::Leaf(Leaf { mime: mime.into(), ..Leaf::default() });
89            &single
90        }
91    };
92    // Boundaries no kept text holds: their own mark, and a hash of the texts.
93    let mark = format!("{:016x}", crate::labels::fnv64(texts.iter().map(|(_, t)| *t).collect::<String>().as_bytes()));
94    let mut out = without_mime_fields(header);
95    out.extend_from_slice(b"MIME-Version: 1.0\r\n");
96    write_part(&mut out, structure, &mut Vec::new(), &texts, &mark);
97    out
98}
99
100fn write_part(out: &mut Vec<u8>, node: &Node, path: &mut Vec<u32>, texts: &[(&[u32], &str)], mark: &str) {
101    match node {
102        Node::Multipart { subtype, parts } => {
103            let level: Vec<String> = path.iter().map(u32::to_string).collect();
104            let boundary = format!("=_sioul-learn-{mark}-{}", level.join("."));
105            out.extend_from_slice(format!("Content-Type: multipart/{}; boundary=\"{boundary}\"\r\n\r\n", token(subtype)).as_bytes());
106            for (i, part) in parts.iter().enumerate() {
107                out.extend_from_slice(format!("--{boundary}\r\n").as_bytes());
108                path.push(i as u32 + 1);
109                write_part(out, part, path, texts, mark);
110                path.pop();
111                out.extend_from_slice(b"\r\n");
112            }
113            out.extend_from_slice(format!("--{boundary}--\r\n").as_bytes());
114        }
115        Node::Leaf(leaf) => {
116            let at = if path.is_empty() { vec![1] } else { path.clone() };
117            let text = texts.iter().find(|(p, _)| *p == at.as_slice()).map(|(_, t)| *t);
118            let mime = if leaf.mime.contains('/') { token(&leaf.mime) } else { "text/plain".to_string() };
119            let mut content_type = format!("Content-Type: {mime}");
120            if mime.starts_with("text/") {
121                // What is kept is UTF-8, whatever the part was.
122                content_type.push_str("; charset=\"utf-8\"");
123            }
124            if let Some(name) = &leaf.name {
125                content_type.push_str(&format!("; name=\"{}\"", quoted(name)));
126            }
127            out.extend_from_slice(content_type.as_bytes());
128            out.extend_from_slice(b"\r\n");
129            if let Some(disposition) = &leaf.disposition {
130                let mut line = format!("Content-Disposition: {}", token(disposition));
131                if let Some(filename) = &leaf.filename {
132                    line.push_str(&format!("; filename=\"{}\"", quoted(filename)));
133                }
134                out.extend_from_slice(line.as_bytes());
135                out.extend_from_slice(b"\r\n");
136            }
137            // An attached message keeps its encoding (mail-parser opens it only
138            // unencoded); everything else is written as it is kept.
139            let encoding = if mime.starts_with("message/") && !leaf.encoding.is_empty() { token(&leaf.encoding) } else { "8bit".to_string() };
140            out.extend_from_slice(format!("Content-Transfer-Encoding: {encoding}\r\n\r\n").as_bytes());
141            out.extend_from_slice(text.unwrap_or("").as_bytes());
142            out.extend_from_slice(b"\r\n");
143        }
144    }
145}
146
147/// The header block without its MIME fields (Content-*, MIME-Version), which
148/// the message made again writes from its structure; lines end in CRLF.
149fn without_mime_fields(header: &[u8]) -> Vec<u8> {
150    let mut out = Vec::with_capacity(header.len());
151    let mut skipping = false;
152    for line in header.split(|&b| b == b'\n') {
153        let line = line.strip_suffix(b"\r").unwrap_or(line);
154        if line.is_empty() {
155            break;
156        }
157        if !matches!(line[0], b' ' | b'\t') {
158            let name = line.split(|&b| b == b':').next().unwrap_or_default().trim_ascii().to_ascii_lowercase();
159            skipping = name.starts_with(b"content-") || name == b"mime-version";
160        }
161        if !skipping {
162            out.extend_from_slice(line);
163            out.extend_from_slice(b"\r\n");
164        }
165    }
166    out
167}
168
169/// A MIME token as written: no quote, space, control or separator.
170fn token(text: &str) -> String {
171    text.chars().filter(|c| c.is_ascii_graphic() && !matches!(c, '"' | ';' | ',' | '\\' | '(' | ')' | '<' | '>' | '@' | '[' | ']' | '?' | '=')).collect::<String>().to_ascii_lowercase()
172}
173
174/// A quoted string's inside: quotes and backslashes escaped, line ends out.
175fn quoted(text: &str) -> String {
176    text.chars().filter(|c| !c.is_control()).flat_map(|c| if matches!(c, '"' | '\\') { vec!['\\', c] } else { vec![c] }).collect()
177}
178
179#[cfg(test)]
180mod tests {
181    use super::*;
182    use sioul_core::folders::Role;
183
184    fn leaf(mime: &str) -> Leaf {
185        Leaf { mime: mime.into(), charset: mime.starts_with("text/").then(|| "utf-8".into()), encoding: "7bit".into(), octets: 10, ..Leaf::default() }
186    }
187
188    fn record(header: &str, structure: Node, plain: Option<(&[u32], &str)>, html: Option<(&[u32], &str)>) -> Record {
189        let text = |(at, text): (&[u32], &str)| Text { at: at.to_vec(), text: text.to_string() };
190        Record {
191            account: "home".into(),
192            folder: "INBOX".into(),
193            role: Role::Inbox,
194            uidvalidity: 1,
195            uid: 1,
196            date: 0,
197            flags: vec![],
198            size: 0,
199            header: header.into(),
200            header_latin1: false,
201            structure: Some(structure),
202            plain: plain.map(text),
203            html: html.map(text),
204            checked: None,
205            fetched: 0,
206        }
207    }
208
209    /// The same message, whole (the phone's card) and as a corpus record
210    /// (training's card): the same excerpt, the same shape, the same headers.
211    #[test]
212    fn a_record_reads_as_its_message() {
213        let raw = b"From: Alice <alice@example.org>\r\nSubject: Invoice\r\nMessage-ID: <1@example.org>\r\nMIME-Version: 1.0\r\nContent-Type: multipart/mixed; boundary=\"b1\"\r\n\r\n--b1\r\nContent-Type: multipart/alternative; boundary=\"b2\"\r\n\r\n--b2\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nPlease find the invoice attached.\r\n--b2\r\nContent-Type: text/html; charset=utf-8\r\n\r\n<p>Please find the invoice attached.</p>\r\n--b2--\r\n--b1\r\nContent-Type: application/pdf; name=\"invoice.pdf\"\r\nContent-Disposition: attachment; filename=\"invoice.pdf\"\r\nContent-Transfer-Encoding: base64\r\n\r\nJVBERi0xLjQK\r\n--b1\r\nContent-Type: image/png\r\nContent-Disposition: inline\r\nContent-Transfer-Encoding: base64\r\n\r\niVBORw0KGgo=\r\n--b1--\r\n";
214        let whole = Card::from_bytes(raw).unwrap();
215        let structure = Node::Multipart {
216            subtype: "mixed".into(),
217            parts: vec![
218                Node::Multipart { subtype: "alternative".into(), parts: vec![Node::Leaf(leaf("text/plain")), Node::Leaf(leaf("text/html"))] },
219                Node::Leaf(Leaf { name: Some("invoice.pdf".into()), disposition: Some("attachment".into()), filename: Some("invoice.pdf".into()), encoding: "base64".into(), ..leaf("application/pdf") }),
220                Node::Leaf(Leaf { disposition: Some("inline".into()), encoding: "base64".into(), ..leaf("image/png") }),
221            ],
222        };
223        assert_eq!(bodies(&structure), (Some(vec![1, 1]), Some(vec![1, 2])));
224        let header = "From: Alice <alice@example.org>\r\nSubject: Invoice\r\nMessage-ID: <1@example.org>\r\nMIME-Version: 1.0\r\nContent-Type: multipart/mixed; boundary=\"b1\"\r\n\r\n";
225        let card = card_of(&record(header, structure, Some((&[1, 1], "Please find the invoice attached.")), None)).unwrap();
226        assert_eq!(card.excerpt.trim(), whole.excerpt.trim());
227        assert_eq!(card.shape, whole.shape);
228        assert_eq!(card.attachments, whole.attachments);
229        assert_eq!((card.subject.as_str(), card.message_id.as_deref()), ("Invoice", whole.message_id.as_deref()));
230        assert_eq!(card.headers.first("Content-Type"), whole.headers.first("Content-Type"), "the record's own headers");
231    }
232
233    /// HTML alone: mail-parser lends it as the text, and the excerpt is its visible text.
234    #[test]
235    fn html_alone_and_stand_ins() {
236        let html_only = Node::Leaf(leaf("text/html"));
237        assert_eq!(bodies(&html_only), (None, Some(vec![1])));
238        let page = format!("<html><head><style>p {{ color: red }}</style></head><body><p>{}</p></body></html>", "Real words here. ".repeat(20));
239        let card = card_of(&record("Subject: News\r\n", html_only, None, Some((&[1], &page)))).unwrap();
240        assert!(card.excerpt.trim().starts_with("Real words here.") && !card.excerpt.contains("color"), "{}", card.excerpt);
241        assert!(card.shape.html && !card.shape.text);
242        // A text part that only says "view it in your browser": the Porch reads the HTML after it.
243        let alternative = Node::Multipart { subtype: "alternative".into(), parts: vec![Node::Leaf(leaf("text/plain")), Node::Leaf(leaf("text/html"))] };
244        let card = card_of(&record("Subject: News\r\n", alternative, Some((&[1], "View it in your browser")), Some((&[2], &page)))).unwrap();
245        assert!(card.excerpt.starts_with("View it in your browser\n\nReal words here."), "{}", card.excerpt);
246        // A named text part, second in a mixed message: an attachment, not the text.
247        let mixed = Node::Multipart {
248            subtype: "mixed".into(),
249            parts: vec![Node::Leaf(leaf("text/plain")), Node::Leaf(Leaf { name: Some("notes.txt".into()), ..leaf("text/plain") })],
250        };
251        assert_eq!(bodies(&mixed), (Some(vec![1]), None));
252        let card = card_of(&record("Subject: Notes\r\n", mixed, Some((&[1], "See the notes.")), None)).unwrap();
253        assert_eq!(card.shape.parts, vec![("notes.txt".to_string(), "text/plain".to_string())]);
254    }
255
256    #[test]
257    fn mime_fields_are_written_again() {
258        let header = b"From: a@example.org\r\nContent-Type: multipart/mixed;\r\n\tboundary=\"x\"\r\nSubject: Hi\r\nMIME-Version: 1.0\r\nContent-Transfer-Encoding: 7bit\r\n\r\nbody";
259        assert_eq!(without_mime_fields(header), b"From: a@example.org\r\nSubject: Hi\r\n".to_vec());
260        assert_eq!(token("Multipart/Mixed\"; x"), "multipart/mixedx");
261        assert_eq!(quoted("a \"b\"\\c\r\n"), "a \\\"b\\\"\\\\c");
262    }
263}
sioul_learn/spamcore.rs

sioul_learn/
spamcore.rs
