sioul_core/
words.rs

1// SPDX-License-Identifier: GPL-3.0-or-later
2// Copyright © 2026 Aurélien Pierre
3
4//! The words Sioul looks for (docs/words.md): every list a recogniser
5//! matches text against (one-time codes, bills, letters, folders, brands,
6//! the spam filter's tokenizer…), out of the code and into data that you
7//! can change.
8//!
9//! Three layers make each list:
10//! 1. **language packs**, shipped: `crates/sioul-core/data/words/<language>.toml`,
11//!    a file per language, as the `.ftl` files are;
12//! 2. **country packs**, shipped: `data/words/countries/<code>.toml`, the
13//!    names of a country (its public bodies, its brands, its banks'
14//!    services), and `data/words/international.toml`, always in use: the
15//!    words of mail programs, servers and apps in any language, brands of
16//!    every country, shared mail providers;
17//! 3. **your changes**, in config.toml under `[words]`: for each list, the
18//!    words you added (`add`) and the shipped ones you took away (`remove`),
19//!    never a copy of the list, so that a newer Sioul's words still reach you.
20//!
21//! A list as Sioul uses it: the words of the packs in use, less the
22//! shipped words you took away, plus yours; compared folded (`text::fold`:
23//! capitals and accents aside), each once, the first spelling kept. A word
24//! both taken away and added is there: you put it back.
25//!
26//! The packs in use: the languages of `[words] languages`, else the
27//! interface's and English; the countries of `[words] countries`, else
28//! `[contacts] region`, else the locale's; and the international pack. A
29//! newer pack on disk (`$XDG_DATA_HOME/sioul/words/`, then each of
30//! `$XDG_DATA_DIRS`' `sioul/words/`) wins over the built-in one by its
31//! `checked` date, as the site presets do (`presets.rs`): a distribution
32//! updates them without building Sioul; your own changes never go there.
33//!
34//! ```toml
35//! [words]
36//! languages = ["fr", "en"]
37//! countries = ["FR"]
38//!
39//! [words.codes.code]
40//! add = ["Bestätigungscode"]
41//! remove = ["code temporaire"]
42//!
43//! [words.brands.brands]
44//! add = { "Ma Banque" = ["mabanque.example"] }
45//! remove = ["Banque Exemple"]
46//! ```
47
48use crate::config::Config;
49use serde::{Deserialize, Serialize};
50use std::path::{Path, PathBuf};
51use std::sync::{Arc, LazyLock, Mutex};
52
53/// Named lists, in their order: brands and their domains, bodies and the
54/// words their letters carry, months by their number.
55pub type Named = Vec<(String, Vec<String>)>;
56
57/// The packs built into Sioul: their file, under `data/words/`, and their text.
58const SHIPPED: &[(&str, &str)] = &[
59    ("en.toml", include_str!("../data/words/en.toml")),
60    ("fr.toml", include_str!("../data/words/fr.toml")),
61    // The lists whose wording is public and stable; letters, papers, bank
62    // exports and payments wait for real examples (docs/words.md).
63    ("de.toml", include_str!("../data/words/de.toml")),
64    ("es.toml", include_str!("../data/words/es.toml")),
65    ("it.toml", include_str!("../data/words/it.toml")),
66    ("countries/FR.toml", include_str!("../data/words/countries/FR.toml")),
67    ("international.toml", include_str!("../data/words/international.toml")),
68];
69
70/// The lists that are named lists (`Named`): their entries are names, not words.
71pub const MAPS: &[&str] = &["brands.brands", "letters.senders", "letters.months", "letters.numbers", "letters.fixed_delays", "capture.months"];
72
73/// The languages read before the configuration says: Sioul's first two, as every list was until 8 October 2026.
74pub const FIRST_LANGUAGES: [&str; 2] = ["fr", "en"];
75/// The country whose names were read before the configuration said: France's.
76pub const FIRST_COUNTRIES: [&str; 1] = ["FR"];
77
78// ---------------------------------------------------------------- the lists
79
80/// One-time codes, passwords, resets, sign-in links, addresses to confirm (`codes`).
81#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
82#[serde(default)]
83pub struct Codes {
84    /// Names a code: a code right beside it is taken.
85    pub code: Vec<String>,
86    /// Points at a code without naming it: counts in a message about signing in.
87    pub pointing: Vec<String>,
88    /// Says the message is about signing in or checking who you are.
89    pub sign_in: Vec<String>,
90    /// Between "your" and "code", makes it no secret of yours ("your zip code").
91    pub not_yours: Vec<String>,
92    /// A shop's code: the message hands no secret over.
93    pub promo: Vec<String>,
94    pub password: Vec<String>,
95    pub reset: Vec<String>,
96    pub link: Vec<String>,
97    /// What a sign-in link's message is about, in its subject or first lines.
98    pub intents: Vec<String>,
99    /// What may expire or work once: the link, its button, the code.
100    pub link_words: Vec<String>,
101    pub validity: Vec<String>,
102    /// Lines answering a request of yours ("if you did not ask for this").
103    pub request: Vec<String>,
104    pub confirm: Vec<String>,
105    /// Hands a code over, the code right after ("your code: K7Q-2M9").
106    pub giving: Vec<String>,
107    /// Whose code, a few words before the word for a code.
108    pub owners: Vec<String>,
109    /// Says a code is yours, right after it ("123456 is your code").
110    pub owned_after: Vec<String>,
111    /// May stand between a phrase and its code ("is").
112    pub linking: Vec<String>,
113    pub once: Vec<String>,
114    /// Words before a validity ("valid 10 minutes").
115    pub expiry_context: Vec<String>,
116    /// The word for a code itself.
117    pub noun: Vec<String>,
118    /// Starts of the units of a validity, after its number.
119    pub unit_minutes: Vec<String>,
120    pub unit_hours: Vec<String>,
121    pub unit_days: Vec<String>,
122}
123
124/// Other apps' notifications that hand a code over or ask for an approval now (`appnotes`).
125#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
126#[serde(default)]
127pub struct Approvals {
128    pub phrases: Vec<String>,
129    /// "Is this you?": counts beside what it is about (`asked_about`).
130    pub asks: Vec<String>,
131    pub asked_about: Vec<String>,
132    /// Words in the name of a channel for codes or sign-ins.
133    pub channels: Vec<String>,
134}
135
136/// Amounts in mail and letters (`money`).
137#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
138#[serde(default)]
139pub struct MoneyWords {
140    /// Words after which the amount a message is about comes.
141    pub keywords: Vec<String>,
142    /// Sentences stating a payment; the amount follows them.
143    pub stated: Vec<String>,
144    /// Labels of the line that holds what was paid.
145    pub totals: Vec<String>,
146    /// Labels of a subtotal, which is no total.
147    pub subtotal: Vec<String>,
148    /// Lines of a legal footer, whose amounts are never a payment.
149    pub legal: Vec<String>,
150}
151
152/// Mail about money (`payments`).
153#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
154#[serde(default)]
155pub struct Payments {
156    pub not_payments: Vec<String>,
157    pub refund: Vec<String>,
158    pub paid: Vec<String>,
159    pub received: Vec<String>,
160    pub order: Vec<String>,
161    pub bill: Vec<String>,
162    /// The label above who was paid, in a receipt.
163    pub to: Vec<String>,
164    /// The label above who paid, in a receipt.
165    pub from: Vec<String>,
166    /// Words before the party's name, after an amount ("de Jean").
167    pub leads: Vec<String>,
168    /// A subject naming an order and saying it is confirmed.
169    pub order_words: Vec<String>,
170    pub confirmed_words: Vec<String>,
171}
172
173/// A CSV export's columns, by their names (`bank`).
174#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
175#[serde(default)]
176pub struct BankColumns {
177    pub date: Vec<String>,
178    pub label: Vec<String>,
179    pub name: Vec<String>,
180    pub net: Vec<String>,
181    pub amount: Vec<String>,
182    pub debit: Vec<String>,
183    pub credit: Vec<String>,
184    pub kind: Vec<String>,
185    pub status: Vec<String>,
186    pub impact: Vec<String>,
187    pub balance: Vec<String>,
188    pub time: Vec<String>,
189    pub currency: Vec<String>,
190    pub id: Vec<String>,
191}
192
193/// Lines above a CSV export's header.
194#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
195#[serde(default)]
196pub struct BankAbove {
197    pub balance: Vec<String>,
198    pub date: Vec<String>,
199    pub account: Vec<String>,
200}
201
202/// Bank and payment-processor exports (`bank`).
203#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
204#[serde(default)]
205pub struct BankWords {
206    /// Words that name nothing in a movement's label.
207    pub filler: Vec<String>,
208    pub columns: BankColumns,
209    pub above: BankAbove,
210    /// Statuses of rows left out.
211    pub skipped: Vec<String>,
212    /// A balance impact saying the row moves no money.
213    pub memo: Vec<String>,
214}
215
216/// Money moved between your own accounts (`accounts`).
217#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
218#[serde(default)]
219pub struct AccountWords {
220    pub between: Vec<String>,
221}
222
223/// What a letter is, the gravest first.
224#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
225#[serde(default)]
226pub struct LetterKinds {
227    pub formal_notice: Vec<String>,
228    pub tax_notice: Vec<String>,
229    pub decision: Vec<String>,
230    /// With `decision`: a way of appeal.
231    pub decision_appeal: Vec<String>,
232    pub reminder: Vec<String>,
233    pub appointment: Vec<String>,
234    pub bill: Vec<String>,
235    pub acknowledgment: Vec<String>,
236    pub attestation: Vec<String>,
237    pub contract: Vec<String>,
238}
239
240/// Paper letters, read by OCR (`letters`).
241#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
242#[serde(default)]
243pub struct LetterWords {
244    /// Bodies that write to everyone: the name shown, the words their letters carry.
245    pub senders: Named,
246    pub kinds: LetterKinds,
247    pub registered: Vec<String>,
248    pub registered_receipt: Vec<String>,
249    /// A line that asks for an amount.
250    pub asks: Vec<String>,
251    /// Month names, by the month's number.
252    pub months: Named,
253    pub ordinals: Vec<String>,
254    /// Numbers written as words, by their value.
255    pub numbers: Named,
256    /// Words before a date by which something is asked.
257    pub deadline: Vec<String>,
258    pub delay_words: Vec<String>,
259    pub delay_of: Vec<String>,
260    pub unit_months: Vec<String>,
261    pub unit_days: Vec<String>,
262    pub unit_weeks: Vec<String>,
263    /// Delays said in words, by their days.
264    pub fixed_delays: Named,
265    /// Says a delay counts from the letter's own date.
266    pub from_letter: Vec<String>,
267    /// The word before an appointment's time.
268    pub at: Vec<String>,
269    /// Labels of your number with them.
270    pub references: Vec<String>,
271}
272
273/// The kinds of paper, by the words a file's name or a subject carries (`papers`).
274#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
275#[serde(default)]
276pub struct PaperKinds {
277    pub passport: Vec<String>,
278    pub identity: Vec<String>,
279    pub residence: Vec<String>,
280    pub driving: Vec<String>,
281    pub health_cover: Vec<String>,
282    pub health_card: Vec<String>,
283    pub tax_notice: Vec<String>,
284    pub rent_receipt: Vec<String>,
285    pub bank_details: Vec<String>,
286    pub payslip: Vec<String>,
287    pub warranty: Vec<String>,
288    pub insurance: Vec<String>,
289    pub certificate: Vec<String>,
290}
291
292#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
293#[serde(default)]
294pub struct PaperWords {
295    pub kinds: PaperKinds,
296}
297
298/// The kinds of contract, by the words a label, a subject or a sender carries (`contracts`).
299#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
300#[serde(default)]
301pub struct ContractKinds {
302    pub rent: Vec<String>,
303    pub energy: Vec<String>,
304    pub health: Vec<String>,
305    pub insurance: Vec<String>,
306    pub telecom: Vec<String>,
307    pub hosting: Vec<String>,
308    pub subscription: Vec<String>,
309    pub bank: Vec<String>,
310}
311
312#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
313#[serde(default)]
314pub struct ContractWords {
315    pub kinds: ContractKinds,
316}
317
318/// Automatic senders (`porch`).
319#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
320#[serde(default)]
321pub struct SenderWords {
322    /// An address holding one of these is filed.
323    pub automatic: Vec<String>,
324}
325
326/// Prefixes of a subject already answered or forwarded (`compose`).
327#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
328#[serde(default)]
329pub struct Replies {
330    pub forward: Vec<String>,
331    pub reply: Vec<String>,
332}
333
334/// A message's quoted history (`reading`).
335#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
336#[serde(default)]
337pub struct Quotes {
338    /// Lines that open a forwarded or answered message.
339    pub openings: Vec<String>,
340    pub header_names: Vec<String>,
341    /// The first header of a quoted block.
342    pub from_names: Vec<String>,
343    /// One of these makes a block of headers a quoted message.
344    pub enough_names: Vec<String>,
345    /// Endings of an attribution line ("Jane wrote:").
346    pub wrote: Vec<String>,
347    /// Starts of an attribution broken over two lines.
348    pub attribution_starts: Vec<String>,
349}
350
351/// Mail folders by their names (`folders`).
352#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
353#[serde(default)]
354pub struct FolderWords {
355    pub sent: Vec<String>,
356    pub drafts: Vec<String>,
357    pub junk: Vec<String>,
358    pub trash: Vec<String>,
359    pub archive: Vec<String>,
360}
361
362/// Senders who borrow a name (`lookalike`).
363#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
364#[serde(default)]
365pub struct BrandWords {
366    /// Brands and public services, and their domains.
367    pub brands: Named,
368    /// Mail providers whose domains millions share.
369    pub shared: Vec<String>,
370    /// Words of a service, which a fake name wraps a brand or your domain in
371    /// ("PayPal Service", "janedoe.example Mail Admin"): an everyday brand's
372    /// name beside these alone is borrowed (`lookalike::impersonation`).
373    pub service_words: Vec<String>,
374    /// The brands whose name is also an everyday word or a place ("Orange",
375    /// "Apple", "La Poste"), by name: borrowed only alone or beside the words
376    /// of a service; any other brand counts anywhere in a name.
377    pub everyday: Vec<String>,
378}
379
380/// A site's notification that someone calls (`sites`).
381#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
382#[serde(default)]
383pub struct CallWords {
384    pub missed: Vec<String>,
385    pub ringing: Vec<String>,
386}
387
388/// Voicemail an operator sends by mail (`voicemail`).
389#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
390#[serde(default)]
391pub struct VoicemailWords {
392    /// The operators' domains: only their mail is read as voicemail.
393    pub operators: Vec<String>,
394    pub words: Vec<String>,
395    pub hidden: Vec<String>,
396    /// Words before the caller's number, matched as whole words.
397    pub caller_leads: Vec<String>,
398    pub own_line: Vec<String>,
399    pub duration: Vec<String>,
400}
401
402/// A topic's words, for the shield.
403#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
404#[serde(default)]
405pub struct ShieldTopics {
406    pub work: Vec<String>,
407    pub support: Vec<String>,
408    pub press: Vec<String>,
409    pub thanks: Vec<String>,
410    pub donation: Vec<String>,
411}
412
413/// The shield of a public address (`shield`).
414#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
415#[serde(default)]
416pub struct ShieldWords {
417    pub threats: Vec<String>,
418    pub insults: Vec<String>,
419    pub rude: Vec<String>,
420    pub you: Vec<String>,
421    pub topics: ShieldTopics,
422}
423
424/// Words of your tasks' categories.
425#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
426#[serde(default)]
427pub struct TaskWords {
428    /// Never proposed, never scheduled (`plan`).
429    pub optional: Vec<String>,
430    /// The Tasks page's own fold.
431    pub joy: Vec<String>,
432    /// Left out of Free time's offers when movement is off (`pause`).
433    pub movement: Vec<String>,
434    /// In view in quiet time, when `[quiet] personal` is unset.
435    pub personal: Vec<String>,
436    /// Work, never in view in quiet time, when `[quiet] work` is unset.
437    pub work: Vec<String>,
438    /// Leisure, among the personal ones (`areas`).
439    pub leisure: Vec<String>,
440    /// Yours, outside work, before your settings are read (`areas`).
441    pub usual_personal: Vec<String>,
442    /// A notes folder of these names holds the Pause's sounds.
443    pub sounds_folders: Vec<String>,
444}
445
446/// The days of the week.
447#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
448#[serde(default)]
449pub struct Weekdays {
450    pub monday: Vec<String>,
451    pub tuesday: Vec<String>,
452    pub wednesday: Vec<String>,
453    pub thursday: Vec<String>,
454    pub friday: Vec<String>,
455    pub saturday: Vec<String>,
456    pub sunday: Vec<String>,
457}
458
459/// Words by the kind of task they say (`tasks::KINDS`).
460#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
461#[serde(default)]
462pub struct TaskKinds {
463    pub call: Vec<String>,
464    pub write: Vec<String>,
465    pub online: Vec<String>,
466    pub out: Vec<String>,
467    pub read: Vec<String>,
468    pub think: Vec<String>,
469    pub make: Vec<String>,
470}
471
472/// Quick capture (`capture`).
473#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
474#[serde(default)]
475pub struct CaptureWords {
476    pub weekdays: Weekdays,
477    /// Month names, by the month's number.
478    pub months: Named,
479    /// Words after "@" that name a kind.
480    pub kind_words: TaskKinds,
481    /// A first word that says a kind plainly.
482    pub kind_verbs: TaskKinds,
483    pub today: Vec<String>,
484    pub tomorrow: Vec<String>,
485    pub next_week: Vec<String>,
486    /// "in 3 days".
487    #[serde(rename = "in")]
488    pub in_: Vec<String>,
489    pub unit_days: Vec<String>,
490    pub unit_weeks: Vec<String>,
491    pub ordinals: Vec<String>,
492}
493
494/// The spam filter's tokenizer (`spam::tokenize`). A table keeps the words it
495/// was trained with, and reads every message with them (`spam::table::Meta`).
496#[derive(Debug, Clone, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
497#[serde(default)]
498pub struct SpamWords {
499    /// A provider's spam mark at the start of a subject, taken off first.
500    pub provider_tags: Vec<String>,
501    pub months: Vec<String>,
502    pub keys: Vec<String>,
503    pub elisions: Vec<String>,
504    pub number_words: Vec<String>,
505    pub stop_words: Vec<String>,
506}
507
508/// Reading scans.
509#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
510#[serde(default)]
511pub struct OcrWords {
512    /// Tesseract's names of the languages' models.
513    pub tesseract: Vec<String>,
514}
515
516/// Every list Sioul looks for, as the packs in use and your changes make them.
517#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
518#[serde(default)]
519pub struct Words {
520    pub codes: Codes,
521    pub approvals: Approvals,
522    pub money: MoneyWords,
523    pub payments: Payments,
524    pub bank: BankWords,
525    pub accounts: AccountWords,
526    pub letters: LetterWords,
527    pub papers: PaperWords,
528    pub contracts: ContractWords,
529    pub senders: SenderWords,
530    pub replies: Replies,
531    pub quotes: Quotes,
532    pub folders: FolderWords,
533    pub brands: BrandWords,
534    pub calls: CallWords,
535    pub voicemail: VoicemailWords,
536    pub shield: ShieldWords,
537    pub tasks: TaskWords,
538    pub capture: CaptureWords,
539    pub spam: SpamWords,
540    pub ocr: OcrWords,
541    /// The languages whose packs made these words.
542    #[serde(skip)]
543    pub languages: Vec<String>,
544    /// The countries whose packs made these words.
545    #[serde(skip)]
546    pub countries: Vec<String>,
547}
548
549// ---------------------------------------------------------------- the packs
550
551/// A pack's lists, or a list's words: tables keep their order, as the file writes them.
552#[derive(Debug, Clone, PartialEq)]
553pub enum Node {
554    List(Vec<String>),
555    Table(Vec<(String, Node)>),
556}
557
558impl Node {
559    fn table() -> Node {
560        Node::Table(Vec::new())
561    }
562
563    /// The node at a dotted path ("codes.code"), if there is one.
564    pub fn get(&self, path: &str) -> Option<&Node> {
565        path.split('.').filter(|p| !p.is_empty()).try_fold(self, |node, part| match node {
566            Node::Table(entries) => entries.iter().find(|(k, _)| k == part).map(|(_, n)| n),
567            Node::List(_) => None,
568        })
569    }
570
571    /// The words of the list at a dotted path; none when there is no such list.
572    pub fn list(&self, path: &str) -> &[String] {
573        match self.get(path) {
574            Some(Node::List(words)) => words,
575            _ => &[],
576        }
577    }
578
579    fn entry_mut(&mut self, key: &str) -> Option<&mut Node> {
580        match self {
581            Node::Table(entries) => entries.iter_mut().find(|(k, _)| k == key).map(|(_, n)| n),
582            Node::List(_) => None,
583        }
584    }
585
586    /// Every list with its dotted path, in order; a named list's entries as "path.name".
587    pub fn flatten(&self) -> Vec<(String, Vec<String>)> {
588        let mut out = Vec::new();
589        fn walk(node: &Node, path: &str, out: &mut Vec<(String, Vec<String>)>) {
590            match node {
591                Node::List(words) => out.push((path.to_string(), words.clone())),
592                Node::Table(entries) => {
593                    for (key, child) in entries {
594                        let path = if path.is_empty() { key.clone() } else { format!("{path}.{key}") };
595                        walk(child, &path, out);
596                    }
597                }
598            }
599        }
600        walk(self, "", &mut out);
601        out
602    }
603}
604
605fn node_of_value(value: &toml_edit::Value) -> Option<Node> {
606    match value {
607        toml_edit::Value::Array(array) => Some(Node::List(array.iter().filter_map(|v| v.as_str().map(str::to_string)).collect())),
608        toml_edit::Value::InlineTable(table) => Some(Node::Table(table.iter().filter_map(|(k, v)| node_of_value(v).map(|n| (k.to_string(), n))).collect())),
609        // A pack's own fields ("language", "checked") are no list.
610        _ => None,
611    }
612}
613
614fn node_of_item(item: &toml_edit::Item) -> Option<Node> {
615    match item {
616        toml_edit::Item::Table(table) => Some(Node::Table(table.iter().filter_map(|(k, v)| node_of_item(v).map(|n| (k.to_string(), n))).collect())),
617        toml_edit::Item::Value(value) => node_of_value(value),
618        _ => None,
619    }
620}
621
622/// One pack: what it is for, when it was checked, its lists.
623#[derive(Debug, Clone)]
624pub struct Pack {
625    /// Its file under `data/words/`: "fr.toml", "countries/FR.toml", "international.toml".
626    pub file: String,
627    /// "2026-10-08": the newest copy wins.
628    pub checked: String,
629    pub lists: Node,
630}
631
632impl Pack {
633    /// A pack's text read; none when it is no TOML.
634    pub fn parse(file: &str, text: &str) -> Option<Pack> {
635        let doc: toml_edit::DocumentMut = text.parse().ok()?;
636        let checked = doc.get("checked").and_then(toml_edit::Item::as_str).unwrap_or("").to_string();
637        let lists = node_of_item(doc.as_item()).unwrap_or_else(Node::table);
638        Some(Pack { file: file.to_string(), checked, lists })
639    }
640
641    /// The language a language pack is for ("fr"), none for the others.
642    pub fn language(&self) -> Option<&str> {
643        (!self.file.contains('/') && self.file != "international.toml").then(|| self.file.trim_end_matches(".toml"))
644    }
645
646    /// The country a country pack is for ("FR"), none for the others.
647    pub fn country(&self) -> Option<&str> {
648        self.file.strip_prefix("countries/").map(|f| f.trim_end_matches(".toml"))
649    }
650}
651
652/// The packs built in, as shipped: no copy on disk read.
653fn shipped() -> &'static [Pack] {
654    static SHIPPED_PACKS: LazyLock<Vec<Pack>> = LazyLock::new(|| SHIPPED.iter().filter_map(|(file, text)| Pack::parse(file, text)).collect());
655    &SHIPPED_PACKS
656}
657
658/// Where newer packs may be: yours first, then the system's (as `presets::places`).
659fn places() -> Vec<PathBuf> {
660    let mut places = vec![crate::config::data_dir().join("words")];
661    let system = std::env::var_os("XDG_DATA_DIRS").unwrap_or_else(|| if cfg!(windows) { Default::default() } else { "/usr/local/share:/usr/share".into() });
662    places.extend(std::env::split_paths(&system).filter(|d| !d.as_os_str().is_empty()).map(|d| d.join("sioul").join("words")));
663    places
664}
665
666/// The packs, each the newest of the built-in one and the copies on disk, by `checked`.
667fn newest() -> &'static [Pack] {
668    static NEWEST: LazyLock<Vec<Pack>> = LazyLock::new(|| {
669        let places = places();
670        shipped()
671            .iter()
672            .map(|pack| {
673                let mut best = pack.clone();
674                for place in &places {
675                    if let Some(found) = std::fs::read_to_string(place.join(&pack.file)).ok().and_then(|t| Pack::parse(&pack.file, &t))
676                        && found.checked > best.checked
677                    {
678                        best = found;
679                    }
680                }
681                best
682            })
683            .collect()
684    });
685    &NEWEST
686}
687
688/// The languages Sioul has a pack for: "en", "fr".
689pub fn available_languages() -> Vec<&'static str> {
690    shipped().iter().filter_map(Pack::language).collect()
691}
692
693/// The countries Sioul has a pack for: "FR".
694pub fn available_countries() -> Vec<&'static str> {
695    shipped().iter().filter_map(Pack::country).collect()
696}
697
698// ---------------------------------------------------------------- what is in use
699
700fn strings(value: Option<&toml::Value>) -> Option<Vec<String>> {
701    value?.as_array().map(|list| list.iter().filter_map(|v| v.as_str().map(|s| s.trim().to_string())).filter(|s| !s.is_empty()).collect())
702}
703
704fn deduplicated(list: Vec<String>) -> Vec<String> {
705    let mut out: Vec<String> = Vec::with_capacity(list.len());
706    for word in list {
707        if !out.contains(&word) {
708            out.push(word);
709        }
710    }
711    out
712}
713
714/// The languages read in your mail: `[words] languages`, else the interface's and English.
715pub fn languages(config: &Config) -> Vec<String> {
716    let said = strings(config.words.get("languages")).map(|list| list.iter().map(|l| base_language(l)).filter(|l| !l.is_empty()).collect::<Vec<_>>());
717    deduplicated(said.unwrap_or_else(|| {
718        let interface = config.language.clone().unwrap_or_else(crate::i18n::system_language);
719        vec![base_language(&interface), "en".to_string()]
720    }))
721}
722
723fn base_language(code: &str) -> String {
724    code.split(['_', '-', '.', '@']).next().unwrap_or("").trim().to_ascii_lowercase()
725}
726
727/// The countries whose names are read: `[words] countries`, else `[contacts] region`, else the locale's.
728pub fn countries(config: &Config) -> Vec<String> {
729    let said = strings(config.words.get("countries")).map(|list| list.iter().map(|c| c.trim().to_ascii_uppercase()).filter(|c| !c.is_empty()).collect::<Vec<_>>());
730    deduplicated(said.unwrap_or_else(|| {
731        let region = config.contacts.region.as_deref().map(str::trim).filter(|r| !r.is_empty()).map(str::to_ascii_uppercase);
732        let locale = || {
733            ["LC_ALL", "LC_TELEPHONE", "LANG"]
734                .iter()
735                .filter_map(|v| std::env::var(v).ok())
736                .find_map(|v| v.split(['.', '@']).next().and_then(|b| b.split(['_', '-']).nth(1)).map(str::to_ascii_uppercase).filter(|c| c.len() == 2))
737        };
738        region.or_else(locale).into_iter().collect()
739    }))
740}
741
742/// The packs in use, in their order: the languages', the countries', the international one.
743fn in_use<'a>(packs: &'a [Pack], languages: &[String], countries: &[String]) -> Vec<&'a Pack> {
744    let mut out: Vec<&Pack> = Vec::new();
745    for language in languages {
746        out.extend(packs.iter().filter(|p| p.language() == Some(language.as_str())));
747    }
748    for country in countries {
749        out.extend(packs.iter().filter(|p| p.country() == Some(country.as_str())));
750    }
751    out.extend(packs.iter().filter(|p| p.file == "international.toml"));
752    out
753}
754
755/// Each pack's lists added to `into`, in order: a list's words after those already there.
756fn merge(into: &mut Node, from: &Node) {
757    let (Node::Table(target), Node::Table(source)) = (into, from) else { return };
758    for (key, node) in source {
759        match target.iter_mut().find(|(k, _)| k == key) {
760            Some((_, existing)) => match (existing, node) {
761                (Node::List(words), Node::List(more)) => words.extend(more.iter().cloned()),
762                (existing @ Node::Table(_), Node::Table(_)) => merge(existing, node),
763                // A list in one pack, a table in another: the first pack's stays.
764                _ => {}
765            },
766            None => target.push((key.clone(), node.clone())),
767        }
768    }
769}
770
771/// A word as lists compare it: trimmed, lowercase, without its accents (`text::fold`).
772pub fn folded(text: &str) -> String {
773    crate::text::fold(text.trim()).into_iter().collect()
774}
775
776/// Whether a folded text holds one of a list's words as a whole word, as
777/// docs/words.md says words are matched: no letter or digit right before it,
778/// nor right after, where the word itself begins or ends with one ("du" in
779/// "part du 01…", never in "durée"; "de :" before anything). A word added in
780/// `[words]` is trimmed, and needs no spaces. The voicemail's callers' leads
781/// and the sites' call words are read so.
782pub fn holds_word(list: &[String], text: &str) -> bool {
783    let alnum = |c: Option<char>| c.is_some_and(char::is_alphanumeric);
784    list.iter().map(|w| folded(w)).filter(|w| !w.is_empty()).any(|word| {
785        let (starts, ends) = (alnum(word.chars().next()), alnum(word.chars().next_back()));
786        text.match_indices(word.as_str()).any(|(at, _)| (!starts || !alnum(text[..at].chars().next_back())) && (!ends || !alnum(text[at + word.len()..].chars().next())))
787    })
788}
789
790/// Whether a word (a category, a folder's name) is one of a list's, folded.
791pub fn is_named(word: &str, list: &[String]) -> bool {
792    let word = folded(word);
793    !word.is_empty() && list.iter().any(|w| folded(w) == word)
794}
795
796/// Whether one of these words (a task's categories) is one of a list's, folded.
797pub fn named_in(words: &[String], list: &[String]) -> bool {
798    let list: Vec<String> = list.iter().map(|w| folded(w)).filter(|w| !w.is_empty()).collect();
799    words.iter().any(|w| list.contains(&folded(w)))
800}
801
802/// Your changes put on the merged lists: `[words.<list>]` tables with `add`
803/// and `remove`. A list's `add` is words; a named list's `add` is names and
804/// their words, its `remove` names. The shipped words you took away go
805/// first, then yours come: one both taken away and added is there.
806fn apply(into: &mut Node, changes: &toml::Table) {
807    for (key, value) in changes {
808        // `languages`, `countries`: settings of the words, no list.
809        let toml::Value::Table(change) = value else { continue };
810        let is_change = change.contains_key("add") || change.contains_key("remove");
811        if is_change {
812            apply_one(into, key, change);
813        }
814        let deeper: toml::Table = change.iter().filter(|(k, _)| !matches!(k.as_str(), "add" | "remove")).map(|(k, v)| (k.clone(), v.clone())).collect();
815        if deeper.is_empty() {
816            continue;
817        }
818        if into.entry_mut(key).is_none()
819            && let Node::Table(entries) = into
820        {
821            entries.push((key.clone(), Node::table()));
822        }
823        if let Some(child @ Node::Table(_)) = into.entry_mut(key) {
824            apply(child, &deeper);
825        }
826    }
827}
828
829fn apply_one(into: &mut Node, key: &str, change: &toml::Table) {
830    let removed: Vec<String> = strings(change.get("remove")).unwrap_or_default().iter().map(|w| folded(w)).collect();
831    let add = change.get("add");
832    if into.entry_mut(key).is_none() {
833        let Node::Table(entries) = into else { return };
834        let node = match add {
835            Some(toml::Value::Table(_)) => Node::table(),
836            Some(toml::Value::Array(_)) => Node::List(Vec::new()),
837            _ => return,
838        };
839        entries.push((key.to_string(), node));
840    }
841    match (into.entry_mut(key), add) {
842        (Some(Node::List(words)), add) => {
843            words.retain(|w| !removed.contains(&folded(w)));
844            if let Some(add) = strings(add) {
845                words.extend(add);
846            }
847        }
848        (Some(Node::Table(entries)), add) => {
849            entries.retain(|(name, _)| !removed.contains(&folded(name)));
850            if let Some(toml::Value::Table(add)) = add {
851                for (name, words) in add {
852                    let words = strings(Some(words)).unwrap_or_default();
853                    match entries.iter_mut().find(|(n, _)| folded(n) == folded(name)) {
854                        Some((_, Node::List(existing))) => existing.extend(words),
855                        Some(_) => {}
856                        None => entries.push((name.clone(), Node::List(words))),
857                    }
858                }
859            }
860        }
861        _ => {}
862    }
863}
864
865/// Each list's words once, compared folded, the first spelling kept; empty words dropped.
866fn tidy(node: &mut Node) {
867    match node {
868        Node::List(words) => {
869            let mut seen: Vec<String> = Vec::with_capacity(words.len());
870            words.retain(|w| {
871                let key = folded(w);
872                let keep = !key.is_empty() && !seen.contains(&key);
873                seen.push(key);
874                keep
875            });
876        }
877        Node::Table(entries) => entries.iter_mut().for_each(|(_, n)| tidy(n)),
878    }
879}
880
881/// The merged lists as the typed structs read them: a named list as pairs, in order.
882fn value_of(node: &Node, path: &str) -> toml::Value {
883    match node {
884        Node::List(words) => toml::Value::Array(words.iter().cloned().map(toml::Value::String).collect()),
885        Node::Table(entries) if MAPS.contains(&path) => toml::Value::Array(
886            entries
887                .iter()
888                .filter_map(|(name, n)| match n {
889                    Node::List(words) => Some(toml::Value::Array(vec![toml::Value::String(name.clone()), toml::Value::Array(words.iter().cloned().map(toml::Value::String).collect())])),
890                    Node::Table(_) => None,
891                })
892                .collect(),
893        ),
894        Node::Table(entries) => toml::Value::Table(
895            entries
896                .iter()
897                .map(|(key, n)| {
898                    let child = if path.is_empty() { key.clone() } else { format!("{path}.{key}") };
899                    (key.clone(), value_of(n, &child))
900                })
901                .collect(),
902        ),
903    }
904}
905
906static BUILTIN: LazyLock<Arc<Words>> = LazyLock::new(|| {
907    let (languages, countries) = (FIRST_LANGUAGES.map(String::from).to_vec(), FIRST_COUNTRIES.map(String::from).to_vec());
908    Arc::new(typed(&lists(&in_use(shipped(), &languages, &countries), None), languages, countries))
909});
910
911/// The lists made from these packs, with these changes (none: the packs alone).
912fn lists(packs: &[&Pack], changes: Option<&toml::Table>) -> Node {
913    let mut tree = Node::table();
914    for pack in packs {
915        merge(&mut tree, &pack.lists);
916    }
917    if let Some(changes) = changes {
918        apply(&mut tree, changes);
919    }
920    tidy(&mut tree);
921    tree
922}
923
924fn typed(tree: &Node, languages: Vec<String>, countries: Vec<String>) -> Words {
925    let mut words: Words = value_of(tree, "").try_into().unwrap_or_else(|e| {
926        eprintln!("sioul: the words: {e}");
927        Words::default()
928    });
929    words.languages = languages;
930    words.countries = countries;
931    words
932}
933
934impl Words {
935    /// The built-in packs alone, for the languages and countries every list
936    /// was written for until 8 October 2026 (French and English, France):
937    /// what tests and callers without a configuration read.
938    pub fn builtin() -> Arc<Words> {
939        BUILTIN.clone()
940    }
941
942    /// The same, borrowed for the program's life: where a context gives no words.
943    pub fn builtin_ref() -> &'static Words {
944        &BUILTIN
945    }
946
947    /// The words as your configuration makes them: the packs in use, the
948    /// newest of each, then your changes. Made once for each configuration.
949    pub fn of(config: &Config) -> Arc<Words> {
950        type Cache = Mutex<Vec<(String, Arc<Words>)>>;
951        static CACHE: LazyLock<Cache> = LazyLock::new(|| Mutex::new(Vec::new()));
952        let (languages, countries) = (languages(config), countries(config));
953        // Keyed by all it is made from: the older settings too (`older_settings`),
954        // else two configurations alike but for `filed_words` shared one list.
955        let key = format!("{languages:?}|{countries:?}|{}|{:?}|{:?}|{:?}", toml::to_string(&config.words).unwrap_or_default(), config.filed_words, config.quiet.personal, config.quiet.work);
956        if let Ok(cache) = CACHE.lock()
957            && let Some((_, words)) = cache.iter().find(|(k, _)| *k == key)
958        {
959            return words.clone();
960        }
961        let tree = lists(&in_use(newest(), &languages, &countries), Some(&config.words));
962        let mut words = typed(&tree, languages, countries);
963        older_settings(config, &mut words);
964        let words = Arc::new(words);
965        if let Ok(mut cache) = CACHE.lock() {
966            cache.retain(|(k, _)| *k != key);
967            cache.insert(0, (key, words.clone()));
968            cache.truncate(4);
969        }
970        words
971    }
972}
973
974/// The settings that held whole lists before `[words]` (`filed_words`,
975/// `[quiet] personal` and `work`): while written, and while `[words]` says
976/// nothing of the same list, they are the list, as they were.
977fn older_settings(config: &Config, words: &mut Words) {
978    let changed = |recogniser: &str, list: &str| config.words.get(recogniser).and_then(|r| r.get(list)).is_some();
979    if let Some(filed) = config.filed_words.as_ref().filter(|f| !f.is_empty())
980        && !changed("senders", "automatic")
981    {
982        words.senders.automatic = filed.clone();
983    }
984    if let Some(personal) = &config.quiet.personal
985        && !changed("tasks", "personal")
986    {
987        words.tasks.personal = personal.clone();
988    }
989    if let Some(work) = &config.quiet.work
990        && !changed("tasks", "work")
991    {
992        words.tasks.work = work.clone();
993    }
994}
995
996/// The words of the configuration this program loaded last (`set_current`).
997static CURRENT: std::sync::RwLock<Option<Arc<Words>>> = std::sync::RwLock::new(None);
998
999/// The words of the configuration this program loaded last, else the
1000/// built-in ones: for the few readers deep in other crates that see no
1001/// configuration (a server's folder names, `folders::role_of`). The window
1002/// and the command line say them (`set_current`) when they read the
1003/// configuration; tests never do, so they read the built-in ones.
1004pub fn current() -> Arc<Words> {
1005    CURRENT.read().ok().and_then(|c| c.clone()).unwrap_or_else(Words::builtin)
1006}
1007
1008/// Says which words this program uses now (`current`).
1009pub fn set_current(words: Arc<Words>) {
1010    if let Ok(mut current) = CURRENT.write() {
1011        *current = Some(words);
1012    }
1013}
1014
1015/// The packs built in, with one payment processor's wording added as its
1016/// users would add it under `[words]` (the shipped packs leave it out, as
1017/// one person's): what the tests of payments read.
1018#[cfg(test)]
1019pub(crate) fn with_processor() -> Arc<Words> {
1020    let config: Config = toml::from_str(
1021        "[words.payments.paid]\nadd = [\"recu pour votre paiement\", \"recu de votre paiement\", \"vous avez autorise un paiement\", \"vous avez envoye un paiement\"]\n\
1022         [words.payments.not_payments]\nadd = [\"rapport d'activite\"]\n[words.payments.to]\nadd = [\"paiement a\"]\n[words.payments.from]\nadd = [\"paiement de\"]\n\
1023         [words.payments.leads]\nadd = [\"en faveur de\"]\n[words.money.stated]\nadd = [\"vous avez autorise un paiement de\"]\n",
1024    )
1025    .unwrap_or_default();
1026    let (languages, countries) = (FIRST_LANGUAGES.map(String::from).to_vec(), FIRST_COUNTRIES.map(String::from).to_vec());
1027    Arc::new(typed(&lists(&in_use(shipped(), &languages, &countries), Some(&config.words)), languages, countries))
1028}
1029
1030/// The lists in use, as dotted paths: the packs' alone (`shipped`) and with
1031/// your changes (`yours`), for the settings that show what you changed.
1032pub struct Compared {
1033    pub shipped: Node,
1034    pub yours: Node,
1035}
1036
1037/// The lists of a configuration, shipped and changed.
1038pub fn compared(config: &Config) -> Compared {
1039    let packs = in_use(newest(), &languages(config), &countries(config));
1040    Compared { shipped: lists(&packs, None), yours: lists(&packs, Some(&config.words)) }
1041}
1042
1043// ---------------------------------------------------------------- writing your changes
1044
1045/// The table of a list's changes in the configuration (`[words.codes.code]`),
1046/// made with the tables above it, those implicit, so the file says only it.
1047fn changes_table<'a>(doc: &'a mut toml_edit::DocumentMut, path: &str) -> Option<&'a mut toml_edit::Table> {
1048    let mut table = doc.as_table_mut();
1049    if !table.contains_key("words") {
1050        let mut words = toml_edit::Table::new();
1051        words.set_implicit(true);
1052        table.insert("words", toml_edit::Item::Table(words));
1053    }
1054    table = table.get_mut("words")?.as_table_mut()?;
1055    for part in path.split('.').filter(|p| !p.is_empty()) {
1056        if !table.contains_key(part) {
1057            let mut next = toml_edit::Table::new();
1058            next.set_implicit(true);
1059            table.insert(part, toml_edit::Item::Table(next));
1060        }
1061        table = table.get_mut(part)?.as_table_mut()?;
1062    }
1063    Some(table)
1064}
1065
1066/// Takes a list's changes out, and the tables above them left empty.
1067fn drop_changes(doc: &mut toml_edit::DocumentMut, path: &str) {
1068    fn prune(table: &mut toml_edit::Table, parts: &[&str]) {
1069        let Some((first, rest)) = parts.split_first() else { return };
1070        let Some(child) = table.get_mut(first).and_then(toml_edit::Item::as_table_mut) else { return };
1071        if rest.is_empty() {
1072            child.remove("add");
1073            child.remove("remove");
1074        } else {
1075            prune(child, rest);
1076        }
1077        if child.is_empty() {
1078            table.remove(first);
1079        }
1080    }
1081    let parts: Vec<&str> = std::iter::once("words").chain(path.split('.').filter(|p| !p.is_empty())).collect();
1082    prune(doc.as_table_mut(), &parts);
1083}
1084
1085fn array(words: &[String]) -> toml_edit::Item {
1086    toml_edit::value(words.iter().map(String::as_str).collect::<toml_edit::Array>())
1087}
1088
1089/// Writes a list's changes so that it holds `wanted`, compared with what the
1090/// packs alone give (`shipped`): the words not shipped go to `add`, the
1091/// shipped ones not wanted to `remove`. Wanting the shipped list takes the
1092/// changes out. A named list (`MAPS`) is not written here: see `write_names`.
1093pub fn write_list(config_path: &Path, path: &str, wanted: &[String], shipped: &[String]) -> Result<(), String> {
1094    // Read, changed and written under the file's lock, which the sharing takes too.
1095    crate::filelock::with_lock(config_path, || {
1096        let shipped_folded: Vec<String> = shipped.iter().map(|w| folded(w)).collect();
1097        let wanted_folded: Vec<String> = wanted.iter().map(|w| folded(w)).collect();
1098        let add: Vec<String> = deduplicated(wanted.iter().filter(|w| !folded(w).is_empty() && !shipped_folded.contains(&folded(w))).map(|w| w.trim().to_string()).collect());
1099        let remove: Vec<String> = shipped.iter().filter(|w| !wanted_folded.contains(&folded(w))).cloned().collect();
1100        let mut doc = crate::config::read_document(config_path)?;
1101        drop_changes(&mut doc, path);
1102        if !add.is_empty() || !remove.is_empty() {
1103            let table = changes_table(&mut doc, path).ok_or_else(|| format!("{}: words.{path} is not a table", config_path.display()))?;
1104            table.set_implicit(false);
1105            if !add.is_empty() {
1106                table.insert("add", array(&add));
1107            }
1108            if !remove.is_empty() {
1109                table.insert("remove", array(&remove));
1110            }
1111        }
1112        crate::config::write_document(config_path, &doc)
1113    })
1114}
1115
1116/// Takes every change of a recogniser out ("codes": `[words.codes…]`): back to the shipped lists.
1117pub fn reset(config_path: &Path, recogniser: &str) -> Result<(), String> {
1118    // Read, changed and written under the file's lock, which the sharing takes too.
1119    crate::filelock::with_lock(config_path, || {
1120        let mut doc = crate::config::read_document(config_path)?;
1121        if let Some(words) = doc.get_mut("words").and_then(toml_edit::Item::as_table_mut) {
1122            words.remove(recogniser);
1123            if words.is_empty() {
1124                doc.remove("words");
1125            }
1126        }
1127        crate::config::write_document(config_path, &doc)
1128    })
1129}
1130
1131/// Takes the changes of these lists out ("folders.trash"…): one line of the
1132/// words tab back to the shipped lists, the other lists of its recognisers kept.
1133pub fn reset_lists(config_path: &Path, paths: &[&str]) -> Result<(), String> {
1134    // Read, changed and written under the file's lock, which the sharing takes too.
1135    crate::filelock::with_lock(config_path, || {
1136        let mut doc = crate::config::read_document(config_path)?;
1137        for path in paths {
1138            drop_changes(&mut doc, path);
1139        }
1140        crate::config::write_document(config_path, &doc)
1141    })
1142}
1143
1144/// Writes a named list's changes (`MAPS`: brands and their domains, bodies
1145/// and their words) so that it holds `wanted`, compared with what the packs
1146/// alone give (`shipped`), names and words folded: a name not shipped goes to
1147/// `add` with its words; a shipped name not wanted to `remove`; a shipped name
1148/// given more words, those words to `add`; a shipped name given fewer, to
1149/// `remove` and back in `add` with the words wanted (added wins). Wanting the
1150/// shipped list takes the changes out.
1151pub fn write_names(config_path: &Path, path: &str, wanted: &[(String, Vec<String>)], shipped: &[(String, Vec<String>)]) -> Result<(), String> {
1152    // Read, changed and written under the file's lock, which the sharing takes too.
1153    crate::filelock::with_lock(config_path, || {
1154        let folded_all = |words: &[String]| words.iter().map(|w| folded(w)).filter(|w| !w.is_empty()).collect::<Vec<_>>();
1155        let trimmed = |words: &[String]| deduplicated(words.iter().map(|w| w.trim().to_string()).filter(|w| !w.is_empty()).collect());
1156        let mut add: Vec<(String, Vec<String>)> = Vec::new();
1157        let mut remove: Vec<String> = Vec::new();
1158        for (name, words) in shipped {
1159            let Some((_, wanted_words)) = wanted.iter().find(|(n, _)| folded(n) == folded(name)) else {
1160                remove.push(name.clone());
1161                continue;
1162            };
1163            let (have, want) = (folded_all(words), folded_all(wanted_words));
1164            if have.iter().all(|w| want.contains(w)) {
1165                let more: Vec<String> = trimmed(wanted_words).into_iter().filter(|w| !have.contains(&folded(w))).collect();
1166                if !more.is_empty() {
1167                    add.push((name.clone(), more));
1168                }
1169            } else {
1170                remove.push(name.clone());
1171                add.push((name.clone(), trimmed(wanted_words)));
1172            }
1173        }
1174        for (name, words) in wanted {
1175            if !folded(name).is_empty() && !shipped.iter().any(|(n, _)| folded(n) == folded(name)) && !add.iter().any(|(n, _)| folded(n) == folded(name)) {
1176                add.push((name.trim().to_string(), trimmed(words)));
1177            }
1178        }
1179        let mut doc = crate::config::read_document(config_path)?;
1180        drop_changes(&mut doc, path);
1181        if !add.is_empty() || !remove.is_empty() {
1182            let table = changes_table(&mut doc, path).ok_or_else(|| format!("{}: words.{path} is not a table", config_path.display()))?;
1183            table.set_implicit(false);
1184            if !add.is_empty() {
1185                let mut names = toml_edit::InlineTable::new();
1186                for (name, words) in &add {
1187                    names.insert(name.as_str(), toml_edit::Value::Array(words.iter().map(String::as_str).collect()));
1188                }
1189                table.insert("add", toml_edit::value(names));
1190            }
1191            if !remove.is_empty() {
1192                table.insert("remove", array(&remove));
1193            }
1194        }
1195        crate::config::write_document(config_path, &doc)
1196    })
1197}
1198
1199/// Writes `[words] languages` (codes, "fr") or `countries` ("FR"), the order kept.
1200pub fn write_setting(config_path: &Path, key: &str, values: &[String]) -> Result<(), String> {
1201    // Read, changed and written under the file's lock, which the sharing takes too.
1202    crate::filelock::with_lock(config_path, || {
1203        if !matches!(key, "languages" | "countries") {
1204            return Err(format!("words.{key}: no such setting"));
1205        }
1206        let mut doc = crate::config::read_document(config_path)?;
1207        let table = changes_table(&mut doc, "").ok_or_else(|| format!("{}: words is not a table", config_path.display()))?;
1208        table.set_implicit(false);
1209        table.insert(key, array(&deduplicated(values.to_vec())));
1210        crate::config::write_document(config_path, &doc)
1211    })
1212}
1213
1214/// For a configuration older than these words (a config.toml without a
1215/// `[words]` table, written before 8 October 2026): writes the languages
1216/// and the country every list was written for until then, so that nothing
1217/// changes until you change it. Returns whether it wrote.
1218pub fn keep_first(config_path: &Path) -> Result<bool, String> {
1219    // Looked at and written under the file's lock, which the sharing takes too.
1220    crate::filelock::with_lock(config_path, || {
1221        let Ok(text) = std::fs::read_to_string(config_path) else { return Ok(false) };
1222        let doc: toml_edit::DocumentMut = text.parse().map_err(|e| format!("{}: {e}", config_path.display()))?;
1223        if doc.contains_key("words") {
1224            return Ok(false);
1225        }
1226        write_setting(config_path, "languages", &FIRST_LANGUAGES.map(String::from))?;
1227        write_setting(config_path, "countries", &FIRST_COUNTRIES.map(String::from))?;
1228        Ok(true)
1229    })
1230}
1231
1232/// `keep_first`, once on each device: at the first start of a Sioul with these
1233/// words. A configuration from before keeps its French, English and France;
1234/// one made afterwards (a new person's, by this Sioul) is left to its
1235/// interface's language and English (`languages`). `marker` says it was done.
1236pub fn keep_first_once(config_path: &Path, marker: &Path) -> Result<bool, String> {
1237    if marker.exists() {
1238        return Ok(false);
1239    }
1240    let wrote = keep_first(config_path)?;
1241    if let Some(parent) = marker.parent() {
1242        std::fs::create_dir_all(parent).map_err(|e| format!("{}: {e}", parent.display()))?;
1243    }
1244    std::fs::write(marker, b"").map_err(|e| format!("{}: {e}", marker.display()))?;
1245    Ok(wrote)
1246}
1247
1248#[cfg(test)]
1249mod tests {
1250    use super::*;
1251
1252    fn config(text: &str) -> Config {
1253        toml::from_str(text).unwrap()
1254    }
1255
1256    /// Every list the typed structs know, as dotted paths, and the named lists.
1257    fn known() -> Vec<String> {
1258        let value = toml::Value::try_from(Words::default()).unwrap();
1259        fn walk(value: &toml::Value, path: &str, out: &mut Vec<String>) {
1260            match value {
1261                toml::Value::Table(table) => {
1262                    for (key, child) in table {
1263                        walk(child, &if path.is_empty() { key.clone() } else { format!("{path}.{key}") }, out);
1264                    }
1265                }
1266                _ => out.push(path.to_string()),
1267            }
1268        }
1269        let mut out = Vec::new();
1270        walk(&value, "", &mut out);
1271        out
1272    }
1273
1274    #[test]
1275    fn every_pack_reads_and_holds_known_lists_only() {
1276        let known = known();
1277        for (file, text) in SHIPPED {
1278            let pack = Pack::parse(file, text).unwrap_or_else(|| panic!("{file} is no TOML"));
1279            assert_eq!(pack.checked.len(), 10, "{file}: its checked date");
1280            for (path, words) in pack.lists.flatten() {
1281                let map = MAPS.iter().find(|m| path.starts_with(&format!("{m}.")));
1282                assert!(known.contains(&path) || map.is_some(), "{file}: {path} is no list Sioul reads");
1283                for word in &words {
1284                    assert!(!folded(word).is_empty(), "{file}: {path} holds an empty word");
1285                }
1286            }
1287        }
1288        assert_eq!(available_languages(), ["en", "fr", "de", "es", "it"]);
1289        assert_eq!(available_countries(), ["FR"]);
1290    }
1291
1292    /// The packs merged give every list back as the code held it on 8 October
1293    /// 2026 (v0.0.3), a person's own words left out (tests/fixtures/words-golden.json).
1294    #[test]
1295    fn the_packs_give_back_the_lists_of_before() {
1296        let golden: serde_json::Value = serde_json::from_str(include_str!("../tests/fixtures/words-golden.json")).unwrap();
1297        let golden = golden["lists"].as_object().unwrap();
1298        let (languages, countries) = (FIRST_LANGUAGES.map(String::from).to_vec(), FIRST_COUNTRIES.map(String::from).to_vec());
1299        let tree = lists(&in_use(shipped(), &languages, &countries), None);
1300        let ours: std::collections::BTreeMap<String, Vec<String>> = tree.flatten().into_iter().collect();
1301        let set = |words: &[String]| words.iter().map(|w| folded(w)).collect::<std::collections::BTreeSet<_>>();
1302        for (path, words) in golden {
1303            let words: Vec<String> = words.as_array().unwrap().iter().map(|w| w.as_str().unwrap().to_string()).collect();
1304            let ours_path = if path == "ocr.languages" { "ocr.tesseract" } else { path.as_str() };
1305            let found = ours.get(ours_path).cloned().unwrap_or_default();
1306            assert_eq!(set(&found), set(&words), "{path}");
1307        }
1308        // And nothing more: every list of the packs is one of before.
1309        for path in ours.keys() {
1310            let before = if path == "ocr.tesseract" { "ocr.languages" } else { path.as_str() };
1311            assert!(golden.contains_key(before), "{path} is new");
1312        }
1313    }
1314
1315    #[test]
1316    fn typed_lists_and_named_lists_in_their_order() {
1317        let words = Words::builtin();
1318        assert!(words.codes.code.iter().any(|w| w == "code de verification") && words.codes.code.iter().any(|w| w == "verification code"));
1319        assert_eq!(words.letters.months.iter().find(|(n, _)| n == "10").map(|(_, w)| w.len()), Some(3));
1320        // Named lists keep the packs' order: France's names, then the international ones.
1321        let brands: Vec<&str> = words.brands.brands.iter().map(|(n, _)| n.as_str()).collect();
1322        assert_eq!(brands.first(), Some(&"Ameli"));
1323        assert!(brands.contains(&"Amazon") && brands.contains(&"Boursorama"));
1324        assert_eq!(words.capture.in_, ["dans", "in"]);
1325        assert_eq!(words.ocr.tesseract, ["fra", "eng"]);
1326        assert_eq!((words.languages.as_slice(), words.countries.as_slice()), (&["fr".to_string(), "en".into()][..], &["FR".to_string()][..]));
1327    }
1328
1329    #[test]
1330    fn your_changes_merge_with_the_packs() {
1331        let config = config(
1332            "[words]\nlanguages = [\"fr\", \"en\"]\ncountries = [\"FR\"]\n\n\
1333             [words.codes.code]\nadd = [\"Bestätigungscode\", \"código de verificación\"]\nremove = [\"Code Temporaire\"]\n\n\
1334             [words.folders.trash]\nadd = [\"Papierkorb\"]\nremove = [\"bin\", \"corbeille\"]\n\n\
1335             [words.senders.automatic]\nadd = [\"noreply\", \"nicht-antworten\"]\nremove = [\"noreply\"]\n\n\
1336             [words.brands.brands]\nadd = { \"Ma Banque\" = [\"mabanque.example\"], Amazon = [\"amazon.example\"] }\nremove = [\"boursorama\"]\n\n\
1337             [words.letters.senders.\"Ma Caisse\"]\nadd = [\"caisse exemple\"]\n\n\
1338             [words.newer.list]\nadd = [\"anything\"]\n",
1339        );
1340        let words = Words::of(&config);
1341        let has = |list: &[String], w: &str| list.iter().any(|x| x == w);
1342        assert!(has(&words.codes.code, "Bestätigungscode") && has(&words.codes.code, "código de verificación"));
1343        assert!(!has(&words.codes.code, "code temporaire"), "taken away, capitals aside");
1344        assert!(has(&words.folders.trash, "Papierkorb") && !has(&words.folders.trash, "bin") && !has(&words.folders.trash, "corbeille"));
1345        assert!(has(&words.folders.trash, "trash"));
1346        assert!(has(&words.senders.automatic, "noreply"), "taken away and added: put back");
1347        assert_eq!(words.senders.automatic.iter().filter(|w| *w == "noreply").count(), 1);
1348        let brand = |name: &str| words.brands.brands.iter().find(|(n, _)| n == name).map(|(_, d)| d.clone());
1349        assert_eq!(brand("Ma Banque"), Some(vec!["mabanque.example".to_string()]));
1350        assert!(brand("Amazon").unwrap().contains(&"amazon.example".to_string()) && brand("Amazon").unwrap().contains(&"amazon.fr".to_string()));
1351        assert_eq!(brand("Boursorama"), None, "a named list's entry taken away by its name, capitals aside");
1352        assert!(words.letters.senders.iter().any(|(n, w)| n == "Ma Caisse" && w == &["caisse exemple"]));
1353        // The rest as shipped.
1354        assert_eq!(words.codes.reset, Words::builtin().codes.reset);
1355    }
1356
1357    #[test]
1358    fn languages_and_countries_in_use() {
1359        let english = config("language = \"en\"\n[words]\nlanguages = [\"en\"]\ncountries = []\n");
1360        let words = Words::of(&english);
1361        assert!(words.codes.code.iter().all(|w| !w.starts_with("code de")), "no French");
1362        assert!(words.brands.brands.iter().all(|(n, _)| n != "Ameli"), "no French names");
1363        assert!(words.brands.brands.iter().any(|(n, _)| n == "Amazon"), "the international pack, always");
1364        assert!(words.folders.sent.iter().any(|w| w == "envoyés"), "folders' names, whatever you read");
1365        assert_eq!(words.ocr.tesseract, ["eng"]);
1366        let unsaid = config("language = \"fr\"\n[contacts]\nregion = \"fr\"\n");
1367        assert_eq!((languages(&unsaid), countries(&unsaid)), (vec!["fr".to_string(), "en".into()], vec!["FR".to_string()]));
1368        let english_first = config("language = \"en_GB\"\n[words]\ncountries = [\"fr\", \"FR\"]\n");
1369        assert_eq!((languages(&english_first), countries(&english_first)), (vec!["en".to_string()], vec!["FR".to_string()]));
1370    }
1371
1372    /// German, Spanish and Italian, once read: a code, the bin's name, a day typed; none of it in English alone.
1373    #[test]
1374    fn german_spanish_and_italian_once_read() {
1375        let today: jiff::civil::Date = "2026-10-03".parse().unwrap();
1376        for (language, subject, body, bin, friday) in [
1377            ("de", "Ihr Bestätigungscode", "Ihr Bestätigungscode lautet 482913. Er ist 10 Minuten gültig.", "Papierkorb", "Freitag"),
1378            ("es", "Tu código de verificación", "Tu código de verificación es 482913. Caduca en 10 minutos.", "Papelera", "viernes"),
1379            ("it", "Il tuo codice di verifica", "Il tuo codice di verifica è 482913. Scade tra 10 minuti.", "Cestino", "venerdi"),
1380        ] {
1381            let words = Words::of(&config(&format!("[words]\nlanguages = [\"{language}\"]\ncountries = []\n")));
1382            let found = crate::codes::detect(&words.codes, subject, body).unwrap_or_else(|| panic!("{language}: no code"));
1383            assert_eq!(found.code.as_deref(), Some("482913"), "{language}");
1384            assert_eq!(crate::folders::role_in(&words.folders, bin, None), crate::folders::Role::Trash, "{language}");
1385            let day = crate::capture::parse_day_with(&words.capture, friday, today).map(|d| d.to_string());
1386            assert_eq!(day.as_deref(), Some("2026-10-09"), "{language}");
1387            assert_eq!(words.ocr.tesseract.len(), 1, "{language}");
1388        }
1389        let english = Words::of(&config("[words]\nlanguages = [\"en\"]\ncountries = []\n"));
1390        assert!(crate::codes::detect(&english.codes, "Ihr Bestätigungscode", "Ihr Bestätigungscode lautet 482913.").is_none());
1391        assert_eq!(crate::folders::role_in(&english.folders, "Papierkorb", None), crate::folders::Role::Other);
1392    }
1393
1394    fn scratch(name: &str) -> PathBuf {
1395        let dir = std::env::temp_dir().join(format!("sioul-words-{name}-{}", std::process::id()));
1396        let _ = std::fs::remove_dir_all(&dir);
1397        std::fs::create_dir_all(&dir).unwrap();
1398        dir.join("config.toml")
1399    }
1400
1401    #[test]
1402    fn writing_keeps_only_the_differences() {
1403        let path = scratch("write");
1404        std::fs::write(&path, "# Mine\nfetch_minutes = 15\n\n[mail]\nthreads = true # by conversation\n").unwrap();
1405        let shipped = Words::builtin().folders.trash.clone();
1406        let mut wanted: Vec<String> = shipped.iter().filter(|w| *w != "bin").cloned().collect();
1407        wanted.push("Papierkorb".into());
1408        write_list(&path, "folders.trash", &wanted, &shipped).unwrap();
1409        let text = std::fs::read_to_string(&path).unwrap();
1410        assert!(text.contains("# Mine") && text.contains("# by conversation"), "comments kept: {text}");
1411        assert!(text.contains("[words.folders.trash]") && !text.contains("[words]\n") && !text.contains("[words.folders]\n"), "{text}");
1412        let config = Config::load(&path).unwrap();
1413        let words = Words::of(&config);
1414        assert!(words.folders.trash.contains(&"Papierkorb".to_string()) && !words.folders.trash.contains(&"bin".to_string()));
1415        // Back to the shipped list: no change left, no empty table.
1416        write_list(&path, "folders.trash", &shipped, &shipped).unwrap();
1417        let text = std::fs::read_to_string(&path).unwrap();
1418        assert!(!text.contains("words"), "{text}");
1419        // Languages written, then reset of a recogniser.
1420        write_setting(&path, "languages", &["fr".into(), "en".into()]).unwrap();
1421        write_list(&path, "codes.code", &["mon code".into()], &[]).unwrap();
1422        reset(&path, "codes").unwrap();
1423        let config = Config::load(&path).unwrap();
1424        assert_eq!(languages(&config), ["fr", "en"]);
1425        assert!(!std::fs::read_to_string(&path).unwrap().contains("codes"));
1426        let _ = std::fs::remove_dir_all(path.parent().unwrap());
1427    }
1428
1429    #[test]
1430    fn an_older_configuration_keeps_what_it_read() {
1431        let path = scratch("first");
1432        std::fs::write(&path, "language = \"en\"\n").unwrap();
1433        assert!(keep_first(&path).unwrap());
1434        let config = Config::load(&path).unwrap();
1435        assert_eq!((languages(&config), countries(&config)), (vec!["fr".to_string(), "en".into()], vec!["FR".to_string()]));
1436        assert!(!keep_first(&path).unwrap(), "once");
1437        let none = path.with_file_name("none.toml");
1438        assert!(!keep_first(&none).unwrap(), "no configuration, nothing written");
1439        let _ = std::fs::remove_dir_all(path.parent().unwrap());
1440    }
1441
1442    /// Once on each device: a configuration made after the first start keeps its interface's language.
1443    #[test]
1444    fn the_first_start_keeps_an_older_configuration_once() {
1445        let path = scratch("once");
1446        let marker = path.with_file_name("state").join("words-first-start");
1447        std::fs::write(&path, "language = \"en\"\n").unwrap();
1448        assert!(keep_first_once(&path, &marker).unwrap());
1449        assert!(marker.exists());
1450        std::fs::write(&path, "language = \"en\"\n").unwrap();
1451        assert!(!keep_first_once(&path, &marker).unwrap());
1452        assert!(!std::fs::read_to_string(&path).unwrap().contains("words"));
1453        // A new device: no configuration yet, the marker written all the same.
1454        let new = path.with_file_name("new.toml");
1455        let new_marker = path.with_file_name("new-state").join("words-first-start");
1456        assert!(!keep_first_once(&new, &new_marker).unwrap());
1457        assert!(new_marker.exists());
1458        let _ = std::fs::remove_dir_all(path.parent().unwrap());
1459    }
1460
1461    /// A named list is written as its differences: a brand of yours, a domain
1462    /// more, a domain fewer, a brand taken away; back to the shipped list, nothing.
1463    #[test]
1464    fn a_named_list_is_written_as_its_differences() {
1465        let path = scratch("names");
1466        std::fs::write(&path, "[words]\nlanguages = [\"fr\", \"en\"]\ncountries = [\"FR\"]\n").unwrap();
1467        let shipped = Words::builtin().brands.brands.clone();
1468        let mut wanted = shipped.clone();
1469        wanted.retain(|(name, _)| name != "Chronopost");
1470        if let Some((_, domains)) = wanted.iter_mut().find(|(name, _)| name == "Colissimo") {
1471            domains.push("colissimo.example".into());
1472        }
1473        wanted.push(("Ma Banque".into(), vec!["mabanque.example".into()]));
1474        write_names(&path, "brands.brands", &wanted, &shipped).unwrap();
1475        let brands = Words::of(&Config::load(&path).unwrap()).brands.brands.clone();
1476        assert!(brands.iter().any(|(n, d)| n == "Ma Banque" && d == &["mabanque.example".to_string()]), "{brands:?}");
1477        assert!(!brands.iter().any(|(n, _)| n == "Chronopost"));
1478        let colissimo = |brands: &[(String, Vec<String>)]| brands.iter().find(|(n, _)| n == "Colissimo").map(|(_, d)| d.clone()).unwrap_or_default();
1479        assert!(colissimo(&brands).contains(&"colissimo.example".to_string()) && colissimo(&brands).contains(&"laposte.fr".to_string()));
1480        let mut fewer = brands.clone();
1481        if let Some((_, domains)) = fewer.iter_mut().find(|(name, _)| name == "Colissimo") {
1482            domains.retain(|d| d != "laposte.fr");
1483        }
1484        write_names(&path, "brands.brands", &fewer, &shipped).unwrap();
1485        let brands = Words::of(&Config::load(&path).unwrap()).brands.brands.clone();
1486        assert!(!colissimo(&brands).contains(&"laposte.fr".to_string()) && colissimo(&brands).contains(&"colissimo.example".to_string()), "{brands:?}");
1487        write_names(&path, "brands.brands", &shipped, &shipped).unwrap();
1488        assert!(!std::fs::read_to_string(&path).unwrap().contains("brands"));
1489        // A line of lists taken back at once, the languages kept.
1490        write_list(&path, "folders.trash", &["Papierkorb".into()], &Words::builtin().folders.trash).unwrap();
1491        write_list(&path, "folders.sent", &["Gesendet".into()], &Words::builtin().folders.sent).unwrap();
1492        reset_lists(&path, &["folders.trash", "folders.sent"]).unwrap();
1493        let text = std::fs::read_to_string(&path).unwrap();
1494        assert!(!text.contains("folders") && text.contains("languages"), "{text}");
1495        let _ = std::fs::remove_dir_all(path.parent().unwrap());
1496    }
1497}
sioul_core/words.rs

sioul_core/
words.rs
