Files
warmbly/internal/app/replyclassify/languages.go
T
Matthew Meszaros 3a04b3f606 Merge pull request #1074 from 555aaditya/feat/replyclassify-positive-de-fr-es
feat: read interest in German, French and Spanish replies for workspaces that choose those tagging languages
2026-10-31 14:38:53 +00:00

701 lines
31 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package replyclassify
import (
"regexp"
"slices"
"sort"
"strings"
"sync"
"unicode"
"unicode/utf8"
)
// The offline layers read every reply with the base vocabulary below, and a
// workspace's tagging languages (models.MailLanguageNames codes) add theirs.
// A language nobody chose adds no pattern, so it cannot misread anyone's mail.
// The one exception is the opt-out scan, which always cuts the quote markers
// of the languages its phrases are in (optOutLanguages in lexicon.go).
// langRules is one language's vocabulary, as its mail clients and servers
// write it.
type langRules struct {
// quote begins the quoted history under a reply.
quote []quoteMarker
// separators are the words inside a dashed original or forwarded line.
separators []string
// ooo are away-message subject prefixes, in their own spelling.
ooo []string
// cues introduce a return date; inclusive ones name the last day away.
// Written accent-folded, as the return-date reader sees the text.
cues, inclusive []string
months map[int][]string
// bounce are non-delivery subject prefixes, lower case.
bounce []string
// declined turns down interest and is read before any interest phrase;
// negative is any other clear no. Both are phrases, never single words.
declined, negative []string
// positive is clear interest. A chosen language's is read after its no
// phrases, and never after a negator or condition in its sentence. positiveAsWritten
// keeps its accents, where folding would change the meaning.
positive, positiveAsWritten []string
// positiveRequests express a request; a trailing qualifier need not negate it.
positiveRequests []string
// postverbalNegation reads French pas after an interest phrase, including intervening adverbs.
postverbalNegation bool
}
var baseRules = langRules{
declined: declinedInterestKeywords,
negative: negativeKeywords,
positive: positiveKeywords,
quote: []quoteMarker{
{re: attribution(`on`, `wrote:`)},
{re: regexp.MustCompile(`(?i)_{10,}`)},
{re: regexp.MustCompile(`(?im)^\s*from:\s.+$`)},
{re: headerBlock(`from`, `sent|date`)},
{re: attribution(`le`, `a écrit\s*:`)},
{re: attribution(`am`, `schrieb\b`)},
{re: attribution(`el`, `escribió\s*:`)},
{re: attribution(`op`, `schreef\b`)},
{re: attribution(`il`, `ha scritto\s*:`)},
},
separators: []string{`original message`, `forwarded message`},
ooo: []string{
// English
"out of office", "out of the office", "automatic reply", "automated reply",
"autoreply", "auto-reply", "auto reply", "auto:", "away:", "on vacation:", "vacation reply",
// German
"abwesenheit", "abwesend", "automatische antwort", "autom. antwort",
"ausser haus", "nicht im buero", "im urlaub:",
// French
"reponse automatique", "absence du bureau", "message d'absence",
// Spanish / Portuguese
"respuesta automatica", "ausencia de la oficina", "ausencia temporal",
"resposta automatica", "fora do escritorio",
// Italian
"risposta automatica", "fuori sede:", "assente dall'ufficio",
// Dutch
"automatisch antwoord", "afwezigheid", "afwezigheidsbericht",
// Nordic / Polish
"automatiskt svar", "automatisk svar", "autosvar", "fravaer", "fravaersmelding",
"automatyczna odpowiedz",
},
cues: []string{
// English
`back on`, `back in the office on`, `back at my desk on`, `be back on`,
`return on`, `returning on`, `will return on`, `i return on`, `my return on`,
`returns on`, `available again on`, `reachable again on`, `back from`,
`until`, `till`, `through`,
// German
`zurueck am`, `zurueck ab`, `wieder am`, `wieder ab`, `ab dem`,
// "ab Montag den 12.10.", "ab Montag, den 12.10." and "ab Montag,
// 12.10.": the weekday needs "den" or a comma before the date.
`ab (?:montag|dienstag|mittwoch|donnerstag|freitag)(?:,? de[nm]|, ?)`,
`wieder erreichbar am`, `wieder erreichbar ab`, `wieder im buero am`,
`bis einschliesslich`, `bis zum`, `bis`,
// A calendar week ("ab KW 42"); "Kalenderwoche" reads as "kw".
`ab kw`, `ab der kw`,
// French
`de retour le`, `jusqu'au`, `jusqu au`, `a partir du`,
// Spanish / Portuguese
`de vuelta el`, `regreso el`, `hasta el`, `a partir del`,
`de volta a`, `ate o dia`, `a partir de`,
// Dutch
`terug op`, `weer aanwezig op`, `tot en met`,
// Italian
`di ritorno il`, `fino al`, `rientro il`,
},
inclusive: []string{"through", "bis einschliesslich", "tot en met"},
months: map[int][]string{
1: {"january", "jan", "januar", "janvier", "enero", "ene", "janeiro", "januari", "gennaio"},
2: {"february", "feb", "februar", "fevrier", "febrero", "fevereiro", "februari", "febbraio"},
3: {"march", "mar", "maerz", "marz", "mars", "marzo", "marco", "maart"},
4: {"april", "apr", "avril", "abril", "aprile"},
5: {"may", "mai", "mayo", "maio", "mei", "maggio"},
6: {"june", "jun", "juni", "juin", "junio", "junho", "giugno"},
7: {"july", "jul", "juli", "juillet", "julio", "julho", "luglio"},
8: {"august", "aug", "aout", "agosto", "augustus", "ago"},
9: {"september", "sep", "sept", "septembre", "septiembre", "setembro", "settembre"},
10: {"october", "oct", "oktober", "octobre", "octubre", "outubro", "okt", "ottobre"},
11: {"november", "nov", "novembre", "noviembre", "novembro"},
12: {"december", "dec", "dezember", "decembre", "diciembre", "dezembro", "dez", "dicembre"},
},
}
// Gmail in Swedish, Danish and Norwegian: "[Den] mån 3 mars 2025 kl 10:12 skrev".
var nordicAttribution = quoteMarker{re: regexp.MustCompile(`(?i)(^|\s)(den\s+)?(\pL{2,4}\.?,?\s+)?\d{1,2}\.?\s*\pL{3,9}\.?\s+\d{4},?\s+kl\.?\s+\d{1,2}[:.]\d{2}\s[^\n]{0,250}?\bskrev\b`)}
// Thunderbird in Russian and Ukrainian: "03.03.2025 10:12, Name пишет:".
var cyrillicAttribution = quoteMarker{re: regexp.MustCompile(`(?i)\d{1,2}[./]\d{1,2}[./]\d{2,4},?\s+\d{1,2}:\d{2},\s[^\n]{0,200}?\s(пишет|пише|написал(\(а\))?|написав(\(ла\))?)\s*:`), lineStart: true}
// gmailSigned is Gmail's attribution in the languages whose verb follows the
// quoted address: "<date> Anna <anna@example.com> kirjutas:".
func gmailSigned(verbs string) quoteMarker {
return quoteMarker{re: regexp.MustCompile(`(?i)@[^\n]{0,120}?[>)]\s*[^\n]{0,30}?(` + verbs + `)\s*[::]`), lineStart: true}
}
var languageRules = map[string]langRules{
"de": {
// GMX and web.de put the date first: "Gesendet: ... Von: ...".
quote: []quoteMarker{{re: headerBlock(`von`, `gesendet|datum`)}, {re: headerBlock(`gesendet`, `von`)}},
separators: []string{`ursprüngliche nachricht`, `original-nachricht`, `weitergeleitete nachricht`},
declined: []string{
"kein interesse", "keinerlei interesse", "nicht interessiert", "nicht mehr interessiert",
"interessiert uns nicht", "interessiert mich nicht",
},
negative: []string{
"nein danke", "nein, danke", "kein bedarf", "keinen bedarf", "falscher ansprechpartner", "falschen ansprechpartner",
"nicht der richtige ansprechpartner", "nicht die richtige ansprechpartnerin", "nicht zuständig",
},
positive: []string{
"klingt gut", "klingt interessant", "klingt spannend", "ich bin interessiert", "wir sind interessiert",
"lassen sie uns telefonieren", "lass uns telefonieren", "können wir telefonieren",
"gerne mehr erfahren", "schicken sie mir mehr informationen", "was kostet das",
},
},
"fr": {
postverbalNegation: true,
quote: []quoteMarker{{re: headerBlock(`de ?`, `envoyé ?|date ?`)}},
separators: []string{`message d'origine`, `message transféré`},
ooo: []string{"absent du bureau", "absente du bureau"},
declined: []string{
"pas intéressé", "pas intéressée", "pas intéressés", "pas intéressées", "aucun intérêt",
"nous intéresse pas", "m'intéresse pas", "ne m'intéresse plus", "ne nous intéresse plus",
"ne suis plus intéressé", "ne suis plus intéressée",
"ne sommes plus intéressés", "ne sommes plus intéressées",
},
negative: []string{
"non merci", "non, merci", "pas le bon interlocuteur", "pas la bonne personne",
"nous avons déjà une solution", "nous avons déjà un prestataire",
},
positive: []string{
"je suis intéressé", "je suis intéressée", "nous sommes intéressés", "nous sommes intéressées",
"ça m'intéresse", "cela m'intéresse", "ça nous intéresse", "cela nous intéresse",
"ça a l'air intéressant", "cela semble intéressant",
},
positiveRequests: []string{
"prenons rendez-vous",
"envoyez-moi plus d'informations", "envoyez-moi une offre", "envoyez-moi un devis",
"quel est le tarif", "quels sont vos tarifs",
},
},
"es": {
quote: []quoteMarker{{re: headerBlock(`de`, `enviado|fecha`)}},
separators: []string{`mensaje original`, `mensaje reenviado`},
ooo: []string{"fuera de la oficina"},
bounce: []string{"no se puede entregar"},
declined: []string{
"no estoy interesado", "no estoy interesada", "no estamos interesados", "no estamos interesadas",
"no me interesa", "no nos interesa", "no tengo interés", "no tenemos interés", "sin interés",
},
negative: []string{
"no gracias", "no, gracias", "no soy la persona", "persona equivocada",
"ya tenemos una solución", "ya tenemos proveedor",
},
positive: []string{
"me interesa mucho", "nos interesa mucho", "estoy muy interesado",
"estoy muy interesada", "estamos muy interesados", "agendemos una llamada",
"envíeme más información", "envíame más información", "cuánto cuesta",
"cuál es el precio",
},
// "Si me interesa, les aviso" is conditional; only the accented "sí" says yes.
positiveAsWritten: []string{"sí me interesa", "sí, me interesa"},
},
"pt": {
quote: []quoteMarker{
{re: attribution(`em`, `escreveu\s*:`)},
{re: headerBlock(`de`, `enviado|enviada em|data`)},
},
separators: []string{`mensagem original`, `mensagem encaminhada`},
bounce: []string{"não é possível entregar"},
declined: []string{
"não tenho interesse", "não temos interesse", "não estou interessado", "não estou interessada",
"não estamos interessados", "não estamos interessadas", "não me interessa", "não nos interessa",
},
negative: []string{
"não obrigado", "não, obrigado", "não obrigada", "não, obrigada", "pessoa errada",
"não sou a pessoa certa", "já temos uma solução", "já temos fornecedor",
},
},
"it": {
quote: []quoteMarker{{re: headerBlock(`da`, `inviato|data`)}},
separators: []string{`messaggio originale`, `messaggio inoltrato`},
ooo: []string{"fuori ufficio"},
bounce: []string{"non recapitabile"},
declined: []string{
"non sono interessato", "non sono interessata", "non siamo interessati", "non siamo interessate",
"non sono più interessato", "non sono più interessata", "non siamo più interessati",
"non mi interessa", "non ci interessa", "nessun interesse",
},
negative: []string{
"no grazie", "no, grazie", "non fa per noi", "persona sbagliata", "non sono la persona giusta",
"abbiamo già una soluzione", "abbiamo già un fornitore",
},
},
"nl": {
quote: []quoteMarker{{re: headerBlock(`van`, `verzonden|datum`)}},
separators: []string{`oorspronkelijk bericht`, `doorgestuurd bericht`},
ooo: []string{"buiten kantoor"},
bounce: []string{"onbestelbaar"},
declined: []string{
"geen interesse", "niet geïnteresseerd", "niet meer geïnteresseerd",
"interesseert ons niet", "interesseert me niet", "interesseert mij niet",
},
negative: []string{
"nee dank je", "nee, dank je", "nee dank u", "nee, dank u", "nee bedankt", "nee, bedankt",
"geen behoefte", "verkeerde persoon", "niet de juiste persoon", "niet de juiste contactpersoon",
"we hebben al een oplossing", "wij hebben al een oplossing",
},
},
"sv": {
quote: []quoteMarker{nordicAttribution, {re: headerBlock(`från`, `skickat|datum`)}},
separators: []string{`ursprungligt meddelande`, `vidarebefordrat meddelande`},
ooo: []string{"frånvaro"},
cues: []string{`tillbaka den`, `tillbaka igen den`, `tillbaka pa kontoret den`, `ater pa kontoret den`, `ater den`, `nas igen den`, `till och med`},
inclusive: []string{"till och med"},
months: map[int][]string{5: {"maj"}, 8: {"augusti"}},
},
"da": {
quote: []quoteMarker{nordicAttribution, {re: headerBlock(`fra`, `sendt|dato`)}},
separators: []string{`oprindelig meddelelse`, `videresendt meddelelse`},
cues: []string{`tilbage den`, `tilbage igen den`, `tilbage pa kontoret den`, `traeffes igen den`, `til og med`},
inclusive: []string{"til og med"},
months: map[int][]string{3: {"marts"}, 5: {"maj"}},
},
"nb": {
quote: []quoteMarker{nordicAttribution, {re: headerBlock(`fra`, `sendt|dato`)}},
separators: []string{`opprinnelig melding`, `videresendt melding`},
cues: []string{`tilbake den`, `tilbake igjen den`, `tilbake pa kontoret den`, `tilgjengelig igjen den`, `til og med`},
inclusive: []string{"til og med"},
months: map[int][]string{12: {"desember"}},
},
"fi": {
quote: []quoteMarker{
{re: regexp.MustCompile(`(?i)\d{4}\s+klo\s+\d{1,2}[:.]\d{2}\s[^\n]{0,250}?\skirjoitti\s*:`), lineStart: true},
{re: headerBlock(`lähettäjä`, `lähetetty|päivämäärä`)},
},
separators: []string{`alkuperäinen viesti`, `välitetty viesti`},
ooo: []string{"automaattinen vastaus", "poissa toimistolta"},
cues: []string{`palaan`, `olen takaisin`, `takaisin toimistolla`, `tavoitettavissa jaelleen`},
months: map[int][]string{
1: {"tammikuuta", "tammikuu"}, 2: {"helmikuuta", "helmikuu"}, 3: {"maaliskuuta", "maaliskuu"},
4: {"huhtikuuta", "huhtikuu"}, 5: {"toukokuuta", "toukokuu"}, 6: {"kesaekuuta", "kesaekuu"},
7: {"heinaekuuta", "heinaekuu"}, 8: {"elokuuta", "elokuu"}, 9: {"syyskuuta", "syyskuu"},
10: {"lokakuuta", "lokakuu"}, 11: {"marraskuuta", "marraskuu"}, 12: {"joulukuuta", "joulukuu"},
},
},
"pl": {
quote: []quoteMarker{
{re: regexp.MustCompile(`(?i)\d{4},?\s+o\s+\d{1,2}:\d{2}\s[^\n]{0,250}?\snapisał(\(a\))?\s*:`), lineStart: true},
{re: attribution(`w dniu`, `(napisał(\(a\))?|pisze)\s*:`)},
{re: headerBlock(`od`, `wysłano|data`)},
},
separators: []string{`oryginalna wiadomość`, `wiadomość oryginalna`, `wiadomość przekazana`},
ooo: []string{"odpowiedź automatyczna", "poza biurem"},
cues: []string{`wracam`, `wroce`, `bede z powrotem`, `z powrotem w biurze`, `do dnia`},
months: map[int][]string{
1: {"stycznia", "styczen"}, 2: {"lutego", "luty"}, 3: {"marca", "marzec"}, 4: {"kwietnia", "kwiecien"},
5: {"maja", "maj"}, 6: {"czerwca", "czerwiec"}, 7: {"lipca", "lipiec"}, 8: {"sierpnia", "sierpien"},
9: {"wrzesnia", "wrzesien"}, 10: {"pazdziernika", "pazdziernik"}, 11: {"listopada", "listopad"},
12: {"grudnia", "grudzien"},
},
},
"cs": {
quote: []quoteMarker{
{re: regexp.MustCompile(`(?i)\d{4}\s+v\s+\d{1,2}:\d{2}\s+odesílatel\s`), lineStart: true},
{re: headerBlock(`od`, `odesláno|datum`)},
},
separators: []string{`původní zpráva`, `přeposlaná zpráva`},
ooo: []string{"automatická odpověď", "mimo kancelář"},
cues: []string{`budu zpet`, `zpet v kancelari`, `vracim se`, `opet k zastizeni`},
months: map[int][]string{
1: {"ledna", "leden"}, 2: {"unora", "unor"}, 3: {"brezna", "brezen"}, 4: {"dubna", "duben"},
5: {"kvetna", "kveten"}, 6: {"cervna", "cerven"}, 7: {"cervence", "cervenec"}, 8: {"srpna", "srpen"},
9: {"zari"}, 10: {"rijna", "rijen"}, 11: {"listopadu", "listopad"}, 12: {"prosince", "prosinec"},
},
},
"ro": {
quote: []quoteMarker{{re: attribution(`pe`, `a scris\s*:`)}, {re: headerBlock(`de la`, `trimis|data`)}},
separators: []string{`mesaj original`, `mesaj redirecționat`},
ooo: []string{"răspuns automat", "în afara biroului"},
},
"hu": {
quote: []quoteMarker{
{re: regexp.MustCompile(`(?i)\sezt írta\s*\(időpont:`), lineStart: true},
{re: regexp.MustCompile(`(?i)\skeltezéssel,[^\n]{0,250}?\sírta\s*:`), lineStart: true},
{re: headerBlock(`feladó`, `elküldve|dátum`)},
},
separators: []string{`eredeti üzenet`, `továbbított üzenet`},
ooo: []string{"automatikus válasz", "házon kívül"},
},
"ru": {
quote: []quoteMarker{
{re: regexp.MustCompile(`(?i)\d{4}\s*г\.\s+в\s+\d{1,2}:\d{2},\s[^\n]{0,200}?<[^<>\n]*@[^<>\n]*>\s*:`), lineStart: true},
cyrillicAttribution,
{re: headerBlock(`от`, `отправлено|дата`)},
},
separators: []string{`исходное сообщение`, `пересылаемое сообщение`},
ooo: []string{"автоматический ответ", "автоответ", "вне офиса"},
},
"uk": {
quote: []quoteMarker{
{re: regexp.MustCompile(`(?i)\d{4}\s*р\.\s+о\s+\d{1,2}:\d{2}\s[^\n]{0,250}?\sпише\s*:`), lineStart: true},
cyrillicAttribution,
{re: headerBlock(`від`, `надіслано|дата`)},
},
separators: []string{`оригінальне повідомлення`, `переслане повідомлення`},
ooo: []string{"автоматична відповідь", "поза офісом"},
},
"el": {
quote: []quoteMarker{
{re: regexp.MustCompile(`(?i)(^|\s)στις\s[^\n]{0,250}?@[^\n]{0,80}?>\s*έγραψε\s*:`)},
{re: headerBlock(`από`, `στάλθηκε|ημερομηνία`)},
},
separators: []string{`αρχικό μήνυμα`, `προωθημένο μήνυμα`},
ooo: []string{"αυτόματη απάντηση", "εκτός γραφείου"},
},
"tr": {
quote: []quoteMarker{
{re: regexp.MustCompile(`(?i)@[^\n]{0,80}?\d{1,2}:\d{2}\s+tarihinde\s+şunu\s+yazdı\s*:`), lineStart: true},
{re: headerBlock(`kimden`, `gönderildi|gönderilme tarihi|tarih`)},
},
separators: []string{`özgün ileti`, `orijinal mesaj`, `iletilen ileti`},
ooo: []string{"otomatik yanıt", "otomatik cevap", "ofis dışında"},
},
"ja": {
quote: []quoteMarker{
{re: regexp.MustCompile(`\d{4}年\s?\d{1,2}月\s?\d{1,2}日\s*[((][^))\n]{1,4}[))]\s*\d{1,2}:\d{2}\s[^\n]{0,200}?<[^<>\n]*@[^<>\n]*>\s*[::]`), lineStart: true},
{re: regexp.MustCompile(`(?s)差出人\s?[::].{1,300}?\s(送信日時|日付)\s?[::]`)},
},
separators: []string{`元のメッセージ`, `転送されたメッセージ`},
ooo: []string{"自動応答", "自動返信", "不在通知", "不在のお知らせ"},
},
"zh": {
quote: []quoteMarker{
{re: regexp.MustCompile(`[于於]\s?\d{4}年\s?\d{1,2}月\s?\d{1,2}日[^\n]{0,40}?[写寫]道\s?[::]`), lineStart: true},
{re: regexp.MustCompile(`(?s)(发件人|寄件者)\s?[::].{1,300}?\s(发送时间|寄件日期|日期)\s?[::]`)},
},
separators: []string{`原始邮件`, `原始郵件`, `转发的邮件`, `已轉寄郵件`},
ooo: []string{"自动回复", "自动答复", "自動回覆", "自動回复"},
},
"ko": {
quote: []quoteMarker{
{re: regexp.MustCompile(`\d{4}년\s*\d{1,2}월\s*\d{1,2}일[^\n]{0,150}?님이\s?작성\s*:`), lineStart: true},
{re: regexp.MustCompile(`(?s)보낸\s?사람\s?:.{1,300}?\s보낸\s?날짜\s?:`)},
},
separators: []string{`원본 메시지`, `전달된 메시지`},
ooo: []string{"자동 회신", "자동 응답", "부재중"},
},
"id": {
quote: []quoteMarker{{re: attribution(`pada`, `menulis\s*:`)}, {re: headerBlock(`dari`, `dikirim|tanggal`)}},
separators: []string{`pesan asli`, `pesan yang diteruskan`},
ooo: []string{"balasan otomatis", "di luar kantor"},
},
"vi": {
quote: []quoteMarker{{re: attribution(`vào`, `đã viết\s*:`)}, {re: headerBlock(`từ`, `đã gửi|ngày`)}},
separators: []string{`tin nhắn gốc`, `thư được chuyển tiếp`},
ooo: []string{"trả lời tự động"},
},
"th": {
quote: []quoteMarker{{re: attribution(`เมื่อ`, `เขียนว่า\s*:`)}},
ooo: []string{"ตอบกลับอัตโนมัติ"},
},
"hi": {
quote: []quoteMarker{{re: regexp.MustCompile(`\d{4}\s+को[^\n]{0,200}?@[^\n]{0,80}?\sने\s+लिखा\s*:`), lineStart: true}},
ooo: []string{"स्वचालित उत्तर"},
},
"ar": {
quote: []quoteMarker{
{re: regexp.MustCompile(`(^|\s)في\s[^\n]{0,250}?\d{1,2}:\d{2}[^\n]{0,40}?\sكتب\s[^\n]{0,120}?@[^\n]{0,80}?>\s*:`)},
{re: headerBlock(`من`, `تاريخ الإرسال|أرسل|التاريخ`)},
},
separators: []string{`الرسالة الأصلية`, `رسالة معاد توجيهها`},
ooo: []string{"رد تلقائي", "خارج المكتب"},
},
"he": {
quote: []quoteMarker{
{re: regexp.MustCompile(`בתאריך[^\n]{0,150}?\d{4}[^\n]{0,80}?\sמאת\s[^\n]{0,120}?@[^\n]{0,80}?>`), lineStart: true},
{re: headerBlock(`מאת`, `נשלח|תאריך`)},
},
separators: []string{`הודעה מקורית`, `הודעה שהועברה`},
ooo: []string{"תשובה אוטומטית", "מחוץ למשרד"},
},
"bg": {
quote: []quoteMarker{gmailSigned(`написа`), {re: headerBlock(`от`, `изпратено|дата`)}},
separators: []string{`оригинално съобщение`, `препратено съобщение`},
},
"bn": {quote: []quoteMarker{gmailSigned(`লিখেছেন`)}},
"ca": {
quote: []quoteMarker{gmailSigned(`va escriure`), {re: headerBlock(`de`, `enviat( el)?|data`)}},
separators: []string{`missatge original`, `missatge reenviat`},
},
"et": {
quote: []quoteMarker{gmailSigned(`kirjutas`), {re: headerBlock(`saatja`, `saadetud|kuupäev`)}},
separators: []string{`algne sõnum`, `edasisaadetud sõnum`},
},
"fa": {quote: []quoteMarker{gmailSigned(`نوشت`), {re: headerBlock(`از`, `ارسال شده|تاریخ`)}}},
"fil": {quote: []quoteMarker{gmailSigned(`sumulat`)}},
"hr": {
quote: []quoteMarker{gmailSigned(`(je\s+)?napisa(o|la)(/la)?(\s+je)?`), {re: headerBlock(`šalje|od`, `poslano|datum`)}},
separators: []string{`izvorna poruka`, `proslijeđena poruka`},
},
"lt": {
quote: []quoteMarker{gmailSigned(`rašė`), {re: headerBlock(`nuo`, `išsiųsta|data`)}},
separators: []string{`pradinis laiškas`, `persiųstas laiškas`},
},
"lv": {
quote: []quoteMarker{gmailSigned(`rakstīja`), {re: headerBlock(`no`, `nosūtīts|datums`)}},
separators: []string{`sākotnējais ziņojums`, `pārsūtīts ziņojums`},
},
"ms": {
quote: []quoteMarker{gmailSigned(`menulis`), {re: headerBlock(`daripada`, `dihantar|tarikh`)}},
separators: []string{`mesej asal`, `mesej dimajukan`},
},
"sk": {
quote: []quoteMarker{gmailSigned(`napísal(\(a\)|a)?`), {re: headerBlock(`od`, `odoslané|dátum`)}},
separators: []string{`pôvodná správa`, `preposlaná správa`},
},
"sl": {
quote: []quoteMarker{gmailSigned(`(je\s+)?napisal(\(-?a\)|a)?`), {re: headerBlock(`od`, `poslano|datum`)}},
separators: []string{`izvirno sporočilo`, `posredovano sporočilo`},
},
"sr": {
quote: []quoteMarker{
gmailSigned(`(је\s+|je\s+)?(написао|написала|napisao|napisala)(/ла|/la)?(\s+је|\s+je)?`),
{re: headerBlock(`од|od`, `послато|послано|poslato|poslano|датум|datum`)},
},
separators: []string{`оригинална порука`, `originalna poruka`, `прослеђена порука`, `prosleđena poruka`},
},
"sw": {quote: []quoteMarker{gmailSigned(`aliandika`)}},
"ta": {quote: []quoteMarker{gmailSigned(`எழுதியது`)}},
"ur": {quote: []quoteMarker{gmailSigned(`نے لکھا`)}},
}
// LanguagesWithRules lists the codes that add offline vocabulary, sorted.
func LanguagesWithRules() []string {
out := make([]string, 0, len(languageRules))
for code := range languageRules {
out = append(out, code)
}
sort.Strings(out)
return out
}
// ruleSet is the compiled vocabulary for one combination of languages.
type ruleSet struct {
quote []quoteMarker
ooo []string
cue *regexp.Regexp
inclusive map[string]bool
months map[string]int
bounce []string
declined []phrase
negative []phrase
positive []phrase
}
// phrase is a sentiment phrase as compiled. The English base set keeps the
// substring match it always had; a chosen language's phrase must stand as
// whole words, so "no gracias" never matches inside "bueno gracias".
type phrase struct {
text string
bounded bool
// negatable phrases do not count after a negator or condition in the same sentence.
negatable bool
// asWritten phrases keep their accents, because folding them changes the meaning ("sí" is not "si").
asWritten bool
postverbalNegation bool
unlessNextWord string
}
// negators are the words that, earlier in the sentence, turn an interest phrase around.
var negators = map[string]bool{
"no": true, "not": true, "nicht": true, "kein": true, "keine": true, "keinen": true, "nie": true,
"pas": true, "ne": true, "non": true, "niet": true, "geen": true, "nao": true, "nunca": true, "jamas": true,
}
// negatedOrConditionalAt keeps a qualified phrase from counting as definite interest.
func negatedOrConditionalAt(text string, start int) bool {
head := text[:start]
if i := strings.LastIndexAny(head, ".!?;"); i >= 0 {
head = head[i+1:]
}
words := strings.FieldsFunc(head, func(r rune) bool { return !unicode.IsLetter(r) && r != '\'' })
for i, word := range words {
word = strings.Trim(word, "'")
if word == "s'il" && i+2 < len(words) && (words[i+1] == "vous" || words[i+1] == "te") && accentFolder.Replace(strings.Trim(words[i+2], "'")) == "plait" {
continue
}
if word == "s'il" || word == "s'ils" {
return true
}
for _, part := range strings.Split(word, "'") {
if negators[accentFolder.Replace(part)] || part == "si" || part == "wenn" || part == "falls" || part == "sofern" {
return true
}
}
}
return false
}
// unfoldedOffset maps a folded match back across replacements that change byte length.
func unfoldedOffset(text string, offset int) int {
for i, r := range text {
if offset <= 0 {
return i
}
offset -= len(accentFolder.Replace(string(r)))
}
return len(text)
}
// contrasts open a reservation; a negator after one takes the interest back.
var contrasts = map[string]bool{
"aber": true, "jedoch": true, "allerdings": true, "mais": true, "cependant": true, "toutefois": true, "pero": true, "aunque": true,
}
// refusedAfter reports a contrast and then a negator later in the reply ("me interesa mucho, pero no ahora").
func refusedAfter(text string, end int) bool {
contrast := false
for _, word := range strings.FieldsFunc(text[end:], func(r rune) bool { return !unicode.IsLetter(r) }) {
if contrast && negators[word] {
return true
}
contrast = contrast || contrasts[word]
}
return false
}
const sentimentWhitespace = `[[:space:]\p{Z}\x{0085}]`
var frenchPostverbalNegation = regexp.MustCompile(`^` + sentimentWhitespace + `+(?:(?:vraiment|absolument|franchement|certainement|tellement|beaucoup)` + sentimentWhitespace + `+)*pas\b(?:` + sentimentWhitespace + `+(\p{L}+))?`)
// frenchNegatedAfter keeps post-verbal pas distinct from the affirmative idiom pas mal.
func frenchNegatedAfter(text string, end int) bool {
match := frenchPostverbalNegation.FindStringSubmatch(text[end:])
return match != nil && match[1] != "mal"
}
func followedByWord(text string, end int, word string) bool {
tail := strings.TrimLeftFunc(text[end:], unicode.IsSpace)
rest, found := strings.CutPrefix(tail, word)
next, _ := utf8.DecodeRuneInString(rest)
return found && !wordRune(next)
}
// in matches an English base phrase against the text as written and a chosen
// language's phrase, whole words only, against its accent-folded form.
func (p phrase) in(text, folded string) bool {
if !p.bounded {
return strings.Contains(text, p.text)
}
original := text
if !p.asWritten {
text = folded
}
for i := 0; i < len(text); {
j := strings.Index(text[i:], p.text)
if j < 0 {
return false
}
start, end := i+j, i+j+len(p.text)
before, _ := utf8.DecodeLastRuneInString(text[:start])
after, _ := utf8.DecodeRuneInString(text[end:])
if !wordRune(before) && !wordRune(after) && !(p.unlessNextWord != "" && followedByWord(text, end, p.unlessNextWord)) {
originalStart := start
if p.negatable && !p.asWritten {
originalStart = unfoldedOffset(original, start)
}
if !p.negatable || !(negatedOrConditionalAt(original, originalStart) || refusedAfter(text, end) || (p.postverbalNegation && frenchNegatedAfter(text, end))) {
return true
}
}
_, size := utf8.DecodeRuneInString(text[start:])
i = start + size
}
return false
}
func wordRune(r rune) bool {
return r != utf8.RuneError && (unicode.IsLetter(r) || unicode.IsDigit(r))
}
var ruleSets sync.Map // sorted codes joined by "," -> *ruleSet
// rulesFor is the base vocabulary plus the given languages'. Unknown codes,
// and codes that add nothing, share the base set.
func rulesFor(langs []string) *ruleSet {
var codes []string
for _, l := range langs {
l = strings.ToLower(strings.TrimSpace(l))
if _, ok := languageRules[l]; ok && !slices.Contains(codes, l) {
codes = append(codes, l)
}
}
sort.Strings(codes)
key := strings.Join(codes, ",")
if rs, ok := ruleSets.Load(key); ok {
return rs.(*ruleSet)
}
parts := []langRules{baseRules}
for _, c := range codes {
parts = append(parts, languageRules[c])
}
rs := compileRules(parts)
actual, _ := ruleSets.LoadOrStore(key, rs)
return actual.(*ruleSet)
}
func compileRules(parts []langRules) *ruleSet {
rs := &ruleSet{inclusive: map[string]bool{}, months: map[string]int{}}
var separators, cues []string
for i, p := range parts {
rs.quote = append(rs.quote, p.quote...)
separators = append(separators, p.separators...)
for _, m := range p.ooo {
rs.ooo = append(rs.ooo, foldAccents(m))
}
cues = append(cues, p.cues...)
for _, c := range p.inclusive {
rs.inclusive[c] = true
}
for month, names := range p.months {
for _, n := range names {
rs.months[n] = month
}
}
rs.bounce = append(rs.bounce, p.bounce...)
// parts[0] is always baseRules.
for _, d := range p.declined {
declined := phrase{text: accentFolder.Replace(strings.ToLower(d)), bounded: i > 0}
if p.postverbalNegation && strings.HasSuffix(declined.text, " pas") {
declined.unlessNextWord = "mal"
}
rs.declined = append(rs.declined, declined)
}
for _, n := range p.negative {
rs.negative = append(rs.negative, phrase{text: accentFolder.Replace(strings.ToLower(n)), bounded: i > 0})
}
for _, pos := range p.positive {
rs.positive = append(rs.positive, phrase{text: accentFolder.Replace(strings.ToLower(pos)), bounded: i > 0, negatable: i > 0, postverbalNegation: p.postverbalNegation})
}
for _, pos := range p.positiveRequests {
rs.positive = append(rs.positive, phrase{text: accentFolder.Replace(strings.ToLower(pos)), bounded: i > 0, negatable: i > 0})
}
for _, pos := range p.positiveAsWritten {
rs.positive = append(rs.positive, phrase{text: strings.ToLower(pos), bounded: true, negatable: true, asWritten: true})
}
}
rs.quote = append(rs.quote, quoteMarker{re: regexp.MustCompile(`(?i)-{2,}\s*(` + strings.Join(separators, `|`) + `)\s*-{2,}`)})
rs.cue = regexp.MustCompile(`(?i)\b(` + strings.Join(longestFirst(cues), `|`) + `)\b`)
return rs
}
// longestFirst orders alternatives so a longer cue wins over a cue it starts
// with: "till och med" is inclusive, "till" is not.
func longestFirst(in []string) []string {
out := append([]string(nil), in...)
sort.SliceStable(out, func(i, j int) bool { return len(out[i]) > len(out[j]) })
return out
}