mirror of
https://github.com/bulwarkmail/native.git
synced 2026-11-01 00:07:01 +00:00
fix: find multi-word, address and long-word searches in the app on Stalwart 1.0
Stalwart 1.0 reads a text filter value that ends in `*` as one prefix over the whole string, not split into words and capped at 16 bytes. The app added `*` to every word it searched for, so on 1.0 `project budget` became the prefix "project* budget", an address was no single word and `Geschäftsführung` (18 bytes) was over the cap: all three found nothing. Quoted phrases broke too, since `"annual report" 2024` went out as one value ending in `*`. The search text now goes out as typed, split the way the webmail does it (5967a1c4): each quoted phrase and each `word*` becomes a condition of its own, ANDed with one condition that holds the other words, so a query without quotes or stars is sent unchanged. The folder, all-folders and tag-view searches, the unified inbox and the from, to, subject and body filters all use it, and `searchEmails` puts `inMailbox` next to the text conditions in one AND instead of on an operator. 0.16's built-in index dropped the `*` anyway, so its results stay the same; there a quoted phrase is an AND of its words. On 1.0 a typed `word*` is now a real prefix. The fake server of the store tests now matches text the way Stalwart's built-in index does, which the old `*` after every word fails.
This commit is contained in:
@@ -272,7 +272,7 @@ describe('email operations', () => {
|
||||
|
||||
const pages = await queryEmailPagesAcrossAccounts(
|
||||
[{ position: 2, sort }, { accountId: 'grp-1', position: 0, sort }],
|
||||
{ limit: 25, filter: { text: 'zephyr*' }, threads: true },
|
||||
{ limit: 25, filter: { text: 'zephyr' }, threads: true },
|
||||
);
|
||||
|
||||
expect(mockRequest).toHaveBeenCalledTimes(1);
|
||||
@@ -281,7 +281,7 @@ describe('email operations', () => {
|
||||
['Email/query', 'acc-1', '0:q'], ['Email/get', 'acc-1', '0:g'], ['Thread/get', 'acc-1', '0:t'],
|
||||
['Email/query', 'grp-1', '1:q'], ['Email/get', 'grp-1', '1:g'], ['Thread/get', 'grp-1', '1:t'],
|
||||
]);
|
||||
expect(calls[0][1]).toMatchObject({ filter: { text: 'zephyr*' }, position: 2, limit: 25, sort });
|
||||
expect(calls[0][1]).toMatchObject({ filter: { text: 'zephyr' }, position: 2, limit: 25, sort });
|
||||
expect(calls[4][1]['#ids']).toEqual({ resultOf: '1:q', name: 'Email/query', path: '/ids' });
|
||||
expect(pages[0]).toEqual({
|
||||
accountId: undefined, ok: true, total: 3,
|
||||
@@ -487,7 +487,9 @@ describe('email operations', () => {
|
||||
});
|
||||
|
||||
describe('searchEmails', () => {
|
||||
it('should search with text filter', async () => {
|
||||
// Stalwart 1.0 reads a value ending in `*` as one prefix over the whole
|
||||
// string, so the text goes out as typed (P1-7).
|
||||
it('should search with the text as typed', async () => {
|
||||
mockRequest.mockResolvedValue({
|
||||
methodResponses: [['Email/query', { ids: ['e1'] }, '0']],
|
||||
});
|
||||
@@ -496,7 +498,7 @@ describe('email operations', () => {
|
||||
|
||||
expect(result).toEqual(['e1']);
|
||||
const call = mockRequest.mock.calls[0][0][0];
|
||||
expect(call[1].filter).toEqual({ text: 'test* query*' });
|
||||
expect(call[1].filter).toEqual({ text: 'test query' });
|
||||
});
|
||||
|
||||
it('should include mailbox filter if specified', async () => {
|
||||
@@ -507,7 +509,24 @@ describe('email operations', () => {
|
||||
await searchEmails('query', 'mb-1');
|
||||
|
||||
const call = mockRequest.mock.calls[0][0][0];
|
||||
expect(call[1].filter).toEqual({ text: 'query*', inMailbox: 'mb-1' });
|
||||
expect(call[1].filter).toEqual({ inMailbox: 'mb-1', text: 'query' });
|
||||
});
|
||||
|
||||
it('ANDs quoted phrases with the folder without mixing operator and property keys', async () => {
|
||||
mockRequest.mockResolvedValue({
|
||||
methodResponses: [['Email/query', { ids: [] }, '0']],
|
||||
});
|
||||
|
||||
await searchEmails('"annual report" 2024', 'mb-1');
|
||||
|
||||
const call = mockRequest.mock.calls[0][0][0];
|
||||
expect(call[1].filter).toEqual({
|
||||
operator: 'AND',
|
||||
conditions: [
|
||||
{ inMailbox: 'mb-1' },
|
||||
{ operator: 'AND', conditions: [{ text: '"annual report"' }, { text: '2024' }] },
|
||||
],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { buildFilter } from '../unified-inbox';
|
||||
import type { Mailbox } from '../types';
|
||||
|
||||
function mailbox(id: string, role: string | null): Mailbox {
|
||||
return {
|
||||
id,
|
||||
name: role ?? id,
|
||||
role,
|
||||
totalEmails: 0,
|
||||
unreadEmails: 0,
|
||||
totalThreads: 0,
|
||||
unreadThreads: 0,
|
||||
myRights: {} as Mailbox['myRights'],
|
||||
};
|
||||
}
|
||||
|
||||
const MAILBOXES = [mailbox('in', 'inbox'), mailbox('sent', 'sent'), mailbox('projects', null), mailbox('bin', 'trash')];
|
||||
|
||||
describe('buildFilter (aggregated views)', () => {
|
||||
it('is just the folder without a query', () => {
|
||||
expect(buildFilter(MAILBOXES, { role: 'inbox' })).toEqual({ inMailbox: 'in' });
|
||||
expect(buildFilter(MAILBOXES, { role: 'archive' })).toBeNull();
|
||||
});
|
||||
|
||||
// Stalwart 1.0 reads a value ending in `*` as one prefix over the whole
|
||||
// string, so a `*` after every word found nothing for two words (P1-7).
|
||||
it('sends a plain query as typed, next to the folder', () => {
|
||||
expect(buildFilter(MAILBOXES, { role: 'inbox', query: ' project budget ' })).toEqual({
|
||||
operator: 'AND',
|
||||
conditions: [{ inMailbox: 'in' }, { text: 'project budget' }],
|
||||
});
|
||||
});
|
||||
|
||||
it('gives quoted phrases and word* conditions of their own in the same AND', () => {
|
||||
expect(buildFilter(MAILBOXES, { view: 'unread', query: '"annual report" budg*' })).toEqual({
|
||||
operator: 'AND',
|
||||
conditions: [
|
||||
{ operator: 'OR', conditions: [{ inMailbox: 'in' }, { inMailbox: 'projects' }] },
|
||||
{ notKeyword: '$seen' },
|
||||
{ text: '"annual report"' },
|
||||
{ text: 'budg*' },
|
||||
],
|
||||
});
|
||||
});
|
||||
});
|
||||
+4
-3
@@ -9,7 +9,7 @@ import {
|
||||
} from './submissions';
|
||||
import { CAPABILITIES } from './types';
|
||||
import type { Attachment, Email, EmailAddress, JMAPMethodCall, Mailbox, Thread } from './types';
|
||||
import { toWildcardQuery } from '../lib/search-utils';
|
||||
import { textFilter } from '../lib/search-utils';
|
||||
import { sanitizeDisplayName } from '../lib/rfc5322-mailbox';
|
||||
import { generateMessageId, stripMessageIdBrackets } from '../lib/email-threading';
|
||||
import { buildMdnMessage, type MdnOptions } from '../lib/mdn';
|
||||
@@ -1350,8 +1350,9 @@ export async function searchEmails(
|
||||
accountIdOverride?: string,
|
||||
): Promise<string[]> {
|
||||
const accountId = accountIdOverride ?? jmapClient.accountId;
|
||||
const filter: Record<string, unknown> = { text: toWildcardQuery(query) };
|
||||
if (mailboxId) filter.inMailbox = mailboxId;
|
||||
// Quoted phrases and `word*` prefixes go out as conditions of their own (see
|
||||
// splitSearchText), and an operator can't carry `inMailbox` next to them.
|
||||
const filter = buildMailboxQueryFilter(mailboxId, textFilter(query));
|
||||
|
||||
const res = await jmapClient.request([
|
||||
['Email/query', {
|
||||
|
||||
@@ -7,7 +7,7 @@ import {
|
||||
import { keywordPointer, mailboxPointer } from './patch-pointer';
|
||||
import { secureFetch } from '../lib/client-cert';
|
||||
import { refreshOAuthAccessToken, type OAuthTokens } from '../lib/oauth';
|
||||
import { toWildcardQuery } from '../lib/search-utils';
|
||||
import { textConditions } from '../lib/search-utils';
|
||||
|
||||
// Aggregated views across accounts ("All inboxes", "All Sent", All mail /
|
||||
// Unread / Starred). Because the JMAP client is a single-account singleton
|
||||
@@ -340,7 +340,12 @@ export function crossIncludedMailboxes(mailboxes: Mailbox[]): Mailbox[] {
|
||||
return mailboxes.filter((m) => !CROSS_EXCLUDED_ROLES.has(m.role ?? ''));
|
||||
}
|
||||
|
||||
function buildFilter(
|
||||
/**
|
||||
* The Email/query filter for one account of an aggregated view: its folder
|
||||
* (or the cross view's folders and flag), AND the search text. Null when the
|
||||
* account has no folder for the view.
|
||||
*/
|
||||
export function buildFilter(
|
||||
mailboxes: Mailbox[],
|
||||
opts: UnifiedFetchOptions,
|
||||
): Record<string, unknown> | null {
|
||||
@@ -360,8 +365,10 @@ function buildFilter(
|
||||
if (!target) return null;
|
||||
conditions.push({ inMailbox: target.id });
|
||||
}
|
||||
// One condition per quoted phrase or `word*` (see splitSearchText), in the
|
||||
// same AND as the folder conditions.
|
||||
const q = opts.query?.trim();
|
||||
if (q) conditions.push({ text: toWildcardQuery(q) });
|
||||
if (q) conditions.push(...textConditions(q));
|
||||
return conditions.length === 1 ? conditions[0] : { operator: 'AND', conditions };
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { splitSearchText, textConditions, textFilter } from '../search-utils';
|
||||
import { textFilterMatches, type StalwartTextVersion } from './stalwart-text-model';
|
||||
|
||||
type Filter = Record<string, unknown>;
|
||||
|
||||
/** Stalwart rejects a filter object with both `operator` and property keys. */
|
||||
function expectNoMixedKeys(filter: Filter): void {
|
||||
if ('operator' in filter) {
|
||||
expect(Object.keys(filter).sort()).toEqual(['conditions', 'operator']);
|
||||
for (const c of filter.conditions as Filter[]) expectNoMixedKeys(c);
|
||||
} else {
|
||||
expect(filter).not.toHaveProperty('conditions');
|
||||
}
|
||||
}
|
||||
|
||||
const MESSAGES = [
|
||||
{ id: 'report', from: 'Anna Muster <anna@firma.de>', to: 'team@firma.de', subject: 'Draft: annual report 2024', body: 'The numbers are final.' },
|
||||
{ id: 'shuffled', from: 'Ben <ben@firma.de>', to: 'team@firma.de', subject: 'Report on annual goals, 2024 draft', body: '' },
|
||||
{ id: 'budget', from: 'Carla <carla@firma.de>', to: 'team@firma.de', subject: 'Project budget', body: 'Numbers for Q3.' },
|
||||
{ id: 'board', from: 'Dieter <dieter@firma.de>', to: 'team@firma.de', subject: 'Protokoll', body: 'Die Geschäftsführung hat entschieden.' },
|
||||
];
|
||||
|
||||
const hits = (version: StalwartTextVersion, filter: Filter) =>
|
||||
MESSAGES.filter((m) => textFilterMatches(version, filter, m)).map((m) => m.id);
|
||||
|
||||
/** What the app sent before: a `*` after every word (the old toWildcardQuery). */
|
||||
const withWildcards = (query: string) => query.trim().split(/\s+/)
|
||||
.map((word) => (word.endsWith('*') || word.endsWith('"') ? word : `${word}*`))
|
||||
.join(' ');
|
||||
|
||||
describe('splitSearchText', () => {
|
||||
it('gives every quoted phrase a part of its own', () => {
|
||||
expect(splitSearchText('"annual report" 2024 "draft"')).toEqual(['"annual report"', '2024', '"draft"']);
|
||||
});
|
||||
|
||||
it('keeps plain queries as one part, as typed', () => {
|
||||
expect(splitSearchText('project budget')).toEqual(['project budget']);
|
||||
expect(splitSearchText('anna@firma.de')).toEqual(['anna@firma.de']);
|
||||
expect(splitSearchText('Geschäftsführung')).toEqual(['Geschäftsführung']);
|
||||
expect(splitSearchText(' project budget ')).toEqual(['project budget']);
|
||||
expect(splitSearchText('e-mail')).toEqual(['e-mail']);
|
||||
});
|
||||
|
||||
it('gives every word ending in * a part of its own', () => {
|
||||
expect(splitSearchText('project budg*')).toEqual(['project', 'budg*']);
|
||||
expect(splitSearchText('"a b" c d*')).toEqual(['"a b"', 'c', 'd*']);
|
||||
});
|
||||
|
||||
it('keeps the other words together in one part where the first of them was typed', () => {
|
||||
expect(splitSearchText('q3 "annual report" budget')).toEqual(['q3 budget', '"annual report"']);
|
||||
expect(splitSearchText('budg* "annual report" project')).toEqual(['budg*', '"annual report"', 'project']);
|
||||
});
|
||||
|
||||
it('closes an unterminated quote at the end of the input', () => {
|
||||
expect(splitSearchText('"half typed')).toEqual(['"half typed"']);
|
||||
expect(splitSearchText('invoice "half typed')).toEqual(['invoice', '"half typed"']);
|
||||
});
|
||||
|
||||
it('drops empty quotes and lone stars', () => {
|
||||
expect(splitSearchText('""')).toEqual([]);
|
||||
expect(splitSearchText('" " budget')).toEqual(['budget']);
|
||||
expect(splitSearchText('*')).toEqual([]);
|
||||
expect(splitSearchText('budget * **')).toEqual(['budget']);
|
||||
});
|
||||
|
||||
it('strips outer single quotes, which would make the words one phrase', () => {
|
||||
expect(splitSearchText("'annual' 2024 'draft'")).toEqual(["annual' 2024 'draft"]);
|
||||
expect(splitSearchText("'annual report'")).toEqual(['annual report']);
|
||||
expect(splitSearchText("''")).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('textConditions / textFilter', () => {
|
||||
it('makes one condition per part on the given property', () => {
|
||||
expect(textConditions('"annual report" 2024 "draft"')).toEqual([
|
||||
{ text: '"annual report"' }, { text: '2024' }, { text: '"draft"' },
|
||||
]);
|
||||
expect(textConditions('anna mül*', 'from')).toEqual([{ from: 'anna' }, { from: 'mül*' }]);
|
||||
});
|
||||
|
||||
it('sends the input as before when no part is left', () => {
|
||||
expect(textConditions('""')).toEqual([{ text: '""' }]);
|
||||
expect(textConditions(' * ')).toEqual([{ text: '*' }]);
|
||||
});
|
||||
|
||||
it('returns a single condition as is and ANDs several, never mixing keys', () => {
|
||||
expect(textFilter('project budget')).toEqual({ text: 'project budget' });
|
||||
const filter = textFilter('"annual report" 2024');
|
||||
expect(filter).toEqual({ operator: 'AND', conditions: [{ text: '"annual report"' }, { text: '2024' }] });
|
||||
expectNoMixedKeys(filter);
|
||||
});
|
||||
});
|
||||
|
||||
describe('text search results (P1-7)', () => {
|
||||
it.each<[string, string[]]>([
|
||||
['project budget', ['budget']],
|
||||
['anna@firma.de', ['report']],
|
||||
['Geschäftsführung', ['board']],
|
||||
])('finds %s on Stalwart 1.0, where the old wildcards found nothing, and the same on 0.16', (query, expected) => {
|
||||
// `project* budget*` is one prefix containing a space; the address is not
|
||||
// one word; `geschäftsführung` is 18 bytes, over the 16-byte prefix cap.
|
||||
expect(hits('1.0', { text: withWildcards(query) })).toEqual([]);
|
||||
expect(textFilter(query)).toEqual({ text: query });
|
||||
expect(hits('1.0', textFilter(query))).toEqual(expected);
|
||||
// 0.16 drops the `*`, so sending the text as typed changes nothing there.
|
||||
expect(hits('0.16', { text: withWildcards(query) })).toEqual(expected);
|
||||
expect(hits('0.16', textFilter(query))).toEqual(expected);
|
||||
});
|
||||
|
||||
it('finds "annual report" 2024 "draft" on Stalwart 1.0 as three conditions', () => {
|
||||
expect(hits('1.0', { text: withWildcards('"annual report" 2024') })).toEqual([]);
|
||||
expect(hits('1.0', { text: '"annual report" 2024 "draft"' })).toEqual([]);
|
||||
expect(hits('1.0', textFilter('"annual report" 2024 "draft"'))).toEqual(['report']);
|
||||
});
|
||||
|
||||
it('keeps the 0.16 hits for "annual report" 2024 "draft": an AND of the words', () => {
|
||||
expect(hits('0.16', { text: '"annual report" 2024 "draft"' })).toEqual(['report', 'shuffled']);
|
||||
expect(hits('0.16', textFilter('"annual report" 2024 "draft"'))).toEqual(['report', 'shuffled']);
|
||||
});
|
||||
|
||||
it('finds project budg* on Stalwart 1.0 and changes nothing on 0.16', () => {
|
||||
expect(hits('1.0', { text: 'project budg*' })).toEqual([]);
|
||||
expect(hits('1.0', textFilter('project budg*'))).toEqual(['budget']);
|
||||
expect(hits('0.16', textFilter('project budg*'))).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,88 @@
|
||||
// A small model of how Stalwart's built-in search index matches one text
|
||||
// filter value against one field, so tests can check what a filter finds on
|
||||
// each version, not only its shape. It leaves out stemming and language
|
||||
// detection. Same model as the webmail's lib/__tests__/search-utils.test.ts.
|
||||
// - 1.0 (store/src/search/document.rs `has_text`): a value wrapped in `"` or
|
||||
// `'` is an exact phrase (word positions exist for subject and body only;
|
||||
// address fields fall back to all words); a value ending in `*` is one
|
||||
// prefix over the rest, lower-cased and not split into words, at most 16
|
||||
// bytes, matching words of at most 16 bytes; anything else needs all its
|
||||
// words in the same field.
|
||||
// - 0.16: no phrases and no prefixes; every value needs all its words in the
|
||||
// same field, and `*` only separates words.
|
||||
|
||||
export type StalwartTextVersion = '0.16' | '1.0';
|
||||
|
||||
const wordsOf = (value: string): string[] => value.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? [];
|
||||
const byteLength = (value: string) => new TextEncoder().encode(value).length;
|
||||
|
||||
/** Whether `value` matches `field`; `hasPositions` for subject and body. */
|
||||
export function textFieldMatches(
|
||||
version: StalwartTextVersion,
|
||||
value: string,
|
||||
field: string,
|
||||
hasPositions: boolean,
|
||||
): boolean {
|
||||
const tokens = wordsOf(field);
|
||||
if (version === '1.0') {
|
||||
const exact = /^"([\s\S]*)"$/.exec(value) ?? /^'([\s\S]*)'$/.exec(value);
|
||||
if (exact) {
|
||||
const phrase = wordsOf(exact[1]);
|
||||
if (phrase.length === 0) return false;
|
||||
if (!hasPositions) return phrase.every((w) => tokens.includes(w));
|
||||
return tokens.some((_, i) => phrase.every((w, j) => tokens[i + j] === w));
|
||||
}
|
||||
if (value.endsWith('*')) {
|
||||
const prefix = value.slice(0, -1).trim().toLowerCase();
|
||||
if (!prefix || byteLength(prefix) > 16) return false;
|
||||
return tokens.some((t) => byteLength(t) <= 16 && t.startsWith(prefix));
|
||||
}
|
||||
}
|
||||
const words = wordsOf(value);
|
||||
return words.length > 0 && words.every((w) => tokens.includes(w));
|
||||
}
|
||||
|
||||
export interface TextModelMessage {
|
||||
from?: string;
|
||||
to?: string;
|
||||
subject?: string;
|
||||
body?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a message matches an Email/query filter made of text conditions
|
||||
* (`text`, `from`, `to`, `subject`, `body`) and AND/OR/NOT operators. `text`
|
||||
* is an OR over from, to, subject and body. Other keys are handed to
|
||||
* `other`, or throw.
|
||||
*/
|
||||
export function textFilterMatches(
|
||||
version: StalwartTextVersion,
|
||||
filter: Record<string, unknown>,
|
||||
message: TextModelMessage,
|
||||
other?: (key: string, value: unknown) => boolean,
|
||||
): boolean {
|
||||
if ('operator' in filter) {
|
||||
const results = (filter.conditions as Record<string, unknown>[])
|
||||
.map((c) => textFilterMatches(version, c, message, other));
|
||||
if (filter.operator === 'AND') return results.every(Boolean);
|
||||
if (filter.operator === 'OR') return results.some(Boolean);
|
||||
return !results.some(Boolean);
|
||||
}
|
||||
return Object.entries(filter).every(([key, value]) => {
|
||||
const field = (name: keyof TextModelMessage, positions: boolean) =>
|
||||
textFieldMatches(version, String(value), message[name] ?? '', positions);
|
||||
switch (key) {
|
||||
case 'text':
|
||||
return field('from', false) || field('to', false) || field('subject', true) || field('body', true);
|
||||
case 'from':
|
||||
case 'to':
|
||||
return field(key, false);
|
||||
case 'subject':
|
||||
case 'body':
|
||||
return field(key, true);
|
||||
default:
|
||||
if (other) return other(key, value);
|
||||
throw new Error(`the text model has no ${key} condition`);
|
||||
}
|
||||
});
|
||||
}
|
||||
+71
-7
@@ -1,8 +1,72 @@
|
||||
export function toWildcardQuery(query: string): string {
|
||||
return query
|
||||
.trim()
|
||||
.split(/\s+/)
|
||||
.filter(Boolean)
|
||||
.map((word) => (word.endsWith('*') || word.endsWith('"') ? word : word + '*'))
|
||||
.join(' ');
|
||||
// Free-text search conditions for Email/query, ported from the webmail's
|
||||
// lib/jmap/search-utils.ts (5967a1c4) so both clients send the same filters.
|
||||
|
||||
/** Filter properties whose value the server reads as free text. */
|
||||
export type SearchTextProperty = 'text' | 'from' | 'to' | 'subject' | 'body';
|
||||
|
||||
// A "quoted phrase" (the closing quote may still be missing while the user
|
||||
// types) or a run of non-space characters.
|
||||
const SEARCH_PART_RE = /"([^"]*)"?|(\S+)/g;
|
||||
|
||||
/**
|
||||
* Splits typed search text into the values to send as separate text
|
||||
* conditions, ANDed together.
|
||||
*
|
||||
* Stalwart 1.0 reads a whole filter value that starts and ends with `"` (or
|
||||
* `'`) as one exact phrase, and a value ending in `*` as one prefix over
|
||||
* everything before it (not split into words, at most 16 bytes). Sent as one
|
||||
* value, `"annual report" 2024 "draft"` is a single phrase and
|
||||
* `project budg*` a prefix containing a space: both find nothing. So:
|
||||
* - every `"phrase"` is a part of its own, kept quoted; empty quotes are
|
||||
* dropped;
|
||||
* - every word ending in `*` is a part of its own; a lone `*` is dropped;
|
||||
* - all other words stay together in one part, joined by single spaces, so
|
||||
* they still have to match within one field and the server detects the
|
||||
* language on the whole string. Without quotes or stars, that part is the
|
||||
* query as typed.
|
||||
* Outer `'` quotes are stripped from that last part, since they would make
|
||||
* it a phrase too. The built-in index of 0.16 has neither phrases nor
|
||||
* prefixes: a quoted part is an AND of its words, matched without stemming,
|
||||
* and `*` only separates words.
|
||||
*/
|
||||
export function splitSearchText(raw: string): string[] {
|
||||
const parts: string[] = [];
|
||||
const words: string[] = [];
|
||||
let wordsAt = -1;
|
||||
for (const [, phrase, word] of raw.matchAll(SEARCH_PART_RE)) {
|
||||
if (phrase !== undefined) {
|
||||
const trimmed = phrase.trim();
|
||||
if (trimmed) parts.push(`"${trimmed}"`);
|
||||
} else if (word.endsWith('*')) {
|
||||
if (!/^\*+$/.test(word)) parts.push(word);
|
||||
} else {
|
||||
if (wordsAt < 0) wordsAt = parts.length;
|
||||
words.push(word);
|
||||
}
|
||||
}
|
||||
let bare = words.join(' ');
|
||||
while (bare.length > 1 && bare.startsWith("'") && bare.endsWith("'")) {
|
||||
bare = bare.slice(1, -1).trim();
|
||||
}
|
||||
// Keep the parts in the order they were typed.
|
||||
if (bare) parts.splice(wordsAt, 0, bare);
|
||||
return parts;
|
||||
}
|
||||
|
||||
/**
|
||||
* One `{ [property]: part }` condition per part of `raw` (see
|
||||
* splitSearchText). When no part is left (`""`, a lone `*`), the input is
|
||||
* sent as one condition as before: dropping it would widen the search to
|
||||
* every message.
|
||||
*/
|
||||
export function textConditions(raw: string, property: SearchTextProperty = 'text'): Record<string, string>[] {
|
||||
const parts = splitSearchText(raw);
|
||||
if (parts.length === 0) return [{ [property]: raw.trim() }];
|
||||
return parts.map((part) => ({ [property]: part }));
|
||||
}
|
||||
|
||||
/** `textConditions` as one filter: the single condition, or an AND of them. */
|
||||
export function textFilter(raw: string, property: SearchTextProperty = 'text'): Record<string, unknown> {
|
||||
const conditions = textConditions(raw, property);
|
||||
return conditions.length === 1 ? conditions[0] : { operator: 'AND', conditions };
|
||||
}
|
||||
|
||||
@@ -799,8 +799,10 @@ export default function EmailListScreen({ onEmailPress, onComposePress }: EmailL
|
||||
};
|
||||
|
||||
// Local input state for uninterrupted typing. The search runs on submit,
|
||||
// or after a 600 ms pause once at least two characters are typed — with
|
||||
// the `*` wildcard a single letter matches the whole mailbox.
|
||||
// or after a 600 ms pause once at least two characters are typed. The text
|
||||
// goes out as typed and the server matches whole words, so a half-typed
|
||||
// word finds nothing until it is complete (a typed `*` makes it a prefix on
|
||||
// Stalwart 1.0).
|
||||
const [searchInput, setSearchInput] = React.useState(storeSearchQuery);
|
||||
const [searchFocused, setSearchFocused] = React.useState(false);
|
||||
React.useEffect(() => {
|
||||
|
||||
@@ -190,7 +190,7 @@ describe('"All folders" search across the own and the team account (#1082)', ()
|
||||
expect(server.current!.requests).toHaveLength(1);
|
||||
const queries = server.current!.callsOf('Email/query');
|
||||
expect(queries.map(([, args]) => args.accountId)).toEqual(['c', 'team']);
|
||||
for (const [, args] of queries) expect(args.filter).toEqual({ text: 'zephyr*' });
|
||||
for (const [, args] of queries) expect(args.filter).toEqual({ text: 'zephyr' });
|
||||
expect(server.current!.callsOf('Email/get').map(([, args]) => args.accountId)).toEqual(['c', 'team']);
|
||||
});
|
||||
|
||||
@@ -211,7 +211,7 @@ describe('"All folders" search across the own and the team account (#1082)', ()
|
||||
|
||||
const queries = server.current!.callsOf('Email/query');
|
||||
expect(queries).toHaveLength(1);
|
||||
expect(queries[0][1]).toMatchObject({ accountId: 'team', filter: { inMailbox: 't-inbox', text: 'zephyr*' } });
|
||||
expect(queries[0][1]).toMatchObject({ accountId: 'team', filter: { inMailbox: 't-inbox', text: 'zephyr' } });
|
||||
expect(ids()).toEqual(['m1', 'm2', 'm3', 'm4']);
|
||||
expect(useEmailStore.getState().emails[0].jmapAccountId).toBeUndefined();
|
||||
});
|
||||
@@ -541,15 +541,41 @@ describe('tag view across the own and the team account (#1038)', () => {
|
||||
it('narrows the tag with a search across both accounts', async () => {
|
||||
await openTag('inbox');
|
||||
useEmailStore.getState().setSearchQuery('red 2');
|
||||
// The fake matches text like Stalwart 1.0, where the old `red* 2*` was
|
||||
// one prefix containing a space and found nothing (P1-7).
|
||||
await vi.waitFor(() => expect(ids()).toEqual(['m2', 'o2']));
|
||||
|
||||
const last = server.current!.requests.at(-1)!.filter(([n]) => n === 'Email/query');
|
||||
expect(last.map(([, a]) => a.filter)).toEqual([
|
||||
{ operator: 'AND', conditions: [{ text: 'red* 2*' }, { hasKeyword: RED }] },
|
||||
{ operator: 'AND', conditions: [{ text: 'red* 2*' }, { hasKeyword: RED }] },
|
||||
{ operator: 'AND', conditions: [{ text: 'red 2' }, { hasKeyword: RED }] },
|
||||
{ operator: 'AND', conditions: [{ text: 'red 2' }, { hasKeyword: RED }] },
|
||||
]);
|
||||
});
|
||||
|
||||
// 1.0 matches the phrase in order; 0.16 has no phrases and reads it as its
|
||||
// words.
|
||||
it.each([['0.16', ['m1', 'o1']], ['1.0', ['o1']]] as const)(
|
||||
'Stalwart %s: narrows the tag with a quoted phrase as a condition of its own',
|
||||
async (textModel, expected) => {
|
||||
server.current = createFakeJmap({
|
||||
c: [mail('o1', 'inbox', 20, 'own red 2 notes', { keywords: { [RED]: true } })],
|
||||
team: [mail('m1', 't-inbox', 22, 'red notes 2', { keywords: { [RED]: true } })],
|
||||
}, { textModel });
|
||||
await openTag('inbox');
|
||||
useEmailStore.getState().setSearchQuery('"red 2" notes');
|
||||
await vi.waitFor(() => {
|
||||
const last = server.current!.requests.at(-1)!.filter(([n]) => n === 'Email/query');
|
||||
expect(last[0][1].filter).toEqual({
|
||||
operator: 'AND',
|
||||
conditions: [{ text: '"red 2"' }, { text: 'notes' }, { hasKeyword: RED }],
|
||||
});
|
||||
});
|
||||
await vi.waitFor(() => expect(useEmailStore.getState().loading).toBe(false));
|
||||
|
||||
expect(ids()).toEqual(expected);
|
||||
},
|
||||
);
|
||||
|
||||
it('untags a team row in the team account and keeps it in view', async () => {
|
||||
await openTag('inbox');
|
||||
await useEmailStore.getState().setKeywordForEmails(['team:m2'], RED, false);
|
||||
|
||||
@@ -708,7 +708,32 @@ describe('email-store', () => {
|
||||
|
||||
const [mailboxId, opts] = mockQueryEmails.mock.calls[0];
|
||||
expect(mailboxId).toBeUndefined();
|
||||
expect(opts.filter).toEqual({ text: 'invoice*' });
|
||||
// As typed: Stalwart 1.0 reads a trailing `*` as one prefix over the
|
||||
// whole value (P1-7).
|
||||
expect(opts.filter).toEqual({ text: 'invoice' });
|
||||
});
|
||||
|
||||
it('sends quoted phrases and word* as conditions of their own, the field filters too (Stalwart 1.0)', async () => {
|
||||
useEmailStore.setState({
|
||||
currentMailboxId: 'mb-1',
|
||||
searchQuery: '"annual report" 2024 budg*',
|
||||
filters: { from: 'anna mül*', subject: 'project budget' },
|
||||
});
|
||||
mockQueryEmails.mockResolvedValue({ ids: [], total: 0, queryState: 'q' });
|
||||
|
||||
await useEmailStore.getState().refreshEmails();
|
||||
|
||||
expect(mockQueryEmails.mock.calls[0][1].filter).toEqual({
|
||||
operator: 'AND',
|
||||
conditions: [
|
||||
{ text: '"annual report"' },
|
||||
{ text: '2024' },
|
||||
{ text: 'budg*' },
|
||||
{ from: 'anna' },
|
||||
{ from: 'mül*' },
|
||||
{ subject: 'project budget' },
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it('scopes a search to the open folder or a picked folder when asked', async () => {
|
||||
@@ -758,7 +783,7 @@ describe('email-store', () => {
|
||||
|
||||
expect(useEmailStore.getState().searchQuery).toBe('invoice');
|
||||
expect(mockQueryEmails.mock.calls[0][0]).toBe('mb-2');
|
||||
expect(mockQueryEmails.mock.calls[0][1].filter).toEqual({ text: 'invoice*' });
|
||||
expect(mockQueryEmails.mock.calls[0][1].filter).toEqual({ text: 'invoice' });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1230,7 +1255,7 @@ describe('email-store', () => {
|
||||
await vi.waitFor(() => expect(useEmailStore.getState().threadCounts).toEqual({ 'acc-1:t1': 3, 'acc-1:t2': 1 }));
|
||||
|
||||
expect(mockQueryAcross).toHaveBeenCalledTimes(1);
|
||||
expect(mockQueryAcross.mock.calls[0][1]).toMatchObject({ filter: { text: 'invoice*' }, threads: true });
|
||||
expect(mockQueryAcross.mock.calls[0][1]).toMatchObject({ filter: { text: 'invoice' }, threads: true });
|
||||
expect(mockQueryEmailPage).toHaveBeenCalledTimes(1);
|
||||
expect(mockGetThreads).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -3,45 +3,38 @@
|
||||
// newest first, position + limit), Email/get and Thread/get (including
|
||||
// result references), and applies Email/set updates and destroys. Every
|
||||
// request is recorded so a test can check which method calls went out, and
|
||||
// in how many requests.
|
||||
// in how many requests. Text conditions are matched the way Stalwart's
|
||||
// built-in index does (see lib/__tests__/stalwart-text-model), 1.0 unless
|
||||
// asked for 0.16, against the subject and the preview (standing in for the
|
||||
// body).
|
||||
|
||||
import type { Email, JMAPMethodCall } from '../../api/types';
|
||||
import { textFilterMatches, type StalwartTextVersion } from '../../lib/__tests__/stalwart-text-model';
|
||||
|
||||
export type FakeEmail = Pick<Email, 'id' | 'threadId' | 'mailboxIds' | 'keywords' | 'receivedAt' | 'subject'>
|
||||
& Partial<Email>;
|
||||
|
||||
type Filter = Record<string, unknown>;
|
||||
|
||||
function matches(email: FakeEmail, filter: Filter | undefined): boolean {
|
||||
function matches(email: FakeEmail, filter: Filter | undefined, textModel: StalwartTextVersion): boolean {
|
||||
if (!filter) return true;
|
||||
if ('operator' in filter) {
|
||||
const results = (filter.conditions as Filter[]).map((c) => matches(email, c));
|
||||
if (filter.operator === 'AND') return results.every(Boolean);
|
||||
if (filter.operator === 'OR') return results.some(Boolean);
|
||||
return !results.some(Boolean);
|
||||
}
|
||||
for (const [key, value] of Object.entries(filter)) {
|
||||
if (key === 'text') {
|
||||
const needle = String(value).replace(/\*/g, '').toLowerCase();
|
||||
if (!`${email.subject ?? ''} ${email.preview ?? ''}`.toLowerCase().includes(needle)) return false;
|
||||
} else if (key === 'inMailbox') {
|
||||
if (!email.mailboxIds[value as string]) return false;
|
||||
} else if (key === 'hasKeyword') {
|
||||
if (!email.keywords[value as string]) return false;
|
||||
} else if (key === 'notKeyword') {
|
||||
if (email.keywords[value as string]) return false;
|
||||
} else {
|
||||
throw new Error(`fake server: unsupported filter ${key}`);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
return textFilterMatches(textModel, filter, { subject: email.subject ?? '', body: email.preview ?? '' }, (key, value) => {
|
||||
if (key === 'inMailbox') return !!email.mailboxIds[value as string];
|
||||
if (key === 'hasKeyword') return !!email.keywords[value as string];
|
||||
if (key === 'notKeyword') return !email.keywords[value as string];
|
||||
throw new Error(`fake server: unsupported filter ${key}`);
|
||||
});
|
||||
}
|
||||
|
||||
function unescapePointer(segment: string): string {
|
||||
return segment.replace(/~1/g, '/').replace(/~0/g, '~');
|
||||
}
|
||||
|
||||
export function createFakeJmap(accounts: Record<string, FakeEmail[]>) {
|
||||
export function createFakeJmap(
|
||||
accounts: Record<string, FakeEmail[]>,
|
||||
options: { textModel?: StalwartTextVersion } = {},
|
||||
) {
|
||||
const textModel = options.textModel ?? '1.0';
|
||||
const requests: JMAPMethodCall[][] = [];
|
||||
/** JMAP account id → error type every call for it is answered with. */
|
||||
const failing = new Map<string, string>();
|
||||
@@ -72,7 +65,7 @@ export function createFakeJmap(accounts: Record<string, FakeEmail[]>) {
|
||||
response = ['error', { type: 'invalidResultReference' }, callId];
|
||||
} else if (name === 'Email/query') {
|
||||
const all = store
|
||||
.filter((e) => matches(e, args.filter as Filter))
|
||||
.filter((e) => matches(e, args.filter as Filter, textModel))
|
||||
.sort((a, b) => b.receivedAt.localeCompare(a.receivedAt));
|
||||
const position = (args.position as number) ?? 0;
|
||||
const limit = (args.limit as number) ?? 50;
|
||||
|
||||
@@ -41,7 +41,7 @@ import { JMAPMethodError } from '../api/jmap-result';
|
||||
import {
|
||||
mailboxesForSiblingOf, mailboxesOfAccount, findJunkMailbox, findArchiveMailbox, findTrashMailbox, ownMailboxes,
|
||||
} from '../lib/mailbox-tree';
|
||||
import { toWildcardQuery } from '../lib/search-utils';
|
||||
import { textConditions } from '../lib/search-utils';
|
||||
import { collapseThreads, rowKeyOf } from '../lib/thread-utils';
|
||||
import { compareEmails, levelKeyword, orderForMailbox, sanitizeSortLevels, type SortLevel } from '../lib/message-list-order';
|
||||
import { buildListSort, markKeywordSortUnsupported } from '../lib/keyword-sort-polarity';
|
||||
@@ -438,14 +438,17 @@ function buildJmapFilter(
|
||||
): Record<string, unknown> | undefined {
|
||||
const conditions: Record<string, unknown>[] = [];
|
||||
|
||||
// Sent as typed: quoted phrases and `word*` prefixes become conditions of
|
||||
// their own, the other words one condition (see splitSearchText). The field
|
||||
// filters follow the same rules on the server.
|
||||
const trimmed = searchQuery.trim();
|
||||
if (trimmed) conditions.push({ text: toWildcardQuery(trimmed) });
|
||||
if (trimmed) conditions.push(...textConditions(trimmed));
|
||||
|
||||
if (filters.keyword) conditions.push({ hasKeyword: filters.keyword });
|
||||
if (filters.from) conditions.push({ from: filters.from });
|
||||
if (filters.to) conditions.push({ to: filters.to });
|
||||
if (filters.subject) conditions.push({ subject: filters.subject });
|
||||
if (filters.body) conditions.push({ body: filters.body });
|
||||
if (filters.from) conditions.push(...textConditions(filters.from, 'from'));
|
||||
if (filters.to) conditions.push(...textConditions(filters.to, 'to'));
|
||||
if (filters.subject) conditions.push(...textConditions(filters.subject, 'subject'));
|
||||
if (filters.body) conditions.push(...textConditions(filters.body, 'body'));
|
||||
|
||||
if (filters.dateAfter) {
|
||||
const d = new Date(filters.dateAfter);
|
||||
|
||||
Reference in New Issue
Block a user