improve inbox rule handling and run spam prediction after inbox rules

Instead of applying inbox rules based on the unread mail state in the
inbox folder, we introduce the new ProcessingState enum on
the mail type. If a mail has been processed by the leader client, which
is checking for matching inbox rules, the ProcessingState is
updated. If there is a matching rule the flag is updated through the
MoveMailService, if there is no matching rule, the flag is updated
using the ClientClassifierResultService. Both requests are
throttled / debounced. After processing inbox rules, spam prediction
is conducted for mails that have not yet been moved by an inbox rule.
The ProcessingState for not matching ham mails is also updated using
the ClientClassifierResultService.

This new inbox rule handing solves the following two problems:
 - when clicking on a notification it could still happen,
   that sometimes the inbox rules where not applied
 - when the inbox folder had a lot of unread mails, the loading time did
   massively increase, since inbox rules were re-applied on every load

Co-authored-by: amm <amm@tutao.de>
Co-authored-by: Nick <nif@tutao.de>
Co-authored-by: das <das@tutao.de>
Co-authored-by: abp <abp@tutao.de>
Co-authored-by: jhm <17314077+jomapp@users.noreply.github.com>
Co-authored-by: map <mpfau@users.noreply.github.com>
Co-authored-by: Kinan <104761667+kibibytium@users.noreply.github.com>

buildSrc/RollupConfig.js

@@ -35,7 +35,7 @@ export const allowedImports = {
 	boot: ["polyfill-helpers", "common-min"],
 	common: ["polyfill-helpers", "common-min"],
 	"gui-base": ["polyfill-helpers", "common-min", "common", "boot"],
-	main: ["polyfill-helpers", "common-min", "common", "boot", "gui-base", "date"],
+	main: ["polyfill-helpers", "common-min", "common", "boot", "gui-base", "date", "spam-classifier"],
 	sanitizer: ["polyfill-helpers", "common-min", "common", "boot", "gui-base"],
 	date: ["polyfill-helpers", "common-min", "common"],
 	"date-gui": ["polyfill-helpers", "common-min", "common", "boot", "gui-base", "main", "sharing", "date", "contacts", "ui-extra"],
@@ -47,7 +47,7 @@ export const allowedImports = {
 	"calendar-view": ["polyfill-helpers", "common-min", "common", "boot", "gui-base", "main", "date", "date-gui", "sharing", "contacts"],
 	login: ["polyfill-helpers", "common-min", "common", "boot", "gui-base", "main"],
 	"spam-classifier": ["polyfill-helpers", "common", "common-min"],
-	worker: ["polyfill-helpers", "common-min", "common", "native-common", "native-worker", "wasm", "wasm-fallback", "spam-classifier"],
+	worker: ["polyfill-helpers", "common-min", "common", "native-common", "native-worker", "wasm", "wasm-fallback"],
 	"pow-worker": [],
 	settings: [
 		"polyfill-helpers",

buildSrc/updateLibs.js

@@ -172,7 +172,7 @@ async function rollupTensorFlow(src, target, banner) {
 					},
 				],
 			}),
-			logResolvePlugin,
+			// logResolvePlugin,
 			nodeResolve(),
 			commonjs(),
 		],

src/calendar-app/calendar-app.ts

@@ -112,6 +112,7 @@ import("../mail-app/translations/en.js")
 					if (isApp()) {
 						calendarLocator.fileApp.clearFileData().catch((e) => console.log("Failed to clean file data", e))
 					}
+					return { asyncAction: Promise.resolve() }
 				},
 				async onFullLoginSuccess() {},
 			}
@@ -333,7 +334,13 @@ import("../mail-app/translations/en.js")
 				},
 				calendarLocator.logins,
 			),
-			webauthnmobile: makeViewResolver<MobileWebauthnAttrs, MobileWebauthnView, { browserWebauthn: BrowserWebauthn }>(
+			webauthnmobile: makeViewResolver<
+				MobileWebauthnAttrs,
+				MobileWebauthnView,
+				{
+					browserWebauthn: BrowserWebauthn
+				}
+			>(
 				{
 					prepareRoute: async () => {
 						const { MobileWebauthnView } = await import("../common/login/MobileWebauthnView.js")

src/calendar-app/workerUtils/worker/CalendarWorkerLocator.ts

@@ -521,7 +521,6 @@ export async function initLocator(worker: CalendarWorkerImpl, browserData: Brows
 			locator.login,
 			locator.keyLoader,
 			locator.publicEncryptionKeyProvider,
-			null,
 		)
 	})
 	const nativePushFacade = new NativePushFacadeSendDispatcher(worker)

src/common/api/common/ClientClassifierType.ts

@@ -1,4 +1,5 @@
 // Keep in sync with server: ClassifierType
 export enum ClientClassifierType {
 	CLIENT_CLASSIFICATION = "20",
+	CUSTOMER_INBOX_RULES = "21",
 }

src/common/api/common/TutanotaConstants.ts

@@ -3,13 +3,13 @@
 import { DAY_IN_MILLIS, downcast } from "@tutao/tutanota-utils"
 import type { CertificateInfo, CreditCard, EmailSenderListElement, GroupMembership } from "../entities/sys/TypeRefs.js"
 import { AccountingInfo, Customer } from "../entities/sys/TypeRefs.js"
-import type { CalendarEventAttendee, ContactCustomDate, ContactRelationship, UserSettingsGroupRoot } from "../entities/tutanota/TypeRefs.js"
+import type { CalendarEventAttendee, ContactCustomDate, ContactRelationship, Mail, UserSettingsGroupRoot } from "../entities/tutanota/TypeRefs.js"
 import { ContactSocialId, MailFolder } from "../entities/tutanota/TypeRefs.js"
 import { isApp, isElectronClient, isIOSApp } from "./Env"
 import type { Country } from "./CountryList"
 import { ProgrammingError } from "./error/ProgrammingError"
 
-export const MAX_NBR_MOVE_DELETE_MAIL_SERVICE = 50
+export const MAX_NBR_OF_MAILS_SYNC_OPERATION = 50
 export const MAX_NBR_OF_CONVERSATIONS = 50
 
 // visible for testing
@@ -1386,4 +1386,22 @@ export enum DeactivationReason {
 	MassSignup,
 }
 
+export enum SpamDecision {
+	NONE = "0",
+	WHITELIST = "1",
+	BLACKLIST = "2",
+	DISCARD = "3",
+}
+
+export enum ProcessingState {
+	INBOX_RULE_PROCESSED_AND_SPAM_PREDICTION_MADE = "0",
+	INBOX_RULE_NOT_PROCESSED = "1",
+	INBOX_RULE_APPLIED = "2",
+	INBOX_RULE_PROCESSED_AND_SPAM_PREDICTION_PENDING = "3",
+}
+
+export function getProcessingState(mail: Mail): ProcessingState {
+	return mail.processingState as ProcessingState
+}
+
 export const PLAN_SELECTOR_SELECTED_BOX_SCALE = "1.03"

src/common/api/common/utils/ErrorUtils.ts

@@ -199,3 +199,11 @@ export function objToError(o: Record<string, any>): Error {
 	e.data = o.data
 	return e
 }
+
+/**
+ * Returns whether the error is expected for the cases where our local state might not be up-to-date with the server yet. E.g. we might be processing an update
+ * for the instance that was already deleted. Normally this would be optimized away but it might still happen due to timing.
+ */
+export function isExpectedErrorForSynchronization(e: Error): boolean {
+	return e instanceof NotFoundError || e instanceof NotAuthorizedError
+}

src/common/api/common/utils/IndexUtils.ts

@@ -87,6 +87,8 @@ export function _createNewIndexUpdate(typeInfo: TypeInfo): IndexUpdate {
 	}
 }
 
+// Removes most html tags from a text.
+// NOTE: This function is not covering all edge-cases.
 export function htmlToText(html: string | null): string {
 	if (html == null) return ""
 	let text = html.replace(/<[^>]*>?/gm, " ")

src/common/api/main/LoginController.ts

@@ -20,7 +20,7 @@ assertMainOrNodeBoot()
 
 export interface PostLoginAction {
 	/** Partial login is achieved with getting the user, can happen offline. The login will wait for the returned promise. */
-	onPartialLoginSuccess(loggedInEvent: LoggedInEvent): Promise<void>
+	onPartialLoginSuccess(loggedInEvent: LoggedInEvent): Promise<{ asyncAction: Promise<void> }>
 
 	/** Full login is achieved with getting group keys. Can do service calls from this point on. */
 	onFullLoginSuccess(loggedInEvent: LoggedInEvent): Promise<void>

src/common/api/worker/EventInstancePrefetcher.ts

@@ -5,10 +5,10 @@ import { assertNotNull, getTypeString, groupBy, isNotNull, isSameTypeRef, parseT
 import { parseKeyVersion } from "./facades/KeyLoaderFacade"
 import { VersionedEncryptedKey } from "./crypto/CryptoWrapper"
 import { OperationType } from "../common/TutanotaConstants"
-import { NotAuthorizedError, NotFoundError } from "../common/error/RestError"
 import { ElementEntity, ListElementEntity, SomeEntity } from "../common/EntityTypes"
 import { CacheMode, type EntityRestInterface } from "./rest/EntityRestClient"
 import { ProgressMonitorDelegate } from "./ProgressMonitorDelegate"
+import { isExpectedErrorForSynchronization } from "../common/utils/ErrorUtils"
 
 export class EventInstancePrefetcher {
 	constructor(private readonly entityCache: EntityRestInterface) {}
@@ -191,11 +191,3 @@ export class EventInstancePrefetcher {
 		}
 	}
 }
-
-/**
- * Returns whether the error is expected for the cases where our local state might not be up-to-date with the server yet. E.g. we might be processing an update
- * for the instance that was already deleted. Normally this would be optimized away but it might still happen due to timing.
- */
-function isExpectedErrorForSynchronization(e: Error): boolean {
-	return e instanceof NotFoundError || e instanceof NotAuthorizedError
-}

src/common/api/worker/facades/lazy/MailFacade.ts

@@ -1,6 +1,7 @@
 import type { CryptoFacade } from "../../crypto/CryptoFacade.js"
 import {
 	ApplyLabelService,
+	ClientClassifierResultService,
 	DraftService,
 	ExternalUserService,
 	ListUnsubscribeService,
@@ -27,10 +28,11 @@ import {
 	MailMethod,
 	MailReportType,
 	MailSetKind,
-	MAX_NBR_MOVE_DELETE_MAIL_SERVICE,
 	MAX_NBR_OF_CONVERSATIONS,
+	MAX_NBR_OF_MAILS_SYNC_OPERATION,
 	OperationType,
 	PhishingMarkerStatus,
+	ProcessingState,
 	PublicKeyIdentifierType,
 	ReportedMailFieldType,
 	SimpleMoveMailTarget,
@@ -40,6 +42,7 @@ import {
 	Contact,
 	createApplyLabelServicePostIn,
 	createAttachmentKeyData,
+	createClientClassifierResultPostIn,
 	createCreateExternalUserGroupData,
 	createCreateMailFolderData,
 	createDeleteMailData,
@@ -155,11 +158,8 @@ import { EntityUpdateData, isUpdateForTypeRef } from "../../../common/utils/Enti
 import { Entity } from "../../../common/EntityTypes"
 import { KeyVerificationMismatchError } from "../../../common/error/KeyVerificationMismatchError"
 import { VerifiedPublicEncryptionKey } from "./KeyVerificationFacade"
-import { SpamClassifier, SpamPredMailDatum } from "../../../../../mail-app/workerUtils/spamClassification/SpamClassifier"
-import { isDraft } from "../../../../../mail-app/mail/model/MailChecks"
 import { Nullable } from "@tutao/tutanota-utils/dist/Utils"
 import { ClientClassifierType } from "../../../common/ClientClassifierType"
-import { getMailBodyText } from "../../../common/CommonMailUtils"
 
 assertWorkerOrNode()
 type Attachments = ReadonlyArray<TutanotaFile | DataFile | FileReference>
@@ -208,7 +208,6 @@ export class MailFacade {
 		private readonly loginFacade: LoginFacade,
 		private readonly keyLoaderFacade: KeyLoaderFacade,
 		private readonly publicEncryptionKeyProvider: PublicEncryptionKeyProvider,
-		private readonly spamClassifier: SpamClassifier | null,
 	) {}
 
 	async createMailFolder(name: string, parent: IdTuple | null, ownerGroupId: Id): Promise<void> {
@@ -392,7 +391,12 @@ export class MailFacade {
 	/**
 	 * Move mails from {@param targetFolder} except those that are in {@param excludeMailSet}.
 	 */
-	async moveMails(mails: readonly IdTuple[], targetFolder: IdTuple, excludeMailSet: IdTuple | null): Promise<MovedMails[]> {
+	async moveMails(
+		mails: readonly IdTuple[],
+		targetFolder: IdTuple,
+		excludeMailSet: IdTuple | null,
+		moveReason: ClientClassifierType | null = null,
+	): Promise<MovedMails[]> {
 		if (isEmpty(mails)) {
 			return []
 		}
@@ -401,7 +405,7 @@ export class MailFacade {
 		const mailsPerList = groupBy(mails, (mailId) => listIdPart(mailId))
 		const movedMails: MovedMails[] = []
 		for (const [_, mailsInList] of mailsPerList) {
-			const mailChunks = splitInChunks(MAX_NBR_MOVE_DELETE_MAIL_SERVICE, mailsInList)
+			const mailChunks = splitInChunks(MAX_NBR_OF_MAILS_SYNC_OPERATION, mailsInList)
 			for (const mails of mailChunks) {
 				const moveMailPostOut = await this.serviceExecutor.post(
 					MoveMailService,
@@ -409,7 +413,7 @@ export class MailFacade {
 						mails,
 						excludeMailSet,
 						targetFolder,
-						moveReason: null,
+						moveReason,
 					}),
 				)
 				movedMails.push(...moveMailPostOut.movedMails)
@@ -421,13 +425,13 @@ export class MailFacade {
 	async simpleMoveMails(
 		mails: readonly IdTuple[],
 		targetFolderKind: SimpleMoveMailTarget,
-		clientSpamClassifier: Nullable<ClientClassifierType>,
+		moveReason: Nullable<ClientClassifierType>,
 	): Promise<MovedMails[]> {
 		if (isEmpty(mails)) {
 			return []
 		}
 
-		const mailChunks = splitInChunks(MAX_NBR_MOVE_DELETE_MAIL_SERVICE, mails)
+		const mailChunks = splitInChunks(MAX_NBR_OF_MAILS_SYNC_OPERATION, mails)
 		const movedMails: MovedMails[] = []
 		for (const mails of mailChunks) {
 			const simpleMove = await this.serviceExecutor.post(
@@ -435,7 +439,7 @@ export class MailFacade {
 				createSimpleMoveMailPostIn({
 					mails,
 					destinationSetType: targetFolderKind,
-					moveReason: clientSpamClassifier,
+					moveReason,
 				}),
 			)
 			movedMails.push(...simpleMove.movedMails)
@@ -453,28 +457,6 @@ export class MailFacade {
 		await this.serviceExecutor.post(ReportMailService, postData)
 	}
 
-	public isSpamClassificationEnabled(ownerGroup: Id): boolean {
-		return this.spamClassifier != null && this.spamClassifier.getEnabledSpamClassifierForOwnerGroup(ownerGroup) != null
-	}
-
-	async predictSpamResult(mail: Mail): Promise<Nullable<boolean>> {
-		if (isDraft(mail)) {
-			return null
-		} else {
-			const spamClassifier = this.spamClassifier?.getEnabledSpamClassifierForOwnerGroup(assertNotNull(mail._ownerGroup)) ?? null
-			if (isNotNull(spamClassifier)) {
-				const mailDetails = await this.loadMailDetailsBlob(mail)
-				const spamPredMailDatum: SpamPredMailDatum = {
-					subject: mail.subject,
-					body: getMailBodyText(mailDetails.body),
-					ownerGroup: assertNotNull(mail._ownerGroup),
-				}
-				return await assertNotNull(this.spamClassifier).predict(spamPredMailDatum)
-			}
-			return null
-		}
-	}
-
 	async deleteMails(mails: readonly IdTuple[], filterMailSet: IdTuple | null): Promise<void> {
 		if (isEmpty(mails)) {
 			return
@@ -483,7 +465,7 @@ export class MailFacade {
 		// Must be split by list (mailbag)
 		const mailsGrouped = groupBy(mails, listIdPart)
 		for (const [_, mails] of mailsGrouped) {
-			const mailChunks = splitInChunks(MAX_NBR_MOVE_DELETE_MAIL_SERVICE, mails)
+			const mailChunks = splitInChunks(MAX_NBR_OF_MAILS_SYNC_OPERATION, mails)
 			for (const mailChunk of mailChunks) {
 				const deleteMailData = createDeleteMailData({
 					mails: mailChunk,
@@ -1197,7 +1179,7 @@ export class MailFacade {
 	 */
 	async markMails(mails: readonly IdTuple[], unread: boolean) {
 		await promiseMap(
-			splitInChunks(MAX_NBR_MOVE_DELETE_MAIL_SERVICE, mails),
+			splitInChunks(MAX_NBR_OF_MAILS_SYNC_OPERATION, mails),
 			async (mails) =>
 				this.serviceExecutor.post(
 					UnreadMailStateService,
@@ -1210,6 +1192,27 @@ export class MailFacade {
 		)
 	}
 
+	/**
+	 * Mark the given mails as read/unread
+	 * @param mails mail ids to mark as unread
+	 * @param processingState
+	 */
+	async updateMailPredictionState(mails: readonly IdTuple[], processingState: ProcessingState) {
+		const isPredictionMade = processingState === ProcessingState.INBOX_RULE_PROCESSED_AND_SPAM_PREDICTION_MADE
+		await promiseMap(
+			splitInChunks(MAX_NBR_OF_MAILS_SYNC_OPERATION, mails),
+			async (mails) =>
+				this.serviceExecutor.post(
+					ClientClassifierResultService,
+					createClientClassifierResultPostIn({
+						mails,
+						isPredictionMade: isPredictionMade,
+					}),
+				),
+			{ concurrency: 5 },
+		)
+	}
+
 	/** Resolve conversation list ids to the IDs of mails in those conversations. */
 	async resolveConversations(conversationListIds: readonly Id[]): Promise<IdTuple[]> {
 		const result = await promiseMap(

src/common/api/worker/offline/OfflineStorage.ts

@@ -47,6 +47,8 @@ import { TypeModelResolver } from "../../common/EntityFunctions"
 import { collapseId, expandId } from "../rest/RestClientIdUtils"
 import { Category, syncMetrics } from "../utils/SyncMetrics"
 import { hasError } from "../../common/utils/ErrorUtils"
+import { SpamClassificationModel, SpamTrainMailDatum } from "../../../../mail-app/workerUtils/spamClassification/SpamClassifier"
+import { Mail } from "../../entities/tutanota/TypeRefs"
 
 /**
  * this is the value of SQLITE_MAX_VARIABLE_NUMBER in sqlite3.c
@@ -102,6 +104,7 @@ export interface OfflineDbMeta {
 	// offline db schema version
 	"offline-version": number
 	lastTrainedTime: number
+	lastTrainedFromScratchTime: number
 }
 
 export const TableDefinitions = Object.freeze({
@@ -719,6 +722,14 @@ export class OfflineStorage implements CacheStorage {
 		await this.putMetadata("lastTrainedTime", ms)
 	}
 
+	async getLastTrainedFromScratchTime(): Promise<number> {
+		return (await this.getMetadata("lastTrainedFromScratchTime")) ?? Date.now()
+	}
+
+	async setLastTrainedFromScratchTime(ms: number): Promise<void> {
+		await this.putMetadata("lastTrainedFromScratchTime", ms)
+	}
+
 	async purgeStorage(): Promise<void> {
 		if (this.userId == null || this.databaseKey == null) {
 			console.warn("not purging storage since we don't have an open db")

src/common/api/worker/rest/CacheStorageProxy.ts

@@ -1,11 +1,10 @@
 import { CacheStorage, LastUpdateTime, Range } from "./DefaultEntityRestCache.js"
 import { ProgrammingError } from "../../common/error/ProgrammingError"
 import { Entity, ListElementEntity, ServerModelParsedInstance, SomeEntity } from "../../common/EntityTypes"
-import { TypeRef } from "@tutao/tutanota-utils"
+import { Nullable, TypeRef } from "@tutao/tutanota-utils"
 import { OfflineStorage, OfflineStorageInitArgs } from "../offline/OfflineStorage.js"
 import { EphemeralCacheStorage, EphemeralStorageInitArgs } from "./EphemeralCacheStorage"
 import { CustomCacheHandlerMap } from "./cacheHandler/CustomCacheHandler.js"
-import { Nullable } from "@tutao/tutanota-utils"
 
 export interface EphemeralStorageArgs extends EphemeralStorageInitArgs {
 	type: "ephemeral"
@@ -194,6 +193,14 @@ export class LateInitializedCacheStorageImpl implements CacheStorageLateInitiali
 		return this.inner.getLastTrainedTime()
 	}
 
+	setLastTrainedFromScratchTime(ms: number): Promise<void> {
+		return this.inner.setLastTrainedFromScratchTime(ms)
+	}
+
+	getLastTrainedFromScratchTime(): Promise<number> {
+		return this.inner.getLastTrainedFromScratchTime() ?? Date.now()
+	}
+
 	setLowerRangeForList<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id, id: Id): Promise<void> {
 		return this.inner.setLowerRangeForList(typeRef, listId, id)
 	}

src/common/api/worker/rest/DefaultEntityRestCache.ts

@@ -8,7 +8,7 @@ import {
 	OwnerEncSessionKeyProvider,
 } from "./EntityRestClient"
 import { OperationType } from "../../common/TutanotaConstants"
-import { assertNotNull, downcast, getFirstOrThrow, getTypeString, isNotEmpty, isSameTypeRef, lastThrow, TypeRef } from "@tutao/tutanota-utils"
+import { assertNotNull, downcast, getFirstOrThrow, getTypeString, isNotEmpty, isSameTypeRef, lastThrow, Nullable, TypeRef } from "@tutao/tutanota-utils"
 import {
 	AuditLogEntryTypeRef,
 	BucketPermissionTypeRef,
@@ -25,7 +25,7 @@ import {
 	UserGroupRootTypeRef,
 } from "../../entities/sys/TypeRefs.js"
 import { ValueType } from "../../common/EntityConstants.js"
-import { Body, CalendarEventUidIndexTypeRef, Mail, MailDetailsBlobTypeRef, MailSetEntryTypeRef, MailTypeRef } from "../../entities/tutanota/TypeRefs.js"
+import { CalendarEventUidIndexTypeRef, MailDetailsBlobTypeRef, MailSetEntryTypeRef, MailTypeRef } from "../../entities/tutanota/TypeRefs.js"
 import {
 	CUSTOM_MAX_ID,
 	CUSTOM_MIN_ID,
@@ -47,9 +47,7 @@ import { TypeModelResolver } from "../../common/EntityFunctions"
 import { AttributeModel } from "../../common/AttributeModel"
 import { collapseId, expandId } from "./RestClientIdUtils"
 import { PatchMerger } from "../offline/PatchMerger"
-import { NotAuthorizedError, NotFoundError } from "../../common/error/RestError"
+import { hasError, isExpectedErrorForSynchronization } from "../../common/utils/ErrorUtils"
-import { Nullable } from "@tutao/tutanota-utils"
-import { hasError } from "../../common/utils/ErrorUtils"
 
 assertWorkerOrNode()
 
@@ -259,6 +257,10 @@ export interface CacheStorage extends ExposedCacheStorage {
 
 	setLastTrainedTime(value: number): Promise<void>
 
+	getLastTrainedFromScratchTime(): Promise<number>
+
+	setLastTrainedFromScratchTime(value: number): Promise<void>
+
 	getUserId(): Id
 
 	deleteAllOwnedBy(owner: Id): Promise<void>
@@ -954,14 +956,6 @@ export class DefaultEntityRestCache implements EntityRestCache {
 	}
 }
 
-/**
- * Returns whether the error is expected for the cases where our local state might not be up-to-date with the server yet. E.g. we might be processing an update
- * for the instance that was already deleted. Normally this would be optimized away but it might still happen due to timing.
- */
-function isExpectedErrorForSynchronization(e: Error): boolean {
-	return e instanceof NotFoundError || e instanceof NotAuthorizedError
-}
-
 /**
  * Check if a range request begins inside an existing range
  */

src/common/api/worker/rest/EphemeralCacheStorage.ts

@@ -43,6 +43,7 @@ export class EphemeralCacheStorage implements CacheStorage {
 	private readonly blobEntities: Map<string, BlobElementTypeCache> = new Map()
 	private lastUpdateTime: number | null = null
 	private lastTrainedTime: number | null = null
+	private lastTrainedFromScratchTime: number | null = null
 	private userId: Id | null = null
 	private lastBatchIdPerGroup = new Map<Id, Id>()
 
@@ -426,6 +427,14 @@ export class EphemeralCacheStorage implements CacheStorage {
 		this.lastTrainedTime = value
 	}
 
+	async getLastTrainedFromScratchTime(): Promise<number> {
+		return this.lastTrainedFromScratchTime ?? Date.now()
+	}
+
+	async setLastTrainedFromScratchTime(ms: number): Promise<void> {
+		this.lastTrainedFromScratchTime = ms
+	}
+
 	async getWholeList<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id): Promise<Array<T>> {
 		const parsedInstances = await this.getWholeListParsed(typeRef, listId)
 		return await this.modelMapper.mapToInstances(typeRef, parsedInstances)

src/common/api/worker/rest/cacheHandler/CustomMailEventCacheHandler.ts

@@ -1,27 +1,14 @@
-import { Mail, MailFolder, MailFolderTypeRef, MailSetEntryTypeRef, MailTypeRef } from "../../../entities/tutanota/TypeRefs"
+import { Mail, MailDetailsBlobTypeRef } from "../../../entities/tutanota/TypeRefs"
-import { assertNotNull, isSameTypeRef, lazy, lazyAsync } from "@tutao/tutanota-utils"
+import { assertNotNull, lazy, lazyAsync } from "@tutao/tutanota-utils"
 import { MailIndexer } from "../../../../../mail-app/workerUtils/index/MailIndexer"
 import { CustomCacheHandler } from "./CustomCacheHandler"
-import { EntityUpdateData } from "../../../common/utils/EntityUpdateUtils"
-import { MailFacade } from "../../facades/lazy/MailFacade"
 import { OfflineStoragePersistence } from "../../../../../mail-app/workerUtils/index/OfflineStoragePersistence"
-import { MailSetKind } from "../../../common/TutanotaConstants"
-import { CacheStorage } from "../DefaultEntityRestCache"
-import { elementIdPart, isSameId, listIdPart } from "../../../common/utils/EntityUtils"
-import { ClientClassifierType } from "../../../common/ClientClassifierType"
-import { MailWithDetailsAndAttachments } from "../../../../../mail-app/workerUtils/index/MailIndexerBackend"
-import { getMailBodyText } from "../../../common/CommonMailUtils"
-import { SpamTrainMailDatum } from "../../../../../mail-app/workerUtils/spamClassification/SpamClassifier"
 
 /**
  * Handles telling the indexer to index or un-index mail data on updates.
  */
 export class CustomMailEventCacheHandler implements CustomCacheHandler<Mail> {
-	constructor(
+	constructor(private readonly indexer: lazyAsync<MailIndexer>) {}
-		private readonly indexerAndMailFacade: lazyAsync<{ mailIndexer: MailIndexer; mailFacade: MailFacade }>,
-		private readonly offlineStoragePersistence: lazy<Promise<OfflineStoragePersistence>>,
-		private readonly storage: CacheStorage,
-	) {}
 
 	shouldLoadOnCreateEvent(): boolean {
 		// New emails should be pre-cached.
@@ -33,127 +20,17 @@ export class CustomMailEventCacheHandler implements CustomCacheHandler<Mail> {
 	}
 
 	async onBeforeCacheDeletion(id: IdTuple): Promise<void> {
-		const { mailIndexer } = await this.indexerAndMailFacade()
+		const indexer = await this.indexer()
-		return mailIndexer.beforeMailDeleted(id)
+		return indexer.beforeMailDeleted(id)
 	}
 
-	async onEntityEventCreate(id: IdTuple, events: EntityUpdateData[]) {
+	async onEntityEventCreate(id: IdTuple) {
-		const { mailIndexer, mailFacade } = await this.indexerAndMailFacade()
+		const indexer = await this.indexer()
-		// At this point, the mail entity, itself, is cached, so when we go to download it again, it will come from cache
+		return indexer.afterMailCreated(id)
-		const newMailData = await mailIndexer.downloadNewMailData(id)
-		await mailIndexer.afterMailCreated(id, newMailData)
-		await this.processSpam(newMailData, mailFacade, id)
 	}
 
-	async onEntityEventUpdate(id: IdTuple, events: EntityUpdateData[]) {
+	async onEntityEventUpdate(id: IdTuple) {
-		const { mailIndexer } = await this.indexerAndMailFacade()
+		const indexer = await this.indexer()
-		await mailIndexer.afterMailUpdated(id)
+		return indexer.afterMailUpdated(id)
-		await this.updateSpamClassificationData(events, id)
-	}
-
-	private async processSpam(newMailData: MailWithDetailsAndAttachments | null, mailFacade: MailFacade, id: readonly [string, string]) {
-		const usedClientSpamClassifier = ClientClassifierType.CLIENT_CLASSIFICATION
-		if (newMailData == null) {
-			return
-		}
-
-		// update spam classification table
-		const mail = newMailData.mail
-		const allFolders = await this.storage.getWholeList(MailFolderTypeRef, listIdPart(mail.sets[0]))
-		const spamFolder = allFolders.find((folder) => folder.folderType === MailSetKind.SPAM)!
-
-		const isStoredInSpamFolder = mail.sets.some((folderId) => isSameId(folderId, spamFolder._id))
-		const { isStoredInTrashFolder, confidence } = this.getSpamConfidence(allFolders, mail)
-
-		// isStoredInSpamFolder is true
-		// this might be run multiple times for a single user if they use multiple devices
-		const predictedSpam = isStoredInTrashFolder ? null : await mailFacade.predictSpamResult(mail)
-		// use the server classification for initial training, mixed with data from when user moves mails in and out of spam
-		const isSpam = predictedSpam ?? isStoredInSpamFolder
-		const offlineStoragePersistence = await this.offlineStoragePersistence()
-
-		const spamTrainMailDatum: SpamTrainMailDatum = {
-			mailId: mail._id,
-			subject: mail.subject,
-			body: getMailBodyText(newMailData.mailDetails.body),
-			isSpam,
-			isSpamConfidence: confidence,
-			ownerGroup: assertNotNull(mail._ownerGroup),
-		}
-
-		let moveServiceCall
-		if (!isStoredInTrashFolder && isSpam && !isStoredInSpamFolder) {
-			spamTrainMailDatum.isSpamConfidence = 1
-			moveServiceCall = mailFacade.simpleMoveMails([id], MailSetKind.SPAM, usedClientSpamClassifier)
-		} else if (!isStoredInTrashFolder && !isSpam && isStoredInSpamFolder) {
-			spamTrainMailDatum.isSpamConfidence = 0
-			moveServiceCall = mailFacade.simpleMoveMails([id], MailSetKind.INBOX, usedClientSpamClassifier)
-		}
-		await offlineStoragePersistence.storeSpamClassification(spamTrainMailDatum)
-		await moveServiceCall
-	}
-
-	private async updateSpamClassificationData(events: EntityUpdateData[], id: readonly [string, string]) {
-		const mail = assertNotNull(await this.storage.get(MailTypeRef, listIdPart(id), elementIdPart(id)))
-		const changedMailSetEntry = events.some((ev) => isSameTypeRef(ev.typeRef, MailSetEntryTypeRef))
-		const mailHasBeenRead = !mail.unread
-
-		if (!mailHasBeenRead && !changedMailSetEntry) {
-			return
-		}
-
-		const allFolders = await this.storage.getWholeList(MailFolderTypeRef, listIdPart(mail.sets[0]))
-		const spamFolder = allFolders.find((folder) => folder.folderType === MailSetKind.SPAM)!
-		const isSpam = mail.sets.some((folderId) => isSameId(folderId, spamFolder._id))
-		let { confidence: isSpamConfidence, isStoredInTrashFolder } = this.getSpamConfidence(allFolders, mail)
-
-		const offlineStoragePersistence = await this.offlineStoragePersistence()
-		const storedClassification = await offlineStoragePersistence.getStoredClassification(mail)
-
-		if (storedClassification != null) {
-			// email is in classification data
-
-			const wasDeletedFromSpamFolder = isStoredInTrashFolder && storedClassification.isSpam
-			if (wasDeletedFromSpamFolder) {
-				// This is the case if we delete from spam Folder, in that case we do not need any change in storedClassification
-			} else if (isSpam !== storedClassification.isSpam || isSpamConfidence !== storedClassification.isSpamConfidence) {
-				// the model has trained on the mail but the spamFlag was wrong so we refit with higher isSpamConfidence
-				await offlineStoragePersistence.updateSpamClassificationData(id, isSpam, isSpamConfidence)
-			}
-		} else {
-			const { mailIndexer } = await this.indexerAndMailFacade()
-			// At this point, the mail entity, itself, is cached, so when we go to download it again, it will come from cache
-			const newMailData = await mailIndexer.downloadNewMailData(id)
-			if (newMailData) {
-				const spamTrainMailDatum: SpamTrainMailDatum = {
-					mailId: mail._id,
-					subject: mail.subject,
-					body: getMailBodyText(newMailData.mailDetails.body),
-					isSpam,
-					isSpamConfidence,
-					ownerGroup: assertNotNull(mail._ownerGroup),
-				}
-
-				await offlineStoragePersistence.storeSpamClassification(spamTrainMailDatum)
-			} else {
-				// race: mail deleted in meantime
-			}
-		}
-	}
-
-	// visible for testing
-	public getSpamConfidence(allFolders: Array<MailFolder>, mail: Mail): { confidence: number; isStoredInTrashFolder: boolean } {
-		const spamFolder = allFolders.find((folder) => folder.folderType === MailSetKind.SPAM)!
-		const trashFolder = allFolders.find((folder) => folder.folderType === MailSetKind.TRASH)!
-
-		const isStoredInSpamFolder = mail.sets.some((folderId) => isSameId(folderId, spamFolder._id))
-		const isStoredInTrashFolder = mail.sets.some((folderId) => isSameId(folderId, trashFolder._id))
-
-		const isReadAndNotInSpamAndNotInTrash = !mail.unread && !isStoredInSpamFolder && !isStoredInTrashFolder
-		if (isStoredInSpamFolder || isReadAndNotInSpamAndNotInTrash) {
-			return { confidence: 1, isStoredInTrashFolder }
-		} else {
-			return { confidence: 0, isStoredInTrashFolder }
-		}
 	}
 }

src/common/desktop/DesktopPostLoginActions.ts

@@ -11,9 +11,10 @@ export class DesktopPostLoginActions implements PostLoginAction {
 		private readonly windowId: number,
 	) {}
 
-	async onPartialLoginSuccess({ userId }: LoggedInEvent): Promise<void> {
+	async onPartialLoginSuccess({ userId }: LoggedInEvent): Promise<{ asyncAction: Promise<void> }> {
 		this.wm.get(this.windowId)?.setUserId(userId)
 		await this.notifier.clearUserNotifications(userId)
+		return { asyncAction: Promise.resolve() }
 	}
 
 	async onFullLoginSuccess({ userId }: LoggedInEvent): Promise<void> {

src/common/login/PostLoginActions.ts

@@ -57,7 +57,7 @@ export class PostLoginActions implements PostLoginAction {
 		private readonly updateClient: () => unknown,
 	) {}
 
-	async onPartialLoginSuccess(loggedInEvent: LoggedInEvent): Promise<void> {
+	async onPartialLoginSuccess(loggedInEvent: LoggedInEvent): Promise<{ asyncAction: Promise<void> }> {
 		// We establish websocket connection even for temporary sessions because we need to get updates e.g. during signup
 		windowFacade.addOnlineListener(() => {
 			console.log(new Date().toISOString(), "online - try reconnect")
@@ -80,7 +80,7 @@ export class PostLoginActions implements PostLoginAction {
 				document.title = "Tuta Mail"
 			}
 
-			return
+			return { asyncAction: Promise.resolve() }
 		} else {
 			let postLoginTitle = document.title === LOGIN_TITLE ? "Tuta Mail" : document.title
 			document.title = neverNull(this.logins.getUserController().userGroupInfo.mailAddress) + " - " + postLoginTitle
@@ -106,6 +106,7 @@ export class PostLoginActions implements PostLoginAction {
 		if (isApp() || isDesktop()) {
 			await this.storeNewCustomThemes()
 		}
+		return { asyncAction: Promise.resolve() }
 	}
 
 	async onFullLoginSuccess(loggedInEvent: LoggedInEvent): Promise<void> {

src/common/offline/CachePostLoginAction.ts

@@ -41,12 +41,13 @@ export class CachePostLoginAction implements PostLoginAction {
 		progressMonitor.completed()
 	}
 
-	async onPartialLoginSuccess(event: LoggedInEvent): Promise<void> {
+	async onPartialLoginSuccess(event: LoggedInEvent): Promise<{ asyncAction: Promise<void> }> {
 		if (event.sessionType === SessionType.Persistent && this.offlineStorageSettings != null) {
 			await this.offlineStorageSettings.init()
 
 			// Clear the excluded data (i.e. trash and spam lists, old data) in the offline storage.
 			await this.cacheStorage.clearExcludedData(this.offlineStorageSettings.getTimeRange())
 		}
+		return { asyncAction: Promise.resolve() }
 	}
 }

src/common/settings/whitelabel/CustomColorEditorPreview.ts

@@ -10,6 +10,7 @@ import { ToggleButton } from "../../gui/base/buttons/ToggleButton.js"
 import { isApp, isDesktop } from "../../api/common/Env.js"
 import { LoginButton } from "../../gui/base/buttons/LoginButton.js"
 import { lang } from "../../misc/LanguageViewModel.js"
+import { ProcessingState } from "../../api/common/TutanotaConstants"
 
 export const BUTTON_WIDTH = 270
 
@@ -94,6 +95,8 @@ export class CustomColorEditorPreview implements Component {
 			phishingStatus: "0",
 			recipientCount: "0",
 			sets: [],
+			processingState: ProcessingState.INBOX_RULE_NOT_PROCESSED,
+			clientSpamClassifierResult: null,
 		} satisfies Partial<Mail>
 		const mail = createMail({
 			sender: createMailAddress({

src/mail-app/app.ts

@@ -155,13 +155,14 @@ import("./translations/en.js")
 							const canSync = await syncManager.canSync()
 							if (!canSync) {
 								await syncManager.disableSync()
-								return
+								return { asyncAction: Promise.resolve() }
 							}
 						}
 						syncManager.syncContacts()
 					}
 					await mailLocator.mailboxModel.init()
 					await mailLocator.mailModel.init()
+					return { asyncAction: Promise.resolve() }
 				},
 				async onFullLoginSuccess() {
 					// We might have outdated Customer features, force reload the customer to make sure the customizations are up-to-date
@@ -235,7 +236,9 @@ import("./translations/en.js")
 		if (isDesktop()) {
 			mailLocator.logins.addPostLoginAction(async () => {
 				return {
-					onPartialLoginSuccess: async () => {},
+					onPartialLoginSuccess: async () => {
+						return { asyncAction: Promise.resolve() }
+					},
 					onFullLoginSuccess: async (event) => {
 						// not a temporary aka signup login
 						if (event.sessionType === SessionType.Persistent) {

src/mail-app/mail/editor/OpenLocallySavedDraftAction.ts

@@ -30,9 +30,10 @@ export class OpenLocallySavedDraftAction implements PostLoginAction {
 
 	async onFullLoginSuccess(_: LoggedInEvent): Promise<void> {}
 
-	async onPartialLoginSuccess(_: LoggedInEvent): Promise<void> {
+	async onPartialLoginSuccess(_: LoggedInEvent): Promise<{ asyncAction: Promise<void> }> {
 		// fire and forget; this might take some time
-		this._loadAutosavedDraft()
+		const asyncAction = this._loadAutosavedDraft()
+		return { asyncAction }
 	}
 
 	/**

src/mail-app/mail/model/InboxRuleHandler.ts

@@ -1,31 +1,34 @@
-import type { InboxRule, Mail, MailFolder, MoveMailData } from "../../../common/api/entities/tutanota/TypeRefs.js"
+import { createMoveMailData, InboxRule, Mail, MailFolder, MoveMailData } from "../../../common/api/entities/tutanota/TypeRefs.js"
-import { createMoveMailData } from "../../../common/api/entities/tutanota/TypeRefs.js"
+import { InboxRuleType, MailSetKind, MAX_NBR_OF_MAILS_SYNC_OPERATION, ProcessingState } from "../../../common/api/common/TutanotaConstants"
-import { InboxRuleType, MailSetKind, MAX_NBR_MOVE_DELETE_MAIL_SERVICE } from "../../../common/api/common/TutanotaConstants"
 import { isDomainName, isRegularExpression } from "../../../common/misc/FormatValidator"
-import { assertNotNull, asyncFind, ofClass, promiseMap, splitInChunks, throttleStart } from "@tutao/tutanota-utils"
+import { assertNotNull, asyncFind, debounce, ofClass, promiseMap, splitInChunks, throttleStart } from "@tutao/tutanota-utils"
 import { lang } from "../../../common/misc/LanguageViewModel"
 import type { MailboxDetail } from "../../../common/mailFunctionality/MailboxModel.js"
 import { LockedError, PreconditionFailedError } from "../../../common/api/common/error/RestError"
 import type { SelectorItemList } from "../../../common/gui/base/DropDownSelector.js"
 import { elementIdPart, isSameId } from "../../../common/api/common/utils/EntityUtils"
-import { assertMainOrNode } from "../../../common/api/common/Env"
+import { assertMainOrNode, isWebClient } from "../../../common/api/common/Env"
 import { MailFacade } from "../../../common/api/worker/facades/lazy/MailFacade.js"
 import { LoginController } from "../../../common/api/main/LoginController.js"
 import { getMailHeaders } from "./MailUtils.js"
 import { MailModel } from "./MailModel"
+import { ClientClassifierType } from "../../../common/api/common/ClientClassifierType"
 
 assertMainOrNode()
+
 const moveMailDataPerFolder: MoveMailData[] = []
-const DEBOUNCE_FIRST_MOVE_MAIL_REQUEST_MS = 200
+let noRuleMatchMailIds: IdTuple[] = []
-let applyingRules = false // used to avoid concurrent application of rules (-> requests to locked service)
+
+const THROTTLE_MOVE_MAIL_SERVICE_REQUESTS_MS = 200
+const DEBOUNCE_CLIENT_CLASSIFIER_RESULT_SERVICE_REQUESTS_MS = 1000
 
 async function sendMoveMailRequest(mailFacade: MailFacade): Promise<void> {
 	if (moveMailDataPerFolder.length) {
 		const moveToTargetFolder = assertNotNull(moveMailDataPerFolder.shift())
-		const mailChunks = splitInChunks(MAX_NBR_MOVE_DELETE_MAIL_SERVICE, moveToTargetFolder.mails)
+		const mailChunks = splitInChunks(MAX_NBR_OF_MAILS_SYNC_OPERATION, moveToTargetFolder.mails)
 		await promiseMap(mailChunks, (mailChunk) => {
 			moveToTargetFolder.mails = mailChunk
-			return mailFacade.moveMails(mailChunk, moveToTargetFolder.targetFolder, null)
+			return mailFacade.moveMails(mailChunk, moveToTargetFolder.targetFolder, null, ClientClassifierType.CUSTOMER_INBOX_RULES)
 		})
 			.catch(
 				ofClass(LockedError, (e) => {
@@ -40,20 +43,25 @@ async function sendMoveMailRequest(mailFacade: MailFacade): Promise<void> {
 				}),
 			)
 			.finally(() => {
-				return sendMoveMailRequest(mailFacade)
+				return processMatchingRules(mailFacade)
 			})
-	} //We are done and unlock for future requests
+	}
 }
 
-// We throttle the moveMail requests to a rate of 200ms
+const processMatchingRules = throttleStart(THROTTLE_MOVE_MAIL_SERVICE_REQUESTS_MS, async (mailFacade: MailFacade) => {
-// Each target folder requires one request
+	// Each target folder requires one request,
-const applyMatchingRules = throttleStart(DEBOUNCE_FIRST_MOVE_MAIL_REQUEST_MS, async (mailFacade: MailFacade) => {
+	// We debounce the requests to a rate of THROTTLE_MOVE_MAIL_SERVICE_REQUESTS_MS
-	if (applyingRules) return
+	return sendMoveMailRequest(mailFacade)
-	// We lock to avoid concurrent requests
+})
-	applyingRules = true
+
-	sendMoveMailRequest(mailFacade).finally(() => {
+const processNotMatchingRules = debounce(DEBOUNCE_CLIENT_CLASSIFIER_RESULT_SERVICE_REQUESTS_MS, async (mailFacade: MailFacade) => {
-		applyingRules = false
+	// Each update to ClientClassifierResultService (for mails that did not move) requires one request
-	})
+	// We debounce the requests to a rate of DEBOUNCE_CLIENT_CLASSIFIER_RESULT_SERVICE_REQUESTS_MS
+	if (noRuleMatchMailIds.length) {
+		const mailIds = noRuleMatchMailIds
+		noRuleMatchMailIds = []
+		return mailFacade.updateMailPredictionState(mailIds, ProcessingState.INBOX_RULE_PROCESSED_AND_SPAM_PREDICTION_PENDING)
+	}
 })
 
 export function getInboxRuleTypeNameMapping(): SelectorItemList<string> {
@@ -101,16 +109,8 @@ export class InboxRuleHandler {
 	 * Checks the mail for an existing inbox rule and moves the mail to the target folder of the rule.
 	 * @returns true if a rule matches otherwise false
 	 */
-	async findAndApplyMatchingRule(
+	async findAndApplyMatchingRule(mailboxDetail: MailboxDetail, mail: Readonly<Mail>, applyRulesOnServer: boolean): Promise<MailFolder | null> {
-		mailboxDetail: MailboxDetail,
+		const shouldApply = mail.processingState === ProcessingState.INBOX_RULE_NOT_PROCESSED
-		mail: Mail,
-		applyRulesOnServer: boolean,
-		applyIfRead: boolean,
-	): Promise<{
-		folder: MailFolder
-		mail: Mail
-	} | null> {
-		const shouldApply = applyIfRead || mail.unread
 
 		if (
 			mail._errors ||
@@ -138,19 +138,25 @@ export class InboxRuleHandler {
 							targetFolder: inboxRule.targetFolder,
 							mails: [mail._id],
 							excludeMailSet: null,
-							moveReason: null,
+							moveReason: ClientClassifierType.CUSTOMER_INBOX_RULES,
 						})
 						moveMailDataPerFolder.push(moveMailData)
 					}
-
-					applyMatchingRules(this.mailFacade)
 				}
 
-				return { folder: targetFolder, mail }
+				processMatchingRules(this.mailFacade)
+
+				return targetFolder
 			} else {
 				return null
 			}
 		} else {
+			// if we are not on the webapp this is handled in SpamClassificationHandler
+			if (isWebClient()) {
+				noRuleMatchMailIds.push(mail._id)
+				processNotMatchingRules(this.mailFacade)
+			}
+
 			return null
 		}
 	}

src/mail-app/mail/model/MailModel.ts

@@ -5,12 +5,13 @@ import { FolderSystem } from "../../../common/api/common/mail/FolderSystem.js"
 import {
 	assertNotNull,
 	collectToMap,
+	downcast,
 	getFirstOrThrow,
 	groupBy,
 	groupByAndMap,
 	isNotNull,
 	lazyMemoized,
-	noOp,
+	Nullable,
 	ofClass,
 	partition,
 	promiseMap,
@@ -31,8 +32,9 @@ import {
 	isLabel,
 	MailReportType,
 	MailSetKind,
-	MAX_NBR_MOVE_DELETE_MAIL_SERVICE,
+	MAX_NBR_OF_MAILS_SYNC_OPERATION,
 	OperationType,
+	ProcessingState,
 	ReportMovedMailsType,
 	SimpleMoveMailTarget,
 	SystemFolderType,
@@ -54,6 +56,9 @@ import { LoginController } from "../../../common/api/main/LoginController.js"
 import { MailFacade } from "../../../common/api/worker/facades/lazy/MailFacade.js"
 import { assertSystemFolderOfType } from "./MailUtils.js"
 import { TutanotaError } from "@tutao/tutanota-error"
+import { SpamClassificationHandler } from "../../workerUtils/spamClassification/SpamClassificationHandler"
+import { isWebClient } from "../../../common/api/common/Env"
+import { isExpectedErrorForSynchronization } from "../../../common/api/common/utils/ErrorUtils"
 
 interface MailboxSets {
 	folders: FolderSystem
@@ -90,6 +95,7 @@ export class MailModel {
 		private readonly logins: LoginController,
 		private readonly mailFacade: MailFacade,
 		private readonly connectivityModel: WebsocketConnectivityModel | null,
+		private readonly spamHandler: () => SpamClassificationHandler,
 		private readonly inboxRuleHandler: () => InboxRuleHandler | null,
 	) {}
 
@@ -182,53 +188,103 @@ export class MailModel {
 	}
 
 	// visibleForTesting
-	async entityEventsReceived(updates: ReadonlyArray<EntityUpdateData>): Promise<void> {
+	async entityEventsReceived(updates: ReadonlyArray<EntityUpdateData>): Promise<{ processingDone: Promise<void> }> {
 		for (const update of updates) {
 			if (isUpdateForTypeRef(MailFolderTypeRef, update)) {
 				await this.init()
 				m.redraw()
-			} else if (isUpdateForTypeRef(MailTypeRef, update) && update.operation === OperationType.CREATE) {
+			} else if (isUpdateForTypeRef(MailTypeRef, update) && update.operation === OperationType.UPDATE) {
-				if (this.inboxRuleHandler && this.connectivityModel) {
 				const mailId: IdTuple = [update.instanceListId, update.instanceId]
-					try {
+				const mail = await this.loadMail(mailId)
-						const mail = await this.entityClient.load(MailTypeRef, mailId)
+				if (mail == null) {
-						const folder = this.getMailFolderForMail(mail)
+					return { processingDone: Promise.resolve() }
+				}
+				const spamHandler = this.spamHandler()
+				await spamHandler.updateSpamClassificationData(mail)
+			} else if (isUpdateForTypeRef(MailTypeRef, update) && update.operation === OperationType.CREATE) {
+				const mailId: IdTuple = [update.instanceListId, update.instanceId]
+				const mail = await this.loadMail(mailId)
+				if (mail == null) {
+					return { processingDone: Promise.resolve() }
+				}
 
-						if (folder && folder.folderType === MailSetKind.INBOX) {
+				// If an inbox rule has been applied or a spam prediction has been made
-							// If we don't find another delete operation on this email in the batch, then it should be a create operation,
+				// we can return, because those are the two final processing states
-							// otherwise it's a move
+				if (
-							await this.getMailboxDetailsForMail(mail)
+					mail.processingState === ProcessingState.INBOX_RULE_APPLIED ||
-								.then((mailboxDetail) => {
+					mail.processingState === ProcessingState.INBOX_RULE_PROCESSED_AND_SPAM_PREDICTION_MADE
+				) {
+					return { processingDone: Promise.resolve() }
+				}
+
+				// The webapp currently does not support spam prediction, and the inbox rule has been processed
+				if (isWebClient() && mail.processingState === ProcessingState.INBOX_RULE_PROCESSED_AND_SPAM_PREDICTION_PENDING) {
+					return { processingDone: Promise.resolve() }
+				}
+
+				const sourceMailFolder = this.getMailFolderForMail(mail)
+				if (sourceMailFolder == null) {
+					return { processingDone: Promise.resolve() }
+				}
+
+				const isLeaderClient = this.connectivityModel?.isLeader() ?? false
+				if (sourceMailFolder.folderType === MailSetKind.INBOX) {
+					const isInboxRuleTargetFolder = await this.getMailboxDetailsForMail(mail).then((mailboxDetail) => {
 						// We only apply rules on server if we are the leader in case of incoming messages
-									return (
+						return mailboxDetail && this.inboxRuleHandler()?.findAndApplyMatchingRule(mailboxDetail, mail, isLeaderClient)
-										mailboxDetail &&
-										this.inboxRuleHandler()?.findAndApplyMatchingRule(
-											mailboxDetail,
-											mail,
-											this.connectivityModel ? this.connectivityModel.isLeader() : false,
-											false,
-										)
-									)
 					})
-								.then((newFolderAndMail) => {
+
-									if (newFolderAndMail) {
+					if (isWebClient()) {
-										this._showNotification(newFolderAndMail.folder, newFolderAndMail.mail)
+						// we only need to show notifications explicitly on the webapp
+						this._showNotification(isInboxRuleTargetFolder ?? sourceMailFolder, mail)
 					} else {
-										this._showNotification(folder, mail)
+						const mailDetails = await this.mailFacade.loadMailDetailsBlob(mail)
+						this.spamHandler().storeTrainingDatum(mail, mailDetails)
+
+						if (isInboxRuleTargetFolder) {
+							return { processingDone: Promise.resolve() }
+						} else if (
+							(isLeaderClient && mail.processingState === ProcessingState.INBOX_RULE_NOT_PROCESSED) ||
+							mail.processingState === ProcessingState.INBOX_RULE_PROCESSED_AND_SPAM_PREDICTION_PENDING
+						) {
+							const folderSystem = this.getFolderSystemByGroupId(assertNotNull(mail._ownerGroup))
+							if (sourceMailFolder && folderSystem) {
+								const predictPromise = this.spamHandler().predictSpamForNewMail(mail, mailDetails, sourceMailFolder, folderSystem)
+								return { processingDone: downcast(predictPromise) }
 							}
-								})
-								.catch(noOp)
 						}
-					} catch (e) {
+					}
-						if (e instanceof NotFoundError) {
+				} else if (sourceMailFolder.folderType === MailSetKind.SPAM) {
-							console.log(`Could not find updated mail ${JSON.stringify(mailId)}`)
+					const mailDetails = await this.mailFacade.loadMailDetailsBlob(mail)
-						} else {
+					this.spamHandler().storeTrainingDatum(mail, mailDetails)
+
+					if (
+						(isLeaderClient && mail.processingState === ProcessingState.INBOX_RULE_NOT_PROCESSED) ||
+						mail.processingState === ProcessingState.INBOX_RULE_PROCESSED_AND_SPAM_PREDICTION_PENDING
+					) {
+						const folderSystem = this.getFolderSystemByGroupId(assertNotNull(mail._ownerGroup))
+						if (sourceMailFolder && folderSystem) {
+							const predictPromise = this.spamHandler().predictSpamForNewMail(mail, mailDetails, sourceMailFolder, folderSystem)
+							return { processingDone: downcast(predictPromise) }
+						}
+					}
+				}
+			} else if (isUpdateForTypeRef(MailTypeRef, update) && update.operation === OperationType.DELETE) {
+				const mailId: IdTuple = [update.instanceListId, update.instanceId]
+				await this.spamHandler().dropClassificationData(mailId)
+			}
+		}
+		return { processingDone: Promise.resolve() }
+	}
+
+	public async loadMail(mailId: IdTuple): Promise<Nullable<Mail>> {
+		return await this.entityClient.load(MailTypeRef, mailId).catch((e) => {
+			if (isExpectedErrorForSynchronization(e)) {
+				console.log(`Could not find mail ${JSON.stringify(mailId)}`)
+				return null
+			}
 			throw e
-						}
+		})
-					}
-				}
-			}
-		}
 	}
 
 	async applyInboxRuleToMail(mail: Mail) {
@@ -236,7 +292,7 @@ export class MailModel {
 		if (inboxRuleHandler) {
 			const mailboxDetail = await this.getMailboxDetailsForMail(mail)
 			if (mailboxDetail) {
-				inboxRuleHandler.findAndApplyMatchingRule(mailboxDetail, mail, true, true)
+				return inboxRuleHandler.findAndApplyMatchingRule(mailboxDetail, mail, true)
 			}
 		}
 	}
@@ -370,10 +426,6 @@ export class MailModel {
 		return await this.mailFacade.moveMails(mails, targetFolder._id, excludeFolder)
 	}
 
-	async trashMails(mails: readonly IdTuple[]): Promise<MovedMails[]> {
-		return await this.mailFacade.simpleMoveMails(mails, MailSetKind.TRASH, null)
-	}
-
 	/**
 	 * Finally deletes all given mails. Caller must ensure that all mails are in folders that allows final delete operation.
 	 * @param mailIds mails to delete
@@ -436,7 +488,7 @@ export class MailModel {
 	async applyLabels(mails: readonly IdTuple[], addedLabels: readonly MailFolder[], removedLabels: readonly MailFolder[]): Promise<void> {
 		const groupedByListIds = groupBy(mails, (mailId) => listIdPart(mailId))
 		for (const [_, groupedMails] of groupedByListIds) {
-			const mailChunks = splitInChunks(MAX_NBR_MOVE_DELETE_MAIL_SERVICE, groupedMails)
+			const mailChunks = splitInChunks(MAX_NBR_OF_MAILS_SYNC_OPERATION, groupedMails)
 			for (const mailChunk of mailChunks) {
 				await this.mailFacade.applyLabels(mailChunk, addedLabels, removedLabels)
 			}

src/mail-app/mail/model/MailSetListModel.ts

@@ -290,7 +290,7 @@ export async function applyInboxRulesToEntries(
 		return entries
 	}
 	return await promiseFilter(entries, async (entry) => {
-		const ruleApplied = await inboxRuleHandler.findAndApplyMatchingRule(mailboxDetail, entry.mail, true, false)
+		const ruleApplied = await inboxRuleHandler.findAndApplyMatchingRule(mailboxDetail, entry.mail, true)
 		return ruleApplied == null
 	})
 }

src/mail-app/mailLocator.ts

@@ -151,10 +151,11 @@ import { IdentityKeyCreator } from "../common/api/worker/facades/lazy/IdentityKe
 import { PublicIdentityKeyProvider } from "../common/api/worker/facades/PublicIdentityKeyProvider"
 import { WhitelabelThemeGenerator } from "../common/gui/WhitelabelThemeGenerator"
 import { UndoModel } from "./UndoModel"
-import { SpamClassifier } from "./workerUtils/spamClassification/SpamClassifier"
 import { GroupSettingsModel } from "../common/sharing/model/GroupSettingsModel"
 import { AutosaveFacade } from "../common/api/worker/facades/lazy/AutosaveFacade"
 import { lang } from "../common/misc/LanguageViewModel.js"
+import { SpamClassificationHandler } from "./workerUtils/spamClassification/SpamClassificationHandler"
+import { SpamClassifier } from "./workerUtils/spamClassification/SpamClassifier"
 
 assertMainOrNode()
 
@@ -300,6 +301,10 @@ class MailLocator implements CommonLocator {
 		return new InboxRuleHandler(this.mailFacade, this.logins, this.mailModel)
 	})
 
+	readonly spamClassificationHandler = lazyMemoized(() => {
+		return new SpamClassificationHandler(this.mailFacade, this.spamClassifier)
+	})
+
 	async searchViewModelFactory(): Promise<() => SearchViewModel> {
 		const { SearchViewModel } = await import("../mail-app/search/view/SearchViewModel.js")
 		const conversationViewModelFactory = await this.conversationViewModelFactory()
@@ -836,6 +841,7 @@ class MailLocator implements CommonLocator {
 			this.logins,
 			this.mailFacade,
 			this.connectivityModel,
+			this.spamClassificationHandler,
 			this.inboxRuleHandler,
 		)
 		this.operationProgressTracker = new OperationProgressTracker()
@@ -886,6 +892,7 @@ class MailLocator implements CommonLocator {
 			const openSettingsHandler = new OpenSettingsHandler(this.logins)
 
 			this.webMobileFacade = new WebMobileFacade(this.connectivityModel, MAIL_PREFIX)
+			this.spamClassifier = spamClassifier
 
 			this.nativeInterfaces = createNativeInterfaces(
 				this.webMobileFacade,
@@ -1032,7 +1039,6 @@ class MailLocator implements CommonLocator {
 		if (selectedThemeFacade instanceof WebThemeFacade) {
 			selectedThemeFacade.addDarkListener(() => mailLocator.themeController.reloadTheme())
 		}
-		this.spamClassifier = spamClassifier
 	}
 
 	readonly calendarModel: () => Promise<CalendarModel> = lazyMemoized(async () => {

src/mail-app/search/model/MailIndexAndSpamClassificationPostLoginAction.ts

@@ -21,11 +21,11 @@ export class MailIndexAndSpamClassificationPostLoginAction implements PostLoginA
 		private readonly customerFacade: CustomerFacade,
 	) {}
 
-	async onPartialLoginSuccess(event: LoggedInEvent): Promise<void> {
+	async onPartialLoginSuccess(event: LoggedInEvent): Promise<{ asyncAction: Promise<void> }> {
 		if (event.sessionType === SessionType.Persistent) {
 			await this.offlineStorageSettings.init()
 			// noinspection ES6MissingAwait
-			this.indexer.resizeMailIndex(this.offlineStorageSettings.getTimeRange().getTime()).then(async () => {
+			const resizeMailIndex = this.indexer.resizeMailIndex(this.offlineStorageSettings.getTimeRange().getTime()).then(async () => {
 				// spamClassification
 				// Wait until indexing is done, as its populate offlineDb
 
@@ -40,7 +40,9 @@ export class MailIndexAndSpamClassificationPostLoginAction implements PostLoginA
 					}
 				}
 			})
+			return { asyncAction: resizeMailIndex }
 		}
+		return { asyncAction: Promise.resolve() }
 	}
 
 	async onFullLoginSuccess(_: LoggedInEvent): Promise<void> {}

src/mail-app/workerUtils/index/IndexedDbIndexer.ts

@@ -642,7 +642,7 @@ export class IndexedDbIndexer implements Indexer {
 							await this.mailIndexer.afterMailUpdated(mailId)
 							break
 						case OperationType.CREATE:
-							await this.mailIndexer.afterMailCreated(mailId, await this.mailIndexer.downloadNewMailData(mailId))
+							await this.mailIndexer.afterMailCreated(mailId)
 							break
 					}
 				} catch (e) {

src/mail-app/workerUtils/index/MailIndexer.ts

@@ -606,14 +606,17 @@ export class MailIndexer {
 	 * @throws NotAuthorizedError if the mail cannot be accessed (and has not been cached)
 	 * @throws NotFoundError if the mail no longer exists (and has not been cached)
 	 */
-	async afterMailCreated(mailId: IdTuple, newMailData: MailWithDetailsAndAttachments | null) {
+	async afterMailCreated(mailId: IdTuple) {
 		await this.initialized.promise
 		if (!this._mailIndexingEnabled) return
 
-		const mail = newMailData?.mail
+		const newMail = await this.entityClient.load(MailTypeRef, mailId)
-		if (mail == null || !this.canIndexMail(mail)) {
+		if (!this.canIndexMail(newMail)) {
 			return
 		}
+
+		// At this point, the mail entity, itself, is cached, so when we go to download it again, it will come from cache
+		const newMailData = await this.downloadNewMailData(mailId)
 		if (newMailData) {
 			await this.backend.onMailCreated(newMailData)
 		}

src/mail-app/workerUtils/index/OfflineStoragePersistence.ts

@@ -70,7 +70,9 @@ export const SpamClassificationDefinitions: Record<string, OfflineStorageTable>
 	// Spam classification training data
 	spam_classification_training_data: {
 		definition:
-			"CREATE TABLE IF NOT EXISTS spam_classification_training_data (listId TEXT NOT NULL, elementId TEXT NOT NULL, ownerGroup TEXT NOT NULL, subject TEXT NOT NULL, body TEXT NOT NULL, isSpam NUMBER, lastModified NUMBER NOT NULL, isSpamConfidence NUMBER NOT NULL, PRIMARY KEY (listId, elementId))",
+			"CREATE TABLE IF NOT EXISTS spam_classification_training_data (listId TEXT NOT NULL, elementId TEXT NOT NULL," +
+			" ownerGroup TEXT NOT NULL, subject TEXT NOT NULL, body TEXT NOT NULL, isSpam NUMBER, " +
+			"lastModified NUMBER NOT NULL, isSpamConfidence NUMBER NOT NULL, PRIMARY KEY (listId, elementId))",
 		purgedWithCache: true,
 	},
 
@@ -200,24 +202,43 @@ export class OfflineStoragePersistence {
 		await this.sqlCipherFacade.run(query, params)
 	}
 
-	async updateSpamClassificationData(id: IdTuple, isSpam: boolean, isSpamConfidence: number): Promise<void> {
+	async deleteSpamClassification(mailId: IdTuple): Promise<void> {
+		const mailListId = listIdPart(mailId)
+		const mailElementId = elementIdPart(mailId)
+		const { query, params } = sql`
+            DELETE
+            FROM spam_classification_training_data
+            where listId = ${mailListId}
+              AND elementId = ${mailElementId}`
+		await this.sqlCipherFacade.run(query, params)
+	}
+
+	async deleteSpamClassificationTrainingDataBeforeCutoff(cutoffTimestamp: number, ownerGroupId: Id): Promise<void> {
+		const { query, params } = sql`DELETE
+                                    FROM spam_classification_training_data
+                                    WHERE lastModified < ${cutoffTimestamp}
+                                      AND ownerGroup = ${ownerGroupId}`
+		await this.sqlCipherFacade.run(query, params)
+	}
+
+	async updateSpamClassification(mailId: IdTuple, isSpam: boolean, isSpamConfidence: number): Promise<void> {
 		const { query, params } = sql`
             UPDATE spam_classification_training_data
             SET lastModified=${Date.now()},
                 isSpamConfidence=${isSpamConfidence},
                 isSpam=${isSpam ? 1 : 0}
-            WHERE listId = ${listIdPart(id)}
+            WHERE listId = ${listIdPart(mailId)}
-              AND elementId = ${elementIdPart(id)}
+              AND elementId = ${elementIdPart(mailId)}
         `
 		await this.sqlCipherFacade.run(query, params)
 	}
 
-	async getStoredClassification(mail: Mail): Promise<Nullable<{ isSpam: boolean; isSpamConfidence: number }>> {
+	async getSpamClassification(mailId: IdTuple): Promise<Nullable<{ isSpam: boolean; isSpamConfidence: number }>> {
 		const { query, params } = sql`
             SELECT isSpam, isSpamConfidence
             FROM spam_classification_training_data
-            where listId = ${listIdPart(mail._id)}
+            where listId = ${listIdPart(mailId)}
-              AND elementId = ${elementIdPart(mail._id)} `
+              AND elementId = ${elementIdPart(mailId)} `
 		const result = await this.sqlCipherFacade.get(query, params)
 		if (!result) {
 			return null
@@ -369,10 +390,6 @@ export class OfflineStoragePersistence {
 		}
 		return untagSqlObject(rowIdResult).rowid
 	}
-
-	public async tokenize(text: string): Promise<ReadonlyArray<string>> {
-		return this.sqlCipherFacade.tokenize(text)
-	}
 }
 
 function serializeMailAddresses(recipients: readonly MailAddress[]): string {

src/mail-app/workerUtils/spamClassification/SpamClassificationHandler.ts

@@ -0,0 +1,147 @@
+import { createMoveMailData, Mail, MailDetails, MailFolder, MoveMailData } from "../../../common/api/entities/tutanota/TypeRefs"
+import { MailSetKind, ProcessingState, SpamDecision } from "../../../common/api/common/TutanotaConstants"
+import { SpamClassifier, SpamPredMailDatum, SpamTrainMailDatum } from "./SpamClassifier"
+import { getMailBodyText } from "../../../common/api/common/CommonMailUtils"
+import { assertNotNull, debounce, isNotNull, Nullable, ofClass } from "@tutao/tutanota-utils"
+import { MailFacade } from "../../../common/api/worker/facades/lazy/MailFacade"
+import { ClientClassifierType } from "../../../common/api/common/ClientClassifierType"
+import { FolderSystem } from "../../../common/api/common/mail/FolderSystem"
+import { LockedError, PreconditionFailedError } from "../../../common/api/common/error/RestError"
+
+const DEBOUNCE_MOVE_MAIL_SERVICE_REQUESTS_MS = 500
+const DEBOUNCE_CLIENT_CLASSIFIER_RESULT_SERVICE_REQUESTS_MS = 1000
+
+const DEFAULT_IS_SPAM_CONFIDENCE = 1
+const DEFAULT_IS_SPAM = false
+
+export class SpamClassificationHandler {
+	public constructor(
+		private readonly mailFacade: MailFacade,
+		private readonly spamClassifier: Nullable<SpamClassifier>,
+	) {}
+
+	hamMoveMailData: MoveMailData | null = null
+	spamMoveMailData: MoveMailData | null = null
+	classifierResultServiceMailIds: IdTuple[] = []
+
+	sendClassifierResultServiceRequest = debounce(DEBOUNCE_CLIENT_CLASSIFIER_RESULT_SERVICE_REQUESTS_MS, async (mailFacade: MailFacade) => {
+		// Each update to ClientClassifierResultService (for mails that did not move) requires one request
+		// We debounce the requests to a rate of DEBOUNCE_CLIENT_CLASSIFIER_RESULT_SERVICE_REQUESTS_MS
+		if (this.classifierResultServiceMailIds.length) {
+			const mailIds = this.classifierResultServiceMailIds
+			this.classifierResultServiceMailIds = []
+			return mailFacade.updateMailPredictionState(mailIds, ProcessingState.INBOX_RULE_PROCESSED_AND_SPAM_PREDICTION_MADE)
+		}
+	})
+
+	sendMoveMailServiceRequest = debounce(DEBOUNCE_MOVE_MAIL_SERVICE_REQUESTS_MS, async (mailFacade: MailFacade) => {
+		// Each update to MoveMailService (for ham or spam mails that did move) requires one request
+		// We debounce the requests to a rate of DEBOUNCE_MOVE_MAIL_SERVICE_REQUESTS_MS
+		if (this.hamMoveMailData) {
+			const moveMailData = this.hamMoveMailData
+			this.hamMoveMailData = null
+			await this.sendMoveMailRequest(mailFacade, moveMailData)
+		}
+		if (this.spamMoveMailData) {
+			const moveMailData = this.spamMoveMailData
+			this.spamMoveMailData = null
+			await this.sendMoveMailRequest(mailFacade, moveMailData)
+		}
+	})
+
+	async sendMoveMailRequest(mailFacade: MailFacade, moveMailData: MoveMailData): Promise<void> {
+		mailFacade
+			.moveMails(moveMailData.mails, moveMailData.targetFolder, null, ClientClassifierType.CLIENT_CLASSIFICATION)
+			.catch(
+				ofClass(LockedError, (e) => {
+					// LockedError should no longer be thrown!?!
+					console.log("moving mails failed", e, moveMailData.targetFolder)
+				}),
+			)
+			.catch(
+				ofClass(PreconditionFailedError, (e) => {
+					// move mail operation may have been locked by other process
+					console.log("moving mails failed", e, moveMailData.targetFolder)
+				}),
+			)
+	}
+
+	public async predictSpamForNewMail(mail: Mail, mailDetails: MailDetails, sourceFolder: MailFolder, folderSystem: FolderSystem): Promise<MailFolder> {
+		const spamPredMailDatum: SpamPredMailDatum = {
+			subject: mail.subject,
+			body: getMailBodyText(mailDetails.body),
+			ownerGroup: assertNotNull(mail._ownerGroup),
+		}
+		const isSpam = (await this.spamClassifier?.predict(spamPredMailDatum)) ?? null
+
+		if (isSpam && sourceFolder.folderType === MailSetKind.INBOX) {
+			const spamFolder = assertNotNull(folderSystem.getSystemFolderByType(MailSetKind.SPAM))
+			if (this.spamMoveMailData) {
+				this.spamMoveMailData.mails.push(mail._id)
+			} else {
+				this.spamMoveMailData = createMoveMailData({
+					targetFolder: spamFolder?._id,
+					mails: [mail._id],
+					excludeMailSet: null,
+					moveReason: ClientClassifierType.CLIENT_CLASSIFICATION,
+				})
+			}
+			await this.sendMoveMailServiceRequest(this.mailFacade)
+			return spamFolder
+		} else if (!isSpam && sourceFolder.folderType === MailSetKind.SPAM) {
+			const hamFolder = assertNotNull(folderSystem.getSystemFolderByType(MailSetKind.INBOX))
+			if (this.hamMoveMailData) {
+				this.hamMoveMailData.mails.push(mail._id)
+			} else {
+				this.hamMoveMailData = createMoveMailData({
+					targetFolder: hamFolder?._id,
+					mails: [mail._id],
+					excludeMailSet: null,
+					moveReason: ClientClassifierType.CLIENT_CLASSIFICATION,
+				})
+			}
+			await this.sendMoveMailServiceRequest(this.mailFacade)
+			return hamFolder
+		} else if (mail.processingState !== ProcessingState.INBOX_RULE_PROCESSED_AND_SPAM_PREDICTION_MADE) {
+			this.classifierResultServiceMailIds.push(mail._id)
+			await this.sendClassifierResultServiceRequest(this.mailFacade)
+			return sourceFolder
+		} else {
+			return sourceFolder
+		}
+	}
+
+	public async updateSpamClassificationData(mail: Mail) {
+		if (this.spamClassifier == null || mail.clientSpamClassifierResult == null) {
+			return
+		}
+		const storedClassification = await this.spamClassifier.getSpamClassification(mail._id)
+		const isSpam = mail.clientSpamClassifierResult.spamDecision === SpamDecision.BLACKLIST
+		const isSpamConfidence = getSpamConfidence(mail)
+
+		if (isNotNull(storedClassification) && (isSpam !== storedClassification.isSpam || isSpamConfidence !== storedClassification.isSpamConfidence)) {
+			// the model has trained on the mail but the spamFlag was wrong so we refit with higher isSpamConfidence
+			await this.spamClassifier.updateSpamClassification(mail._id, isSpam, isSpamConfidence)
+		}
+	}
+
+	public async dropClassificationData(mailId: IdTuple) {
+		await this.spamClassifier?.deleteSpamClassification(mailId)
+	}
+
+	public async storeTrainingDatum(mail: Mail, mailDetails: MailDetails) {
+		const spamTrainMailDatum: SpamTrainMailDatum = {
+			mailId: mail._id,
+			subject: mail.subject,
+			body: getMailBodyText(mailDetails.body),
+			isSpam: DEFAULT_IS_SPAM,
+			isSpamConfidence: DEFAULT_IS_SPAM_CONFIDENCE,
+			ownerGroup: assertNotNull(mail._ownerGroup),
+		}
+		await this.spamClassifier?.storeSpamClassification(spamTrainMailDatum)
+	}
+}
+
+export function getSpamConfidence(mail: Mail): number {
+	return Number(mail.clientSpamClassifierResult?.confidence ?? DEFAULT_IS_SPAM_CONFIDENCE)
+}

src/mail-app/workerUtils/spamClassification/SpamClassificationInitializer.ts

@@ -1,5 +1,4 @@
 import { EntityClient } from "../../../common/api/common/EntityClient"
-import { UserFacade } from "../../../common/api/worker/facades/UserFacade"
 import { assertNotNull, isNotNull, lazyAsync } from "@tutao/tutanota-utils"
 import { MailBag, MailboxGroupRootTypeRef, MailBoxTypeRef, MailFolder, MailFolderTypeRef, MailTypeRef } from "../../../common/api/entities/tutanota/TypeRefs"
 import { getMailSetKind, MailSetKind } from "../../../common/api/common/TutanotaConstants"
@@ -9,6 +8,7 @@ import { getMailBodyText } from "../../../common/api/common/CommonMailUtils"
 import { BulkMailLoader, MailWithMailDetails } from "../index/BulkMailLoader"
 import { hasError } from "../../../common/api/common/utils/ErrorUtils"
 import { SpamTrainMailDatum } from "./SpamClassifier"
+import { getSpamConfidence } from "./SpamClassificationHandler"
 
 const INITIAL_SPAM_CLASSIFICATION_INDEX_INTERVAL_DAYS = 28
 
@@ -22,7 +22,6 @@ export class SpamClassificationInitializer {
 
 	constructor(
 		private readonly entityClient: EntityClient,
-		private readonly userFacade: UserFacade,
 		private readonly offlineStorage: OfflineStoragePersistence,
 		private readonly bulkMailLoader: lazyAsync<BulkMailLoader>,
 	) {}
@@ -91,16 +90,14 @@ export class SpamClassificationInitializer {
 
 	private mailWithDetailsToMailDatum(spamFolder: MailFolder, inboxFolder: MailFolder, { mail, mailDetails }: MailWithMailDetails): SpamTrainMailDatum {
 		const isSpam = mail.sets.some((folderId) => isSameId(folderId, spamFolder._id))
-		const isCertain = !mail.unread || !mail.sets.some((folderId) => isSameId(folderId, inboxFolder._id))
 		return {
 			mailId: mail._id,
 			subject: mail.subject,
 			body: getMailBodyText(mailDetails.body),
 			isSpam: isSpam,
-			isSpamConfidence: isCertain ? 1 : 0,
+			isSpamConfidence: getSpamConfidence(mail),
 			listId: listIdPart(mail._id),
 			elementId: elementIdPart(mail._id),
-			// todo: when owner group is null?
 			ownerGroup: assertNotNull(mail._ownerGroup),
 		} as SpamTrainMailDatum
 	}

src/mail-app/workerUtils/spamClassification/SpamClassifier.ts

@@ -1,5 +1,5 @@
 import { assertWorkerOrNode } from "../../../common/api/common/Env"
-import { assertNotNull, defer, groupByAndMap, isNotNull, Nullable, promiseMap } from "@tutao/tutanota-utils"
+import { assertNotNull, defer, groupByAndMap, isNotNull, Nullable, promiseMap, tokenize } from "@tutao/tutanota-utils"
 import { DynamicTfVectorizer } from "./DynamicTfVectorizer"
 import { HashingVectorizer } from "./HashingVectorizer"
 import {
@@ -22,11 +22,9 @@ import {
 } from "./PreprocessPatterns"
 import { SpamClassificationInitializer } from "./SpamClassificationInitializer"
 import { CacheStorage } from "../../../common/api/worker/rest/DefaultEntityRestCache"
-import { OfflineStoragePersistence } from "../index/OfflineStoragePersistence"
+import { htmlToText } from "../../../common/api/common/utils/IndexUtils"
-import { filterMailMemberships, htmlToText } from "../../../common/api/common/utils/IndexUtils"
 import {
 	dense,
-	dropout,
 	fromMemory,
 	glorotUniform,
 	LayersModel,
@@ -39,6 +37,7 @@ import {
 import type { Tensor } from "@tensorflow/tfjs-core"
 import type { ModelArtifacts } from "@tensorflow/tfjs-core/dist/io/types"
 import type { ModelFitArgs } from "@tensorflow/tfjs-layers"
+import { OfflineStoragePersistence } from "../index/OfflineStoragePersistence"
 
 assertWorkerOrNode()
 
@@ -64,7 +63,7 @@ export type SpamPredMailDatum = {
 	ownerGroup: Id
 }
 
-const PREDICTION_THRESHOLD = 0.5
+const PREDICTION_THRESHOLD = 0.55
 
 export type PreprocessConfiguration = {
 	isPreprocessMails: boolean
@@ -92,18 +91,21 @@ export const DEFAULT_PREPROCESS_CONFIGURATION: PreprocessConfiguration = {
 	isRemoveSpaceBeforeNewLine: true,
 }
 
-const TRAINING_INTERVAL = 1000 * 60 * 10
+const TRAINING_INTERVAL = 1000 * 60 * 10 // 10 minutes
+const FULL_RETRAINING_INTERVAL = 1000 * 60 * 60 * 24 * 7 // 1 week
 
 type TrainingPerformance = {
 	trainingTime: number
 	vectorizationTime: number
 }
 
+export const spamClassifierTokenizer = (text: string): string[] => tokenize(text)
+
 export class SpamClassifier {
 	private readonly classifier: Map<Id, { model: LayersModel; isEnabled: boolean }>
 
 	constructor(
-		private readonly offlineStorage: OfflineStoragePersistence | null,
+		private readonly offlineStorage: OfflineStoragePersistence,
 		private readonly offlineStorageCache: CacheStorage,
 		private readonly initializer: SpamClassificationInitializer,
 		private readonly deterministic: boolean = false,
@@ -113,18 +115,18 @@ export class SpamClassifier {
 		this.classifier = new Map()
 	}
 
-	public getEnabledSpamClassifierForOwnerGroup(ownerGroup: Id): Nullable<LayersModel> {
-		const classifier = this.classifier.get(ownerGroup) ?? null
-		if (classifier && classifier.isEnabled) {
-			return classifier.model
-		}
-		return null
-	}
-
 	public async initialize(ownerGroup: Id): Promise<void> {
 		const loadedModel = await this.loadModel(ownerGroup)
 
 		const storage = assertNotNull(this.offlineStorageCache)
+		setInterval(async () => {
+			const cutoffDate = Date.now() - FULL_RETRAINING_INTERVAL
+			const lastFullTrainingTime = await storage.getLastTrainedFromScratchTime()
+
+			if (cutoffDate > lastFullTrainingTime) {
+				await this.retrainModelFromScratch(storage, ownerGroup, cutoffDate)
+			}
+		}, FULL_RETRAINING_INTERVAL)
 		if (isNotNull(loadedModel)) {
 			console.log("Loaded existing spam classification model from database")
 
@@ -138,14 +140,19 @@ export class SpamClassifier {
 		}
 
 		console.log("No existing model found. Training from scratch...")
-		const data = await this.initializer.init(ownerGroup)
+		await this.trainFromScratch(storage, ownerGroup)
-		await this.initialTraining(data)
-		await this.saveModel(ownerGroup)
 		setInterval(async () => {
 			await this.updateAndSaveModel(storage, ownerGroup)
 		}, TRAINING_INTERVAL)
 	}
 
+	private async trainFromScratch(storage: CacheStorage, ownerGroup: string) {
+		const data = await this.initializer.init(ownerGroup)
+		await this.initialTraining(data)
+		await this.saveModel(ownerGroup)
+		await storage.setLastTrainedFromScratchTime(Date.now())
+	}
+
 	// VisibleForTesting
 	public async updateAndSaveModel(storage: CacheStorage, ownerGroup: Id) {
 		const isModelUpdated = await this.updateModelFromCutoff(await storage.getLastTrainedTime(), ownerGroup)
@@ -216,9 +223,11 @@ export class SpamClassifier {
 	}
 
 	public async initialTraining(mails: SpamTrainMailDatum[]): Promise<TrainingPerformance> {
-		const vectorizationStart = performance.now()
+		const preprocessingStart = performance.now()
+		const tokenizedMails = await promiseMap(mails, (mail) => spamClassifierTokenizer(this.preprocessMail(mail)))
+		const preprocessingTime = performance.now() - preprocessingStart
 
-		const tokenizedMails = await promiseMap(mails, (mail) => assertNotNull(this.offlineStorage).tokenize(this.preprocessMail(mail)))
+		const vectorizationStart = performance.now()
 		if (this.vectorizer instanceof DynamicTfVectorizer) {
 			this.vectorizer.buildInitialTokenVocabulary(tokenizedMails)
 		}
@@ -237,11 +246,13 @@ export class SpamClassifier {
 			epochs: 16,
 			batchSize: 32,
 			shuffle: !this.deterministic,
-			callbacks: {
+			// callbacks: {
-				onEpochEnd: async (epoch, logs) => {
+			// 	onEpochEnd: async (epoch, logs) => {
-					console.log(`Epoch ${epoch + 1} - Loss: ${logs!.loss.toFixed(4)}`)
+			// 		if (logs) {
-				},
+			// 			console.log(`Epoch ${epoch + 1} - Loss: ${logs.loss.toFixed(4)}`)
-			},
+			// 		}
+			// 	},
+			// },
 		})
 		const trainingTime = performance.now() - trainingStart
 
@@ -251,7 +262,9 @@ export class SpamClassifier {
 
 		this.classifier.set(mails[0].ownerGroup, { model: classifier, isEnabled: true })
 
-		console.log(`### Finished Initial Training ### (total trained mails: ${mails.length})`)
+		console.log(
+			`### Finished Initial Training ### (total trained mails: ${mails.length}, preprocessing time: ${preprocessingTime}, vectorization time: ${vectorizationTime}ms, training time: ${trainingTime})`,
+		)
 
 		return { vectorizationTime, trainingTime }
 	}
@@ -283,10 +296,9 @@ export class SpamClassifier {
 		const retrainingStart = performance.now()
 
 		const modelToUpdate = assertNotNull(this.classifier.get(ownerGroup))
-		const offlineStorage = assertNotNull(this.offlineStorage)
 		const tokenizedMailsArray = await promiseMap(newTrainingMails, async (mail) => {
 			const preprocessedMail = this.preprocessMail(mail)
-			const tokenizedMail = await offlineStorage.tokenize(preprocessedMail)
+			const tokenizedMail = spamClassifierTokenizer(preprocessedMail)
 			return { tokenizedMail, isSpamConfidence: mail.isSpamConfidence, isSpam: mail.isSpam ? 1 : 0 }
 		})
 
@@ -319,11 +331,11 @@ export class SpamClassifier {
 					epochs: 8,
 					batchSize: 32,
 					shuffle: !this.deterministic,
-					callbacks: {
+					// callbacks: {
-						onEpochEnd: async (epoch, logs) => {
+					// 	onEpochEnd: async (epoch, logs) => {
-							console.log(`Epoch ${epoch + 1} - Loss: ${logs!.loss.toFixed(4)}`)
+					// 		console.log(`Epoch ${epoch + 1} - Loss: ${logs!.loss.toFixed(4)}`)
-						},
+					// 	},
-					},
+					// },
 				}
 				for (let i = 0; i <= isSpamConfidence; i++) {
 					await modelToUpdate.model.fit(xs, ys, modelFitArgs)
@@ -349,7 +361,7 @@ export class SpamClassifier {
 		}
 
 		const preprocessedMail = this.preprocessMail(spamPredMailDatum)
-		const tokenizedMail = await assertNotNull(this.offlineStorage).tokenize(preprocessedMail)
+		const tokenizedMail = spamClassifierTokenizer(preprocessedMail)
 		const vectors = await assertNotNull(this.vectorizer).transform([tokenizedMail])
 
 		const xs = tensor2d(vectors, [vectors.length, assertNotNull(this.vectorizer).dimension], undefined)
@@ -357,7 +369,7 @@ export class SpamClassifier {
 		const predictionData = await predictionTensor.data()
 		const prediction = predictionData[0]
 
-		console.log(`predicted new mail to be with probability ${prediction.toFixed(2)} spam. Owner Group: ${spamPredMailDatum.ownerGroup}`)
+		// console.log(`predicted new mail to be with probability ${prediction.toFixed(2)} spam. Owner Group: ${spamPredMailDatum.ownerGroup}`)
 
 		// When using the webgl backend we need to manually dispose @tensorflow tensors
 		xs.dispose()
@@ -366,97 +378,20 @@ export class SpamClassifier {
 		return prediction > PREDICTION_THRESHOLD
 	}
 
-	/*
+	public getSpamClassification(mailId: IdTuple) {
-	 * TODO: Only for internal release
+		return this.offlineStorage.getSpamClassification(mailId)
-	 *
-	 * Allows to check the accuracy of your currently trained classifier against the content of mailbox itself
-	 * How-to:
-	 * 1) Open console and switch context to worker-bootstrap.js
-	 * 2) Execute this method in console: `locator.spamClassifier.getSpamMetricsForCurrentMailBox()`
-	 * 3) Let execution continue from breakpoint
-	 *
-	 * Since we change constant of this.initializer,
-	 * it's better to restart the client to not have unexpected effect
-	 */
-	public async getSpamMetricsForCurrentMailBox(ownerGroup?: Id): Promise<void> {
-		const { LocalTimeDateProvider } = await import("../../../common/api/worker/DateProvider.js")
-		const dateProvider = new LocalTimeDateProvider()
-
-		const getIdOfClassificationMail = (classificationData: any) => {
-			return ((classificationData.listId as Id) + "/" + classificationData.elementId) as Id
-		}
-		const user = assertNotNull((this.initializer as any).userFacade.getUser())
-		const firstOwnerGroup = ownerGroup ?? filterMailMemberships(user)[0]._id
-		console.log(`Testing with ownergroup: ${firstOwnerGroup}`)
-
-		const readingAllSpamStart = performance.now()
-		const trainedMails = await assertNotNull(this.offlineStorage)
-			.getCertainSpamClassificationTrainingDataAfterCutoff(0, firstOwnerGroup)
-			.then((mails) => new Set(mails.map(getIdOfClassificationMail)))
-		console.log(`Done reading ${trainedMails.size} certain training mail data in: ${performance.now() - readingAllSpamStart}ms`)
-
-		// since we train with last -28 days, we can test with last -90
-		;(this.initializer as any).TIME_LIMIT = dateProvider.getStartOfDayShiftedBy(-90)
-		// if exists, try to test with at 5xleast same number of mails as in training sample
-		;(this.initializer as any).MIN_MAILS_COUNT = trainedMails.size * 5
-		// to avoid putting stuff into offline storage
-		;(this.initializer as any).offlineStorage.storeSpamClassification = async () => {
-			console.log("not putting classification datum into offline storage")
 	}
 
-		const downloadingExtraMailsStart = performance.now()
+	public updateSpamClassification(mailId: IdTuple, isSpam: boolean, isSpamConfidence: number) {
-		const testingMails = (await this.initializer.init(firstOwnerGroup))
+		return this.offlineStorage.updateSpamClassification(mailId, isSpam, isSpamConfidence)
-			// do not test with the same mails that was used to train
-			.filter((classificationData) => !trainedMails.has(getIdOfClassificationMail(classificationData)))
-		console.log(`Done downloading extra ${testingMails.length} of last 90 days mail data in: ${performance.now() - downloadingExtraMailsStart}ms`)
-
-		const testingAllSamplesStart = performance.now()
-		await this.test(testingMails)
-		console.log(`Done testing all extra mails sample in: ${performance.now() - testingAllSamplesStart}ms`)
 	}
 
-	public async test(mails: SpamTrainMailDatum[]): Promise<void> {
+	public storeSpamClassification(spamTrainMailDatum: SpamTrainMailDatum) {
-		if (!this.classifier) {
+		return this.offlineStorage.storeSpamClassification(spamTrainMailDatum)
-			throw new Error("Model has not been loaded")
 	}
 
-		let predictionArray: number[] = []
+	public deleteSpamClassification(mailId: IdTuple) {
-		for (let mail of mails) {
+		return this.offlineStorage.deleteSpamClassification(mailId)
-			const prediction = await this.predict(mail)
-			predictionArray.push(prediction ? 1 : 0)
-		}
-		const ysArray = mails.map((mail) => mail.isSpam)
-
-		let tp = 0,
-			tn = 0,
-			fp = 0,
-			fn = 0
-
-		for (let i = 0; i < predictionArray.length; i++) {
-			const predictedSpam = predictionArray[i] > 0.5
-			const isActuallyASpam = ysArray[i]
-			if (predictedSpam && isActuallyASpam) tp++
-			else if (!predictedSpam && !isActuallyASpam) tn++
-			else if (predictedSpam && !isActuallyASpam) fp++
-			else if (!predictedSpam && isActuallyASpam) fn++
-		}
-
-		const total = tp + tn + fp + fn
-		const accuracy = (tp + tn) / total
-		const precision = tp / (tp + fp + 1e-7)
-		const recall = tp / (tp + fn + 1e-7)
-		const f1 = 2 * ((precision * recall) / (precision + recall + 1e-7))
-
-		console.log("\n--- Evaluation Metrics ---")
-		console.log(`Accuracy: \t${(accuracy * 100).toFixed(2)}%`)
-		console.log(`Precision:\t${(precision * 100).toFixed(2)}%`)
-		console.log(`Recall:   \t${(recall * 100).toFixed(2)}%`)
-		console.log(`F1 Score: \t${(f1 * 100).toFixed(2)}%`)
-		console.log("\nConfusion Matrix:")
-		console.log({
-			Predicted_Spam: { True_Positive: tp, False_Positive: fp },
-			Predicted_Ham: { False_Negative: fn, True_Negative: tn },
-		})
 	}
 
 	// visibleForTesting
@@ -538,7 +473,7 @@ export class SpamClassifier {
 		}
 	}
 
-	// VisibleForTesting
+	// visibleForTesting
 	public async loadModel(ownerGroup: Id): Promise<Nullable<LayersModel>> {
 		const model = await assertNotNull(this.offlineStorage).getSpamClassificationModel(ownerGroup)
 		if (model) {
@@ -565,7 +500,19 @@ export class SpamClassifier {
 		return concatenated.length > 0 ? concatenated : " "
 	}
 
-	// === Testing methods
+	private async retrainModelFromScratch(storage: CacheStorage, ownerGroup: Id, cutoffTimestamp: number) {
+		console.log("Model is being re-trained from scratch, deleting old data")
+		try {
+			await assertNotNull(this.offlineStorage).deleteSpamClassificationTrainingDataBeforeCutoff(cutoffTimestamp, ownerGroup)
+		} catch (e) {
+			console.error("Failed delete old training data: ", e)
+			return
+		}
+
+		await this.trainFromScratch(storage, ownerGroup)
+	}
+
+	// visibleForTesting
 	public async cloneClassifier(): Promise<SpamClassifier> {
 		const newClassifier = new SpamClassifier(
 			this.offlineStorage,
@@ -587,4 +534,9 @@ export class SpamClassifier {
 
 		return newClassifier
 	}
+
+	// visibleForTesting
+	public addSpamClassifierForOwner(ownerGroup: Id, model: LayersModel, isEnabled: boolean) {
+		this.classifier.set(ownerGroup, { model, isEnabled })
+	}
 }

src/mail-app/workerUtils/worker/WorkerLocator.ts

@@ -20,6 +20,7 @@ import {
 	isIOSApp,
 	isOfflineStorageAvailable,
 	isTest,
+	isWebClient,
 } from "../../../common/api/common/Env.js"
 import { Const } from "../../../common/api/common/TutanotaConstants.js"
 import type { BrowserData } from "../../../common/misc/ClientConstants.js"
@@ -269,7 +270,7 @@ export async function initLocator(worker: WorkerImpl, browserData: BrowserData)
 		return new IndexerCore(await db(), browserData)
 	})
 
-	const mailIndexerAndMailFacade = lazyMemoized(async () => {
+	const mailIndexer = lazyMemoized(async () => {
 		const { IndexedDbMailIndexerBackend } = await import("../index/IndexedDbMailIndexerBackend")
 		const { OfflineStorageMailIndexerBackend } = await import("../index/OfflineStorageMailIndexerBackend")
 		const { MailIndexer } = await import("../index/MailIndexer.js")
@@ -278,30 +279,24 @@ export async function initLocator(worker: WorkerImpl, browserData: BrowserData)
 		const mailFacade = await locator.mail()
 		if (isOfflineStorageAvailable()) {
 			const persistence = await offlineStorageIndexerPersistence()
-			return {
+			return new MailIndexer(
-				mailIndexer: new MailIndexer(
 				mainInterface.infoMessageHandler,
 				bulkLoaderFactory,
 				locator.cachingEntityClient,
 				dateProvider,
 				mailFacade,
 				() => new OfflineStorageMailIndexerBackend(persistence),
-				),
+			)
-				mailFacade: mailFacade,
-			}
 		} else {
 			const core = await indexerCore()
-			return {
+			return new MailIndexer(
-				mailIndexer: new MailIndexer(
 				mainInterface.infoMessageHandler,
 				bulkLoaderFactory,
 				locator.cachingEntityClient,
 				dateProvider,
 				mailFacade,
 				(userId) => new IndexedDbMailIndexerBackend(core, userId, typeModelResolver),
-				),
+			)
-				mailFacade: mailFacade,
-			}
 		}
 	})
 
@@ -335,18 +330,12 @@ export async function initLocator(worker: WorkerImpl, browserData: BrowserData)
 	if (isOfflineStorageAvailable() && !isAdminClient()) {
 		locator.sqlCipherFacade = new SqlCipherFacadeSendDispatcher(locator.native)
 		offlineStorageProvider = async () => {
-			if (isDesktop()) {
 			const { SpamClassifier } = await import("../spamClassification/SpamClassifier")
 			const { SpamClassificationInitializer } = await import("../spamClassification/SpamClassificationInitializer")
 			const offlineStorage = await offlineStorageIndexerPersistence()
-				const spamClassifierInitializer = new SpamClassificationInitializer(
+			const spamClassifierInitializer = new SpamClassificationInitializer(locator.cachingEntityClient, offlineStorage, locator.bulkMailLoader)
-					locator.cachingEntityClient,
-					locator.user,
-					offlineStorage,
-					locator.bulkMailLoader,
-				)
 			locator.spamClassifier = new SpamClassifier(offlineStorage, locator.cacheStorage, spamClassifierInitializer)
-			}
+
 			const { KeyVerificationTableDefinitions } = await import("../../../common/api/worker/facades/IdentityKeyTrustDatabase.js")
 			const { SearchTableDefinitions, SpamClassificationDefinitions } = await import("../index/OfflineStoragePersistence.js")
 			const { AutosaveDraftsTableDefinitions } = await import("../../../common/api/worker/facades/lazy/OfflineStorageAutosaveFacade.js")
@@ -358,7 +347,7 @@ export async function initLocator(worker: WorkerImpl, browserData: BrowserData)
 				},
 				{
 					ref: MailTypeRef,
-					handler: new CustomMailEventCacheHandler(mailIndexerAndMailFacade, offlineStorageIndexerPersistence, locator.cacheStorage),
+					handler: new CustomMailEventCacheHandler(mailIndexer),
 				},
 				{ ref: UserTypeRef, handler: new CustomUserCacheHandler(locator.cacheStorage) },
 			)
@@ -452,12 +441,11 @@ export async function initLocator(worker: WorkerImpl, browserData: BrowserData)
 
 	locator.indexer = lazyMemoized(async () => {
 		const contact = await contactIndexer()
-		const { mailIndexer, mailFacade } = await mailIndexerAndMailFacade()
 
 		if (isOfflineStorageAvailable()) {
 			const { OfflineStorageIndexer } = await import("../index/OfflineStorageIndexer.js")
 			const persistence = await offlineStorageIndexerPersistence()
-			return new OfflineStorageIndexer(locator.user, persistence, mailIndexer, mainInterface.infoMessageHandler, contact)
+			return new OfflineStorageIndexer(locator.user, persistence, await mailIndexer(), mainInterface.infoMessageHandler, contact)
 		} else {
 			const { IndexedDbIndexer } = await import("../index/IndexedDbIndexer.js")
 			const core = await indexerCore()
@@ -467,7 +455,7 @@ export async function initLocator(worker: WorkerImpl, browserData: BrowserData)
 				core,
 				mainInterface.infoMessageHandler,
 				locator.cachingEntityClient,
-				mailIndexer,
+				await mailIndexer(),
 				contact,
 				typeModelResolver,
 				locator.keyLoader,
@@ -670,16 +658,15 @@ export async function initLocator(worker: WorkerImpl, browserData: BrowserData)
 	)
 
 	locator.search = lazyMemoized(async () => {
-		const { mailIndexer } = await mailIndexerAndMailFacade()
 		if (isOfflineStorageAvailable()) {
 			const { OfflineStorageSearchFacade } = await import("../index/OfflineStorageSearchFacade.js")
-			return new OfflineStorageSearchFacade(locator.sqlCipherFacade, mailIndexer, await contactIndexer())
+			return new OfflineStorageSearchFacade(locator.sqlCipherFacade, await mailIndexer(), await contactIndexer())
 		} else {
 			const { IndexedDbSearchFacade } = await import("../index/IndexedDbSearchFacade.js")
 			return new IndexedDbSearchFacade(
 				locator.user,
 				await db(),
-				mailIndexer,
+				await mailIndexer(),
 				await contactSuggestionFacade(),
 				browserData,
 				locator.cachingEntityClient,
@@ -744,6 +731,7 @@ export async function initLocator(worker: WorkerImpl, browserData: BrowserData)
 	})
 	locator.mail = lazyMemoized(async () => {
 		const { MailFacade } = await import("../../../common/api/worker/facades/lazy/MailFacade.js")
+
 		return new MailFacade(
 			locator.user,
 			locator.cachingEntityClient,
@@ -754,7 +742,6 @@ export async function initLocator(worker: WorkerImpl, browserData: BrowserData)
 			locator.login,
 			locator.keyLoader,
 			locator.publicEncryptionKeyProvider,
-			locator.spamClassifier,
 		)
 	})
 	const nativePushFacade = new NativePushFacadeSendDispatcher(worker)

test/tests/Suite.ts

@@ -68,7 +68,6 @@ import "./api/worker/rest/EntityRestClientTest.js"
 import "./api/worker/rest/EphemeralCacheStorageTest.js"
 import "./api/worker/rest/PatchGeneratorTest.js"
 import "./api/worker/rest/ServiceExecutorTest.js"
-import "./api/worker/rest/cacheHandler/CustomMailEventCacheHandlerTest.js"
 import "./api/worker/search/BulkMailLoaderTest.js"
 import "./api/worker/search/ContactIndexerTest.js"
 import "./api/worker/search/EventQueueTest.js"
@@ -85,7 +84,6 @@ import "./api/worker/facades/KeyVerificationFacadeTest.js"
 import "./api/worker/utils/SleepDetectorTest.js"
 import "./api/worker/utils/spamClassification/TfIdfVectorizerTest.js"
 import "./api/worker/utils/spamClassification/HashingVectorizerTest.js"
-import "./api/worker/utils/spamClassification/SpamClassifierTest.js"
 import "./api/worker/utils/spamClassification/PreprocessPatternsTest.js"
 import "./calendar/AlarmSchedulerTest.js"
 import "./calendar/CalendarAgendaViewTest.js"
@@ -182,6 +180,7 @@ import "./misc/parsing/ParserCombinatorTest.js"
 import "./sharing/GroupSettingsModelTest.js"
 import "./mail/editor/OpenLocallySavedDraftActionTest.js"
 import "./mail/search/MailIndexAndSpamClassificationPostLoginActionTest.js"
+import "./mail/SpamClassificationHandlerTest.js"
 
 import * as td from "testdouble"
 import { random } from "@tutao/tutanota-crypto"
@@ -213,6 +212,7 @@ async function setupSuite({ integration }: { integration?: boolean }) {
 	if (typeof process !== "undefined") {
 		// setup the Entropy for all testcases
 		await random.addEntropy([{ data: 36, entropy: 256, source: "key" }])
+		await import("./api/worker/utils/spamClassification/SpamClassifierTest.js")
 		await import("./api/worker/offline/OfflineStorageMigratorTest.js")
 		await import("./api/worker/offline/OfflineStorageTest.js")
 		await import("./api/worker/rest/RestClientTest.js")

test/tests/api/worker/crypto/CryptoFacadeTest.ts

@@ -10,6 +10,7 @@ import {
 	GroupType,
 	PermissionType,
 	PresentableKeyVerificationState,
+	ProcessingState,
 	PublicKeyIdentifierType,
 } from "../../../../../src/common/api/common/TutanotaConstants.js"
 import {
@@ -1854,6 +1855,8 @@ o.spec("CryptoFacadeTest", function () {
 			mailDetailsDraft: null,
 			sets: [],
 			keyVerificationState: null,
+			processingState: ProcessingState.INBOX_RULE_APPLIED,
+			clientSpamClassifierResult: null,
 		})
 
 		// casting here is fine, since we just want to mimic server response data

test/tests/api/worker/facades/MailFacadeTest.ts

@@ -18,7 +18,7 @@ import {
 import {
 	CryptoProtocolVersion,
 	MailAuthenticationStatus,
-	MAX_NBR_MOVE_DELETE_MAIL_SERVICE,
+	MAX_NBR_OF_MAILS_SYNC_OPERATION,
 	ReportedMailFieldType,
 } from "../../../../../src/common/api/common/TutanotaConstants.js"
 import { matchers, object, when } from "testdouble"
@@ -72,7 +72,6 @@ o.spec("MailFacade test", function () {
 		loginFacade = object()
 		keyLoaderFacade = object()
 		publicEncryptionKeyProvider = object()
-		spamClassifier = object()
 		facade = new MailFacade(
 			userFacade,
 			entityClient,
@@ -83,7 +82,6 @@ o.spec("MailFacade test", function () {
 			loginFacade,
 			keyLoaderFacade,
 			publicEncryptionKeyProvider,
-			spamClassifier,
 		)
 	})
 
@@ -491,7 +489,7 @@ o.spec("MailFacade test", function () {
 		o.test("batches large amounts of mails", async () => {
 			const expectedBatches = 4
 			const testIds: IdTuple[] = []
-			for (let i = 0; i < MAX_NBR_MOVE_DELETE_MAIL_SERVICE * expectedBatches; i++) {
+			for (let i = 0; i < MAX_NBR_OF_MAILS_SYNC_OPERATION * expectedBatches; i++) {
 				testIds.push([`${i}`, `${i}`])
 			}
 			await facade.markMails(testIds, true)
@@ -500,7 +498,7 @@ o.spec("MailFacade test", function () {
 					serviceExecutor.post(
 						UnreadMailStateService,
 						matchers.contains({
-							mails: testIds.slice(i * MAX_NBR_MOVE_DELETE_MAIL_SERVICE, (i + 1) * MAX_NBR_MOVE_DELETE_MAIL_SERVICE),
+							mails: testIds.slice(i * MAX_NBR_OF_MAILS_SYNC_OPERATION, (i + 1) * MAX_NBR_OF_MAILS_SYNC_OPERATION),
 							unread: true,
 						}),
 					),

test/tests/api/worker/rest/cacheHandler/CustomMailEventCacheHandlerTest.ts

@@ -1,320 +0,0 @@
-import o from "@tutao/otest"
-import { func, matchers, object, verify, when } from "testdouble"
-import { lazy, lazyAsync } from "@tutao/tutanota-utils"
-import { MailIndexer } from "../../../../../../src/mail-app/workerUtils/index/MailIndexer"
-import { MailFacade } from "../../../../../../src/common/api/worker/facades/lazy/MailFacade"
-import { OfflineStoragePersistence } from "../../../../../../src/mail-app/workerUtils/index/OfflineStoragePersistence"
-import { CacheStorage } from "../../../../../../src/common/api/worker/rest/DefaultEntityRestCache"
-import { CustomMailEventCacheHandler } from "../../../../../../src/common/api/worker/rest/cacheHandler/CustomMailEventCacheHandler"
-import { Body, Mail, MailDetails, MailFolderTypeRef, MailSetEntryTypeRef, MailTypeRef } from "../../../../../../src/common/api/entities/tutanota/TypeRefs"
-import { MailSetKind } from "../../../../../../src/common/api/common/TutanotaConstants"
-import { ClientClassifierType } from "../../../../../../src/common/api/common/ClientClassifierType"
-import { EntityUpdateData } from "../../../../../../src/common/api/common/utils/EntityUpdateUtils"
-import { SpamTrainMailDatum } from "../../../../../../src/mail-app/workerUtils/spamClassification/SpamClassifier"
-import { getMailBodyText } from "../../../../../../src/common/api/common/CommonMailUtils"
-import { createTestEntity } from "../../../../TestUtils"
-
-/**
- * These tests should verify that the following are obeyed:
- * - All Mails in Spam have isSpamConfidence of 1 (during create)
- * - Moved Mails have isSpamConfidence of 1 (event update)
- * - Read Mails have isSpamConfidence of  1 (event update)
- * - Mails in Inbox have isSpamConfidence of 0.
- */
-o.spec("CustomMailEventCacheHandler", function () {
-	let cacheStorageMock: CacheStorage
-	let offlineStorageMock: lazy<Promise<OfflineStoragePersistence>>
-	let indexerAndMailFacadeMock: lazyAsync<{ mailIndexer: MailIndexer; mailFacade: MailFacade }>
-
-	const inboxFolder = createTestEntity(MailFolderTypeRef, { _id: ["listId", "inbox"], folderType: MailSetKind.INBOX })
-	const trashFolder = createTestEntity(MailFolderTypeRef, { _id: ["listId", "trash"], folderType: MailSetKind.TRASH })
-	const spamFolder = createTestEntity(MailFolderTypeRef, { _id: ["listId", "spam"], folderType: MailSetKind.SPAM })
-	const allFolders = [inboxFolder, trashFolder, spamFolder]
-
-	o.beforeEach(function () {
-		cacheStorageMock = object() as CacheStorage
-		offlineStorageMock = func() as lazy<Promise<OfflineStoragePersistence>>
-		indexerAndMailFacadeMock = func() as lazyAsync<{ mailIndexer: MailIndexer; mailFacade: MailFacade }>
-
-		when(cacheStorageMock.getWholeList(MailFolderTypeRef, matchers.anything())).thenResolve(allFolders)
-	})
-
-	o.spec("onEntityEventCreate", function () {
-		let mailIndexer = object() as MailIndexer
-		let mailFacade = object() as MailFacade
-		let body: Body
-		let mailDetails: MailDetails
-		let mail: Mail
-
-		o.beforeEach(function () {
-			when(indexerAndMailFacadeMock()).thenResolve({ mailIndexer, mailFacade })
-
-			body = object({ text: "Body Text" }) as Body
-			mailDetails = object({ body }) as MailDetails
-			mail = createTestEntity(MailTypeRef, {
-				_id: ["listId", "elementId"],
-				sets: [spamFolder._id],
-				subject: "subject",
-				_ownerGroup: "owner",
-				unread: false,
-			})
-			when(mailIndexer.downloadNewMailData(matchers.anything())).thenResolve({
-				mail,
-				mailDetails,
-			})
-		})
-
-		o("does not process spam e-mails when it fails to download new mail", async function () {
-			when(mailIndexer.downloadNewMailData(matchers.anything())).thenResolve(null)
-
-			const offlineStorage = object() as OfflineStoragePersistence
-			when(offlineStorageMock()).thenResolve(offlineStorage)
-
-			const cacheHandler = new CustomMailEventCacheHandler(indexerAndMailFacadeMock, offlineStorageMock, cacheStorageMock)
-			await cacheHandler.onEntityEventCreate(["listId", "elementId"], [])
-
-			verify(cacheStorageMock.getWholeList(MailFolderTypeRef, matchers.anything()), { times: 0 })
-		})
-
-		o("processSpam maintains server classification when client classification is not enabled", async function () {
-			const offlineStorage = object() as OfflineStoragePersistence
-			when(offlineStorageMock()).thenResolve(offlineStorage)
-			when(mailFacade.predictSpamResult(mail)).thenResolve(null)
-
-			const cacheHandler = new CustomMailEventCacheHandler(indexerAndMailFacadeMock, offlineStorageMock, cacheStorageMock)
-			await cacheHandler.onEntityEventCreate(["listId", "elementId"], [])
-
-			const spamTrainMailDatum: SpamTrainMailDatum = {
-				mailId: mail._id,
-				subject: mail.subject,
-				body: getMailBodyText(body),
-				isSpam: true,
-				isSpamConfidence: 1,
-				ownerGroup: "owner",
-			}
-
-			verify(offlineStorage.storeSpamClassification(spamTrainMailDatum), { times: 1 })
-		})
-
-		o("processSpam uses client classification when enabled", async function () {
-			const offlineStorage = object() as OfflineStoragePersistence
-			when(offlineStorageMock()).thenResolve(offlineStorage)
-			when(mailFacade.predictSpamResult(mail)).thenResolve(false)
-
-			const cacheHandler = new CustomMailEventCacheHandler(indexerAndMailFacadeMock, offlineStorageMock, cacheStorageMock)
-			await cacheHandler.onEntityEventCreate(["listId", "elementId"], [])
-
-			const spamTrainMailDatum: SpamTrainMailDatum = {
-				mailId: mail._id,
-				subject: mail.subject,
-				body: getMailBodyText(body),
-				isSpam: false,
-				isSpamConfidence: 0,
-				ownerGroup: "owner",
-			}
-
-			verify(offlineStorage.storeSpamClassification(spamTrainMailDatum), { times: 1 })
-		})
-
-		o("processSpam correctly verifies if email is stored in spam folder", async function () {
-			mail.sets = [spamFolder._id]
-			mail.unread = true
-
-			const offlineStorage = object() as OfflineStoragePersistence
-			when(offlineStorageMock()).thenResolve(offlineStorage)
-			when(mailFacade.predictSpamResult(mail)).thenResolve(false)
-
-			const cacheHandler = new CustomMailEventCacheHandler(indexerAndMailFacadeMock, offlineStorageMock, cacheStorageMock)
-			await cacheHandler.onEntityEventCreate(["listId", "elementId"], [])
-
-			const spamTrainMailDatum: SpamTrainMailDatum = {
-				mailId: mail._id,
-				subject: mail.subject,
-				body: getMailBodyText(body),
-				isSpam: false,
-				ownerGroup: "owner",
-				isSpamConfidence: 0,
-			}
-
-			verify(offlineStorage.storeSpamClassification(spamTrainMailDatum), { times: 1 })
-		})
-
-		o("getSpamConfidence is 0 for mail in trash folder ", async function () {
-			mail.unread = false
-			mail.sets = [["listId", "trash"]]
-
-			const cacheHandler = new CustomMailEventCacheHandler(indexerAndMailFacadeMock, offlineStorageMock, cacheStorageMock)
-			o(cacheHandler.getSpamConfidence(allFolders, mail).confidence).equals(0)
-		})
-
-		o("getSpamConfidence is 1 for mail in spam folder ", async function () {
-			mail.unread = true
-			mail.sets = [spamFolder._id]
-
-			const cacheHandler = new CustomMailEventCacheHandler(indexerAndMailFacadeMock, offlineStorageMock, cacheStorageMock)
-			o(cacheHandler.getSpamConfidence(allFolders, mail).confidence).equals(1)
-		})
-
-		o("getSpamConfidence for inbox folder depends on read status", async function () {
-			const cacheHandler = new CustomMailEventCacheHandler(indexerAndMailFacadeMock, offlineStorageMock, cacheStorageMock)
-			mail.sets = [inboxFolder._id]
-
-			mail.unread = true
-			o(cacheHandler.getSpamConfidence(allFolders, mail).confidence).equals(0)
-			mail.unread = false
-			o(cacheHandler.getSpamConfidence(allFolders, mail).confidence).equals(1)
-		})
-
-		o("processSpam moves mail to spam when detected as such and its not already in spam", async function () {
-			mail.sets = [inboxFolder._id]
-			const offlineStorage = object() as OfflineStoragePersistence
-			when(offlineStorageMock()).thenResolve(offlineStorage)
-			when(mailFacade.predictSpamResult(mail)).thenResolve(true)
-			when(mailFacade.isSpamClassificationEnabled("owner")).thenReturn(true)
-
-			const cacheHandler = new CustomMailEventCacheHandler(indexerAndMailFacadeMock, offlineStorageMock, cacheStorageMock)
-			await cacheHandler.onEntityEventCreate(["listId", "elementId"], [])
-
-			const spamTrainMailDatum: SpamTrainMailDatum = {
-				mailId: mail._id,
-				subject: mail.subject,
-				body: getMailBodyText(body),
-				isSpam: true,
-				isSpamConfidence: 1,
-				ownerGroup: "owner",
-			}
-
-			verify(offlineStorage.storeSpamClassification(spamTrainMailDatum), { times: 1 })
-			verify(mailFacade.simpleMoveMails([["listId", "elementId"]], MailSetKind.SPAM, ClientClassifierType.CLIENT_CLASSIFICATION))
-		})
-
-		o("processSpam moves mail to inbox when detected as such and its not already in inbox", async function () {
-			const offlineStorage = object() as OfflineStoragePersistence
-			when(offlineStorageMock()).thenResolve(offlineStorage)
-			mail.sets = [spamFolder._id] // the mail is in spam folder
-			when(mailFacade.predictSpamResult(mail)).thenResolve(false)
-
-			const cacheHandler = new CustomMailEventCacheHandler(indexerAndMailFacadeMock, offlineStorageMock, cacheStorageMock)
-			await cacheHandler.onEntityEventCreate(["listId", "elementId"], [])
-
-			const spamTrainMailDatum: SpamTrainMailDatum = {
-				mailId: mail._id,
-				subject: mail.subject,
-				body: getMailBodyText(body),
-				isSpam: false,
-				isSpamConfidence: 0,
-				ownerGroup: "owner",
-			}
-
-			verify(offlineStorage.storeSpamClassification(spamTrainMailDatum), { times: 1 })
-			verify(mailFacade.simpleMoveMails([["listId", "elementId"]], MailSetKind.INBOX, ClientClassifierType.CLIENT_CLASSIFICATION))
-		})
-	})
-
-	o.spec("onEntityEventUpdate", function () {
-		let mailIndexer = object() as MailIndexer
-		let mailFacade = object() as MailFacade
-		let mail: Mail
-		let body: Body
-		let mailDetails: MailDetails
-
-		o.beforeEach(function () {
-			when(indexerAndMailFacadeMock()).thenResolve({ mailIndexer, mailFacade })
-
-			body = object({ text: "Body Text" }) as Body
-			mailDetails = object({ body }) as MailDetails
-			mail = createTestEntity(MailTypeRef, {
-				_id: ["listId", "elementId"],
-				subject: "subject",
-				sets: [inboxFolder._id],
-				_ownerGroup: "owner",
-			})
-			when(mailIndexer.downloadNewMailData(matchers.anything())).thenResolve({
-				mail,
-				mailDetails,
-			})
-		})
-
-		o("does nothing if mail has not been read and not moved or had label applied.", async function () {
-			const offlineStorage = object() as OfflineStoragePersistence
-			when(offlineStorageMock()).thenResolve(offlineStorage)
-			mail.unread = true
-			when(cacheStorageMock.get(MailTypeRef, "listId", "elementId")).thenResolve(mail)
-
-			const cacheHandler = new CustomMailEventCacheHandler(indexerAndMailFacadeMock, offlineStorageMock, cacheStorageMock)
-			await cacheHandler.onEntityEventUpdate(["listId", "elementId"], [])
-
-			verify(offlineStorage.updateSpamClassificationData(matchers.anything(), matchers.anything(), matchers.anything()), { times: 0 })
-		})
-
-		o("does nothing if we delete a mail from spam folder", async function () {
-			const offlineStorage = object() as OfflineStoragePersistence
-			when(offlineStorageMock()).thenResolve(offlineStorage)
-			when(cacheStorageMock.get(MailTypeRef, "listId", "elementId")).thenResolve(mail)
-			const cacheHandler = new CustomMailEventCacheHandler(indexerAndMailFacadeMock, offlineStorageMock, cacheStorageMock)
-
-			mail.sets = [spamFolder._id]
-			await cacheHandler.onEntityEventCreate(["listId", "elementId"], [])
-			verify(offlineStorage.storeSpamClassification(matchers.anything()), { times: 1 })
-
-			mail.sets = [trashFolder._id]
-			await cacheHandler.onEntityEventUpdate(["listId", "elementId"], [])
-
-			verify(offlineStorage.updateSpamClassificationData(matchers.anything(), matchers.anything(), matchers.anything()), { times: 0 })
-		})
-
-		o("does update spam classification data if mail has been read in inbox and not moved", async function () {
-			mail.sets = [inboxFolder._id]
-			const offlineStorage = object() as OfflineStoragePersistence
-			when(offlineStorage.getStoredClassification(mail)).thenResolve({ isSpam: false, isSpamConfidence: 0 })
-			when(offlineStorageMock()).thenResolve(offlineStorage)
-			mail.unread = false
-			when(cacheStorageMock.get(MailTypeRef, "listId", "elementId")).thenResolve(mail)
-
-			const cacheHandler = new CustomMailEventCacheHandler(indexerAndMailFacadeMock, offlineStorageMock, cacheStorageMock)
-			await cacheHandler.onEntityEventUpdate(["listId", "elementId"], [])
-
-			verify(offlineStorage.updateSpamClassificationData(["listId", "elementId"], false, 1), { times: 1 })
-			verify(mailFacade.predictSpamResult(mail), { times: 0 })
-		})
-
-		o("does update spam classification data if mail has not been read but moved", async function () {
-			mail.sets = [spamFolder._id]
-			const offlineStorage = object() as OfflineStoragePersistence
-			when(offlineStorage.getStoredClassification(mail)).thenResolve({ isSpam: false, isSpamConfidence: 0 })
-			when(offlineStorageMock()).thenResolve(offlineStorage)
-			mail.unread = true
-			when(cacheStorageMock.get(MailTypeRef, "listId", "elementId")).thenResolve(mail)
-			const event = object({ typeRef: MailSetEntryTypeRef }) as unknown as EntityUpdateData
-
-			const cacheHandler = new CustomMailEventCacheHandler(indexerAndMailFacadeMock, offlineStorageMock, cacheStorageMock)
-			await cacheHandler.onEntityEventUpdate(["listId", "elementId"], [event])
-
-			verify(offlineStorage.updateSpamClassificationData(["listId", "elementId"], true, 1), { times: 1 })
-		})
-
-		o("does update spam classification data if mail was not previously included", async function () {
-			mail.sets = [inboxFolder._id]
-			const offlineStorage = object() as OfflineStoragePersistence
-			when(offlineStorage.getStoredClassification(mail)).thenResolve(null)
-			when(offlineStorageMock()).thenResolve(offlineStorage)
-			mail.unread = true
-			when(cacheStorageMock.get(MailTypeRef, "listId", "elementId")).thenResolve(mail)
-			const event = object({ typeRef: MailSetEntryTypeRef }) as unknown as EntityUpdateData
-
-			const cacheHandler = new CustomMailEventCacheHandler(indexerAndMailFacadeMock, offlineStorageMock, cacheStorageMock)
-			await cacheHandler.onEntityEventUpdate(["listId", "elementId"], [event])
-
-			const spamTrainMailDatum: SpamTrainMailDatum = {
-				mailId: mail._id,
-				subject: mail.subject,
-				body: getMailBodyText(body),
-				isSpam: false,
-				isSpamConfidence: 0,
-				ownerGroup: "owner",
-			}
-
-			verify(offlineStorage.storeSpamClassification(spamTrainMailDatum), { times: 1 })
-		})
-	})
-})

test/tests/api/worker/search/IndexUtilsTest.ts

@@ -220,7 +220,7 @@ o.spec("Index Utils", () => {
 			"this string has <i>html</i> code <!-- ignore comments-->i want to <b>remove</b><br>Link Number 1 -><a href='http://www.bbc.co.uk'>BBC</a> Link Number 1<br><p>Now back to normal text and stuff</p>"
 		let plain = "this string has  html  code  i want to  remove  Link Number 1 -> BBC  Link Number 1  Now back to normal text and stuff "
 		o(htmlToText(html)).equals(plain)
-		o(htmlToText("<img src='>' >")).equals(" ' >") // TODO handle this case
+		o(htmlToText("<img src='>' >")).equals(" ' >")
 
 		o(htmlToText("&nbsp;&amp;&lt;&gt;")).equals(" &<>")
 		o(htmlToText("&ouml;")).equals("ö")

test/tests/api/worker/search/IndexedDbIndexerTest.ts

@@ -1186,9 +1186,9 @@ o.spec("IndexedDbIndexer", () => {
 
 			o.test("create", async () => {
 				await indexer._processEntityEvents(testBatch)
-				verify(mailIndexer.afterMailCreated(["create", "id-1"], matchers.anything()))
+				verify(mailIndexer.afterMailCreated(["create", "id-1"]))
-				verify(mailIndexer.afterMailCreated(["create", "id-3"], matchers.anything()))
+				verify(mailIndexer.afterMailCreated(["create", "id-3"]))
-				verify(mailIndexer.afterMailCreated(matchers.anything(), matchers.anything()), { times: 2 })
+				verify(mailIndexer.afterMailCreated(matchers.anything()), { times: 2 })
 				verify(core.writeGroupDataBatchId(testBatch.groupId, testBatch.batchId))
 			})
 			o.test("update", async () => {
@@ -1207,13 +1207,13 @@ o.spec("IndexedDbIndexer", () => {
 			})
 
 			o.test("gracefully handles not found errors", async () => {
-				when(mailIndexer.afterMailCreated(["create", "id-1"], matchers.anything())).thenReject(new NotFoundError("Not found :("))
+				when(mailIndexer.afterMailCreated(["create", "id-1"])).thenReject(new NotFoundError("Not found :("))
-				when(mailIndexer.afterMailCreated(["update", "id-4"], matchers.anything())).thenReject(new NotFoundError("Not found :("))
+				when(mailIndexer.afterMailCreated(["update", "id-4"])).thenReject(new NotFoundError("Not found :("))
 				await indexer._processEntityEvents(testBatch)
 
-				verify(mailIndexer.afterMailCreated(["create", "id-1"], matchers.anything()))
+				verify(mailIndexer.afterMailCreated(["create", "id-1"]))
-				verify(mailIndexer.afterMailCreated(["create", "id-3"], matchers.anything()))
+				verify(mailIndexer.afterMailCreated(["create", "id-3"]))
-				verify(mailIndexer.afterMailCreated(matchers.anything(), matchers.anything()), { times: 2 })
+				verify(mailIndexer.afterMailCreated(matchers.anything()), { times: 2 })
 
 				verify(mailIndexer.afterMailUpdated(["update", "id-4"]))
 				verify(mailIndexer.afterMailUpdated(["update", "id-6"]))
@@ -1227,13 +1227,13 @@ o.spec("IndexedDbIndexer", () => {
 			})
 
 			o.test("gracefully handles not authorized errors", async () => {
-				when(mailIndexer.afterMailCreated(["create", "id-1"], matchers.anything())).thenReject(new NotAuthorizedError("You shall not pass :("))
+				when(mailIndexer.afterMailCreated(["create", "id-1"])).thenReject(new NotAuthorizedError("You shall not pass :("))
-				when(mailIndexer.afterMailCreated(["update", "id-4"], matchers.anything())).thenReject(new NotAuthorizedError("You shall not pass :("))
+				when(mailIndexer.afterMailCreated(["update", "id-4"])).thenReject(new NotAuthorizedError("You shall not pass :("))
 				await indexer._processEntityEvents(testBatch)
 
-				verify(mailIndexer.afterMailCreated(["create", "id-1"], matchers.anything()))
+				verify(mailIndexer.afterMailCreated(["create", "id-1"]))
-				verify(mailIndexer.afterMailCreated(["create", "id-3"], matchers.anything()))
+				verify(mailIndexer.afterMailCreated(["create", "id-3"]))
-				verify(mailIndexer.afterMailCreated(matchers.anything(), matchers.anything()), { times: 2 })
+				verify(mailIndexer.afterMailCreated(matchers.anything()), { times: 2 })
 
 				verify(mailIndexer.afterMailUpdated(["update", "id-4"]))
 				verify(mailIndexer.afterMailUpdated(["update", "id-6"]))

test/tests/api/worker/search/MailIndexerTest.ts

@@ -446,21 +446,21 @@ o.spec("MailIndexer", () => {
 		o.spec("afterMailCreated", () => {
 			o.test("no-op if mailIndexing is disabled", async () => {
 				await initWithEnabled(false)
-				await indexer.afterMailCreated(mailIdTuple, null)
+				await indexer.afterMailCreated(mailIdTuple)
 				verify(backend.onMailCreated(matchers.anything()), { times: 0 })
 			})
 			o.test("no-op if new email is out of index range", async () => {
 				addEntities()
 				setCurrentIndexTimestamp(now + 1)
 				await initWithEnabled(true)
-				await indexer.afterMailCreated(mailIdTuple, null)
+				await indexer.afterMailCreated(mailIdTuple)
 				verify(backend.onMailCreated(matchers.anything()), { times: 0 })
 			})
 			o.test("creates if mailIndexing is enabled", async () => {
 				const entities = addEntities()
 				setCurrentIndexTimestamp(now)
 				await initWithEnabled(true)
-				await indexer.afterMailCreated(mailIdTuple, entities)
+				await indexer.afterMailCreated(mailIdTuple)
 				verify(backend.onMailCreated(entities))
 			})
 			o.test("no-op if draft details fail to download", async () => {

test/tests/api/worker/utils/spamClassification/HashingVectorizerTest.ts

@@ -1,12 +1,7 @@
 import o from "@tutao/otest"
 import { HashingVectorizer } from "../../../../../../src/mail-app/workerUtils/spamClassification/HashingVectorizer"
 import { arrayEquals } from "@tutao/tutanota-utils"
-
+import { spamClassifierTokenizer } from "../../../../../../src/mail-app/workerUtils/spamClassification/SpamClassifier"
-export const tokenize = (text: string): string[] =>
-	text
-		.toLowerCase()
-		.split(/\s+/)
-		.filter((t) => t.length > 1)
 
 o.spec("HashingVectorizer", () => {
 	const rawDocuments = [
@@ -17,7 +12,7 @@ o.spec("HashingVectorizer", () => {
 		"Millions of people choose Tuta to protect their personal and professional communication.",
 	]
 
-	const tokenizedDocuments = rawDocuments.map(tokenize)
+	const tokenizedDocuments = rawDocuments.map(spamClassifierTokenizer)
 
 	o("vectorize creates same vector for same tokens", async () => {
 		const vectorizer = new HashingVectorizer()

test/tests/api/worker/utils/spamClassification/SpamClassifierTest.ts

@@ -4,9 +4,9 @@ import { parseCsv } from "../../../../../../src/common/misc/parsing/CsvParser"
 import {
 	DEFAULT_PREPROCESS_CONFIGURATION,
 	SpamClassifier,
+	spamClassifierTokenizer as testTokenize,
 	SpamTrainMailDatum,
 } from "../../../../../../src/mail-app/workerUtils/spamClassification/SpamClassifier"
-import { tokenize as testTokenize } from "./HashingVectorizerTest"
 import { OfflineStoragePersistence } from "../../../../../../src/mail-app/workerUtils/index/OfflineStoragePersistence"
 import { matchers, object, when } from "testdouble"
 import { assertNotNull, promiseMap } from "@tutao/tutanota-utils"
@@ -16,7 +16,11 @@ import { mockAttribute } from "@tutao/tutanota-test-utils"
 import "@tensorflow/tfjs-backend-cpu"
 import { HashingVectorizer } from "../../../../../../src/mail-app/workerUtils/spamClassification/HashingVectorizer"
 import { LayersModel, tensor1d } from "../../../../../../src/mail-app/workerUtils/spamClassification/tensorflow-custom"
+import { createTestEntity } from "../../../../TestUtils"
+import { MailTypeRef } from "../../../../../../src/common/api/entities/tutanota/TypeRefs"
+import { Sequential } from "@tensorflow/tfjs-layers"
 
+const { anything } = matchers
 export const DATASET_FILE_PATH: string = "./tests/api/worker/utils/spamClassification/spam_classification_test_mails.csv"
 
 export async function readMailDataFromCSV(filePath: string): Promise<{
@@ -50,7 +54,7 @@ export async function readMailDataFromCSV(filePath: string): Promise<{
 }
 
 // Initial training (cutoff by day or amount)
-o.spec("SpamClassifier", () => {
+o.spec("SpamClassifierTest", () => {
 	const mockOfflineStorageCache = object<CacheStorage>()
 	const mockOfflineStorage = object<OfflineStoragePersistence>()
 	const mockSpamClassificationInitializer = object<SpamClassificationInitializer>()
@@ -68,9 +72,6 @@ o.spec("SpamClassifier", () => {
 		dataSlice = spamData.concat(hamData)
 		seededShuffle(dataSlice, 42)
 
-		mockOfflineStorage.tokenize = async (text) => {
-			return testTokenize(text)
-		}
 		mockSpamClassificationInitializer.init = async () => {
 			return dataSlice
 		}
@@ -86,6 +87,48 @@ o.spec("SpamClassifier", () => {
 		)
 	})
 
+	o("processSpam maintains server classification when client classification is not enabled", async function () {
+		const mail = createTestEntity(MailTypeRef, {
+			_id: ["mailListId", "mailId"],
+			sets: [["folderList", "serverFolder"]],
+		})
+		const spamTrainMailDatum: SpamTrainMailDatum = {
+			mailId: mail._id,
+			subject: mail.subject,
+			body: "some body",
+			isSpam: true,
+			isSpamConfidence: 1,
+			ownerGroup: "owner",
+		}
+		const layersModel = object<Sequential>()
+		spamClassifier.addSpamClassifierForOwner(spamTrainMailDatum.ownerGroup, layersModel, false)
+
+		const predictedSpam = await spamClassifier.predict(spamTrainMailDatum)
+		o(predictedSpam).equals(null)
+	})
+
+	o("processSpam uses client classification when enabled", async function () {
+		const mail = createTestEntity(MailTypeRef, {
+			_id: ["mailListId", "mailId"],
+			sets: [["folderList", "serverFolder"]],
+		})
+		const spamTrainMailDatum: SpamTrainMailDatum = {
+			mailId: mail._id,
+			subject: mail.subject,
+			body: "some body",
+			isSpam: false,
+			isSpamConfidence: 0,
+			ownerGroup: "owner",
+		}
+
+		const layersModel = object<Sequential>()
+		when(layersModel.predict(anything())).thenReturn(tensor1d([1]))
+		spamClassifier.addSpamClassifierForOwner(spamTrainMailDatum.ownerGroup, layersModel, true)
+
+		const predictedSpam = await spamClassifier.predict(spamTrainMailDatum)
+		o(predictedSpam).equals(true)
+	})
+
 	o("Initial training only", async () => {
 		o.timeout(20_000)
 
@@ -94,13 +137,12 @@ o.spec("SpamClassifier", () => {
 		const testSet = dataSlice.slice(trainTestSplit)
 
 		await spamClassifier.initialTraining(trainSet)
-		await spamClassifier.test(testSet)
+		await testClassifier(spamClassifier, testSet)
 	})
 
 	o("Initial training and refitting in multi step", async () => {
 		o.timeout(20_000)
 
-		const testStart = Date.now()
 		const trainTestSplit = dataSlice.length * 0.8
 		const trainSet = dataSlice.slice(0, trainTestSplit)
 		const testSet = dataSlice.slice(trainTestSplit)
@@ -112,15 +154,15 @@ o.spec("SpamClassifier", () => {
 		o(await mockSpamClassificationInitializer.init("owner")).deepEquals(trainSetFirstHalf)
 		await spamClassifier.initialTraining(dataSlice)
 		console.log(`==> Result when testing with mails in two steps (first step).`)
-		await spamClassifier.test(testSet)
+		await testClassifier(spamClassifier, testSet)
 
 		await spamClassifier.updateModel("owner", trainSetSecondHalf)
 		console.log(`==> Result when testing with mails in two steps (second step).`)
-		await spamClassifier.test(testSet)
+		await testClassifier(spamClassifier, testSet)
 	})
 
 	o("preprocessMail outputs expected tokens for mail content", async () => {
-		const classifier = new SpamClassifier(null, object(), object())
+		const classifier = new SpamClassifier(object(), object(), object())
 		const mail = {
 			subject: `Sample Tokens and values`,
 			// prettier-ignore
@@ -336,18 +378,14 @@ if (DO_RUN_PERFORMANCE_ANALYSIS) {
 	o.spec("SpamClassifier - Performance Analysis", () => {
 		const mockOfflineStorageCache = object<CacheStorage>()
 		const mockOfflineStorage = object<OfflineStoragePersistence>()
-		let classifier = object<SpamClassifier>()
+		let spamClassifier = object<SpamClassifier>()
 		let dataSlice: SpamTrainMailDatum[]
 		o.beforeEach(() => {
-			mockOfflineStorage.tokenize = async (text) => {
-				return testTokenize(text)
-			}
-
 			const mockSpamClassificationInitializer = object<SpamClassificationInitializer>()
 			mockSpamClassificationInitializer.init = async () => {
 				return dataSlice
 			}
-			classifier = new SpamClassifier(mockOfflineStorage, mockOfflineStorageCache, mockSpamClassificationInitializer)
+			spamClassifier = new SpamClassifier(mockOfflineStorage, mockOfflineStorageCache, mockSpamClassificationInitializer)
 		})
 
 		o("time to refit", async () => {
@@ -359,7 +397,7 @@ if (DO_RUN_PERFORMANCE_ANALYSIS) {
 			seededShuffle(dataSlice, 42)
 
 			const start = performance.now()
-			await classifier.initialTraining(dataSlice)
+			await spamClassifier.initialTraining(dataSlice)
 			const initialTrainingDuration = performance.now() - start
 			console.log(`initial training time ${initialTrainingDuration}ms`)
 
@@ -367,7 +405,7 @@ if (DO_RUN_PERFORMANCE_ANALYSIS) {
 				const nowSpam = [hamSlice[0]]
 				nowSpam.map((formerHam) => (formerHam.isSpam = true))
 				const retrainingStart = performance.now()
-				await classifier.updateModel("owner", nowSpam)
+				await spamClassifier.updateModel("owner", nowSpam)
 				const retrainingDuration = performance.now() - retrainingStart
 				console.log(`retraining time ${retrainingDuration}ms`)
 			}
@@ -381,17 +419,17 @@ if (DO_RUN_PERFORMANCE_ANALYSIS) {
 			dataSlice = hamSlice.concat(spamSlice)
 			// seededShuffle(dataSlice, 42)
 
-			await classifier.initialTraining(dataSlice)
+			await spamClassifier.initialTraining(dataSlice)
 			const falseNegatives = spamData
 				.slice(10)
-				.filter(async (mailDatum) => mailDatum.isSpam !== (await classifier.predict(mailDatum)))
+				.filter(async (mailDatum) => mailDatum.isSpam !== (await spamClassifier.predict(mailDatum)))
 				.sort()
 				.slice(0, 10)
 
 			let retrainingNeeded = new Array<number>(falseNegatives.length).fill(0)
 			for (let i = 0; i < falseNegatives.length; i++) {
 				const sample = falseNegatives[i]
-				const copiedClassifier = await classifier.cloneClassifier()
+				const copiedClassifier = await spamClassifier.cloneClassifier()
 
 				let retrainCount = 0
 				let predictedSpam = false
@@ -458,15 +496,15 @@ if (DO_RUN_PERFORMANCE_ANALYSIS) {
 			dataSlice = hamSlice.concat(spamSlice)
 			// seededShuffle(dataSlice, 42)
 
-			await classifier.initialTraining(dataSlice)
+			await spamClassifier.initialTraining(dataSlice)
 			const falsePositive = hamData
 				.slice(10)
-				.filter(async (mailDatum) => mailDatum.isSpam !== (await classifier.predict(mailDatum)))
+				.filter(async (mailDatum) => mailDatum.isSpam !== (await spamClassifier.predict(mailDatum)))
 				.slice(0, 10)
 			let retrainingNeeded = new Array<number>(falsePositive.length).fill(0)
 			for (let i = 0; i < falsePositive.length; i++) {
 				const sample = falsePositive[i]
-				const copiedClassifier = await classifier.cloneClassifier()
+				const copiedClassifier = await spamClassifier.cloneClassifier()
 
 				let retrainCount = 0
 				let predictedSpam = false
@@ -492,16 +530,16 @@ if (DO_RUN_PERFORMANCE_ANALYSIS) {
 			dataSlice = hamSlice.concat(spamSlice)
 			seededShuffle(dataSlice, 42)
 
-			await classifier.initialTraining(dataSlice)
+			await spamClassifier.initialTraining(dataSlice)
 			const falseNegatives = spamData
 				.slice(10)
-				.filter(async (mailDatum) => mailDatum.isSpam !== (await classifier.predict(mailDatum)))
+				.filter(async (mailDatum) => mailDatum.isSpam !== (await spamClassifier.predict(mailDatum)))
 				.slice(0, 10)
 
 			let retrainingNeeded = new Array<number>(falseNegatives.length).fill(0)
 			for (let i = 0; i < falseNegatives.length; i++) {
 				const sample = falseNegatives[i]
-				const copiedClassifier = await classifier.cloneClassifier()
+				const copiedClassifier = await spamClassifier.cloneClassifier()
 
 				let retrainCount = 0
 				let predictedSpam = false
@@ -532,7 +570,7 @@ if (DO_RUN_PERFORMANCE_ANALYSIS) {
 			await promiseMap(
 				new Array<number>(ITERATION_COUNT).fill(0),
 				async () => {
-					const { vectorizationTime, trainingTime } = await classifier.initialTraining(dataSlice)
+					const { vectorizationTime, trainingTime } = await spamClassifier.initialTraining(dataSlice)
 					trainingTimes.push(trainingTime)
 					vectorizationTimes.push(vectorizationTime)
 					trainingWithoutVectorization.push(trainingTime - vectorizationTime)
@@ -560,6 +598,47 @@ if (DO_RUN_PERFORMANCE_ANALYSIS) {
 		})
 	})
 }
+
+async function testClassifier(classifier: SpamClassifier, mails: SpamTrainMailDatum[]): Promise<void> {
+	let predictionArray: number[] = []
+	for (let mail of mails) {
+		const prediction = await classifier.predict(mail)
+		predictionArray.push(prediction ? 1 : 0)
+	}
+	const ysArray = mails.map((mail) => mail.isSpam)
+
+	let tp = 0,
+		tn = 0,
+		fp = 0,
+		fn = 0
+
+	for (let i = 0; i < predictionArray.length; i++) {
+		const predictedSpam = predictionArray[i] > 0.5
+		const isActuallyASpam = ysArray[i]
+		if (predictedSpam && isActuallyASpam) tp++
+		else if (!predictedSpam && !isActuallyASpam) tn++
+		else if (predictedSpam && !isActuallyASpam) fp++
+		else if (!predictedSpam && isActuallyASpam) fn++
+	}
+
+	const total = tp + tn + fp + fn
+	const accuracy = (tp + tn) / total
+	const precision = tp / (tp + fp + 1e-7)
+	const recall = tp / (tp + fn + 1e-7)
+	const f1 = 2 * ((precision * recall) / (precision + recall + 1e-7))
+
+	console.log("\n--- Evaluation Metrics ---")
+	console.log(`Accuracy: \t${(accuracy * 100).toFixed(2)}%`)
+	console.log(`Precision:\t${(precision * 100).toFixed(2)}%`)
+	console.log(`Recall:   \t${(recall * 100).toFixed(2)}%`)
+	console.log(`F1 Score: \t${(f1 * 100).toFixed(2)}%`)
+	console.log("\nConfusion Matrix:")
+	console.log({
+		Predicted_Spam: { True_Positive: tp, False_Positive: fp },
+		Predicted_Ham: { False_Negative: fn, True_Negative: tn },
+	})
+}
+
 // For testing, we need deterministic shuffling which is not provided by tf.util.shuffle(dataSlice)
 // Seeded Fisher-Yates shuffle
 function seededShuffle<T>(array: T[], seed: number): void {

test/tests/mail/MailModelTest.ts

@@ -1,8 +1,18 @@
 import o from "@tutao/otest"
 import { Notifications } from "../../../src/common/gui/Notifications.js"
-import { Spy, spy, verify } from "@tutao/tutanota-test-utils"
+import { mock, Spy, spy, verify } from "@tutao/tutanota-test-utils"
-import { MailSetKind, OperationType } from "../../../src/common/api/common/TutanotaConstants.js"
+import { MailSetKind, OperationType, ProcessingState } from "../../../src/common/api/common/TutanotaConstants.js"
-import { Mail, MailFolderTypeRef, MailSetEntryTypeRef, MailTypeRef } from "../../../src/common/api/entities/tutanota/TypeRefs.js"
+import {
+	BodyTypeRef,
+	ClientSpamClassifierResultTypeRef,
+	Mail,
+	MailDetails,
+	MailDetailsBlob,
+	MailDetailsBlobTypeRef,
+	MailDetailsTypeRef,
+	MailFolderTypeRef,
+	MailTypeRef,
+} from "../../../src/common/api/entities/tutanota/TypeRefs.js"
 import { EntityClient } from "../../../src/common/api/common/EntityClient.js"
 import { EntityRestClientMock } from "../api/worker/rest/EntityRestClientMock.js"
 import { downcast } from "@tutao/tutanota-utils"
@@ -11,23 +21,40 @@ import { instance, matchers, object, when } from "testdouble"
 import { UserController } from "../../../src/common/api/main/UserController.js"
 import { createTestEntity } from "../TestUtils.js"
 import { EntityUpdateData, PrefetchStatus } from "../../../src/common/api/common/utils/EntityUpdateUtils.js"
-import { MailboxModel } from "../../../src/common/mailFunctionality/MailboxModel.js"
+import { MailboxDetail, MailboxModel } from "../../../src/common/mailFunctionality/MailboxModel.js"
 import { getElementId, getListId } from "../../../src/common/api/common/utils/EntityUtils.js"
 import { MailModel } from "../../../src/mail-app/mail/model/MailModel.js"
 import { EventController } from "../../../src/common/api/main/EventController.js"
 import { MailFacade } from "../../../src/common/api/worker/facades/lazy/MailFacade.js"
 import { ClientModelInfo } from "../../../src/common/api/common/EntityFunctions"
+import { InboxRuleHandler } from "../../../src/mail-app/mail/model/InboxRuleHandler"
+import { SpamClassificationHandler } from "../../../src/mail-app/workerUtils/spamClassification/SpamClassificationHandler"
+import { SpamClassifier, SpamTrainMailDatum } from "../../../src/mail-app/workerUtils/spamClassification/SpamClassifier"
+import { WebsocketConnectivityModel } from "../../../src/common/misc/WebsocketConnectivityModel"
+import { FolderSystem } from "../../../src/common/api/common/mail/FolderSystem"
+import { NotAuthorizedError, NotFoundError } from "../../../src/common/api/common/error/RestError"
+
+const { anything } = matchers
 
 o.spec("MailModelTest", function () {
 	let notifications: Partial<Notifications>
 	let showSpy: Spy
 	let model: MailModel
-	const inboxFolder = createTestEntity(MailFolderTypeRef, { _id: ["folderListId", "inboxId"] })
+	const inboxFolder = createTestEntity(MailFolderTypeRef, {
-	inboxFolder.folderType = MailSetKind.INBOX
+		_id: ["folderListId", "inboxId"],
-	const anotherFolder = createTestEntity(MailFolderTypeRef, { _id: ["folderListId", "archiveId"] })
+		folderType: MailSetKind.INBOX,
-	anotherFolder.folderType = MailSetKind.ARCHIVE
+	})
+	const spamFolder = createTestEntity(MailFolderTypeRef, {
+		_id: ["folderListId", "spamId"],
+		folderType: MailSetKind.SPAM,
+	})
+	const anotherFolder = createTestEntity(MailFolderTypeRef, {
+		_id: ["folderListId", "archiveId"],
+		folderType: MailSetKind.ARCHIVE,
+	})
 	let logins: LoginController
 	let mailFacade: MailFacade
+	let connectivityModel: WebsocketConnectivityModel
 	const restClient: EntityRestClientMock = new EntityRestClientMock()
 
 	o.beforeEach(function () {
@@ -41,6 +68,9 @@ o.spec("MailModelTest", function () {
 		when(userController.isUpdateForLoggedInUserInstance(matchers.anything(), matchers.anything())).thenReturn(false)
 		when(logins.getUserController()).thenReturn(userController)
 
+		connectivityModel = object<WebsocketConnectivityModel>()
+		when(connectivityModel.isLeader()).thenReturn(true)
+
 		model = new MailModel(
 			downcast({}),
 			mailboxModel,
@@ -48,40 +78,41 @@ o.spec("MailModelTest", function () {
 			new EntityClient(restClient, ClientModelInfo.getNewInstanceForTestsOnly()),
 			logins,
 			mailFacade,
-			null,
+			connectivityModel,
+			() => object(),
 			() => null,
 		)
 	})
+
 	o("doesn't send notification for another folder", async function () {
-		const mailSetEntry = createTestEntity(MailSetEntryTypeRef, { _id: [anotherFolder.entries, "mailSetEntryId"] })
+		const mail = createTestEntity(MailTypeRef, { _id: ["mailBagListId", "mailId"], sets: [] })
-		restClient.addListInstances(mailSetEntry)
+		restClient.addListInstances(mail)
 		await model.entityEventsReceived([
 			makeUpdate({
-				instanceListId: getListId(mailSetEntry) as NonEmptyString,
+				instanceListId: getListId(mail) as NonEmptyString,
-				instanceId: getElementId(mailSetEntry),
+				instanceId: getElementId(mail),
 				operation: OperationType.CREATE,
 			}),
 		])
 		o(showSpy.invocations.length).equals(0)
 	})
 	o("doesn't send notification for move operation", async function () {
-		const mailSetEntry = createTestEntity(MailSetEntryTypeRef, { _id: [inboxFolder.entries, "mailSetEntryId"] })
+		const mail = createTestEntity(MailTypeRef, { _id: ["mailBagListId", "mailId"], sets: [] })
-		restClient.addListInstances(mailSetEntry)
+		restClient.addListInstances(mail)
 		await model.entityEventsReceived([
 			makeUpdate({
-				instanceListId: getListId(mailSetEntry) as NonEmptyString,
+				instanceListId: getListId(mail) as NonEmptyString,
-				instanceId: getElementId(mailSetEntry),
+				instanceId: getElementId(mail),
 				operation: OperationType.DELETE,
 			}),
 			makeUpdate({
-				instanceListId: getListId(mailSetEntry) as NonEmptyString,
+				instanceListId: getListId(mail) as NonEmptyString,
-				instanceId: getElementId(mailSetEntry),
+				instanceId: getElementId(mail),
 				operation: OperationType.CREATE,
 			}),
 		])
 		o(showSpy.invocations.length).equals(0)
 	})
-
 	o("markMails", async function () {
 		const mailId1: IdTuple = ["mailbag id1", "mail id1"]
 		const mailId2: IdTuple = ["mailbag id2", "mail id2"]
@@ -90,6 +121,200 @@ o.spec("MailModelTest", function () {
 		verify(mailFacade.markMails([mailId1, mailId2, mailId3], true))
 	})
 
+	o.spec("Inbox rule processing and spam prediction", () => {
+		let inboxRuleHandler: InboxRuleHandler
+		let spamClassificationHandler: SpamClassificationHandler
+		let spamClassifier: SpamClassifier
+		let mailboxModel: MailboxModel
+		let modelWithSpamAndInboxRule: MailModel
+		let mail: Mail
+		let mailDetails: MailDetails
+
+		o.beforeEach(async () => {
+			const entityClient = new EntityClient(restClient, ClientModelInfo.getNewInstanceForTestsOnly())
+			mailboxModel = instance(MailboxModel)
+			inboxRuleHandler = object<InboxRuleHandler>()
+			spamClassifier = object<SpamClassifier>()
+			spamClassificationHandler = new SpamClassificationHandler(mailFacade, spamClassifier)
+
+			mailDetails = createTestEntity(MailDetailsTypeRef, {
+				_id: "mailDetail",
+				body: createTestEntity(BodyTypeRef, { text: "some text" }),
+			})
+			mail = createTestEntity(MailTypeRef, {
+				_id: ["mailListId", "mailId"],
+				_ownerGroup: "mailGroup",
+				mailDetails: ["detailsList", mailDetails._id],
+				subject: "subject",
+				sets: [inboxFolder._id],
+				processingState: ProcessingState.INBOX_RULE_NOT_PROCESSED,
+			})
+			const mailDetailsBlob: MailDetailsBlob = createTestEntity(MailDetailsBlobTypeRef, {
+				_id: mail.mailDetails!,
+				details: mailDetails,
+			})
+
+			restClient.addListInstances(mail)
+			restClient.addBlobInstances(mailDetailsBlob)
+
+			when(mailFacade.loadMailDetailsBlob(mail)).thenResolve(mailDetails)
+
+			modelWithSpamAndInboxRule = mock(
+				new MailModel(
+					downcast({}),
+					mailboxModel,
+					instance(EventController),
+					entityClient,
+					logins,
+					mailFacade,
+					connectivityModel,
+					() => spamClassificationHandler,
+					() => inboxRuleHandler,
+				),
+				(m: MailModel) => {
+					m.getFolderSystemByGroupId = (groupId) => {
+						o(groupId).equals("mailGroup")
+						return new FolderSystem([inboxFolder, spamFolder, anotherFolder])
+					}
+					m.getMailboxDetailsForMail = async (_: Mail) => object<MailboxDetail>()
+				},
+			)
+		})
+
+		o("does not re-apply inbox rules or re-classify mail if the mail is in a final processingState", async function () {
+			const alreadyClassifiedMail = createTestEntity(MailTypeRef, {
+				_id: ["mailListId", "maildIdWithFinalProcessingState"],
+				_ownerGroup: "mailGroup",
+				mailDetails: ["detailsList", mailDetails._id],
+				sets: [inboxFolder._id],
+				processingState: ProcessingState.INBOX_RULE_PROCESSED_AND_SPAM_PREDICTION_MADE,
+				clientSpamClassifierResult: createTestEntity(ClientSpamClassifierResultTypeRef),
+			})
+			restClient.addListInstances(alreadyClassifiedMail)
+			when(mailFacade.loadMailDetailsBlob(alreadyClassifiedMail)).thenResolve(mailDetails)
+
+			const alreadyClassifiedMailCreateEvent = makeUpdate({
+				instanceListId: "mailListId",
+				instanceId: "maildIdWithFinalProcessingState",
+				operation: OperationType.CREATE,
+			})
+
+			const { processingDone } = await modelWithSpamAndInboxRule.entityEventsReceived([alreadyClassifiedMailCreateEvent])
+			await processingDone
+
+			verify(inboxRuleHandler.findAndApplyMatchingRule(anything(), anything(), anything()), { times: 0 })
+			verify(spamClassificationHandler.predictSpamForNewMail(anything(), anything(), anything(), anything()), { times: 0 })
+			verify(spamClassifier.storeSpamClassification(anything()), { times: 0 })
+			verify(spamClassifier.predict(anything()), { times: 0 })
+		})
+
+		o("does not try to apply inbox rule when downloading of mail fails on create mail event", async function () {
+			restClient.setListElementException(mail._id, new NotFoundError("Mail not found"))
+
+			const mailCreateEvent = makeUpdate({
+				instanceListId: getListId(mail) as NonEmptyString,
+				instanceId: getElementId(mail),
+				operation: OperationType.CREATE,
+			})
+			await modelWithSpamAndInboxRule.entityEventsReceived([mailCreateEvent])
+
+			verify(inboxRuleHandler.findAndApplyMatchingRule(anything(), anything(), anything()), { times: 0 })
+		})
+
+		o("spam prediction does not happen when inbox rule is applied", async () => {
+			when(spamClassifier.predict(anything())).thenResolve(false)
+
+			const mailCreateEvent = makeUpdate({
+				instanceListId: "mailListId",
+				instanceId: "mailId",
+				operation: OperationType.CREATE,
+			})
+
+			// when inbox rule is applied
+			when(inboxRuleHandler.findAndApplyMatchingRule(anything(), anything(), anything())).thenResolve(inboxFolder)
+			const { processingDone } = await modelWithSpamAndInboxRule.entityEventsReceived([mailCreateEvent])
+			await processingDone
+			const expectedSpamTrainMailDatum: SpamTrainMailDatum = {
+				mailId: ["mailListId", "mailId"],
+				ownerGroup: "mailGroup",
+				body: "some text",
+				subject: "subject",
+				isSpam: false,
+				isSpamConfidence: 1,
+			}
+			verify(spamClassifier.storeSpamClassification(expectedSpamTrainMailDatum), { times: 1 })
+			verify(spamClassifier.predict(anything()), { times: 0 })
+		})
+
+		o("spam prediction happens when inbox rule is not applied", async () => {
+			when(spamClassifier.predict(anything())).thenResolve(false)
+
+			const mailCreateEvent = makeUpdate({
+				instanceListId: "mailListId",
+				instanceId: "mailId",
+				operation: OperationType.CREATE,
+			})
+
+			when(inboxRuleHandler.findAndApplyMatchingRule(anything(), anything(), anything())).thenResolve(null)
+			const { processingDone } = await modelWithSpamAndInboxRule.entityEventsReceived([mailCreateEvent])
+			await processingDone
+
+			const expectedSpamTrainMailDatum: SpamTrainMailDatum = {
+				mailId: ["mailListId", "mailId"],
+				ownerGroup: "mailGroup",
+				body: "some text",
+				subject: "subject",
+				isSpam: false,
+				isSpamConfidence: 1,
+			}
+			verify(spamClassifier.storeSpamClassification(expectedSpamTrainMailDatum), { times: 1 })
+			verify(spamClassifier.predict(anything()), { times: 1 })
+		})
+
+		o("does not try to do spam classification when downloading of mail fails on create mail event", async function () {
+			when(inboxRuleHandler.findAndApplyMatchingRule(anything(), anything(), anything())).thenResolve(null)
+			const mailCreateEvent = makeUpdate({
+				instanceListId: "mailListId",
+				instanceId: "mailId",
+				operation: OperationType.CREATE,
+			})
+
+			// mail not being there
+			restClient.setListElementException(mail._id, new NotAuthorizedError("blah"))
+			const { processingDone: inboxRuleProcessedMailNotThere } = await modelWithSpamAndInboxRule.entityEventsReceived([mailCreateEvent])
+			await inboxRuleProcessedMailNotThere
+			verify(spamClassifier.storeSpamClassification(anything()), { times: 0 })
+			verify(spamClassifier.predict(anything()), { times: 0 })
+
+			// mail being there
+			restClient.addListInstances(mail)
+			const { processingDone: inboxRuleProcessedMailIsThere } = await modelWithSpamAndInboxRule.entityEventsReceived([mailCreateEvent])
+			await inboxRuleProcessedMailIsThere
+			const expectedSpamTrainMailDatum: SpamTrainMailDatum = {
+				mailId: ["mailListId", "mailId"],
+				ownerGroup: "mailGroup",
+				body: "some text",
+				subject: "subject",
+				isSpam: false,
+				isSpamConfidence: 1,
+			}
+			verify(spamClassifier.storeSpamClassification(expectedSpamTrainMailDatum), { times: 1 })
+			verify(spamClassifier.predict(anything()), { times: 1 })
+		})
+
+		o("deletes a training datum for deleted mail event", async () => {
+			const mailDeleteEvent = makeUpdate({
+				instanceListId: "mailListId",
+				instanceId: "mailId",
+				operation: OperationType.DELETE,
+			})
+			const { processingDone } = await modelWithSpamAndInboxRule.entityEventsReceived([mailDeleteEvent])
+			await processingDone
+
+			verify(spamClassifier.deleteSpamClassification(mail._id), { times: 1 })
+		})
+	})
+
 	function makeUpdate({
 		instanceId,
 		instanceListId,

test/tests/mail/SpamClassificationHandlerTest.ts

@@ -0,0 +1,161 @@
+import o from "@tutao/otest"
+import { matchers, object, verify, when } from "testdouble"
+import {
+	Body,
+	BodyTypeRef,
+	ClientSpamClassifierResultTypeRef,
+	Mail,
+	MailDetails,
+	MailDetailsTypeRef,
+	MailFolderTypeRef,
+	MailTypeRef,
+} from "../../../src/common/api/entities/tutanota/TypeRefs"
+import { SpamClassifier, SpamTrainMailDatum } from "../../../src/mail-app/workerUtils/spamClassification/SpamClassifier"
+import { getMailBodyText } from "../../../src/common/api/common/CommonMailUtils"
+import { MailSetKind, ProcessingState, SpamDecision } from "../../../src/common/api/common/TutanotaConstants"
+import { ClientClassifierType } from "../../../src/common/api/common/ClientClassifierType"
+import { assert, assertNotNull } from "@tutao/tutanota-utils"
+import { MailFacade } from "../../../src/common/api/worker/facades/lazy/MailFacade"
+import { createTestEntity } from "../TestUtils"
+import { SpamClassificationHandler } from "../../../src/mail-app/workerUtils/spamClassification/SpamClassificationHandler"
+import { FolderSystem } from "../../../src/common/api/common/mail/FolderSystem"
+import { isSameId } from "../../../src/common/api/common/utils/EntityUtils"
+import { any } from "@tensorflow/tfjs-core"
+
+const { anything } = matchers
+
+o.spec("SpamClassificationHandlerTest", function () {
+	let mailFacade = object<MailFacade>()
+	let body: Body
+	let mail: Mail
+	let spamClassifier: SpamClassifier
+	let spamHandler: SpamClassificationHandler
+	let folderSystem: FolderSystem
+	let mailDetails: MailDetails
+
+	const inboxFolder = createTestEntity(MailFolderTypeRef, { _id: ["listId", "inbox"], folderType: MailSetKind.INBOX })
+	const trashFolder = createTestEntity(MailFolderTypeRef, { _id: ["listId", "trash"], folderType: MailSetKind.TRASH })
+	const spamFolder = createTestEntity(MailFolderTypeRef, { _id: ["listId", "spam"], folderType: MailSetKind.SPAM })
+
+	o.beforeEach(function () {
+		spamClassifier = object<SpamClassifier>()
+
+		body = createTestEntity(BodyTypeRef, { text: "Body Text" })
+		mailDetails = createTestEntity(MailDetailsTypeRef, { _id: "mailDetail", body })
+		mail = createTestEntity(MailTypeRef, {
+			_id: ["listId", "elementId"],
+			sets: [spamFolder._id],
+			subject: "subject",
+			_ownerGroup: "owner",
+			mailDetails: ["detailsList", mailDetails._id],
+			unread: true,
+			processingState: ProcessingState.INBOX_RULE_NOT_PROCESSED,
+			clientSpamClassifierResult: createTestEntity(ClientSpamClassifierResultTypeRef, { spamDecision: SpamDecision.NONE }),
+		})
+		folderSystem = object<FolderSystem>()
+
+		when(mailFacade.moveMails(anything(), anything(), anything(), ClientClassifierType.CLIENT_CLASSIFICATION)).thenResolve([])
+		when(folderSystem.getSystemFolderByType(MailSetKind.SPAM)).thenReturn(spamFolder)
+		when(folderSystem.getSystemFolderByType(MailSetKind.INBOX)).thenReturn(inboxFolder)
+		when(folderSystem.getSystemFolderByType(MailSetKind.TRASH)).thenReturn(trashFolder)
+		when(folderSystem.getFolderByMail(anything())).thenDo((mail: Mail) => {
+			assert(mail.sets.length === 1, "Expected exactly one mail set")
+			const mailFolderId = assertNotNull(mail.sets[0])
+			if (isSameId(mailFolderId, trashFolder._id)) return trashFolder
+			else if (isSameId(mailFolderId, spamFolder._id)) return spamFolder
+			else if (isSameId(mailFolderId, inboxFolder._id)) return inboxFolder
+			else throw new Error("Unknown mail Folder")
+		})
+		when(
+			mailFacade.loadMailDetailsBlob(
+				matchers.argThat((requestedMails: Array<Mail>) => {
+					assert(requestedMails.length === 1, "exactly one mail is requested at a time")
+					return isSameId(requestedMails[0]._id, mail._id)
+				}),
+			),
+			anything(),
+		).thenDo(async () => [{ mail, mailDetails }])
+		spamHandler = new SpamClassificationHandler(mailFacade, spamClassifier)
+	})
+
+	o("predictSpamForNewMail does move mail from inbox to spam folder if mail is spam", async function () {
+		mail.sets = [inboxFolder._id]
+		when(spamClassifier.predict(anything())).thenResolve(true)
+
+		const finalResult = await spamHandler.predictSpamForNewMail(mail, mailDetails, inboxFolder, folderSystem)
+		o(spamHandler.hamMoveMailData).deepEquals(null)
+		o(spamHandler.spamMoveMailData?.mails).deepEquals([mail._id])
+		o(spamHandler.classifierResultServiceMailIds).deepEquals([])
+		o(finalResult).deepEquals(spamFolder)
+	})
+
+	o("predictSpamForNewMail does NOT move mail from inbox to spam folder if mail is ham", async function () {
+		mail.sets = [inboxFolder._id]
+		when(spamClassifier.predict(anything())).thenResolve(false)
+
+		const finalResult = await spamHandler.predictSpamForNewMail(mail, mailDetails, inboxFolder, folderSystem)
+		o(spamHandler.hamMoveMailData).deepEquals(null)
+		o(spamHandler.spamMoveMailData).deepEquals(null)
+		o(spamHandler.classifierResultServiceMailIds).deepEquals([mail._id])
+		o(finalResult).deepEquals(inboxFolder)
+	})
+
+	o("predictSpamForNewMail does NOT move mail from spam to inbox folder if mail is spam", async function () {
+		mail.sets = [spamFolder._id]
+		when(spamClassifier.predict(anything())).thenResolve(true)
+
+		const finalResult = await spamHandler.predictSpamForNewMail(mail, mailDetails, spamFolder, folderSystem)
+		o(spamHandler.hamMoveMailData).deepEquals(null)
+		o(spamHandler.spamMoveMailData).deepEquals(null)
+		o(spamHandler.classifierResultServiceMailIds).deepEquals([mail._id])
+		o(finalResult).deepEquals(spamFolder)
+	})
+
+	o("predictSpamForNewMail moves mail from spam to inbox folder if mail is ham", async function () {
+		mail.sets = [spamFolder._id]
+		when(spamClassifier.predict(anything())).thenResolve(false)
+
+		const finalResult = await spamHandler.predictSpamForNewMail(mail, mailDetails, spamFolder, folderSystem)
+		o(spamHandler.hamMoveMailData?.mails).deepEquals([mail._id])
+		o(spamHandler.spamMoveMailData).deepEquals(null)
+		o(spamHandler.classifierResultServiceMailIds).deepEquals([])
+		o(finalResult).deepEquals(inboxFolder)
+	})
+
+	o("predictSpamForNewMail does NOT move mail from spam to spam folder if mail is spam", async function () {
+		mail.sets = [spamFolder._id]
+		when(spamClassifier.predict(anything())).thenResolve(true)
+
+		const finalResult = await spamHandler.predictSpamForNewMail(mail, mailDetails, spamFolder, folderSystem)
+		o(spamHandler.hamMoveMailData).deepEquals(null)
+		o(spamHandler.spamMoveMailData).deepEquals(null)
+		o(spamHandler.classifierResultServiceMailIds).deepEquals([mail._id])
+		o(finalResult).deepEquals(spamFolder)
+	})
+
+	o(
+		"predictSpamForNewMail does NOT send classifierResultService request if processingState is INBOX_RULE_PROCESSED_AND_SPAM_PREDICTION_MADE",
+		async function () {
+			mail.sets = [inboxFolder._id]
+			mail.processingState = ProcessingState.INBOX_RULE_PROCESSED_AND_SPAM_PREDICTION_MADE
+			when(spamClassifier.predict(anything())).thenResolve(false)
+
+			const finalResult = await spamHandler.predictSpamForNewMail(mail, mailDetails, inboxFolder, folderSystem)
+			o(spamHandler.hamMoveMailData).deepEquals(null)
+			o(spamHandler.spamMoveMailData).deepEquals(null)
+			o(spamHandler.classifierResultServiceMailIds).deepEquals([])
+			o(finalResult).deepEquals(inboxFolder)
+		},
+	)
+
+	o("update spam classification data on every mail update", async function () {
+		when(spamClassifier.getSpamClassification(anything())).thenResolve({ isSpam: false, isSpamConfidence: 0 })
+		mail.clientSpamClassifierResult = createTestEntity(ClientSpamClassifierResultTypeRef, {
+			spamDecision: SpamDecision.BLACKLIST,
+			confidence: "1",
+		})
+
+		await spamHandler.updateSpamClassificationData(mail)
+		verify(spamClassifier.updateSpamClassification(["listId", "elementId"], true, 1), { times: 1 })
+	})
+})

test/tests/mail/model/ConversationListModelTest.ts

@@ -209,7 +209,7 @@ o.spec("ConversationListModel", () => {
 		verify(mailModel.getMailboxDetailsForMailFolder(matchers.anything()), {
 			times: 0,
 		})
-		verify(inboxRuleHandler.findAndApplyMatchingRule(mailboxDetail, matchers.anything(), true, false), {
+		verify(inboxRuleHandler.findAndApplyMatchingRule(mailboxDetail, matchers.anything(), true), {
 			times: 0,
 		})
 	})
@@ -227,7 +227,7 @@ o.spec("ConversationListModel", () => {
 		verify(mailModel.getMailboxDetailsForMailFolder(matchers.anything()), {
 			times: 0,
 		})
-		verify(inboxRuleHandler.findAndApplyMatchingRule(mailboxDetail, matchers.anything(), true, false), {
+		verify(inboxRuleHandler.findAndApplyMatchingRule(mailboxDetail, matchers.anything(), true), {
 			times: 0,
 		})
 	})
@@ -246,7 +246,7 @@ o.spec("ConversationListModel", () => {
 		verify(mailModel.getMailboxDetailsForMailFolder(matchers.anything()), {
 			times: 0,
 		})
-		verify(inboxRuleHandler.findAndApplyMatchingRule(mailboxDetail, matchers.anything(), true, false), {
+		verify(inboxRuleHandler.findAndApplyMatchingRule(mailboxDetail, matchers.anything(), true), {
 			times: 0,
 		})
 		o.check(model.loadingStatus).equals(ListLoadingState.Idle)
@@ -266,7 +266,6 @@ o.spec("ConversationListModel", () => {
 				mailboxDetail,
 				matchers.argThat((mail: Mail) => isSameId(mail._id, makeMailId(25))),
 				true,
-				false,
 			),
 		).thenResolve({})
 
@@ -282,7 +281,7 @@ o.spec("ConversationListModel", () => {
 		verify(mailModel.getMailboxDetailsForMailFolder(matchers.anything()), {
 			times: 1,
 		})
-		verify(inboxRuleHandler.findAndApplyMatchingRule(mailboxDetail, matchers.anything(), true, false), {
+		verify(inboxRuleHandler.findAndApplyMatchingRule(mailboxDetail, matchers.anything(), true), {
 			times: 100,
 		})
 	})

test/tests/mail/model/MailListModelTest.ts

@@ -204,7 +204,7 @@ o.spec("MailListModel", () => {
 		verify(mailModel.getMailboxDetailsForMailFolder(matchers.anything()), {
 			times: 0,
 		})
-		verify(inboxRuleHandler.findAndApplyMatchingRule(mailboxDetail, matchers.anything(), true, false), {
+		verify(inboxRuleHandler.findAndApplyMatchingRule(mailboxDetail, matchers.anything(), true), {
 			times: 0,
 		})
 	})
@@ -222,7 +222,7 @@ o.spec("MailListModel", () => {
 		verify(mailModel.getMailboxDetailsForMailFolder(matchers.anything()), {
 			times: 0,
 		})
-		verify(inboxRuleHandler.findAndApplyMatchingRule(mailboxDetail, matchers.anything(), true, false), {
+		verify(inboxRuleHandler.findAndApplyMatchingRule(mailboxDetail, matchers.anything(), true), {
 			times: 0,
 		})
 	})
@@ -236,7 +236,6 @@ o.spec("MailListModel", () => {
 				mailboxDetail,
 				matchers.argThat((mail: Mail) => isSameId(mail._id, makeMailId(25))),
 				true,
-				false,
 			),
 		).thenResolve({})
 
@@ -252,7 +251,7 @@ o.spec("MailListModel", () => {
 		verify(mailModel.getMailboxDetailsForMailFolder(matchers.anything()), {
 			times: 1,
 		})
-		verify(inboxRuleHandler.findAndApplyMatchingRule(mailboxDetail, matchers.anything(), true, false), {
+		verify(inboxRuleHandler.findAndApplyMatchingRule(mailboxDetail, matchers.anything(), true), {
 			times: 100,
 		})
 	})

test/tests/mail/search/MailIndexAndSpamClassificationPostLoginActionTest.ts

@@ -41,9 +41,8 @@ o.spec("MailIndexAndSpamClassificationPostLoginAction", () => {
 		} as User)
 
 		when(customerFacadeMock.getUser()).thenResolve(user)
-		await postLoginAction.onPartialLoginSuccess(loggedInEvent)
+		const { asyncAction } = await postLoginAction.onPartialLoginSuccess(loggedInEvent)
-		// since the resizeMailIndex.then() is not awaited, we resolve all pending promises manually
+		await asyncAction
-		await new Promise((resolve) => setImmediate(resolve))
 
 		verify(spamClassifierMock.initialize("firstMailGroup"), { times: 1 })
 		verify(spamClassifierMock.initialize("secondMailGroup"), { times: 1 })

tuta-sdk/rust/sdk/src/entities/entity_facade.rs

@@ -1814,7 +1814,13 @@ mod tests {
 				],
 			)]),
 		"1465"=> JsonElement::Array(vec![]),
-		"1677"=> JsonElement::Null
+		"1677"=> JsonElement::Null,
+		"1728"=> JsonElement::String(
+				"1".to_string(),
+			),
+		"1729"=> JsonElement::Array(
+				vec![],
+			),
 		}
 	}
 

tuta-sdk/rust/sdk/test_data/email_response.json

@@ -41,5 +41,7 @@
 	"115": [],
 	"108": "2",
 	"1465": [],
-	"1677": null
+	"1677": null,
+	"1728": "1",
+	"1729": []
 }

tuta-sdk/rust/sdk/test_data/email_response_attachments.json

@@ -41,5 +41,7 @@
 	"426": "AR8zeFN4c98e8Ds8AkusyHbPK0iPTHJsnwisT/nzYPQhyVEMEV9SCk4s20/s5YKWdeU960ddEtcAcCpGRBVSS9Y=",
 	"1021": "0",
 	"896": "1723113273034",
-	"1677": null
+	"1677": null,
+	"1728": "1",
+	"1729": []
 }

tuta-sdk/rust/sdk/test_data/email_response_empty_encrypted_boolean_defaults_to_false.json

@@ -41,5 +41,7 @@
 	"115": [],
 	"108": "2",
 	"1465": [],
-	"1677": null
+	"1677": null,
+	"1728": "1",
+	"1729": []
 }

tuta-sdk/rust/sdk/tests/download_mail_test/mail.json

@@ -41,5 +41,7 @@
 		}
 	],
 	"466": "",
-	"1677": null
+	"1677": null,
+	"1728": "1",
+	"1729": []
 }