Stowage/tutanota
2
0
Fork 0
mirror of https://github.com/tutao/tutanota.git synced 2025-10-19 16:03:43 +00:00
Code Issues Projects Releases Packages Wiki Activity

Delete spam training data on mail deletion

Browse source
This commit is contained in:
das 2025-10-15 13:55:30 +02:00 • committed by abp
parent 5b8b49d316
commit ae570ccf9b
No known key found for this signature in database
GPG key ID: 791D4EC38A7AA7C2
9 changed files with 30 additions and 36 deletions
Download patch file Download diff file Expand all files Collapse all files

1
src/calendar-app/workerUtils/worker/CalendarWorkerLocator.ts
View file

@ -521,7 +521,6 @@ export async function initLocator(worker: CalendarWorkerImpl, browserData: Brows
locator.login,
locator.keyLoader,
locator.publicEncryptionKeyProvider,
null,
)
})
const nativePushFacade = new NativePushFacadeSendDispatcher(worker)

10
src/common/api/worker/facades/lazy/MailFacade.ts
View file

@ -28,8 +28,8 @@ import {
MailMethod,
MailReportType,
MailSetKind,
MAX_NBR_OF_MAILS_SYNC_OPERATION,
MAX_NBR_OF_CONVERSATIONS,
MAX_NBR_OF_MAILS_SYNC_OPERATION,
OperationType,
PhishingMarkerStatus,
PublicKeyIdentifierType,
@ -157,11 +157,8 @@ import { EntityUpdateData, isUpdateForTypeRef } from "../../../common/utils/Enti
import { Entity } from "../../../common/EntityTypes"
import { KeyVerificationMismatchError } from "../../../common/error/KeyVerificationMismatchError"
import { VerifiedPublicEncryptionKey } from "./KeyVerificationFacade"
import { SpamClassifier, SpamPredMailDatum } from "../../../../../mail-app/workerUtils/spamClassification/SpamClassifier"
import { isDraft } from "../../../../../mail-app/mail/model/MailChecks"
import { Nullable } from "@tutao/tutanota-utils/dist/Utils"
import { ClientClassifierType } from "../../../common/ClientClassifierType"
import { getMailBodyText } from "../../../common/CommonMailUtils"
assertWorkerOrNode()
type Attachments = ReadonlyArray<TutanotaFile | DataFile | FileReference>
@ -210,7 +207,6 @@ export class MailFacade {
private readonly loginFacade: LoginFacade,
private readonly keyLoaderFacade: KeyLoaderFacade,
private readonly publicEncryptionKeyProvider: PublicEncryptionKeyProvider,
private readonly spamClassifier: SpamClassifier | null,
) {}
async createMailFolder(name: string, parent: IdTuple | null, ownerGroupId: Id): Promise<void> {
@ -460,10 +456,6 @@ export class MailFacade {
await this.serviceExecutor.post(ReportMailService, postData)
}
public isSpamClassificationEnabled(ownerGroup: Id): boolean {
return this.spamClassifier != null && this.spamClassifier.getEnabledSpamClassifierForOwnerGroup(ownerGroup) != null
}
async deleteMails(mails: readonly IdTuple[], filterMailSet: IdTuple | null): Promise<void> {
if (isEmpty(mails)) {
return

8
src/common/api/worker/rest/DefaultEntityRestCache.ts
View file

@ -8,7 +8,7 @@ import {
OwnerEncSessionKeyProvider,
} from "./EntityRestClient"
import { OperationType } from "../../common/TutanotaConstants"
import { assertNotNull, downcast, getFirstOrThrow, getTypeString, isNotEmpty, isSameTypeRef, lastThrow, TypeRef } from "@tutao/tutanota-utils"
import { assertNotNull, downcast, getFirstOrThrow, getTypeString, isNotEmpty, isSameTypeRef, lastThrow, Nullable, TypeRef } from "@tutao/tutanota-utils"
import {
AuditLogEntryTypeRef,
BucketPermissionTypeRef,
@ -25,7 +25,7 @@ import {
UserGroupRootTypeRef,
} from "../../entities/sys/TypeRefs.js"
import { ValueType } from "../../common/EntityConstants.js"
import { Body, CalendarEventUidIndexTypeRef, Mail, MailDetailsBlobTypeRef, MailSetEntryTypeRef, MailTypeRef } from "../../entities/tutanota/TypeRefs.js"
import { CalendarEventUidIndexTypeRef, MailDetailsBlobTypeRef, MailSetEntryTypeRef, MailTypeRef } from "../../entities/tutanota/TypeRefs.js"
import {
CUSTOM_MAX_ID,
CUSTOM_MIN_ID,
@ -48,7 +48,6 @@ import { AttributeModel } from "../../common/AttributeModel"
import { collapseId, expandId } from "./RestClientIdUtils"
import { PatchMerger } from "../offline/PatchMerger"
import { NotAuthorizedError, NotFoundError } from "../../common/error/RestError"
import { Nullable } from "@tutao/tutanota-utils"
import { hasError } from "../../common/utils/ErrorUtils"
assertWorkerOrNode()
@ -794,6 +793,7 @@ export class DefaultEntityRestCache implements EntityRestCache {
// delete mailDetails if they are available (as we don't send an event for this type)
const mail = await this.storage.get(update.typeRef, update.instanceListId, update.instanceId)
if (mail) {
update.instance = mail as unknown as ServerModelParsedInstance // Can I even do this?
let mailDetailsId = mail.mailDetails
await this.storage.deleteIfExists(update.typeRef, update.instanceListId, update.instanceId)
if (mailDetailsId != null) {
@ -839,7 +839,7 @@ export class DefaultEntityRestCache implements EntityRestCache {
await handler.onEntityEventUpdate?.(id, filteredUpdateEvents)
break
case OperationType.DELETE:
await handler.onEntityEventDelete?.(id)
await handler.onEntityEventDelete?.(id, update) // send mail here
break
}
} catch (e) {

2
src/common/api/worker/rest/cacheHandler/CustomCacheHandler.ts
View file

@ -101,5 +101,5 @@ export interface CustomCacheHandler<T extends SomeEntity> {
*
* @param id ID of the entity
*/
onEntityEventDelete?: (id: T["_id"]) => Promise<void>
onEntityEventDelete?: (id: T["_id"], event: EntityUpdateData) => Promise<void>
}

18
src/common/api/worker/rest/cacheHandler/CustomMailEventCacheHandler.ts
View file

@ -1,13 +1,19 @@
import { Mail } from "../../../entities/tutanota/TypeRefs"
import { lazyAsync } from "@tutao/tutanota-utils"
import { Mail, MailDetailsBlobTypeRef } from "../../../entities/tutanota/TypeRefs"
import { assertNotNull, lazy, lazyAsync } from "@tutao/tutanota-utils"
import { MailIndexer } from "../../../../../mail-app/workerUtils/index/MailIndexer"
import { CustomCacheHandler } from "./CustomCacheHandler"
import { OfflineStoragePersistence } from "../../../../../mail-app/workerUtils/index/OfflineStoragePersistence"
import { EntityUpdateData } from "../../../common/utils/EntityUpdateUtils"
import { elementIdPart, listIdPart } from "../../../common/utils/EntityUtils"
/**
* Handles telling the indexer to index or un-index mail data on updates.
*/
export class CustomMailEventCacheHandler implements CustomCacheHandler<Mail> {
constructor(private readonly indexer: lazyAsync<MailIndexer>) {}
constructor(
private readonly indexer: lazyAsync<MailIndexer>,
private readonly offlineStoragePersistence: OfflineStoragePersistence,
) {}
shouldLoadOnCreateEvent(): boolean {
// New emails should be pre-cached.
@ -32,4 +38,10 @@ export class CustomMailEventCacheHandler implements CustomCacheHandler<Mail> {
const indexer = await this.indexer()
return indexer.afterMailUpdated(id)
}
async onEntityEventDelete(id: IdTuple, event: EntityUpdateData) {
const mail = event.instance as unknown as Mail // how can we do this better?
const ownerGroup = assertNotNull(mail._ownerGroup)
await this.offlineStoragePersistence.deleteSpamClassificationData(ownerGroup, mail._id)
}
}

6
src/mail-app/mail/model/MailModel.ts
View file

@ -243,12 +243,6 @@ export class MailModel {
mailFolderAfterInboxRuleAndSpamProcessing.then((targetFolder) => {
this._showNotification(targetFolder ?? initialMailFolder, mail)
})
} else if (isUpdateForTypeRef(MailTypeRef, update) && update.operation === OperationType.DELETE) {
const mailId: IdTuple = [update.instanceListId, update.instanceListId]
// todo: how can we get the group in case of delete events?
const mailOwnerGroup = this.logins.getUserController().getMailGroupMemberships().at(0)!.group
await this.spamHandler().dropClassificationData(mailOwnerGroup, mailId)
}
}
}

16
src/mail-app/workerUtils/index/OfflineStoragePersistence.ts
View file

@ -212,6 +212,14 @@ export class OfflineStoragePersistence {
await this.sqlCipherFacade.run(query, params)
}
async deleteSpamClassificationTrainingDataBeforeCutoff(cutoffTimestamp: number, ownerGroupId: Id): Promise<void> {
const { query, params } = sql`DELETE
FROM spam_classification_training_data
WHERE lastModified < ${cutoffTimestamp}
AND ownerGroup = ${ownerGroupId}`
await this.sqlCipherFacade.run(query, params)
}
async updateSpamClassificationData(id: IdTuple, isSpam: boolean, isSpamConfidence: number): Promise<void> {
const { query, params } = sql`
UPDATE spam_classification_training_data
@ -250,14 +258,6 @@ export class OfflineStoragePersistence {
return resultRows.map(untagSqlObject).map((row) => row as unknown as SpamTrainMailDatum)
}
async deleteSpamClassificationTrainingDataBeforeCutoff(cutoffTimestamp: number, ownerGroupId: Id): Promise<void> {
const { query, params } = sql`DELETE
FROM spam_classification_training_data
WHERE lastModified < ${cutoffTimestamp}
AND ownerGroup = ${ownerGroupId}`
await this.sqlCipherFacade.run(query, params)
}
async putSpamClassificationModel(model: SpamClassificationModel) {
const { query, params } = sql`INSERT
OR REPLACE INTO

3
src/mail-app/workerUtils/worker/WorkerLocator.ts
View file

@ -346,7 +346,7 @@ export async function initLocator(worker: WorkerImpl, browserData: BrowserData)
},
{
ref: MailTypeRef,
handler: new CustomMailEventCacheHandler(mailIndexer),
handler: new CustomMailEventCacheHandler(mailIndexer, offlineStorage),
},
{ ref: UserTypeRef, handler: new CustomUserCacheHandler(locator.cacheStorage) },
)
@ -752,7 +752,6 @@ export async function initLocator(worker: WorkerImpl, browserData: BrowserData)
locator.login,
locator.keyLoader,
locator.publicEncryptionKeyProvider,
spamClassifier,
)
})
const nativePushFacade = new NativePushFacadeSendDispatcher(worker)

2
test/tests/api/worker/facades/MailFacadeTest.ts
View file

@ -69,7 +69,6 @@ o.spec("MailFacade test", function () {
loginFacade = object()
keyLoaderFacade = object()
publicEncryptionKeyProvider = object()
spamClassifier = object()
facade = new MailFacade(
userFacade,
entity,
@ -80,7 +79,6 @@ o.spec("MailFacade test", function () {
loginFacade,
keyLoaderFacade,
publicEncryptionKeyProvider,
spamClassifier,
)
})