mirror of
https://github.com/tutao/tutanota.git
synced 2025-12-08 06:09:50 +00:00
5293be6a4a
We sync the spam training data encrypted through our server to make sure that all clients for a specific user behave the same when classifying mails. Additionally, this enables the spam classification in the webApp. We compress the training data vectors (see clientSpamTrainingDatum) before uploading to our server using SparseVectorCompressor.ts. When a user has the ClientSpamClassification enabled, the spam training data sync will happen for every mail received. ClientSpamTrainingDatum are not stored in the CacheStorage. No entityEvents are emitted for this type. However, we retrieve creations and updates for ClientSpamTrainingData through the modifiedClientSpamTrainingDataIndex. We calculate a threshold per classifier based on the dataset ham to spam ratio, we also subsample our training data to cap the ham to spam ratio within a certain limit. Co-authored-by: jomapp <17314077+jomapp@users.noreply.github.com> Co-authored-by: das <das@tutao.de> Co-authored-by: abp <abp@tutao.de> Co-authored-by: Kinan <104761667+kibibytium@users.noreply.github.com> Co-authored-by: sug <sug@tutao.de> Co-authored-by: nif <nif@tutao.de> Co-authored-by: map <mpfau@users.noreply.github.com>
61 lines
2.2 KiB
TypeScript
61 lines
2.2 KiB
TypeScript
import { identity } from "@tutao/tutanota-utils"
|
|
import type { LoginController } from "./LoginController"
|
|
import stream from "mithril/stream"
|
|
import Stream from "mithril/stream"
|
|
import { assertMainOrNode } from "../common/Env"
|
|
import { WebsocketCounterData } from "../entities/sys/TypeRefs"
|
|
import { EntityUpdateData } from "../common/utils/EntityUpdateUtils.js"
|
|
|
|
assertMainOrNode()
|
|
|
|
export type ExposedEventController = Pick<EventController, "onEntityUpdateReceived" | "onCountersUpdateReceived">
|
|
|
|
const TAG = "[EventController]"
|
|
|
|
export type EntityEventsListener = (updates: ReadonlyArray<EntityUpdateData>, eventOwnerGroupId: Id) => Promise<unknown>
|
|
|
|
export class EventController {
|
|
private countersStream: Stream<WebsocketCounterData> = stream()
|
|
private entityListeners: Set<EntityEventsListener> = new Set()
|
|
|
|
constructor(private readonly logins: LoginController) {}
|
|
|
|
addEntityListener(listener: EntityEventsListener) {
|
|
if (this.entityListeners.has(listener)) {
|
|
console.warn(TAG, "Adding the same listener twice!")
|
|
} else {
|
|
this.entityListeners.add(listener)
|
|
}
|
|
}
|
|
|
|
removeEntityListener(listener: EntityEventsListener) {
|
|
const wasRemoved = this.entityListeners.delete(listener)
|
|
if (!wasRemoved) {
|
|
console.warn(TAG, "Could not remove listener, possible leak?", listener)
|
|
}
|
|
}
|
|
|
|
getCountersStream(): Stream<WebsocketCounterData> {
|
|
// Create copy so it's never ended
|
|
return this.countersStream.map(identity)
|
|
}
|
|
|
|
async onEntityUpdateReceived(entityUpdates: readonly EntityUpdateData[], eventOwnerGroupId: Id): Promise<void> {
|
|
if (this.logins.isUserLoggedIn()) {
|
|
// the UserController must be notified first as other event receivers depend on it to be up-to-date
|
|
await this.logins.getUserController().entityEventsReceived(entityUpdates, eventOwnerGroupId)
|
|
}
|
|
for (const listener of this.entityListeners) {
|
|
// run listeners async to speed up processing
|
|
// we ran it sequentially before to prevent parallel loading of instances
|
|
// this should not be a problem anymore as we prefetch now
|
|
|
|
// noinspection ES6MissingAwait
|
|
listener(entityUpdates, eventOwnerGroupId)
|
|
}
|
|
}
|
|
|
|
async onCountersUpdateReceived(update: WebsocketCounterData): Promise<void> {
|
|
this.countersStream(update)
|
|
}
|
|
}
|