tutanota/test/tests/api/worker/facades/MailFacadeTest.ts

724 lines
26 KiB
TypeScript
Raw Normal View History

2023-06-29 18:26:45 +02:00
import o from "@tutao/otest"
import { MailFacade, phishingMarkerValue, validateMimeTypesForAttachments } from "../../../../../src/common/api/worker/facades/lazy/MailFacade.js"
import {
FileTypeRef,
InternalRecipientKeyDataTypeRef,
Mail,
MailAddressTypeRef,
MailDetails,
MailDetailsBlobTypeRef,
MailDetailsTypeRef,
MailTypeRef,
ReportedMailFieldMarkerTypeRef,
SecureExternalRecipientKeyDataTypeRef,
SendDraftData,
SendDraftDataTypeRef,
SymEncInternalRecipientKeyDataTypeRef,
} from "../../../../../src/common/api/entities/tutanota/TypeRefs.js"
import {
CryptoProtocolVersion,
MailAuthenticationStatus,
MAX_NBR_MOVE_DELETE_MAIL_SERVICE,
ReportedMailFieldType,
} from "../../../../../src/common/api/common/TutanotaConstants.js"
import { matchers, object, when } from "testdouble"
import { CryptoFacade } from "../../../../../src/common/api/worker/crypto/CryptoFacade.js"
import { IServiceExecutor } from "../../../../../src/common/api/common/ServiceRequest.js"
import { EntityClient } from "../../../../../src/common/api/common/EntityClient.js"
import { BlobFacade } from "../../../../../src/common/api/worker/facades/lazy/BlobFacade.js"
import { UserFacade } from "../../../../../src/common/api/worker/facades/UserFacade"
import { NativeFileApp } from "../../../../../src/common/native/common/FileApp.js"
import { LoginFacade } from "../../../../../src/common/api/worker/facades/LoginFacade.js"
import { DataFile } from "../../../../../src/common/api/common/DataFile.js"
import { downcast, KeyVersion, lazyNumberRange } from "@tutao/tutanota-utils"
import { ProgrammingError } from "../../../../../src/common/api/common/error/ProgrammingError.js"
import { createTestEntity } from "../../../TestUtils.js"
import { KeyLoaderFacade } from "../../../../../src/common/api/worker/facades/KeyLoaderFacade.js"
import { PublicEncryptionKeyProvider } from "../../../../../src/common/api/worker/facades/PublicEncryptionKeyProvider.js"
import { assertThrows, verify } from "@tutao/tutanota-test-utils"
import { UnreadMailStateService } from "../../../../../src/common/api/entities/tutanota/Services"
import { BucketKeyTypeRef, InstanceSessionKey, InstanceSessionKeyTypeRef } from "../../../../../src/common/api/entities/sys/TypeRefs"
import { OwnerEncSessionKeyProvider } from "../../../../../src/common/api/worker/rest/EntityRestClient"
import { elementIdPart, getElementId } from "../../../../../src/common/api/common/utils/EntityUtils"
import { VersionedEncryptedKey } from "../../../../../src/common/api/worker/crypto/CryptoWrapper"
import { Recipient } from "../../../../../src/common/api/common/recipients/Recipient"
import { AesKey } from "@tutao/tutanota-crypto"
import { RecipientsNotFoundError } from "../../../../../src/common/api/common/error/RecipientsNotFoundError"
import { KeyVerificationMismatchError } from "../../../../../src/common/api/common/error/KeyVerificationMismatchError"
[antispam] Add client-side local spam filtering Implement a local machine learning model for client-side spam filtering. The local model is implemented using tensorflow "LayersModel" to train separate models in all available mailboxes, resulting in one model per ownerGroup (i.e. mailbox). Initially, the training data is aggregated from the last 30 days of received mails, and the data is stored in a separate offline database table named spam_classification_training_data. The trained model is stored in the table spam_classification_model. The initial training starts after indexing, with periodic training happening every 30 minutes and on each subsequent login. The model will predict on incoming mails once we have received the entity event for said mail, moving it to either inbox or spam folder. When users move mails, we update the training data labels accordingly, by adjusting the isSpam classification and isSpamConfidence values in the offline database. The MoveMailService now contains a moveReason, which indicates that the mail has been moved by our spam filter. Client-side spam filtering can be activated using the SpamClientClassification feature flag, and is for now only available on the desktop client. Co-authored-by: sug <sug@tutao.de> Co-authored-by: kib <104761667+kibibytium@users.noreply.github.com> Co-authored-by: abp <abp@tutao.de> Co-authored-by: map <mpfau@users.noreply.github.com> Co-authored-by: jhm <17314077+jomapp@users.noreply.github.com> Co-authored-by: frm <frm@tutao.de> Co-authored-by: das <das@tutao.de> Co-authored-by: nif <nif@tutao.de> Co-authored-by: amm <amm@tutao.de>
2025-10-14 12:32:17 +02:00
import { SpamClassifier } from "../../../../../src/mail-app/workerUtils/spamClassification/SpamClassifier"
import { CacheStorage } from "../../../../../src/common/api/worker/rest/DefaultEntityRestCache"
o.spec("MailFacade test", function () {
let facade: MailFacade
let userFacade: UserFacade
2022-03-09 17:43:29 +01:00
let cryptoFacade: CryptoFacade
let serviceExecutor: IServiceExecutor
let entityClient: EntityClient
2022-03-16 10:14:53 +01:00
let blobFacade: BlobFacade
let fileApp: NativeFileApp
let loginFacade: LoginFacade
Support group key rotation (#6588) * Allow groups to have multiple key versions tutadb#1628 * Adapt to model changes * Fix CommonMailUtilsTest * Remove symEncBucketKey from SecureExternalRecipientKeyData * Remove deprecated types Also fix tests that relied on them as dummy types * Add userKeyVersion to RecoverCode * Remove clientKey Seems to be unused. * Remove CreateFolderService Unused. * Remove symEncSessionKey from DraftCreateData Unused. * Remove symEncShareBucketKey from MailBox Unused. * Add userKeyVersion to TutanotaProperties * Remove PasswordRetrievalService type The service itself had been long gone. * Remove userKeyVersion from CustomerAccountCreateData CreateMailGroupData * Fix customer account creation Set the key version that we actually need there: the *system* admin pub key version. The sender key version is not needed, because the system admin only has RSA keys. Also, this is a new customer, so that would be version zero anyway. * Fix resolving bucket key with group reference Get the right versions along the way. * Use current group key when encrypting instance session keys * Remove left-over key getting Also document a couple of current key usages * Pass group key providers to EntityClient instead of group key * Fix types and do not provide sender key version for rsa Fix resolveServiceSessionKey * Rename constant to avoid confusion There is another constant with the same name. * Use TutanotaModelV69 * Introduce client side mechanism to handle key rotation requests see tutadb 1771 * Do not export 128-bit key generator It is only needed for tests within the package. * Remove group key version when creating user area groups Plus some minor clarity improvements. * Fix version handling when updating drafts and sending to secure external * Remove versions when creating external users They are zero. * Fix changing the admin flag * Remove (almost) all local admin related code * Improve readability * Default to user key version zero when loading entropy * Decrypt current groupKey with correct userGroupKey version * Fix system application offline migrations * Fix tutanota application offline migrations * Improve offline migration functions * Use AesKey type * Minor improvements from review * Use AesKey type instead of Aes128Key where possible * Model update after rebase * Fix getting user group key Should never try to get from the cache like a normal group key. * Fix getting former group key Start ID was off-by-one. * Minor changes from review. We just checked all usages of all public methods of KeyLoaderFacade to make sure we're using the correct versions where we need them. * More minor changes from review. * Pass ownerKeyProvider instead of ownerKey when updating with the EntityClient * Pass ownerKeyProvider only when necessary * Document ownerKeyProvider parameter * Fix offline database migration * Fix unlocking the indexer data --------- Co-authored-by: vaf <vaf@tutao.de> Co-authored-by: bedhub <bedhub@users.noreply.github.com> Co-authored-by: bed <bed@tutao.de>
2024-04-17 10:34:33 +02:00
let keyLoaderFacade: KeyLoaderFacade
let publicEncryptionKeyProvider: PublicEncryptionKeyProvider
[antispam] Add client-side local spam filtering Implement a local machine learning model for client-side spam filtering. The local model is implemented using tensorflow "LayersModel" to train separate models in all available mailboxes, resulting in one model per ownerGroup (i.e. mailbox). Initially, the training data is aggregated from the last 30 days of received mails, and the data is stored in a separate offline database table named spam_classification_training_data. The trained model is stored in the table spam_classification_model. The initial training starts after indexing, with periodic training happening every 30 minutes and on each subsequent login. The model will predict on incoming mails once we have received the entity event for said mail, moving it to either inbox or spam folder. When users move mails, we update the training data labels accordingly, by adjusting the isSpam classification and isSpamConfidence values in the offline database. The MoveMailService now contains a moveReason, which indicates that the mail has been moved by our spam filter. Client-side spam filtering can be activated using the SpamClientClassification feature flag, and is for now only available on the desktop client. Co-authored-by: sug <sug@tutao.de> Co-authored-by: kib <104761667+kibibytium@users.noreply.github.com> Co-authored-by: abp <abp@tutao.de> Co-authored-by: map <mpfau@users.noreply.github.com> Co-authored-by: jhm <17314077+jomapp@users.noreply.github.com> Co-authored-by: frm <frm@tutao.de> Co-authored-by: das <das@tutao.de> Co-authored-by: nif <nif@tutao.de> Co-authored-by: amm <amm@tutao.de>
2025-10-14 12:32:17 +02:00
let cacheStorage: CacheStorage
let spamClassifier: SpamClassifier
2022-03-09 17:43:29 +01:00
o.beforeEach(function () {
userFacade = object()
2022-03-16 10:14:53 +01:00
blobFacade = object()
entityClient = object()
2022-03-09 17:43:29 +01:00
cryptoFacade = object()
serviceExecutor = object()
fileApp = object()
loginFacade = object()
Support group key rotation (#6588) * Allow groups to have multiple key versions tutadb#1628 * Adapt to model changes * Fix CommonMailUtilsTest * Remove symEncBucketKey from SecureExternalRecipientKeyData * Remove deprecated types Also fix tests that relied on them as dummy types * Add userKeyVersion to RecoverCode * Remove clientKey Seems to be unused. * Remove CreateFolderService Unused. * Remove symEncSessionKey from DraftCreateData Unused. * Remove symEncShareBucketKey from MailBox Unused. * Add userKeyVersion to TutanotaProperties * Remove PasswordRetrievalService type The service itself had been long gone. * Remove userKeyVersion from CustomerAccountCreateData CreateMailGroupData * Fix customer account creation Set the key version that we actually need there: the *system* admin pub key version. The sender key version is not needed, because the system admin only has RSA keys. Also, this is a new customer, so that would be version zero anyway. * Fix resolving bucket key with group reference Get the right versions along the way. * Use current group key when encrypting instance session keys * Remove left-over key getting Also document a couple of current key usages * Pass group key providers to EntityClient instead of group key * Fix types and do not provide sender key version for rsa Fix resolveServiceSessionKey * Rename constant to avoid confusion There is another constant with the same name. * Use TutanotaModelV69 * Introduce client side mechanism to handle key rotation requests see tutadb 1771 * Do not export 128-bit key generator It is only needed for tests within the package. * Remove group key version when creating user area groups Plus some minor clarity improvements. * Fix version handling when updating drafts and sending to secure external * Remove versions when creating external users They are zero. * Fix changing the admin flag * Remove (almost) all local admin related code * Improve readability * Default to user key version zero when loading entropy * Decrypt current groupKey with correct userGroupKey version * Fix system application offline migrations * Fix tutanota application offline migrations * Improve offline migration functions * Use AesKey type * Minor improvements from review * Use AesKey type instead of Aes128Key where possible * Model update after rebase * Fix getting user group key Should never try to get from the cache like a normal group key. * Fix getting former group key Start ID was off-by-one. * Minor changes from review. We just checked all usages of all public methods of KeyLoaderFacade to make sure we're using the correct versions where we need them. * More minor changes from review. * Pass ownerKeyProvider instead of ownerKey when updating with the EntityClient * Pass ownerKeyProvider only when necessary * Document ownerKeyProvider parameter * Fix offline database migration * Fix unlocking the indexer data --------- Co-authored-by: vaf <vaf@tutao.de> Co-authored-by: bedhub <bedhub@users.noreply.github.com> Co-authored-by: bed <bed@tutao.de>
2024-04-17 10:34:33 +02:00
keyLoaderFacade = object()
publicEncryptionKeyProvider = object()
[antispam] Add client-side local spam filtering Implement a local machine learning model for client-side spam filtering. The local model is implemented using tensorflow "LayersModel" to train separate models in all available mailboxes, resulting in one model per ownerGroup (i.e. mailbox). Initially, the training data is aggregated from the last 30 days of received mails, and the data is stored in a separate offline database table named spam_classification_training_data. The trained model is stored in the table spam_classification_model. The initial training starts after indexing, with periodic training happening every 30 minutes and on each subsequent login. The model will predict on incoming mails once we have received the entity event for said mail, moving it to either inbox or spam folder. When users move mails, we update the training data labels accordingly, by adjusting the isSpam classification and isSpamConfidence values in the offline database. The MoveMailService now contains a moveReason, which indicates that the mail has been moved by our spam filter. Client-side spam filtering can be activated using the SpamClientClassification feature flag, and is for now only available on the desktop client. Co-authored-by: sug <sug@tutao.de> Co-authored-by: kib <104761667+kibibytium@users.noreply.github.com> Co-authored-by: abp <abp@tutao.de> Co-authored-by: map <mpfau@users.noreply.github.com> Co-authored-by: jhm <17314077+jomapp@users.noreply.github.com> Co-authored-by: frm <frm@tutao.de> Co-authored-by: das <das@tutao.de> Co-authored-by: nif <nif@tutao.de> Co-authored-by: amm <amm@tutao.de>
2025-10-14 12:32:17 +02:00
spamClassifier = object()
facade = new MailFacade(
userFacade,
entityClient,
cryptoFacade,
serviceExecutor,
blobFacade,
fileApp,
loginFacade,
keyLoaderFacade,
publicEncryptionKeyProvider,
[antispam] Add client-side local spam filtering Implement a local machine learning model for client-side spam filtering. The local model is implemented using tensorflow "LayersModel" to train separate models in all available mailboxes, resulting in one model per ownerGroup (i.e. mailbox). Initially, the training data is aggregated from the last 30 days of received mails, and the data is stored in a separate offline database table named spam_classification_training_data. The trained model is stored in the table spam_classification_model. The initial training starts after indexing, with periodic training happening every 30 minutes and on each subsequent login. The model will predict on incoming mails once we have received the entity event for said mail, moving it to either inbox or spam folder. When users move mails, we update the training data labels accordingly, by adjusting the isSpam classification and isSpamConfidence values in the offline database. The MoveMailService now contains a moveReason, which indicates that the mail has been moved by our spam filter. Client-side spam filtering can be activated using the SpamClientClassification feature flag, and is for now only available on the desktop client. Co-authored-by: sug <sug@tutao.de> Co-authored-by: kib <104761667+kibibytium@users.noreply.github.com> Co-authored-by: abp <abp@tutao.de> Co-authored-by: map <mpfau@users.noreply.github.com> Co-authored-by: jhm <17314077+jomapp@users.noreply.github.com> Co-authored-by: frm <frm@tutao.de> Co-authored-by: das <das@tutao.de> Co-authored-by: nif <nif@tutao.de> Co-authored-by: amm <amm@tutao.de>
2025-10-14 12:32:17 +02:00
spamClassifier,
)
})
o.spec("checkMailForPhishing", function () {
let mailDetails: MailDetails
let mail: Mail
o.beforeEach(function () {
const mailDetailsListId = "mailDetailsListId"
const mailDetailsElementId = "mailDetailsElementId"
mailDetails = createTestEntity(MailDetailsTypeRef, { authStatus: MailAuthenticationStatus.AUTHENTICATED })
mail = createTestEntity(MailTypeRef, {
mailDetails: [mailDetailsListId, mailDetailsElementId],
subject: "Test",
sender: createTestEntity(MailAddressTypeRef, {
name: "a",
address: "test@example.com",
2022-12-27 15:37:40 +01:00
}),
})
when(entityClient.loadMultiple(MailDetailsBlobTypeRef, mailDetailsListId, [mailDetailsElementId], matchers.anything())).thenResolve([
createTestEntity(MailDetailsBlobTypeRef, { details: mailDetails }),
])
})
o("not phishing if no markers", async function () {
2022-12-27 15:37:40 +01:00
o(await facade.checkMailForPhishing(mail, [{ href: "https://example.com", innerHTML: "link" }])).equals(false)
})
o("not phishing if no matching markers", async function () {
facade.phishingMarkersUpdateReceived([
createTestEntity(ReportedMailFieldMarkerTypeRef, {
marker: phishingMarkerValue(ReportedMailFieldType.SUBJECT, "Test 2"),
}),
createTestEntity(ReportedMailFieldMarkerTypeRef, {
2022-12-27 15:37:40 +01:00
marker: phishingMarkerValue(ReportedMailFieldType.FROM_DOMAIN, "example2.com"),
}),
])
2022-12-27 15:37:40 +01:00
o(await facade.checkMailForPhishing(mail, [{ href: "https://example.com", innerHTML: "link" }])).equals(false)
})
o("not phishing if only from domain matches", async function () {
facade.phishingMarkersUpdateReceived([
createTestEntity(ReportedMailFieldMarkerTypeRef, {
marker: phishingMarkerValue(ReportedMailFieldType.SUBJECT, "Test 2"),
}),
createTestEntity(ReportedMailFieldMarkerTypeRef, {
2022-12-27 15:37:40 +01:00
marker: phishingMarkerValue(ReportedMailFieldType.FROM_DOMAIN, "example.com"),
}),
])
2022-12-27 15:37:40 +01:00
o(await facade.checkMailForPhishing(mail, [{ href: "https://example.com", innerHTML: "link" }])).equals(false)
})
o("not phishing if only subject matches", async function () {
facade.phishingMarkersUpdateReceived([
createTestEntity(ReportedMailFieldMarkerTypeRef, {
marker: phishingMarkerValue(ReportedMailFieldType.SUBJECT, "Test"),
}),
createTestEntity(ReportedMailFieldMarkerTypeRef, {
2022-12-27 15:37:40 +01:00
marker: phishingMarkerValue(ReportedMailFieldType.FROM_DOMAIN, "example2.com"),
}),
])
2022-12-27 15:37:40 +01:00
o(await facade.checkMailForPhishing(mail, [{ href: "https://example.com", innerHTML: "link" }])).equals(false)
})
o("is phishing if subject and sender domain matches", async function () {
facade.phishingMarkersUpdateReceived([
createTestEntity(ReportedMailFieldMarkerTypeRef, {
marker: phishingMarkerValue(ReportedMailFieldType.SUBJECT, "Test"),
}),
createTestEntity(ReportedMailFieldMarkerTypeRef, {
2022-12-27 15:37:40 +01:00
marker: phishingMarkerValue(ReportedMailFieldType.FROM_DOMAIN, "example.com"),
}),
])
2022-12-27 15:37:40 +01:00
o(await facade.checkMailForPhishing(mail, [{ href: "https://example.com", innerHTML: "link" }])).equals(true)
})
o("is phishing if subject with whitespaces and sender domain matches", async function () {
mail.subject = "\tTest spaces \n"
facade.phishingMarkersUpdateReceived([
createTestEntity(ReportedMailFieldMarkerTypeRef, {
marker: phishingMarkerValue(ReportedMailFieldType.SUBJECT, "Testspaces"),
}),
createTestEntity(ReportedMailFieldMarkerTypeRef, {
2022-12-27 15:37:40 +01:00
marker: phishingMarkerValue(ReportedMailFieldType.FROM_DOMAIN, "example.com"),
}),
])
2022-12-27 15:37:40 +01:00
o(await facade.checkMailForPhishing(mail, [{ href: "https://example.com", innerHTML: "link" }])).equals(true)
})
o("is not phishing if subject and sender domain matches but not authenticated", async function () {
mailDetails.authStatus = MailAuthenticationStatus.SOFT_FAIL
facade.phishingMarkersUpdateReceived([
createTestEntity(ReportedMailFieldMarkerTypeRef, {
marker: phishingMarkerValue(ReportedMailFieldType.SUBJECT, "Test"),
}),
createTestEntity(ReportedMailFieldMarkerTypeRef, {
2022-12-27 15:37:40 +01:00
marker: phishingMarkerValue(ReportedMailFieldType.FROM_DOMAIN, "example.com"),
}),
])
2022-12-27 15:37:40 +01:00
o(await facade.checkMailForPhishing(mail, [{ href: "https://example.com", innerHTML: "link" }])).equals(false)
})
o("is phishing if subject and sender address matches", async function () {
facade.phishingMarkersUpdateReceived([
createTestEntity(ReportedMailFieldMarkerTypeRef, {
marker: phishingMarkerValue(ReportedMailFieldType.SUBJECT, "Test"),
}),
createTestEntity(ReportedMailFieldMarkerTypeRef, {
2022-12-27 15:37:40 +01:00
marker: phishingMarkerValue(ReportedMailFieldType.FROM_ADDRESS, "test@example.com"),
}),
])
2022-12-27 15:37:40 +01:00
o(await facade.checkMailForPhishing(mail, [{ href: "https://example.com", innerHTML: "link" }])).equals(true)
})
o("is not phishing if subject and sender address matches but not authenticated", async function () {
mailDetails.authStatus = MailAuthenticationStatus.SOFT_FAIL
facade.phishingMarkersUpdateReceived([
createTestEntity(ReportedMailFieldMarkerTypeRef, {
marker: phishingMarkerValue(ReportedMailFieldType.SUBJECT, "Test"),
}),
createTestEntity(ReportedMailFieldMarkerTypeRef, {
2022-12-27 15:37:40 +01:00
marker: phishingMarkerValue(ReportedMailFieldType.FROM_ADDRESS, "test@example.com"),
}),
])
2022-12-27 15:37:40 +01:00
o(await facade.checkMailForPhishing(mail, [{ href: "https://example.com", innerHTML: "link" }])).equals(false)
})
o("is phishing if subject and non auth sender domain matches", async function () {
mailDetails.authStatus = MailAuthenticationStatus.SOFT_FAIL
facade.phishingMarkersUpdateReceived([
createTestEntity(ReportedMailFieldMarkerTypeRef, {
marker: phishingMarkerValue(ReportedMailFieldType.SUBJECT, "Test"),
}),
createTestEntity(ReportedMailFieldMarkerTypeRef, {
2022-12-27 15:37:40 +01:00
marker: phishingMarkerValue(ReportedMailFieldType.FROM_DOMAIN_NON_AUTH, "example.com"),
}),
])
2022-12-27 15:37:40 +01:00
o(await facade.checkMailForPhishing(mail, [{ href: "https://example.com", innerHTML: "link" }])).equals(true)
})
o("is phishing if subject and non auth sender address matches", async function () {
mailDetails.authStatus = MailAuthenticationStatus.SOFT_FAIL
facade.phishingMarkersUpdateReceived([
createTestEntity(ReportedMailFieldMarkerTypeRef, {
marker: phishingMarkerValue(ReportedMailFieldType.SUBJECT, "Test"),
}),
createTestEntity(ReportedMailFieldMarkerTypeRef, {
2022-12-27 15:37:40 +01:00
marker: phishingMarkerValue(ReportedMailFieldType.FROM_ADDRESS_NON_AUTH, "test@example.com"),
}),
])
2022-12-27 15:37:40 +01:00
o(await facade.checkMailForPhishing(mail, [{ href: "https://example.com", innerHTML: "link" }])).equals(true)
})
o("is phishing if subject and link matches", async function () {
facade.phishingMarkersUpdateReceived([
createTestEntity(ReportedMailFieldMarkerTypeRef, {
marker: phishingMarkerValue(ReportedMailFieldType.SUBJECT, "Test"),
}),
createTestEntity(ReportedMailFieldMarkerTypeRef, {
2022-12-27 15:37:40 +01:00
marker: phishingMarkerValue(ReportedMailFieldType.LINK, "https://example.com"),
}),
])
2022-12-27 15:37:40 +01:00
o(await facade.checkMailForPhishing(mail, [{ href: "https://example.com", innerHTML: "link" }])).equals(true)
})
o("is not phishing if just two links match", async function () {
facade.phishingMarkersUpdateReceived([
createTestEntity(ReportedMailFieldMarkerTypeRef, {
marker: phishingMarkerValue(ReportedMailFieldType.LINK, "https://example.com"),
}),
createTestEntity(ReportedMailFieldMarkerTypeRef, {
2022-12-27 15:37:40 +01:00
marker: phishingMarkerValue(ReportedMailFieldType.LINK, "https://example2.com"),
}),
])
2022-12-27 15:37:40 +01:00
o(
await facade.checkMailForPhishing(mail, [
{ href: "https://example.com", innerHTML: "link1" },
{ href: "https://example2.com", innerHTML: "link2" },
]),
).equals(false)
})
o("is phishing if subject and link domain matches", async function () {
facade.phishingMarkersUpdateReceived([
createTestEntity(ReportedMailFieldMarkerTypeRef, {
marker: phishingMarkerValue(ReportedMailFieldType.SUBJECT, "Test"),
}),
createTestEntity(ReportedMailFieldMarkerTypeRef, {
2022-12-27 15:37:40 +01:00
marker: phishingMarkerValue(ReportedMailFieldType.LINK_DOMAIN, "example.com"),
}),
])
2022-12-27 15:37:40 +01:00
o(await facade.checkMailForPhishing(mail, [{ href: "https://example.com", innerHTML: "link" }])).equals(true)
})
o("does not throw on invalid link", async function () {
facade.phishingMarkersUpdateReceived([
createTestEntity(ReportedMailFieldMarkerTypeRef, {
marker: phishingMarkerValue(ReportedMailFieldType.SUBJECT, "Test"),
}),
createTestEntity(ReportedMailFieldMarkerTypeRef, {
2022-12-27 15:37:40 +01:00
marker: phishingMarkerValue(ReportedMailFieldType.LINK_DOMAIN, "example.com"),
}),
])
2022-12-27 15:37:40 +01:00
o(
await facade.checkMailForPhishing(mail, [
{ href: "/example1", innerHTML: "link1" },
{ href: "example2", innerHTML: "link2" },
{ href: "http:/", innerHTML: "link3" },
]),
).equals(false)
})
o("is phishing if subject and suspicious link", async function () {
facade.phishingMarkersUpdateReceived([
createTestEntity(ReportedMailFieldMarkerTypeRef, {
marker: phishingMarkerValue(ReportedMailFieldType.SUBJECT, "Test"),
}),
])
o(
await facade.checkMailForPhishing(mail, [
{
href: "https://example.com",
innerHTML: "https://evil-domain.com",
},
]),
).equals(true)
})
o("link is not suspicious if on the same domain", async function () {
facade.phishingMarkersUpdateReceived([
createTestEntity(ReportedMailFieldMarkerTypeRef, {
marker: phishingMarkerValue(ReportedMailFieldType.SUBJECT, "Test"),
}),
])
o(
await facade.checkMailForPhishing(mail, [
{
href: "https://example.com",
innerHTML: "https://example.com/test",
},
]),
).equals(false)
})
})
o.spec("verifyMimeTypesForAttachments", () => {
function attach(mimeType, name): DataFile {
return downcast({
mimeType,
name,
_type: "DataFile",
})
}
o("valid mimetypes", () => {
validateMimeTypesForAttachments([attach("application/json", "something.json")])
validateMimeTypesForAttachments([attach("audio/ogg; codec=opus", "something.opus")])
validateMimeTypesForAttachments([attach('video/webm; codecs="vp8, opus"', "something.webm")])
validateMimeTypesForAttachments([attach("something/orrather", "something.somethingorrather")])
validateMimeTypesForAttachments([attach("thisisvalid/technically+this_is-ok_even-if-YOU-dont-like-it", "something.valid")])
validateMimeTypesForAttachments([attach("anotherthing/youcando;ishave=multiple;parameters=in;a=mimetype", "something.technicallyvalidaswell")])
})
o("invalid mimetypes", () => {
o(() => {
validateMimeTypesForAttachments([attach("applicationjson", "something.json")])
}).throws(ProgrammingError)
o(() => {
validateMimeTypesForAttachments([attach("application/json", "something.json"), attach("applicationjson", "something.json")])
}).throws(ProgrammingError)
o(() => {
validateMimeTypesForAttachments([attach("applicationjson", "something.json"), attach("application/json", "something.json")])
}).throws(ProgrammingError)
o(() => {
validateMimeTypesForAttachments([attach("", "bad.json")])
}).throws(ProgrammingError)
o(() => {
validateMimeTypesForAttachments([attach("a/b/c", "no.json")])
}).throws(ProgrammingError)
o(() => {
validateMimeTypesForAttachments([attach("a/b?c", "please stop.json")])
}).throws(ProgrammingError)
o(() => {
validateMimeTypesForAttachments([attach('video/webm; codecs="vp8, opus oh no i forgot the quote; oops=mybad', "why.webm")])
}).throws(ProgrammingError)
o(() => {
validateMimeTypesForAttachments([attach("video/webm; parameterwithoutavalue", "bad.webm")])
}).throws(ProgrammingError)
})
o("isTutaCryptMail", () => {
const pqRecipient = createTestEntity(InternalRecipientKeyDataTypeRef, { protocolVersion: CryptoProtocolVersion.TUTA_CRYPT })
const rsaRecipient = createTestEntity(InternalRecipientKeyDataTypeRef, { protocolVersion: CryptoProtocolVersion.RSA })
const secureExternalRecipient = createTestEntity(SecureExternalRecipientKeyDataTypeRef, {})
const symEncInternalRecipient = createTestEntity(SymEncInternalRecipientKeyDataTypeRef, {})
o(
facade.isTutaCryptMail(
createTestEntity(SendDraftDataTypeRef, {
internalRecipientKeyData: [pqRecipient],
secureExternalRecipientKeyData: [],
symEncInternalRecipientKeyData: [],
}),
),
).equals(true)
o(
facade.isTutaCryptMail(
createTestEntity(SendDraftDataTypeRef, {
internalRecipientKeyData: [pqRecipient, pqRecipient],
secureExternalRecipientKeyData: [],
symEncInternalRecipientKeyData: [],
}),
),
).equals(true)
o(
facade.isTutaCryptMail(
createTestEntity(SendDraftDataTypeRef, {
internalRecipientKeyData: [],
secureExternalRecipientKeyData: [],
symEncInternalRecipientKeyData: [],
}),
),
).equals(false)
o(
facade.isTutaCryptMail(
createTestEntity(SendDraftDataTypeRef, {
internalRecipientKeyData: [pqRecipient, rsaRecipient],
secureExternalRecipientKeyData: [],
symEncInternalRecipientKeyData: [],
}),
),
).equals(false)
o(
facade.isTutaCryptMail(
createTestEntity(SendDraftDataTypeRef, {
internalRecipientKeyData: [pqRecipient],
secureExternalRecipientKeyData: [secureExternalRecipient],
symEncInternalRecipientKeyData: [],
}),
),
).equals(false)
o(
facade.isTutaCryptMail(
createTestEntity(SendDraftDataTypeRef, {
internalRecipientKeyData: [pqRecipient],
secureExternalRecipientKeyData: [],
symEncInternalRecipientKeyData: [symEncInternalRecipient],
}),
),
).equals(false)
})
})
o.spec("markMails", () => {
o.test("test with single mail", async () => {
const testIds: IdTuple[] = [["a", "b"]]
await facade.markMails(testIds, true)
verify(
serviceExecutor.post(
UnreadMailStateService,
matchers.contains({
mails: testIds,
unread: true,
}),
),
)
})
o.test("test with a few mails", async () => {
const testIds: IdTuple[] = [
["a", "b"],
["c", "d"],
]
await facade.markMails(testIds, true)
verify(
serviceExecutor.post(
UnreadMailStateService,
matchers.contains({
mails: testIds,
unread: true,
}),
),
)
})
o.test("batches large amounts of mails", async () => {
const expectedBatches = 4
const testIds: IdTuple[] = []
for (let i = 0; i < MAX_NBR_MOVE_DELETE_MAIL_SERVICE * expectedBatches; i++) {
testIds.push([`${i}`, `${i}`])
}
await facade.markMails(testIds, true)
for (let i = 0; i < expectedBatches; i++) {
verify(
serviceExecutor.post(
UnreadMailStateService,
matchers.contains({
mails: testIds.slice(i * MAX_NBR_MOVE_DELETE_MAIL_SERVICE, (i + 1) * MAX_NBR_MOVE_DELETE_MAIL_SERVICE),
unread: true,
}),
),
)
}
verify(serviceExecutor.post(UnreadMailStateService, matchers.anything()), { times: expectedBatches })
})
})
o.spec("createOwnerEncSessionKeyProviderForAttachments", () => {
function sessionKeyId(mailIndex: number, attachmentIndex: number) {
return `attachmentId_mail_${mailIndex}_attachment_${attachmentIndex}`
}
function setUpMail(mailIndex: number, attachmentCount: number): Mail {
const mail = createTestEntity(MailTypeRef, {
bucketKey: createTestEntity(BucketKeyTypeRef, {
_id: `hey I'm an ID for bucket key #${mailIndex}`,
}),
})
const instanceSessionKeys: InstanceSessionKey[] = []
for (const attachmentIndex of lazyNumberRange(0, attachmentCount)) {
const attachmentId = sessionKeyId(mailIndex, attachmentIndex)
const instanceSessionKey = createTestEntity(InstanceSessionKeyTypeRef, {
instanceId: attachmentId,
symEncSessionKey: new Uint8Array([mailIndex, attachmentIndex, 3, 4]),
symKeyVersion: `${mailIndex}`,
})
instanceSessionKeys.push(instanceSessionKey)
mail.attachments.push(["someListId", attachmentId])
}
when(cryptoFacade.resolveWithBucketKey(mail)).thenResolve({
resolvedSessionKeyForInstance: [],
instanceSessionKeys,
})
return mail
}
async function checkMail(resolver: OwnerEncSessionKeyProvider, fileCount: number, mails: readonly Mail[]) {
for (const [mailIndex, mail] of mails.entries()) {
for (const [attachmentIndex, attachmentId] of mail.attachments.entries()) {
const attachment = createTestEntity(FileTypeRef, {
_id: attachmentId,
name: `file_${attachmentIndex}`,
})
o.check(await resolver(elementIdPart(attachmentId), attachment)).deepEquals({
key: new Uint8Array([mailIndex, attachmentIndex, 3, 4]),
encryptingKeyVersion: mailIndex as KeyVersion,
})(`hellooooo I'm an ID for some file instance #${attachmentId} for mail #${mailIndex}`)
}
}
}
o.test("one mail with no bucket key", async () => {
const mail = createTestEntity(MailTypeRef)
await facade.createOwnerEncSessionKeyProviderForAttachments([mail])
// since our resolver will do nothing, we just need to ensure that cryptoFacade was never called in the first place
verify(cryptoFacade.resolveWithBucketKey(matchers.anything()), { times: 0 })
})
o.test("one mail with one file instance", async () => {
const mails = [setUpMail(0, 1)]
const resolver = await facade.createOwnerEncSessionKeyProviderForAttachments(mails)
await checkMail(resolver, 1, mails)
})
o.test("a lot of mails with one file instance", async () => {
const count = 100
const instanceCount = 1
const mails: Mail[] = []
for (let i = 0; i < count; i++) {
mails.push(setUpMail(i, instanceCount))
}
const resolver = await facade.createOwnerEncSessionKeyProviderForAttachments(mails)
await checkMail(resolver, instanceCount, mails)
})
o.test("one mail with many file instances", async () => {
const instanceCount = 256
const mails = [setUpMail(0, instanceCount)]
const resolver = await facade.createOwnerEncSessionKeyProviderForAttachments(mails)
await checkMail(resolver, instanceCount, mails)
})
o.test("a lot of mails with many file instances", async () => {
const count = 100
const instanceCount = 64
const mails: Mail[] = []
for (let i = 0; i < count; i++) {
mails.push(setUpMail(i, instanceCount))
}
const resolver = await facade.createOwnerEncSessionKeyProviderForAttachments(mails)
await checkMail(resolver, instanceCount, mails)
})
o.test("when already decrypted it just returns the key", async () => {
const mail = setUpMail(0, 1)
mail.bucketKey = null
const resolver = await facade.createOwnerEncSessionKeyProviderForAttachments([mail])
const expectedSK: VersionedEncryptedKey = {
key: new Uint8Array([1, 2, 3, 4]),
encryptingKeyVersion: 10,
}
const attachment = createTestEntity(FileTypeRef, {
_id: mail.attachments[0],
_ownerEncSessionKey: expectedSK.key,
_ownerKeyVersion: String(expectedSK.encryptingKeyVersion),
name: `file_${0}`,
})
o.check(await resolver(getElementId(attachment), attachment)).deepEquals(expectedSK)
})
})
o.spec("addRecipientKeyData", function () {
o("correctly throws RecipientsNotFoundError", async function () {
const bucketKey: AesKey = object()
const sendDraftData: SendDraftData = object()
const senderMailGroupId: Id = object()
const notFoundRecipient1: Recipient = object()
// @ts-ignore
notFoundRecipient1.address = "one@tuta.com"
const notFoundRecipient2: Recipient = object()
// @ts-ignore
notFoundRecipient2.address = "two@tuta.com"
const someRecipient1: Recipient = object()
const someRecipient2: Recipient = object()
const recipients: Array<Recipient> = [someRecipient1, notFoundRecipient1, someRecipient2, notFoundRecipient2]
const captor = matchers.captor()
when(
cryptoFacade.encryptBucketKeyForInternalRecipient(
matchers.anything(),
matchers.anything(),
notFoundRecipient1.address,
captor.capture(),
matchers.anything(),
),
).thenDo(() => {
const notFoundRecipients: string[] = captor.value
notFoundRecipients.push(notFoundRecipient1.address)
})
when(
cryptoFacade.encryptBucketKeyForInternalRecipient(
matchers.anything(),
matchers.anything(),
notFoundRecipient2.address,
captor.capture(),
matchers.anything(),
),
).thenDo(() => {
const notFoundRecipients: string[] = captor.value
notFoundRecipients.push(notFoundRecipient2.address)
})
const err = await assertThrows(RecipientsNotFoundError, async () => {
// @ts-ignore
await facade.addRecipientKeyData(bucketKey, sendDraftData, recipients, senderMailGroupId)
})
o(err.message).equals(`${notFoundRecipient1.address}\n${notFoundRecipient2.address}`)
})
o("correctly throws KeyVerificationMismatchError", async function () {
const bucketKey: AesKey = object()
const sendDraftData: SendDraftData = object()
const senderMailGroupId: Id = object()
const unverifiedRecipient1: Recipient = object()
// @ts-ignore
unverifiedRecipient1.address = "one@tuta.com"
const unverifiedRecipient2: Recipient = object()
// @ts-ignore
unverifiedRecipient2.address = "two@tuta.com"
const someRecipient1: Recipient = object()
const someRecipient2: Recipient = object()
const recipients: Array<Recipient> = [someRecipient1, unverifiedRecipient1, someRecipient2, unverifiedRecipient2]
const captor = matchers.captor()
when(
cryptoFacade.encryptBucketKeyForInternalRecipient(
matchers.anything(),
matchers.anything(),
unverifiedRecipient1.address,
matchers.anything(),
captor.capture(),
),
).thenDo(() => {
const mismatchRecipients: string[] = captor.value
mismatchRecipients.push(unverifiedRecipient1.address)
})
when(
cryptoFacade.encryptBucketKeyForInternalRecipient(
matchers.anything(),
matchers.anything(),
unverifiedRecipient2.address,
matchers.anything(),
captor.capture(),
),
).thenDo(() => {
const mismatchRecipients: string[] = captor.value
mismatchRecipients.push(unverifiedRecipient2.address)
})
const err = await assertThrows(KeyVerificationMismatchError, async () => {
// @ts-ignore
await facade.addRecipientKeyData(bucketKey, sendDraftData, recipients, senderMailGroupId)
})
o(err.data).deepEquals([unverifiedRecipient1.address, unverifiedRecipient2.address])
})
})
2022-12-27 15:37:40 +01:00
})