mirror of
https://github.com/tutao/tutanota.git
synced 2025-12-08 06:09:50 +00:00
Add SQLite search on clients where offline storage is available
- Introduce a separate Indexer for SQLite using FTS5 - Split search backends and use the right one based on client (IndexedDB for Browser, and OfflineStorage everywhere else) - Split SearchFacade into two implementations - Adds a table for storing unindexed metadata for mails - Escape special character for SQLite search To escape special characters from fts5 syntax. However, simply surrounding each token in quotes is sufficient to do this. See section 3.1 "FTS5 Strings" here: https://www.sqlite.org/fts5.html which states that a string may be specified by surrounding it in quotes, and that special string requirements only exist for strings that are not in quotes. - Add EncryptedDbWrapper - Simplify out of sync logic in IndexedDbIndexer - Fix deadlock when initializing IndexedDbIndexer - Cleanup indexedDb index when migrating to offline storage index - Pass contactSuggestionFacade to IndexedDbSearchFacade The only suggestion facade used by IndexedDbSearchFacade was the contact suggestion facade. So we made it clearer. - Remove IndexerCore stats - Split custom cache handlers into separate files We were already doing this with user, so we should do this with the other entity types. - Rewrite IndexedDb tests - Add OfflineStorage indexer tests - Add custom cache handlers tests to OfflineStorageTest - Add tests for custom cache handlers with ephemeral storage - Use dbStub instead of dbMock in IndexedDbIndexerTest - Replace spy with testdouble in IndexedDbIndexerTest Close #8550 Co-authored-by: ivk <ivk@tutao.de> Co-authored-by: paw <paw-hub@users.noreply.github.com> Co-authored-by: wrd <wrd@tutao.de> Co-authored-by: bir <bir@tutao.de> Co-authored-by: hrb-hub <hrb-hub@users.noreply.github.com>
This commit is contained in:
ivk
hrb-hub
parent
94e09ee634
commit
86d5775e16
99 changed files with 7820 additions and 6286 deletions
|
|
@ -9,7 +9,7 @@ import {
|
|||
OwnerEncSessionKeyProvider,
|
||||
} from "./EntityRestClient"
|
||||
import { OperationType } from "../../common/TutanotaConstants"
|
||||
import { assertNotNull, difference, getFirstOrThrow, getTypeString, groupBy, isSameTypeRef, lastThrow, TypeRef } from "@tutao/tutanota-utils"
|
||||
import { assertNotNull, getFirstOrThrow, getTypeString, groupBy, isSameTypeRef, lastThrow, TypeRef } from "@tutao/tutanota-utils"
|
||||
import {
|
||||
AuditLogEntryTypeRef,
|
||||
BucketPermissionTypeRef,
|
||||
|
|
@ -22,10 +22,8 @@ import {
|
|||
RejectedSenderTypeRef,
|
||||
SecondFactorTypeRef,
|
||||
SessionTypeRef,
|
||||
User,
|
||||
UserGroupKeyDistributionTypeRef,
|
||||
UserGroupRootTypeRef,
|
||||
UserTypeRef,
|
||||
} from "../../entities/sys/TypeRefs.js"
|
||||
import { ValueType } from "../../common/EntityConstants.js"
|
||||
import { NotAuthorizedError, NotFoundError } from "../../common/error/RestError"
|
||||
|
|
@ -45,11 +43,11 @@ import { ProgrammingError } from "../../common/error/ProgrammingError"
|
|||
import { assertWorkerOrNode } from "../../common/Env"
|
||||
import type { Entity, ListElementEntity, ServerModelParsedInstance, SomeEntity, TypeModel } from "../../common/EntityTypes"
|
||||
import { ENTITY_EVENT_BATCH_EXPIRE_MS } from "../EventBusClient"
|
||||
import { CustomCacheHandlerMap } from "./CustomCacheHandler.js"
|
||||
import { CustomCacheHandlerMap } from "./cacheHandler/CustomCacheHandler.js"
|
||||
import { containsEventOfType, EntityUpdateData, getEventOfType, isUpdateForTypeRef } from "../../common/utils/EntityUpdateUtils.js"
|
||||
import { isCustomIdType } from "../offline/OfflineStorage.js"
|
||||
import { AttributeModel } from "../../common/AttributeModel"
|
||||
import { TypeModelResolver } from "../../common/EntityFunctions"
|
||||
import { AttributeModel } from "../../common/AttributeModel"
|
||||
|
||||
assertWorkerOrNode()
|
||||
|
||||
|
|
@ -170,9 +168,6 @@ export interface ExposedCacheStorage {
|
|||
* we must maintain the integrity of our list ranges.
|
||||
* */
|
||||
deleteIfExists<T extends SomeEntity>(typeRef: TypeRef<T>, listId: Id | null, id: Id): Promise<void>
|
||||
|
||||
/** delete all instances of the given type that share {@param listId}. also deletes the range of that list. */
|
||||
deleteWholeList<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id): Promise<void>
|
||||
}
|
||||
|
||||
export interface CacheStorage extends ExposedCacheStorage {
|
||||
|
|
@ -206,7 +201,7 @@ export interface CacheStorage extends ExposedCacheStorage {
|
|||
* get a map with cache handlers for the customId types this storage implementation supports
|
||||
* customId types that don't have a custom handler don't get served from the cache
|
||||
*/
|
||||
getCustomCacheHandlerMap(entityRestClient: EntityRestClient): CustomCacheHandlerMap
|
||||
getCustomCacheHandlerMap(): CustomCacheHandlerMap
|
||||
|
||||
isElementIdInCacheRange<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id, id: Id): Promise<boolean>
|
||||
|
||||
|
|
@ -427,7 +422,7 @@ export class DefaultEntityRestCache implements EntityRestCache {
|
|||
reverse: boolean,
|
||||
opts: EntityRestClientLoadOptions = {},
|
||||
): Promise<T[]> {
|
||||
const customHandler = this.storage.getCustomCacheHandlerMap(this.entityRestClient).get(typeRef)
|
||||
const customHandler = this.storage.getCustomCacheHandlerMap().get(typeRef)
|
||||
if (customHandler && customHandler.loadRange) {
|
||||
return await customHandler.loadRange(this.storage, listId, start, count, reverse)
|
||||
}
|
||||
|
|
@ -758,7 +753,7 @@ export class DefaultEntityRestCache implements EntityRestCache {
|
|||
const ids = updates.map((update) => update.instanceId)
|
||||
|
||||
// We only want to load the instances that are in cache range
|
||||
const customHandler = this.storage.getCustomCacheHandlerMap(this.entityRestClient).get(typeRef)
|
||||
const customHandler = this.storage.getCustomCacheHandlerMap().get(typeRef)
|
||||
const idsInCacheRange =
|
||||
customHandler && customHandler.getElementIdsInCacheRange
|
||||
? await customHandler.getElementIdsInCacheRange(this.storage, instanceListId, ids)
|
||||
|
|
@ -791,8 +786,8 @@ export class DefaultEntityRestCache implements EntityRestCache {
|
|||
}
|
||||
}
|
||||
|
||||
const otherEventUpdates: EntityUpdateData[] = []
|
||||
// we need an array of UpdateEntityData
|
||||
const otherEventUpdates: EntityUpdateData[] = []
|
||||
for (let update of regularUpdates) {
|
||||
const { operation, typeRef } = update
|
||||
const { instanceListId, instanceId } = getUpdateInstanceId(update)
|
||||
|
|
@ -868,7 +863,7 @@ export class DefaultEntityRestCache implements EntityRestCache {
|
|||
// If there is a custom handler we follow its decision.
|
||||
// Otherwise, we do a range check to see if we need to keep the range up-to-date.
|
||||
const shouldLoad =
|
||||
(await this.storage.getCustomCacheHandlerMap(this.entityRestClient).get(typeRef)?.shouldLoadOnCreateEvent?.(update)) ??
|
||||
this.storage.getCustomCacheHandlerMap().get(typeRef)?.shouldLoadOnCreateEvent?.(update) ??
|
||||
(await this.storage.isElementIdInCacheRange(typeRef, instanceListId, instanceId))
|
||||
if (shouldLoad) {
|
||||
// No need to try to download something that's not there anymore
|
||||
|
|
@ -927,9 +922,6 @@ export class DefaultEntityRestCache implements EntityRestCache {
|
|||
console.log("DefaultEntityRestCache - processUpdateEvent of type Group:" + instanceId)
|
||||
}
|
||||
const newEntity = await this.entityRestClient.loadParsedInstance(typeRef, collapseId(instanceListId, instanceId))
|
||||
if (isSameTypeRef(typeRef, UserTypeRef)) {
|
||||
await this.handleUpdatedUser(cached, newEntity)
|
||||
}
|
||||
await this.storage.put(typeRef, newEntity)
|
||||
return update
|
||||
} catch (e) {
|
||||
|
|
@ -947,23 +939,6 @@ export class DefaultEntityRestCache implements EntityRestCache {
|
|||
return update
|
||||
}
|
||||
|
||||
private async handleUpdatedUser(cachedUserInstance: ServerModelParsedInstance, newUserInstance: ServerModelParsedInstance) {
|
||||
// When we are removed from a group we just get an update for our user
|
||||
// with no membership on it. We need to clean up all the entities that
|
||||
// belong to that group since we shouldn't be able to access them anymore
|
||||
// and we won't get any update or another chance to clean them up.
|
||||
const oldUser = await this.entityRestClient.mapInstanceToEntity<User>(UserTypeRef, cachedUserInstance)
|
||||
if (oldUser._id !== this.storage.getUserId()) {
|
||||
return
|
||||
}
|
||||
const newUser = await this.entityRestClient.mapInstanceToEntity<User>(UserTypeRef, newUserInstance)
|
||||
const removedShips = difference(oldUser.memberships, newUser.memberships, (l, r) => l._id === r._id)
|
||||
for (const ship of removedShips) {
|
||||
console.log("Lost membership on ", ship._id, ship.groupType)
|
||||
await this.storage.deleteAllOwnedBy(ship.group)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @returns {Array<Id>} the ids that are in cache range and therefore should be cached
|
||||
|
|
@ -1026,7 +1001,10 @@ export function collapseId(listId: Id | null, elementId: Id): Id | IdTuple {
|
|||
}
|
||||
}
|
||||
|
||||
export function getUpdateInstanceId(update: EntityUpdateData): { instanceListId: Id | null; instanceId: Id } {
|
||||
export function getUpdateInstanceId(update: EntityUpdateData): {
|
||||
instanceListId: Id | null
|
||||
instanceId: Id
|
||||
} {
|
||||
let instanceListId
|
||||
if (update.instanceListId === "") {
|
||||
instanceListId = null
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue