store ServerModelParsedInstances in offline db

Co-authored-by: nig <nig@tutao.de>
Co-authored-by: abp <abp@tutao.de>
Co-authored-by: jhm <17314077+jomapp@users.noreply.github.com>
Co-authored-by: das <das@tutao.de>
This commit is contained in:
mac-github 2025-05-06 11:25:24 +02:00 committed by map
parent 593c98b461
commit 279278179b
79 changed files with 1872 additions and 2649 deletions

View file

@ -1,13 +1,16 @@
import { BlobElementEntity, ElementEntity, ListElementEntity, SomeEntity } from "../../common/EntityTypes.js"
import { Entity, ListElementEntity, ServerModelParsedInstance, TypeModel } from "../../common/EntityTypes.js"
import { EntityRestClient } from "./EntityRestClient.js"
import { firstBiggerThanSecond } from "../../common/utils/EntityUtils.js"
import { CacheStorage, expandId, LastUpdateTime } from "./DefaultEntityRestCache.js"
import { assertNotNull, clone, getFromMap, getTypeId, remove, TypeRef } from "@tutao/tutanota-utils"
import { assertNotNull, clone, getFromMap, getTypeString, remove, TypeRef } from "@tutao/tutanota-utils"
import { CustomCacheHandlerMap } from "./CustomCacheHandler.js"
import { resolveClientTypeReference, resolveServerTypeReference } from "../../common/EntityFunctions.js"
import { Type as TypeId } from "../../common/EntityConstants.js"
import { ProgrammingError } from "../../common/error/ProgrammingError.js"
import { customIdToBase64Url, ensureBase64Ext } from "../offline/OfflineStorage.js"
import { AttributeModel } from "../../common/AttributeModel"
import { ModelMapper } from "../crypto/ModelMapper"
import { parseTypeString } from "@tutao/tutanota-utils/dist/TypeRef"
/** Cache for a single list. */
type ListCache = {
@ -16,7 +19,7 @@ type ListCache = {
lowerRangeId: Id
upperRangeId: Id
/** All the entities loaded, inside or outside the range (e.g. load for a single entity). */
elements: Map<Id, ListElementEntity>
elements: Map<Id, ServerModelParsedInstance>
}
/** Map from list id to list cache. */
@ -24,7 +27,7 @@ type ListTypeCache = Map<Id, ListCache>
type BlobElementCache = {
/** All the entities loaded, inside or outside the range (e.g. load for a single entity). */
elements: Map<Id, BlobElementEntity>
elements: Map<Id, ServerModelParsedInstance>
}
/** Map from list id to list cache. */
@ -36,7 +39,7 @@ export interface EphemeralStorageInitArgs {
export class EphemeralCacheStorage implements CacheStorage {
/** Path to id to entity map. */
private readonly entities: Map<string, Map<Id, ElementEntity>> = new Map()
private readonly entities: Map<string, Map<Id, ServerModelParsedInstance>> = new Map()
private readonly lists: Map<string, ListTypeCache> = new Map()
private readonly blobEntities: Map<string, BlobElementTypeCache> = new Map()
private readonly customCacheHandlerMap: CustomCacheHandlerMap = new CustomCacheHandlerMap()
@ -44,6 +47,8 @@ export class EphemeralCacheStorage implements CacheStorage {
private userId: Id | null = null
private lastBatchIdPerGroup = new Map<Id, Id>()
constructor(private readonly modelMapper: ModelMapper) {}
init({ userId }: EphemeralStorageInitArgs) {
this.userId = userId
}
@ -60,155 +65,41 @@ export class EphemeralCacheStorage implements CacheStorage {
/**
* Get a given entity from the cache, expects that you have already checked for existence
*/
async get<T extends SomeEntity>(typeRef: TypeRef<T>, listId: Id | null, elementId: Id): Promise<T | null> {
async getParsed(typeRef: TypeRef<unknown>, listId: Id | null, id: Id): Promise<ServerModelParsedInstance | null> {
// We downcast because we can't prove that map has correct entity on the type level
const type = getTypeId(typeRef)
const typeModel = await resolveClientTypeReference(typeRef)
elementId = ensureBase64Ext(typeModel, elementId)
switch (typeModel.type) {
case TypeId.Element:
return clone((this.entities.get(type)?.get(elementId) as T | undefined) ?? null)
case TypeId.ListElement:
return clone((this.lists.get(type)?.get(assertNotNull(listId))?.elements.get(elementId) as T | undefined) ?? null)
case TypeId.BlobElement:
return clone((this.blobEntities.get(type)?.get(assertNotNull(listId))?.elements.get(elementId) as T | undefined) ?? null)
default:
throw new ProgrammingError("must be a persistent type")
}
}
async deleteIfExists<T>(typeRef: TypeRef<T>, listId: Id | null, elementId: Id): Promise<void> {
const type = getTypeId(typeRef)
const typeModel = await resolveClientTypeReference(typeRef)
elementId = ensureBase64Ext(typeModel, elementId)
switch (typeModel.type) {
case TypeId.Element:
this.entities.get(type)?.delete(elementId)
break
case TypeId.ListElement: {
const cache = this.lists.get(type)?.get(assertNotNull(listId))
if (cache != null) {
cache.elements.delete(elementId)
remove(cache.allRange, elementId)
}
break
}
case TypeId.BlobElement:
this.blobEntities.get(type)?.get(assertNotNull(listId))?.elements.delete(elementId)
break
default:
throw new ProgrammingError("must be a persistent type")
}
}
private addElementEntity<T extends ElementEntity>(typeRef: TypeRef<T>, id: Id, entity: T) {
getFromMap(this.entities, getTypeId(typeRef), () => new Map()).set(id, entity)
}
async isElementIdInCacheRange<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id, elementId: Id): Promise<boolean> {
const typeModel = await resolveClientTypeReference(typeRef)
elementId = ensureBase64Ext(typeModel, elementId)
const cache = this.lists.get(getTypeId(typeRef))?.get(listId)
return cache != null && !firstBiggerThanSecond(elementId, cache.upperRangeId) && !firstBiggerThanSecond(cache.lowerRangeId, elementId)
}
async put(originalEntity: SomeEntity): Promise<void> {
const entity = clone(originalEntity)
const typeRef = entity._type
const type = getTypeString(typeRef)
const typeModel = await resolveServerTypeReference(typeRef)
let { listId, elementId } = expandId(originalEntity._id)
elementId = ensureBase64Ext(typeModel, elementId)
id = ensureBase64Ext(typeModel, id)
switch (typeModel.type) {
case TypeId.Element: {
const elementEntity = entity as ElementEntity
this.addElementEntity(elementEntity._type, elementId, elementEntity)
break
}
case TypeId.ListElement: {
const listElementEntity = entity as ListElementEntity
const listElementTypeRef = typeRef as TypeRef<ListElementEntity>
listId = listId as Id
await this.putListElement(listElementTypeRef, listId, elementId, listElementEntity)
break
}
case TypeId.BlobElement: {
const blobElementEntity = entity as BlobElementEntity
const blobTypeRef = typeRef as TypeRef<BlobElementEntity>
listId = listId as Id
await this.putBlobElement(blobTypeRef, listId, elementId, blobElementEntity)
break
}
case TypeId.Element:
return clone(this.entities.get(type)?.get(id) ?? null)
case TypeId.ListElement:
return clone(this.lists.get(type)?.get(assertNotNull(listId))?.elements.get(id) ?? null)
case TypeId.BlobElement:
return clone(this.blobEntities.get(type)?.get(assertNotNull(listId))?.elements.get(id) ?? null)
default:
throw new ProgrammingError("must be a persistent type")
}
}
private async putBlobElement(typeRef: TypeRef<BlobElementEntity>, listId: Id, elementId: Id, entity: BlobElementEntity) {
const cache = this.blobEntities.get(getTypeId(typeRef))?.get(listId)
if (cache == null) {
// first element in this list
const newCache = {
elements: new Map([[elementId, entity]]),
}
getFromMap(this.blobEntities, getTypeId(typeRef), () => new Map()).set(listId, newCache)
} else {
// if the element already exists in the cache, overwrite it
cache.elements.set(elementId, entity)
}
}
/** prcondition: elementId is converted to base64ext if necessary */
private async putListElement(typeRef: TypeRef<ListElementEntity>, listId: Id, elementId: Id, entity: ListElementEntity) {
const typeId = getTypeId(typeRef)
const cache = this.lists.get(typeId)?.get(listId)
if (cache == null) {
// first element in this list
const newCache = {
allRange: [elementId],
lowerRangeId: elementId,
upperRangeId: elementId,
elements: new Map([[elementId, entity]]),
}
getFromMap(this.lists, typeId, () => new Map()).set(listId, newCache)
} else {
// if the element already exists in the cache, overwrite it
// add new element to existing list if necessary
cache.elements.set(elementId, entity)
const typeModel = await resolveClientTypeReference(typeRef)
if (await this.isElementIdInCacheRange(typeRef, listId, customIdToBase64Url(typeModel, elementId))) {
this.insertIntoRange(cache.allRange, elementId)
}
}
}
/** precondition: elementId is converted to base64ext if necessary */
private insertIntoRange(allRange: Array<Id>, elementId: Id) {
for (let i = 0; i < allRange.length; i++) {
const rangeElement = allRange[i]
if (firstBiggerThanSecond(rangeElement, elementId)) {
allRange.splice(i, 0, elementId)
return
}
if (rangeElement === elementId) {
return
}
}
allRange.push(elementId)
}
async provideFromRange<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id, startElementId: Id, count: number, reverse: boolean): Promise<T[]> {
const typeModel = await resolveClientTypeReference(typeRef)
async provideFromRangeParsed(
typeRef: TypeRef<unknown>,
listId: string,
startElementId: string,
count: number,
reverse: boolean,
): Promise<ServerModelParsedInstance[]> {
const typeModel = await resolveServerTypeReference(typeRef)
startElementId = ensureBase64Ext(typeModel, startElementId)
const listCache = this.lists.get(getTypeId(typeRef))?.get(listId)
const listCache = this.lists.get(getTypeString(typeRef))?.get(listId)
if (listCache == null) {
return []
}
let range = listCache.allRange
let ids: Id[] = []
let ids: Id[]
if (reverse) {
let i
for (i = range.length - 1; i >= 0; i--) {
@ -231,29 +122,180 @@ export class EphemeralCacheStorage implements CacheStorage {
const i = range.findIndex((id) => firstBiggerThanSecond(id, startElementId))
ids = range.slice(i, i + count)
}
let result: T[] = []
let result: ServerModelParsedInstance[] = []
for (let a = 0; a < ids.length; a++) {
result.push(clone(listCache.elements.get(ids[a]) as T))
const cachedInstance = listCache.elements.get(ids[a])
if (cachedInstance != null) {
const clonedInstance = clone(cachedInstance)
result.push(clonedInstance)
}
}
return result
}
async provideMultiple<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id, elementIds: Id[]): Promise<Array<T>> {
const listCache = this.lists.get(getTypeId(typeRef))?.get(listId)
async provideMultipleParsed(typeRef: TypeRef<unknown>, listId: string, elementIds: string[]): Promise<ServerModelParsedInstance[]> {
const listCache = this.lists.get(getTypeString(typeRef))?.get(listId)
const typeModel = await resolveClientTypeReference(typeRef)
const typeModel = await resolveServerTypeReference(typeRef)
elementIds = elementIds.map((el) => ensureBase64Ext(typeModel, el))
if (listCache == null) {
return []
}
let result: T[] = []
let result: Array<ServerModelParsedInstance> = []
for (let a = 0; a < elementIds.length; a++) {
result.push(clone(listCache.elements.get(elementIds[a]) as T))
const cachedItem = listCache.elements.get(elementIds[a])
if (cachedItem) {
const clonedItem = clone(cachedItem)
result.push(clonedItem)
}
}
return result
}
async getWholeListParsed(typeRef: TypeRef<unknown>, listId: string): Promise<ServerModelParsedInstance[]> {
const listCache = this.lists.get(getTypeString(typeRef))?.get(listId)
if (listCache == null) {
return []
}
return listCache.allRange.map((id) => clone(listCache.elements.get(id)!))
}
async get<T extends Entity>(typeRef: TypeRef<T>, listId: string | null, id: string): Promise<T | null> {
const parsedInstance = await this.getParsed(typeRef, listId, id)
if (parsedInstance == null) {
return null
}
return await this.modelMapper.mapToInstance<T>(typeRef, parsedInstance)
}
async deleteIfExists<T>(typeRef: TypeRef<T>, listId: Id | null, elementId: Id): Promise<void> {
const type = getTypeString(typeRef)
const typeModel = await resolveServerTypeReference(typeRef)
elementId = ensureBase64Ext(typeModel, elementId)
switch (typeModel.type) {
case TypeId.Element:
this.entities.get(type)?.delete(elementId)
break
case TypeId.ListElement: {
const cache = this.lists.get(type)?.get(assertNotNull(listId))
if (cache != null) {
cache.elements.delete(elementId)
remove(cache.allRange, elementId)
}
break
}
case TypeId.BlobElement:
this.blobEntities.get(type)?.get(assertNotNull(listId))?.elements.delete(elementId)
break
default:
throw new ProgrammingError("must be a persistent type")
}
}
private putElementEntity(typeRef: TypeRef<unknown>, id: Id, entity: ServerModelParsedInstance) {
getFromMap(this.entities, getTypeString(typeRef), () => new Map()).set(id, entity)
}
async isElementIdInCacheRange(typeRef: TypeRef<unknown>, listId: Id, elementId: Id): Promise<boolean> {
const typeModel = await resolveServerTypeReference(typeRef)
elementId = ensureBase64Ext(typeModel, elementId)
const cache = this.lists.get(getTypeString(typeRef))?.get(listId)
return cache != null && !firstBiggerThanSecond(elementId, cache.upperRangeId) && !firstBiggerThanSecond(cache.lowerRangeId, elementId)
}
async put(typeRef: TypeRef<unknown>, instance: ServerModelParsedInstance): Promise<void> {
const instanceClone = clone(instance)
const typeModel = await resolveServerTypeReference(typeRef)
const instanceId = AttributeModel.getAttribute<IdTuple | Id>(instanceClone, "_id", typeModel)
let { listId, elementId } = expandId(instanceId)
elementId = ensureBase64Ext(typeModel, elementId)
switch (typeModel.type) {
case TypeId.Element: {
this.putElementEntity(typeRef, elementId, instanceClone)
break
}
case TypeId.ListElement: {
listId = listId as Id
await this.putListElement(typeRef, listId, elementId, instanceClone)
break
}
case TypeId.BlobElement: {
listId = listId as Id
await this.putBlobElement(typeRef, listId, elementId, instanceClone)
break
}
default:
throw new ProgrammingError("must be a persistent type")
}
}
private async putBlobElement(typeRef: TypeRef<unknown>, listId: Id, elementId: Id, entity: ServerModelParsedInstance) {
const cache = this.blobEntities.get(getTypeString(typeRef))?.get(listId)
if (cache == null) {
// first element in this list
const newCache = {
elements: new Map([[elementId, entity]]),
}
getFromMap(this.blobEntities, getTypeString(typeRef), () => new Map()).set(listId, newCache)
} else {
// if the element already exists in the cache, overwrite it
cache.elements.set(elementId, entity)
}
}
/** @pre: elementId is converted to base64ext if necessary */
private async putListElement(typeRef: TypeRef<unknown>, listId: Id, elementId: Id, entity: ServerModelParsedInstance) {
const typeId = getTypeString(typeRef)
const cache = this.lists.get(typeId)?.get(listId)
if (cache == null) {
// first element in this list
const newCache = {
allRange: [elementId],
lowerRangeId: elementId,
upperRangeId: elementId,
elements: new Map([[elementId, entity]]),
}
getFromMap(this.lists, typeId, () => new Map()).set(listId, newCache)
} else {
// if the element already exists in the cache, overwrite it
// add new element to existing list if necessary
cache.elements.set(elementId, entity)
const typeModel = await resolveServerTypeReference(typeRef)
if (await this.isElementIdInCacheRange(typeRef, listId, customIdToBase64Url(typeModel, elementId))) {
this.insertIntoRange(cache.allRange, elementId)
}
}
}
/** precondition: elementId is converted to base64ext if necessary */
private insertIntoRange(allRange: Array<Id>, elementId: Id) {
for (let i = 0; i < allRange.length; i++) {
const rangeElement = allRange[i]
if (firstBiggerThanSecond(rangeElement, elementId)) {
allRange.splice(i, 0, elementId)
return
}
if (rangeElement === elementId) {
return
}
}
allRange.push(elementId)
}
async provideFromRange<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id, startElementId: Id, count: number, reverse: boolean): Promise<T[]> {
const parsedInstances = await this.provideFromRangeParsed(typeRef, listId, startElementId, count, reverse)
return await this.modelMapper.mapToInstances(typeRef, parsedInstances)
}
async provideMultiple<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id, elementIds: Id[]): Promise<Array<T>> {
const parsedInstances = await this.provideMultipleParsed(typeRef, listId, elementIds)
return await this.modelMapper.mapToInstances(typeRef, parsedInstances)
}
async getRangeForList<T extends ListElementEntity>(
typeRef: TypeRef<T>,
listId: Id,
@ -261,13 +303,13 @@ export class EphemeralCacheStorage implements CacheStorage {
lower: Id
upper: Id
} | null> {
const listCache = this.lists.get(getTypeId(typeRef))?.get(listId)
const listCache = this.lists.get(getTypeString(typeRef))?.get(listId)
if (listCache == null) {
return null
}
const typeModel = await resolveClientTypeReference(typeRef)
const typeModel = await resolveServerTypeReference(typeRef)
return {
lower: customIdToBase64Url(typeModel, listCache.lowerRangeId),
upper: customIdToBase64Url(typeModel, listCache.upperRangeId),
@ -275,9 +317,9 @@ export class EphemeralCacheStorage implements CacheStorage {
}
async setUpperRangeForList<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id, upperId: Id): Promise<void> {
const typeModel = await resolveClientTypeReference(typeRef)
const typeModel = await resolveServerTypeReference(typeRef)
upperId = ensureBase64Ext(typeModel, upperId)
const listCache = this.lists.get(getTypeId(typeRef))?.get(listId)
const listCache = this.lists.get(getTypeString(typeRef))?.get(listId)
if (listCache == null) {
throw new Error("list does not exist")
}
@ -285,9 +327,9 @@ export class EphemeralCacheStorage implements CacheStorage {
}
async setLowerRangeForList<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id, lowerId: Id): Promise<void> {
const typeModel = await resolveClientTypeReference(typeRef)
const typeModel = await resolveServerTypeReference(typeRef)
lowerId = ensureBase64Ext(typeModel, lowerId)
const listCache = this.lists.get(getTypeId(typeRef))?.get(listId)
const listCache = this.lists.get(getTypeString(typeRef))?.get(listId)
if (listCache == null) {
throw new Error("list does not exist")
}
@ -302,11 +344,11 @@ export class EphemeralCacheStorage implements CacheStorage {
* @param upper
*/
async setNewRangeForList<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id, lower: Id, upper: Id): Promise<void> {
const typeModel = await resolveClientTypeReference(typeRef)
const typeModel = await resolveServerTypeReference(typeRef)
lower = ensureBase64Ext(typeModel, lower)
upper = ensureBase64Ext(typeModel, upper)
const typeId = getTypeId(typeRef)
const typeId = getTypeString(typeRef)
const listCache = this.lists.get(typeId)?.get(listId)
if (listCache == null) {
getFromMap(this.lists, typeId, () => new Map()).set(listId, {
@ -323,10 +365,10 @@ export class EphemeralCacheStorage implements CacheStorage {
}
async getIdsInRange<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id): Promise<Array<Id>> {
const typeModel = await resolveClientTypeReference(typeRef)
const typeModel = await resolveServerTypeReference(typeRef)
return (
this.lists
.get(getTypeId(typeRef))
.get(getTypeString(typeRef))
?.get(listId)
?.allRange.map((elementId) => {
return customIdToBase64Url(typeModel, elementId)
@ -355,16 +397,11 @@ export class EphemeralCacheStorage implements CacheStorage {
}
async getWholeList<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id): Promise<Array<T>> {
const listCache = this.lists.get(getTypeId(typeRef))?.get(listId)
if (listCache == null) {
return []
}
return listCache.allRange.map((id) => clone(listCache.elements.get(id) as T))
const parsedInstances = await this.getWholeListParsed(typeRef, listId)
return await this.modelMapper.mapToInstances(typeRef, parsedInstances)
}
getCustomCacheHandlerMap(entityRestClient: EntityRestClient): CustomCacheHandlerMap {
getCustomCacheHandlerMap(_: EntityRestClient): CustomCacheHandlerMap {
return this.customCacheHandlerMap
}
@ -373,34 +410,42 @@ export class EphemeralCacheStorage implements CacheStorage {
}
async deleteAllOwnedBy(owner: Id): Promise<void> {
for (const typeMap of this.entities.values()) {
for (const [typeString, typeMap] of this.entities.entries()) {
const typeRef = parseTypeString(typeString)
const typeModel = await resolveServerTypeReference(typeRef)
for (const [id, entity] of typeMap.entries()) {
if (entity._ownerGroup === owner) {
const ownerGroup = AttributeModel.getAttribute<Id>(entity, "_ownerGroup", typeModel)
if (ownerGroup === owner) {
typeMap.delete(id)
}
}
}
for (const cacheForType of this.lists.values()) {
this.deleteAllOwnedByFromCache(cacheForType, owner)
for (const [typeString, cacheForType] of this.lists.entries()) {
const typeRef = parseTypeString(typeString)
const typeModel = await resolveServerTypeReference(typeRef)
this.deleteAllOwnedByFromCache(typeModel, cacheForType, owner)
}
for (const cacheForType of this.blobEntities.values()) {
this.deleteAllOwnedByFromCache(cacheForType, owner)
for (const [typeString, cacheForType] of this.blobEntities.entries()) {
const typeRef = parseTypeString(typeString)
const typeModel = await resolveServerTypeReference(typeRef)
this.deleteAllOwnedByFromCache(typeModel, cacheForType, owner)
}
this.lastBatchIdPerGroup.delete(owner)
}
async deleteWholeList<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id): Promise<void> {
this.lists.get(getTypeId(typeRef))?.delete(listId)
this.lists.get(getTypeString(typeRef))?.delete(listId)
}
private deleteAllOwnedByFromCache(cacheForType: Map<Id, ListCache | BlobElementCache>, owner: string) {
private deleteAllOwnedByFromCache(typeModel: TypeModel, cacheForType: Map<Id, ListCache | BlobElementCache>, owner: string) {
// If we find at least one element in the list that is owned by our target owner, we delete the entire list.
// This is OK in most cases because the vast majority of lists are single owner.
// For the other cases, we are just clearing the cache a bit sooner than needed.
const listIdsToDelete: string[] = []
for (const [listId, listCache] of cacheForType.entries()) {
for (const [id, element] of listCache.elements.entries()) {
if (element._ownerGroup === owner) {
for (const [_, element] of listCache.elements.entries()) {
const ownerGroup = AttributeModel.getAttribute<Id>(element, "_ownerGroup", typeModel)
if (ownerGroup === owner) {
listIdsToDelete.push(listId)
break
}
@ -418,17 +463,15 @@ export class EphemeralCacheStorage implements CacheStorage {
/**
* We want to lock the access to the "ranges" db when updating / reading the
* offline available mail list ranges for each mail list (referenced using the listId)
* @param listId the mail list that we want to lock
*/
lockRangesDbAccess(listId: string): Promise<void> {
lockRangesDbAccess(_: string): Promise<void> {
return Promise.resolve()
}
/**
* This is the counterpart to the function "lockRangesDbAccess(listId)"
* @param listId the mail list that we want to unlock
*/
unlockRangesDbAccess(listId: string): Promise<void> {
unlockRangesDbAccess(_: string): Promise<void> {
return Promise.resolve()
}
}