Stowage/tutanota
2
0
Fork 0
mirror of https://github.com/tutao/tutanota.git synced 2025-12-08 06:09:50 +00:00
Code Issues Projects Releases Packages Wiki Activity

allow read only cache part one, #1163

Browse source
This commit is contained in:
ivk 2019-03-20 12:17:50 +01:00
parent f24f5db589
commit 2d70f85273
6 changed files with 139 additions and 47 deletions
Download patch file Download diff file Expand all files Collapse all files

23
src/api/common/EntityFunctions.js
View file

@ -2,7 +2,8 @@
import {base64ToBase64Url, base64ToUint8Array, base64UrlToBase64, stringToUtf8Uint8Array, uint8ArrayToBase64, utf8Uint8ArrayToString} from "./utils/Encoding"
import EC from "./EntityConstants"
import {asyncImport} from "./utils/Utils"
import {last} from "./utils/ArrayUtils" // importing with {} from CJS modules is not supported for dist-builds currently (must be a systemjs builder bug)
import {last} from "./utils/ArrayUtils"
const Type = EC.Type
const ValueType = EC.ValueType
const Cardinality = EC.Cardinality
@ -42,6 +43,8 @@ export const CUSTOM_MIN_ID = ""
export const RANGE_ITEM_LIMIT = 1000
export const LOAD_MULTIPLE_LIMIT = 100
export const READ_ONLY_HEADER = "read-only"
/**
* Attention: TypeRef must be defined as class and not as Flow type. Flow does not respect flow types with generics when checking return values of the generic class. See https://github.com/facebook/flow/issues/3348
*/
@ -182,7 +185,7 @@ export function _loadEntity<T>(typeRef: TypeRef<T>, id: Id | IdTuple, queryParam
}
export function _loadMultipleEntities<T>(typeRef: TypeRef<T>, listId: ?Id, elementIds: Id[], target: EntityRestInterface): Promise<T[]> {
export function _loadMultipleEntities<T>(typeRef: TypeRef<T>, listId: ?Id, elementIds: Id[], target: EntityRestInterface, extraHeaders?: Params): Promise<T[]> {
// split the ids into chunks
let idChunks = [];
for (let i = 0; i < elementIds.length; i += LOAD_MULTIPLE_LIMIT) {
@ -194,14 +197,15 @@ export function _loadMultipleEntities<T>(typeRef: TypeRef<T>, listId: ?Id, eleme
let queryParams = {
ids: idChunk.join(",")
}
return (target.entityRequest(typeRef, HttpMethod.GET, listId, null, null, queryParams): any)
return (target.entityRequest(typeRef, HttpMethod.GET, listId, null, null, queryParams, extraHeaders): any)
}, {concurrency: 1}).then(instanceChunks => {
return Array.prototype.concat.apply([], instanceChunks);
})
})
}
export function _loadEntityRange<T>(typeRef: TypeRef<T>, listId: Id, start: Id, count: number, reverse: boolean, target: EntityRestInterface): Promise<T[]> {
export function _loadEntityRange<T>(typeRef: TypeRef<T>, listId: Id, start: Id, count: number, reverse: boolean, target: EntityRestInterface,
extraHeaders?: Params): Promise<T[]> {
return resolveTypeReference(typeRef).then(typeModel => {
if (typeModel.type !== Type.ListElement) throw new Error("only ListElement types are permitted")
let queryParams = {
@ -209,20 +213,20 @@ export function _loadEntityRange<T>(typeRef: TypeRef<T>, listId: Id, start: Id,
count: count + "",
reverse: reverse.toString()
}
return (target.entityRequest(typeRef, HttpMethod.GET, listId, null, null, queryParams): any)
return (target.entityRequest(typeRef, HttpMethod.GET, listId, null, null, queryParams, extraHeaders): any)
})
}
export function _loadReverseRangeBetween<T: ListElement>(typeRef: TypeRef<T>, listId: Id, start: Id, end: Id, target: EntityRestInterface,
rangeItemLimit: number): Promise<{elements: T[], loadedCompletely: boolean}> {
rangeItemLimit: number, extraHeaders?: Params): Promise<{elements: T[], loadedCompletely: boolean}> {
return resolveTypeReference(typeRef).then(typeModel => {
if (typeModel.type !== Type.ListElement) throw new Error("only ListElement types are permitted")
return _loadEntityRange(typeRef, listId, start, rangeItemLimit, true, target)
return _loadEntityRange(typeRef, listId, start, rangeItemLimit, true, target, extraHeaders)
.then(loadedEntities => {
const filteredEntities = loadedEntities.filter(entity => firstBiggerThanSecond(getLetId(entity)[1], end))
if (filteredEntities.length === rangeItemLimit) {
const lastElementId = getElementId(filteredEntities[loadedEntities.length - 1])
return _loadReverseRangeBetween(typeRef, listId, lastElementId, end, target, rangeItemLimit)
return _loadReverseRangeBetween(typeRef, listId, lastElementId, end, target, rangeItemLimit, extraHeaders)
.then(({elements: remainingEntities, loadedCompletely}) => {
return {elements: filteredEntities.concat(remainingEntities), loadedCompletely}
})
@ -382,3 +386,6 @@ export function customIdToString(customId: string) {
return utf8Uint8ArrayToString(base64ToUint8Array(base64UrlToBase64(customId)));
}
export function readOnlyHeaders(): Params {
return {[READ_ONLY_HEADER]: "true"}
}

4
src/api/worker/EntityWorker.js
View file

@ -99,8 +99,8 @@ export class EntityWorker {
this._target = target
}
load<T>(typeRef: TypeRef<T>, id: Id | IdTuple, queryParams: ?Params): Promise<T> {
return _loadEntity(typeRef, id, queryParams, this._target)
load<T>(typeRef: TypeRef<T>, id: Id | IdTuple, queryParams: ?Params, extraHeaders?: Params): Promise<T> {
return _loadEntity(typeRef, id, queryParams, this._target, extraHeaders)
}
loadRoot<T>(typeRef: TypeRef<T>, groupId: Id): Promise<T> {

88
src/api/worker/rest/EntityRestCache.js
View file

@ -8,10 +8,11 @@ import {
getLetId,
HttpMethod,
isSameTypeRef,
READ_ONLY_HEADER,
resolveTypeReference,
TypeRef
} from "../../common/EntityFunctions"
import {OperationType} from "../../common/TutanotaConstants"
import {MailState, OperationType} from "../../common/TutanotaConstants"
import {flat, remove} from "../../common/utils/ArrayUtils"
import {clone, downcast, neverNull} from "../../common/utils/Utils"
import {PermissionTypeRef} from "../../entities/sys/Permission"
@ -23,11 +24,13 @@ import {StatisticLogEntryTypeRef} from "../../entities/tutanota/StatisticLogEntr
import {BucketPermissionTypeRef} from "../../entities/sys/BucketPermission"
import {SecondFactorTypeRef} from "../../entities/sys/SecondFactor"
import {RecoverCodeTypeRef} from "../../entities/sys/RecoverCode"
import {MailTypeRef} from "../../entities/tutanota/Mail"
const ValueType = EC.ValueType
assertWorkerOrNode()
/**
* This implementation provides a caching mechanism to the rest chain.
* It forwards requests to the entity rest client.
@ -94,21 +97,24 @@ export class EntityRestCache implements EntityRestInterface {
}
entityRequest<T>(typeRef: TypeRef<T>, method: HttpMethodEnum, listId: ?Id, id: ?Id, entity: ?T, queryParameter: ?Params, extraHeaders?: Params): Promise<any> {
let readOnly = false
if (extraHeaders) {
readOnly = extraHeaders[READ_ONLY_HEADER] === "true"
delete extraHeaders[READ_ONLY_HEADER]
}
if (method === HttpMethod.GET && !this._ignoredTypes.find(ref => isSameTypeRef(typeRef, ref))) {
if ((typeRef.app === "monitor") || (queryParameter && queryParameter["version"])) {
// monitor app and version requests are never cached
return this._entityRestClient.entityRequest(typeRef, method, listId, id, entity, queryParameter, extraHeaders)
} else if (!id && queryParameter && queryParameter["ids"]) {
return this._getMultiple(typeRef, method, listId, id, entity, queryParameter, extraHeaders)
} else if (listId && !id && queryParameter && queryParameter["start"] !== null && queryParameter["start"]
!== undefined && queryParameter["count"] !== null && queryParameter["count"] !== undefined
&& queryParameter["reverse"]) { // check for null and undefined because "" and 0 are als falsy
return this._getMultiple(typeRef, method, listId, id, entity, queryParameter, extraHeaders, readOnly)
} else if (this.isRangeRequest(listId, id, queryParameter)) {
// load range
return resolveTypeReference(typeRef).then(typeModel => {
if (typeModel.values["_id"].type === ValueType.GeneratedId) {
let params = neverNull(queryParameter)
return this._loadRange(downcast(typeRef), neverNull(listId), params["start"], Number(params["count"]), params["reverse"]
=== "true")
=== "true", readOnly)
} else {
// we currently only store ranges for generated ids
return this._entityRestClient.entityRequest(typeRef, method, listId, id, entity, queryParameter, extraHeaders)
@ -121,7 +127,9 @@ export class EntityRestCache implements EntityRestInterface {
} else {
return this._entityRestClient.entityRequest(typeRef, method, listId, id, entity, queryParameter, extraHeaders)
.then(entity => {
if (!readOnly) {
this._putIntoCache(entity)
}
return entity
})
}
@ -134,8 +142,15 @@ export class EntityRestCache implements EntityRestInterface {
}
}
isRangeRequest(listId: ?Id, id: ?Id, queryParameter: ?Params) {
// check for null and undefined because "" and 0 are als falsy
return listId && !id
&& queryParameter && queryParameter["start"] !== null && queryParameter["start"] !== undefined && queryParameter["count"] !== null
&& queryParameter["count"] !== undefined && queryParameter["reverse"]
}
_getMultiple<T>(typeRef: TypeRef<T>, method: HttpMethodEnum, listId: ?Id, id: ?Id, entity: ?T, queryParameter: Params,
extraHeaders?: Params): Promise<Array<T>> {
extraHeaders?: Params, readOnly: boolean): Promise<Array<T>> {
const ids = queryParameter["ids"].split(",")
const inCache = [], notInCache = []
ids.forEach((id) => {
@ -150,17 +165,18 @@ export class EntityRestCache implements EntityRestInterface {
this._entityRestClient.entityRequest(typeRef, method, listId, id, entity, newQuery, extraHeaders)
.then((response) => {
const entities: Array<T> = downcast(response)
if (!readOnly) {
entities.forEach((e) => this._putIntoCache(e))
}
return entities
}),
inCache.map(id => this._getFromCache(typeRef, listId, id))
]).then(flat)
}
_loadRange<T: ListElement>(typeRef: TypeRef<T>, listId: Id, start: Id, count: number, reverse: boolean): Promise<T[]> {
_loadRange<T: ListElement>(typeRef: TypeRef<T>, listId: Id, start: Id, count: number, reverse: boolean, readOnly: boolean): Promise<T[]> {
let path = typeRefToPath(typeRef)
let listCache = (this._listEntities[path]
&& this._listEntities[path][listId]) ? this._listEntities[path][listId] : null
const listCache = (this._listEntities[path] && this._listEntities[path][listId]) ? this._listEntities[path][listId] : null
// check which range must be loaded from server
if (!listCache || (start === GENERATED_MAX_ID && reverse && listCache.upperRangeId !== GENERATED_MAX_ID)
|| (start === GENERATED_MIN_ID && !reverse && listCache.lowerRangeId !== GENERATED_MIN_ID)) {
@ -173,23 +189,30 @@ export class EntityRestCache implements EntityRestInterface {
count: String(count),
reverse: String(reverse)
}).then(result => {
let entities = ((result: any): T[])
let entities: Array<T> = downcast(result)
// create the list data path in the cache if not existing
if (readOnly) {
return entities;
}
let newListCache
if (!listCache) {
if (!this._listEntities[path]) {
this._listEntities[path] = {}
}
listCache = {allRange: [], lowerRangeId: start, upperRangeId: start, elements: {}}
this._listEntities[path][listId] = listCache
newListCache = {allRange: [], lowerRangeId: start, upperRangeId: start, elements: {}}
this._listEntities[path][listId] = newListCache
} else {
listCache.allRange = []
listCache.lowerRangeId = start
listCache.upperRangeId = start
newListCache = listCache
newListCache.allRange = []
newListCache.lowerRangeId = start
newListCache.upperRangeId = start
}
return this._handleElementRangeResult(listCache, start, count, reverse, entities, count)
return this._handleElementRangeResult(newListCache, start, count, reverse, entities, count)
})
} else if (!firstBiggerThanSecond(start, listCache.upperRangeId)
&& !firstBiggerThanSecond(listCache.lowerRangeId, start)) { // check if the requested start element is located in the range
// count the numbers of elements that are already in allRange to determine the number of elements to read
let newRequestParams = this._getNumberOfElementsToRead(listCache, start, count, reverse)
if (newRequestParams.newCount > 0) {
@ -197,15 +220,33 @@ export class EntityRestCache implements EntityRestInterface {
start: newRequestParams.newStart,
count: String(newRequestParams.newCount),
reverse: String(reverse)
}).then(entities => {
return this._handleElementRangeResult(neverNull(listCache), start, count, reverse, ((entities: any): T[]), newRequestParams.newCount)
}).then(result => {
let entities: Array<T> = downcast(result)
if (readOnly) {
const cachedEntities = this._provideFromCache(listCache, start, count - newRequestParams.newCount, reverse)
if (reverse) {
return entities.concat(cachedEntities)
} else {
return cachedEntities.concat(entities)
}
} else {
return this._handleElementRangeResult(neverNull(listCache), start, count, reverse, entities, newRequestParams.newCount)
}
})
} else {
// all elements are located in the cache.
return Promise.resolve(this._provideFromCache(listCache, start, count, reverse))
}
} else if ((firstBiggerThanSecond(start, listCache.upperRangeId) && !reverse)
} else if ((firstBiggerThanSecond(start, listCache.upperRangeId) && !reverse) // Start is outside the range.
|| (firstBiggerThanSecond(listCache.lowerRangeId, start) && reverse)) {
if (readOnly) {
// Doesn't make any sense to read from existing range because we know that elements are not in the cache
return this.entityRequest(typeRef, HttpMethod.GET, listId, null, null, {
start: start,
count: String(count),
reverse: String(reverse)
})
}
let loadStartId
if (firstBiggerThanSecond(start, listCache.upperRangeId) && !reverse) {
// start is higher than range. load from upper range id with same count. then, if all available elements have been loaded or the requested number is in cache, return from cache. otherwise load again the same way.
@ -368,6 +409,13 @@ export class EntityRestCache implements EntityRestInterface {
let typeRef = new TypeRef(data.application, data.type)
if (data.operation === OperationType.UPDATE) {
if (this._isInCache(typeRef, data.instanceListId, data.instanceId)) {
if (isSameTypeRef(MailTypeRef, typeRef)) {
// ignore update of owner enc session key to avoid loading mail instance twice.
const cachedMail: Mail = downcast(this._getFromCache(typeRef, data.instanceListId, data.instanceId))
if (cachedMail.state !== MailState.DRAFT) {
return Promise.resolve()
}
}
return this._entityRestClient.entityRequest(typeRef, HttpMethod.GET, data.instanceListId, data.instanceId)
.then(entity => {
this._putIntoCache(entity)

2
src/api/worker/rest/EntityRestClient.js
View file

@ -14,6 +14,8 @@ export function typeRefToPath(typeRef: TypeRef<any>): string {
export type AuthHeadersProvider = () => Params
/**
* Retrieves the instances from the backend (db) and converts them to entities.
*

15
src/api/worker/search/MailIndexer.js
View file

@ -8,7 +8,7 @@ import {MailBoxTypeRef} from "../../entities/tutanota/MailBox"
import {MailFolderTypeRef} from "../../entities/tutanota/MailFolder"
import {_TypeModel as MailModel, MailTypeRef} from "../../entities/tutanota/Mail"
import {ElementDataOS, GroupDataOS, MetaDataOS} from "./DbFacade"
import {elementIdPart, isSameId, listIdPart, TypeRef} from "../../common/EntityFunctions"
import {elementIdPart, isSameId, listIdPart, readOnlyHeaders, TypeRef} from "../../common/EntityFunctions"
import {containsEventOfType, neverNull} from "../../common/utils/Utils"
import {timestampToGeneratedId} from "../../common/utils/Encoding"
import {_createNewIndexUpdate, encryptIndexKeyBase64, filterMailMemberships, getPerformanceTimestamp, htmlToText, typeRefToTypeInfo} from "./IndexUtils"
@ -45,13 +45,11 @@ export class MailIndexer {
_db: Db;
_worker: WorkerImpl;
_entityRestClient: EntityRestInterface;
_noncachingEntity: EntityWorker;
_defaultCachingClient: EntityWorker;
constructor(core: IndexerCore, db: Db, worker: WorkerImpl, entityRestClient: EntityRestInterface, defaultCachingRestClient: EntityRestInterface) {
this._core = core
this._db = db
this._noncachingEntity = new EntityWorker(entityRestClient)
this._defaultCachingClient = new EntityWorker(defaultCachingRestClient)
this._worker = worker
@ -99,10 +97,10 @@ export class MailIndexer {
if (this._isExcluded(event)) {
return Promise.resolve()
}
return this._noncachingEntity.load(MailTypeRef, [event.instanceListId, event.instanceId]).then(mail => {
return this._defaultCachingClient.load(MailTypeRef, [event.instanceListId, event.instanceId], null, readOnlyHeaders()).then(mail => {
return Promise.all([
Promise.map(mail.attachments, attachmentId => this._noncachingEntity.load(FileTypeRef, attachmentId)),
this._noncachingEntity.load(MailBodyTypeRef, mail.body)
Promise.map(mail.attachments, attachmentId => this._defaultCachingClient.load(FileTypeRef, attachmentId, null, readOnlyHeaders())),
this._defaultCachingClient.load(MailBodyTypeRef, mail.body, null, readOnlyHeaders())
]).spread((files, body) => {
let keyToIndexEntries = this.createMailIndexEntries(mail, body, files)
return {mail, keyToIndexEntries}
@ -533,7 +531,7 @@ export class MailIndexer {
return Promise.resolve()
}
return this._noncachingEntity.load(MailTypeRef, [event.instanceListId, event.instanceId]).then(mail => {
return this._defaultCachingClient.load(MailTypeRef, [event.instanceListId, event.instanceId], null, readOnlyHeaders()).then(mail => {
if (mail.state === MailState.DRAFT) {
return Promise.all([
this._core._processDeleted(event, indexUpdate),
@ -655,4 +653,7 @@ class IndexLoader {
}, {concurrency: 2})
.then(entityResults => flat(entityResults))
}
}

44
test/api/rest/EntityRestCacheTest.js
View file

@ -12,6 +12,7 @@ import {
getElementId,
HttpMethod,
isSameTypeRef,
readOnlyHeaders,
stringToCustomId,
TypeRef
} from "../../../src/api/common/EntityFunctions"
@ -408,9 +409,16 @@ o.spec("entity rest cache", function () {
})
})
o("load list elements partly from server - range min to id3 loaded", function (done) {
o("load list elements partly from server - range min to id3 loaded", async function () {
await _loadListElementsPartlyFromServer_RangeMinRoId3Loaded(false)
})
o.only("load list elements partly from server - range min to id3 loaded - readOnlyCache", async function () {
await _loadListElementsPartlyFromServer_RangeMinRoId3Loaded(true)
})
function _loadListElementsPartlyFromServer_RangeMinRoId3Loaded(readOnly: boolean): Promise<*> {
let mail4 = createMailInstance("listId1", "id4", "subject4")
setupMailList(true, false).then(originalMails => {
return setupMailList(true, false).then(originalMails => {
clientEntityRequest = function (typeRef, method, listId, id, entity, queryParameter, extraHeaders) {
o(isSameTypeRef(typeRef, MailTypeRef)).equals(true)
o(method).equals(HttpMethod.GET)
@ -418,20 +426,22 @@ o.spec("entity rest cache", function () {
o(id).equals(null)
o(entity).equals(null)
o(queryParameter).deepEquals({start: originalMails[2]._id[1], count: "1", reverse: "false"})
o(typeof extraHeaders === "undefined").equals(true) // never pass read only parameter to network request
return Promise.resolve([mail4])
}
return cache.entityRequest(MailTypeRef, HttpMethod.GET, "listId1", null, null, {
start: GENERATED_MIN_ID,
count: "4",
reverse: "false"
}).then(mails => {
},
readOnly ? readOnlyHeaders() : {}
).then(mails => {
o(mails).deepEquals([originalMails[0], originalMails[1], originalMails[2], clone(mail4)])
})
}).then(() => {
o(clientSpy.callCount).equals(2) // entities are provided from server
done()
})
})
}
o("load list elements partly from server - range max to id2 loaded - start in middle of range", function (done) {
let mail0 = createMailInstance("listId1", "id0", "subject0")
@ -457,6 +467,30 @@ o.spec("entity rest cache", function () {
done()
})
})
o.only("load list elements partly from server - range max to id2 loaded - start in middle of range - read only cache", async function () {
let mail0 = createMailInstance("listId1", "id0", "subject0")
await setupMailList(false, true).then(originalMails => {
clientEntityRequest = function (typeRef, method, listId, id, entity, queryParameter, extraHeaders) {
o(isSameTypeRef(typeRef, MailTypeRef)).equals(true)
o(method).equals(HttpMethod.GET)
o(listId).equals("listId1")
o(id).equals(null)
o(entity).equals(null)
o(queryParameter).deepEquals({start: originalMails[0]._id[1], count: "3", reverse: "true"})
return Promise.resolve([mail0])
}
return cache.entityRequest(MailTypeRef, HttpMethod.GET, "listId1", null, null, {
start: createId("id2"),
count: "4",
reverse: "true"
}).then(mails => {
o(mails).deepEquals([originalMails[0], clone(mail0)])
})
}).then(() => {
o(clientSpy.callCount).equals(2) // entities are provided from server
})
})
o("load list elements partly from server - range max to id2 loaded - loadMore", function (done) {
let mail0 = createMailInstance("listId1", "id0", "subject0")