make OfflineStorage use base64Ext for storing customIds #7429

we now store custom id entities in the offline storage, which means we need to
make sure storing ranges and comparing ids works for them. in order to achieve
that, we decided to store the normally base64Url-encoded, not lexicographically
sortable ids in the sortable base64Ext format.

the Offline Storage needs to use the "converted" base64Ext ids internally everywhere
for custom id types, but give out ranges and entities in the "raw" base64Url format and
take raw ids as parameters.

to make this easier, we implement the conversion in the public CacheStorage::getRangeForList
implementation and use the private OfflineStorage::getRange method internally.

packages/tutanota-utils/lib/Encoding.ts

@@ -73,7 +73,7 @@ export function base64ToBase64Ext(base64: Base64): Base64Ext {
 	let base64ext = ""
 
 	for (let i = 0; i < base64.length; i++) {
-		let index = base64Lookup[base64.charAt(i)]
+		const index = base64Lookup[base64.charAt(i)]
 		base64ext += base64extAlphabet[index]
 	}
 

src/calendar-app/calendar/search/view/CalendarSearchViewModel.ts

@@ -396,7 +396,6 @@ export class CalendarSearchViewModel {
 		// note in case of refactor: the fact that the list updates the URL every time it changes
 		// its state is a major source of complexity and makes everything very order-dependent
 		return new ListModel<SearchResultListEntry>({
-			topId: GENERATED_MAX_ID,
 			fetch: async (startId: Id, count: number) => {
 				const lastResult = this._searchResult
 				if (lastResult !== this._searchResult) {

src/common/api/common/utils/EntityUtils.ts

@@ -1,4 +1,3 @@
-import type { Hex } from "@tutao/tutanota-utils"
 import {
 	base64ExtToBase64,
 	base64ToBase64Ext,
@@ -7,9 +6,11 @@ import {
 	base64UrlToBase64,
 	clone,
 	compare,
+	Hex,
 	hexToBase64,
 	isSameTypeRef,
 	pad,
+	repeat,
 	stringToUtf8Uint8Array,
 	TypeRef,
 	uint8ArrayToBase64,
@@ -37,19 +38,23 @@ export const GENERATED_MIN_ID = "------------"
  */
 export const GENERATED_ID_BYTES_LENGTH = 9
 
+/**
+ * The byte length of a custom Id used by mail set entries
+ * 4 bytes timestamp (1024ms resolution)
+ * 9 bytes mail element Id
+ */
+export const MAIL_SET_ENTRY_ID_BYTE_LENGTH = 13
+
 /**
  * The minimum ID for elements with custom id stored on the server
  */
 export const CUSTOM_MIN_ID = ""
 /**
- * the maximum custom element id is enforced to be less than 256 bytes on the server. decoding this as b64url gives 255 bytes.
+ * the maximum custom element id is enforced to be less than 256 bytes on the server. decoding this as Base64Url gives 255 bytes.
  *
- * NOTE: this is currently only used as a marker value when caching calendar events.
+ * NOTE: this is currently only used as a marker value when caching CalenderEvent and MailSetEntry.
  */
-export const CUSTOM_MAX_ID =
+export const CUSTOM_MAX_ID = repeat("_", 340)
-	"_______________________________________________________________________________________________________________________________________________________" +
-	"_______________________________________________________________________________________________________________________________________________________" +
-	"______________________________________"
 export const RANGE_ITEM_LIMIT = 1000
 export const LOAD_MULTIPLE_LIMIT = 100
 export const POST_MULTIPLE_LIMIT = 100
@@ -126,7 +131,7 @@ export function firstBiggerThanSecondCustomId(firstId: Id, secondId: Id): boolea
 	return compare(customIdToUint8array(firstId), customIdToUint8array(secondId)) === 1
 }
 
-function customIdToUint8array(id: Id): Uint8Array {
+export function customIdToUint8array(id: Id): Uint8Array {
 	if (id === "") {
 		return new Uint8Array()
 	}
@@ -443,6 +448,27 @@ function removeIdentityFields<E extends Partial<SomeEntity>>(entity: E) {
 	_removeIdentityFields(entity)
 }
 
+/** construct a mail set entry Id for a given mail. see MailFolderHelper.java */
+export function constructMailSetEntryId(receiveDate: Date, mailId: Id): string {
+	const buffer = new DataView(new ArrayBuffer(MAIL_SET_ENTRY_ID_BYTE_LENGTH))
+	const mailIdBytes = base64ToUint8Array(base64ExtToBase64(mailId))
+
+	// shifting the received timestamp by 10 bit reduces the resolution from 1ms to 1024ms.
+	// truncating to 4 bytes leaves us with enough space for epoch + 4_294_967_295 not-quite-seconds
+	// (until around 2109-05-15 15:00)
+	const timestamp: bigint = BigInt(Math.trunc(receiveDate.getTime()))
+	const truncatedReceiveDate = (timestamp >> 10n) & 0xffffffffn
+
+	// we don't need the leading zeroes
+	buffer.setBigUint64(0, truncatedReceiveDate << 32n)
+
+	for (let i = 0; i < mailIdBytes.length; i++) {
+		buffer.setUint8(i + 4, mailIdBytes[i])
+	}
+
+	return uint8arrayToCustomId(new Uint8Array(buffer.buffer))
+}
+
 export const LEGACY_TO_RECIPIENTS_ID = 112
 export const LEGACY_CC_RECIPIENTS_ID = 113
 export const LEGACY_BCC_RECIPIENTS_ID = 114

src/common/api/worker/EventQueue.ts

@@ -4,7 +4,7 @@ import { ConnectionError, ServiceUnavailableError } from "../common/error/RestEr
 import type { EntityUpdate } from "../entities/sys/TypeRefs.js"
 import { CustomerInfoTypeRef } from "../entities/sys/TypeRefs.js"
 import { ProgrammingError } from "../common/error/ProgrammingError.js"
-import { MailTypeRef } from "../entities/tutanota/TypeRefs.js"
+import { MailSetEntryTypeRef, MailTypeRef } from "../entities/tutanota/TypeRefs.js"
 import { isSameId } from "../common/utils/EntityUtils.js"
 import { containsEventOfType, EntityUpdateData, getEventOfType } from "../common/utils/EntityUpdateUtils.js"
 import { ProgressMonitorDelegate } from "./ProgressMonitorDelegate.js"
@@ -26,11 +26,12 @@ type QueueAction = (nextElement: QueuedBatch) => Promise<void>
 const MOVABLE_EVENT_TYPE_REFS = [
 	// moved in MoveMailService
 	MailTypeRef, // moved in SwitchAccountTypeService
+	MailSetEntryTypeRef,
 	CustomerInfoTypeRef,
 ]
 
 /**
- * Whether the entity of the event supports MOVE operation. MOVE is supposed to be immutable so we cannot apply it to all instances.
+ * Whether the entity of the event supports MOVE operation. MOVE is supposed to be immutable, so we cannot apply it to all instances.
  */
 function isMovableEventType(event: EntityUpdate): boolean {
 	return MOVABLE_EVENT_TYPE_REFS.some((typeRef) => isSameTypeRefByAttr(typeRef, event.application, event.type))

src/common/api/worker/offline/OfflineStorage.ts

@@ -1,5 +1,6 @@
 import { ElementEntity, ListElementEntity, SomeEntity, TypeModel } from "../../common/EntityTypes.js"
 import {
+	CUSTOM_MIN_ID,
 	elementIdPart,
 	firstBiggerThanSecond,
 	GENERATED_MAX_ID,
@@ -14,6 +15,10 @@ import { EncodeOptions, Token, Type } from "cborg"
 import {
 	assert,
 	assertNotNull,
+	base64ExtToBase64,
+	base64ToBase64Ext,
+	base64ToBase64Url,
+	base64UrlToBase64,
 	DAY_IN_MILLIS,
 	getTypeId,
 	groupByAndMap,
@@ -44,7 +49,7 @@ import { InterWindowEventFacadeSendDispatcher } from "../../../native/common/gen
 import { SqlCipherFacade } from "../../../native/common/generatedipc/SqlCipherFacade.js"
 import { FormattedQuery, SqlValue, TaggedSqlValue, untagSqlObject } from "./SqlValue.js"
 import { FolderSystem } from "../../common/mail/FolderSystem.js"
-import { Type as TypeId } from "../../common/EntityConstants.js"
+import { Type as TypeId, ValueType } from "../../common/EntityConstants.js"
 import { OutOfSyncError } from "../../common/error/OutOfSyncError.js"
 import { sql, SqlFragment } from "./Sql.js"
 import { isDraft, isSpamOrTrashFolder } from "../../common/CommonMailUtils.js"
@@ -109,7 +114,7 @@ const TableDefinitions = Object.freeze({
 		"type TEXT NOT NULL, listId TEXT NOT NULL, elementId TEXT NOT NULL, ownerGroup TEXT, entity BLOB NOT NULL, PRIMARY KEY (type, listId, elementId)",
 } as const)
 
-type Range = { lower: string; upper: string }
+type Range = { lower: Id; upper: Id }
 
 export interface OfflineStorageInitArgs {
 	userId: Id
@@ -183,6 +188,7 @@ export class OfflineStorage implements CacheStorage, ExposedCacheStorage {
 		const type = getTypeId(typeRef)
 		let typeModel: TypeModel
 		typeModel = await resolveTypeReference(typeRef)
+		elementId = ensureBase64Ext(typeModel, elementId)
 		let formattedQuery
 		switch (typeModel.type) {
 			case TypeId.Element:
@@ -231,6 +237,7 @@ export class OfflineStorage implements CacheStorage, ExposedCacheStorage {
 	async get<T extends SomeEntity>(typeRef: TypeRef<T>, listId: Id | null, elementId: Id): Promise<T | null> {
 		const type = getTypeId(typeRef)
 		const typeModel = await resolveTypeReference(typeRef)
+		elementId = ensureBase64Ext(typeModel, elementId)
 		let formattedQuery
 		switch (typeModel.type) {
 			case TypeId.Element:
@@ -251,6 +258,9 @@ export class OfflineStorage implements CacheStorage, ExposedCacheStorage {
 
 	async provideMultiple<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id, elementIds: Id[]): Promise<Array<T>> {
 		if (elementIds.length === 0) return []
+		const typeModel = await resolveTypeReference(typeRef)
+		elementIds = elementIds.map((el) => ensureBase64Ext(typeModel, el))
+
 		const type = getTypeId(typeRef)
 		const serializedList: ReadonlyArray<Record<string, TaggedSqlValue>> = await this.allChunked(
 			MAX_SAFE_SQL_VARS - 2,
@@ -265,31 +275,44 @@ export class OfflineStorage implements CacheStorage, ExposedCacheStorage {
 
 	async getIdsInRange<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id): Promise<Array<Id>> {
 		const type = getTypeId(typeRef)
-		const range = await this.getRange(type, listId)
+		const range = await this.getRange(typeRef, listId)
 		if (range == null) {
 			throw new Error(`no range exists for ${type} and list ${listId}`)
 		}
-		const { lower, upper } = range
 		const { query, params } = sql`SELECT elementId FROM list_entities
 WHERE type = ${type}
 AND listId = ${listId}
-AND (elementId = ${lower}
+AND (elementId = ${range.lower}
-OR ${firstIdBigger("elementId", lower)})
+OR ${firstIdBigger("elementId", range.lower)})
-AND NOT(${firstIdBigger("elementId", upper)})`
+AND NOT(${firstIdBigger("elementId", range.upper)})`
 		const rows = await this.sqlCipherFacade.all(query, params)
 		return rows.map((row) => row.elementId.value as string)
 	}
 
+	/** don't use this internally in this class, use OfflineStorage::getRange instead. OfflineStorage is
+	 * using converted custom IDs internally which is undone when using this to access the range.
+	 */
 	async getRangeForList<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id): Promise<Range | null> {
-		return this.getRange(getTypeId(typeRef), listId)
+		let range = await this.getRange(typeRef, listId)
+		const typeModel = await resolveTypeReference(typeRef)
+		if (range == null) return range
+		return {
+			lower: customIdToBase64Url(typeModel, range.lower),
+			upper: customIdToBase64Url(typeModel, range.upper),
+		}
 	}
 
-	async isElementIdInCacheRange<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id, id: Id): Promise<boolean> {
+	async isElementIdInCacheRange<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id, elementId: Id): Promise<boolean> {
-		const range = await this.getRangeForList(typeRef, listId)
+		const typeModel = await resolveTypeReference(typeRef)
-		return range != null && !firstBiggerThanSecond(id, range.upper) && !firstBiggerThanSecond(range.lower, id)
+		elementId = ensureBase64Ext(typeModel, elementId)
+
+		const range = await this.getRange(typeRef, listId)
+		return range != null && !firstBiggerThanSecond(elementId, range.upper) && !firstBiggerThanSecond(range.lower, elementId)
 	}
 
 	async provideFromRange<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id, start: Id, count: number, reverse: boolean): Promise<T[]> {
+		const typeModel = await resolveTypeReference(typeRef)
+		start = ensureBase64Ext(typeModel, start)
 		const type = getTypeId(typeRef)
 		let formattedQuery
 		if (reverse) {
@@ -313,10 +336,11 @@ AND NOT(${firstIdBigger("elementId", upper)})`
 
 	async put(originalEntity: SomeEntity): Promise<void> {
 		const serializedEntity = this.serialize(originalEntity)
-		const { listId, elementId } = expandId(originalEntity._id)
+		let { listId, elementId } = expandId(originalEntity._id)
 		const type = getTypeId(originalEntity._type)
 		const ownerGroup = originalEntity._ownerGroup
 		const typeModel = await resolveTypeReference(originalEntity._type)
+		elementId = ensureBase64Ext(typeModel, elementId)
 		let formattedQuery: FormattedQuery
 		switch (typeModel.type) {
 			case TypeId.Element:
@@ -334,19 +358,25 @@ AND NOT(${firstIdBigger("elementId", upper)})`
 		await this.sqlCipherFacade.run(formattedQuery.query, formattedQuery.params)
 	}
 
-	async setLowerRangeForList<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id, id: Id): Promise<void> {
+	async setLowerRangeForList<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id, lowerId: Id): Promise<void> {
+		lowerId = ensureBase64Ext(await resolveTypeReference(typeRef), lowerId)
 		const type = getTypeId(typeRef)
-		const { query, params } = sql`UPDATE ranges SET lower = ${id} WHERE type = ${type} AND listId = ${listId}`
+		const { query, params } = sql`UPDATE ranges SET lower = ${lowerId} WHERE type = ${type} AND listId = ${listId}`
 		await this.sqlCipherFacade.run(query, params)
 	}
 
-	async setUpperRangeForList<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id, id: Id): Promise<void> {
+	async setUpperRangeForList<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id, upperId: Id): Promise<void> {
+		upperId = ensureBase64Ext(await resolveTypeReference(typeRef), upperId)
 		const type = getTypeId(typeRef)
-		const { query, params } = sql`UPDATE ranges SET upper = ${id} WHERE type = ${type} AND listId = ${listId}`
+		const { query, params } = sql`UPDATE ranges SET upper = ${upperId} WHERE type = ${type} AND listId = ${listId}`
 		await this.sqlCipherFacade.run(query, params)
 	}
 
 	async setNewRangeForList<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id, lower: Id, upper: Id): Promise<void> {
+		const typeModel = await resolveTypeReference(typeRef)
+		lower = ensureBase64Ext(typeModel, lower)
+		upper = ensureBase64Ext(typeModel, upper)
+
 		const type = getTypeId(typeRef)
 		const { query, params } = sql`INSERT OR REPLACE INTO ranges VALUES (${type}, ${listId}, ${lower}, ${upper})`
 		return this.sqlCipherFacade.run(query, params)
@@ -506,7 +536,7 @@ AND NOT(${firstIdBigger("elementId", upper)})`
 		for (const mailBox of mailBoxes) {
 			const isMailsetMigrated = mailBox.currentMailBag != null
 			if (isMailsetMigrated) {
-				var mailListIds = [mailBox.currentMailBag!, ...mailBox.archivedMailBags].map((mailbag) => mailbag.mails)
+				const mailListIds = [mailBox.currentMailBag!, ...mailBox.archivedMailBags].map((mailbag) => mailbag.mails)
 				for (const mailListId of mailListIds) {
 					await this.deleteMailList(mailListId, cutoffId)
 				}
@@ -531,9 +561,12 @@ AND NOT(${firstIdBigger("elementId", upper)})`
 		}
 	}
 
-	private async getRange(type: string, listId: Id): Promise<Range | null> {
+	private async getRange(typeRef: TypeRef<ElementEntity | ListElementEntity>, listId: Id): Promise<Range | null> {
+		const type = getTypeId(typeRef)
+
 		const { query, params } = sql`SELECT upper, lower FROM ranges WHERE type = ${type} AND listId = ${listId}`
 		const row = (await this.sqlCipherFacade.get(query, params)) ?? null
+
 		return mapNullable(row, untagSqlObject) as Range | null
 	}
 
@@ -552,6 +585,8 @@ AND NOT(${firstIdBigger("elementId", upper)})`
 	 * 	2. We might need them in the future for showing the whole thread
 	 */
 	private async deleteMailList(listId: Id, cutoffId: Id): Promise<void> {
+		// fixme: do we want to remove mailsetentries as well here?
+		// we don't have the mailsetentry list id
 		// We lock access to the "ranges" db here in order to prevent race conditions when accessing the "ranges" database.
 		await this.lockRangesDbAccess(listId)
 		try {
@@ -647,8 +682,11 @@ AND NOT(${firstIdBigger("elementId", upper)})`
 
 	private async updateRangeForList<T extends ListElementEntity>(typeRef: TypeRef<T>, listId: Id, cutoffId: Id): Promise<void> {
 		const type = getTypeId(typeRef)
+		const typeModel = await resolveTypeReference(typeRef)
+		const isCustomId = isCustomIdType(typeModel)
+		cutoffId = ensureBase64Ext(typeModel, cutoffId)
 
-		const range = await this.getRange(type, listId)
+		const range = await this.getRange(typeRef, listId)
 		if (range == null) {
 			return
 		}
@@ -657,8 +695,9 @@ AND NOT(${firstIdBigger("elementId", upper)})`
 		// saved range if we would be removing elements from the list, in order to not lose the information that the range is complete in storage.
 		// So we have to check how old the oldest element in said range is. If it is newer than cutoffId, then we will not modify the range,
 		// otherwise we will just modify it normally
-		if (range.lower === GENERATED_MIN_ID) {
+		const expectedMinId = isCustomId ? CUSTOM_MIN_ID : GENERATED_MIN_ID
-			const entities = await this.provideFromRange(typeRef, listId, GENERATED_MIN_ID, 1, false)
+		if (range.lower === expectedMinId) {
+			const entities = await this.provideFromRange(typeRef, listId, expectedMinId, 1, false)
 			const id = mapNullable(entities[0], getElementId)
 			const rangeWontBeModified = id == null || firstBiggerThanSecond(id, cutoffId) || id === cutoffId
 			if (rangeWontBeModified) {
@@ -759,3 +798,24 @@ function firstIdBigger(...args: [string, "elementId"] | ["elementId", string]):
 	}
 	return new SqlFragment(`(CASE WHEN length(${l}) > length(${r}) THEN 1 WHEN length(${l}) < length(${r}) THEN 0 ELSE ${l} > ${r} END)`, [v, v, v])
 }
+
+export function isCustomIdType(typeModel: TypeModel): boolean {
+	return typeModel.values._id.type === ValueType.CustomId
+}
+
+/**
+ * We store customIds as base64ext in the db to make them sortable, but we get them as base64url from the server.
+ */
+export function ensureBase64Ext(typeModel: TypeModel, elementId: Id): Id {
+	if (isCustomIdType(typeModel)) {
+		return base64ToBase64Ext(base64UrlToBase64(elementId))
+	}
+	return elementId
+}
+
+export function customIdToBase64Url(typeModel: TypeModel, elementId: Id): Id {
+	if (isCustomIdType(typeModel)) {
+		return base64ToBase64Url(base64ExtToBase64(elementId))
+	}
+	return elementId
+}

src/common/api/worker/rest/DefaultEntityRestCache.ts

@@ -2,7 +2,7 @@ import type { EntityRestInterface, OwnerEncSessionKeyProvider, OwnerKeyProvider
 import { EntityRestClient, EntityRestClientSetupOptions } from "./EntityRestClient"
 import { resolveTypeReference } from "../../common/EntityFunctions"
 import { OperationType } from "../../common/TutanotaConstants"
-import { assertNotNull, difference, getFirstOrThrow, groupBy, isSameTypeRef, lastThrow, TypeRef } from "@tutao/tutanota-utils"
+import { assertNotNull, difference, getFirstOrThrow, getTypeId, groupBy, isEmpty, isSameTypeRef, lastThrow, TypeRef } from "@tutao/tutanota-utils"
 import {
 	BucketPermissionTypeRef,
 	EntityEventBatchTypeRef,
@@ -20,8 +20,8 @@ import {
 } from "../../entities/sys/TypeRefs.js"
 import { ValueType } from "../../common/EntityConstants.js"
 import { NotAuthorizedError, NotFoundError } from "../../common/error/RestError"
-import { CalendarEventUidIndexTypeRef, Mail, MailDetailsBlobTypeRef, MailTypeRef } from "../../entities/tutanota/TypeRefs.js"
+import { CalendarEventUidIndexTypeRef, Mail, MailDetailsBlobTypeRef, MailSetEntryTypeRef, MailTypeRef } from "../../entities/tutanota/TypeRefs.js"
-import { firstBiggerThanSecond, GENERATED_MAX_ID, GENERATED_MIN_ID, getElementId, isSameId } from "../../common/utils/EntityUtils"
+import { CUSTOM_MAX_ID, CUSTOM_MIN_ID, firstBiggerThanSecond, GENERATED_MAX_ID, GENERATED_MIN_ID, getElementId, isSameId } from "../../common/utils/EntityUtils"
 import { ProgrammingError } from "../../common/error/ProgrammingError"
 import { assertWorkerOrNode } from "../../common/Env"
 import type { ListElementEntity, SomeEntity, TypeModel } from "../../common/EntityTypes"
@@ -30,6 +30,7 @@ import { QueuedBatch } from "../EventQueue.js"
 import { ENTITY_EVENT_BATCH_EXPIRE_MS } from "../EventBusClient"
 import { CustomCacheHandlerMap } from "./CustomCacheHandler.js"
 import { containsEventOfType, EntityUpdateData, getEventOfType } from "../../common/utils/EntityUpdateUtils.js"
+import { isCustomIdType } from "../offline/OfflineStorage.js"
 
 assertWorkerOrNode()
 
@@ -377,12 +378,15 @@ export class DefaultEntityRestCache implements EntityRestCache {
 
 		try {
 			const range = await this.storage.getRangeForList(typeRef, listId)
-
+			if (getTypeId(typeRef) == "tutanota/MailSetEntry") {
+				console.log(getTypeId(typeRef), listId, start, count, reverse)
+				console.log("range", range)
+			}
 			if (range == null) {
 				await this.populateNewListWithRange(typeRef, listId, start, count, reverse)
-			} else if (isStartIdWithinRange(range, start)) {
+			} else if (isStartIdWithinRange(range, start, typeModel)) {
 				await this.extendFromWithinRange(typeRef, listId, start, count, reverse)
-			} else if (isRangeRequestAwayFromExistingRange(range, reverse, start)) {
+			} else if (isRangeRequestAwayFromExistingRange(range, reverse, start, typeModel)) {
 				await this.extendAwayFromRange(typeRef, listId, start, count, reverse)
 			} else {
 				await this.extendTowardsRange(typeRef, listId, start, count, reverse)
@@ -512,12 +516,14 @@ export class DefaultEntityRestCache implements EntityRestCache {
 		wasReverseRequest: boolean,
 		receivedEntities: T[],
 	) {
+		const isCustomId = isCustomIdType(await resolveTypeReference(typeRef))
 		let elementsToAdd = receivedEntities
 		if (wasReverseRequest) {
 			// Ensure that elements are cached in ascending (not reverse) order
 			elementsToAdd = receivedEntities.reverse()
 			if (receivedEntities.length < countRequested) {
-				await this.storage.setLowerRangeForList(typeRef, listId, GENERATED_MIN_ID)
+				console.log("finished loading, setting min id")
+				await this.storage.setLowerRangeForList(typeRef, listId, isCustomId ? CUSTOM_MIN_ID : GENERATED_MIN_ID)
 			} else {
 				// After reversing the list the first element in the list is the lower range limit
 				await this.storage.setLowerRangeForList(typeRef, listId, getElementId(getFirstOrThrow(receivedEntities)))
@@ -526,7 +532,8 @@ export class DefaultEntityRestCache implements EntityRestCache {
 			// Last element in the list is the upper range limit
 			if (receivedEntities.length < countRequested) {
 				// all elements have been loaded, so the upper range must be set to MAX_ID
-				await this.storage.setUpperRangeForList(typeRef, listId, GENERATED_MAX_ID)
+				console.log("finished loading, setting max id")
+				await this.storage.setUpperRangeForList(typeRef, listId, isCustomId ? CUSTOM_MAX_ID : GENERATED_MAX_ID)
 			} else {
 				await this.storage.setUpperRangeForList(typeRef, listId, getElementId(lastThrow(receivedEntities)))
 			}
@@ -556,7 +563,13 @@ export class DefaultEntityRestCache implements EntityRestCache {
 		}
 		const { lower, upper } = range
 		let indexOfStart = allRangeList.indexOf(start)
-		if ((!reverse && upper === GENERATED_MAX_ID) || (reverse && lower === GENERATED_MIN_ID)) {
+
+		const typeModel = await resolveTypeReference(typeRef)
+		const isCustomId = isCustomIdType(typeModel)
+		if (
+			(!reverse && (isCustomId ? upper == CUSTOM_MAX_ID : upper === GENERATED_MAX_ID)) ||
+			(reverse && (isCustomId ? lower == CUSTOM_MIN_ID : lower === GENERATED_MIN_ID))
+		) {
 			// we have already loaded the complete range in the desired direction, so we do not have to load from server
 			elementsToRead = 0
 		} else if (allRangeList.length === 0) {
@@ -571,7 +584,7 @@ export class DefaultEntityRestCache implements EntityRestCache {
 				elementsToRead = count - (allRangeList.length - 1 - indexOfStart)
 				startElementId = allRangeList[allRangeList.length - 1] // use the  highest id in allRange as start element
 			}
-		} else if (lower === start || (firstBiggerThanSecond(start, lower) && firstBiggerThanSecond(allRangeList[0], start))) {
+		} else if (lower === start || (firstBiggerThanSecond(start, lower, typeModel) && firstBiggerThanSecond(allRangeList[0], start, typeModel))) {
 			// Start element is not in allRange but has been used has start element for a range request, eg. EntityRestInterface.GENERATED_MIN_ID, or start is between lower range id and lowest element in range
 			if (!reverse) {
 				// if not reverse read only elements that are not in allRange
@@ -579,7 +592,10 @@ export class DefaultEntityRestCache implements EntityRestCache {
 				elementsToRead = count - allRangeList.length
 			}
 			// if reverse read all elements
-		} else if (upper === start || (firstBiggerThanSecond(start, allRangeList[allRangeList.length - 1]) && firstBiggerThanSecond(upper, start))) {
+		} else if (
+			upper === start ||
+			(firstBiggerThanSecond(start, allRangeList[allRangeList.length - 1], typeModel) && firstBiggerThanSecond(upper, start, typeModel))
+		) {
 			// Start element is not in allRange but has been used has start element for a range request, eg. EntityRestInterface.GENERATED_MAX_ID, or start is between upper range id and highest element in range
 			if (reverse) {
 				// if not reverse read only elements that are not in allRange
@@ -606,18 +622,17 @@ export class DefaultEntityRestCache implements EntityRestCache {
 		const regularUpdates: EntityUpdate[] = [] // all updates not resulting from post multiple requests
 		const updatesArray = batch.events
 		for (const update of updatesArray) {
-			if (update.application !== "monitor") {
+			// monitor application is ignored
-				// monitor application is ignored
+			if (update.application === "monitor") continue
-				// mails are ignored because move operations are handled as a special event (and no post multiple is possible)
+			// mails are ignored because move operations are handled as a special event (and no post multiple is possible)
-				if (
+			if (
-					update.operation === OperationType.CREATE &&
+				update.operation === OperationType.CREATE &&
-					getUpdateInstanceId(update).instanceListId != null &&
+				getUpdateInstanceId(update).instanceListId != null &&
-					!isSameTypeRef(new TypeRef(update.application, update.type), MailTypeRef)
+				!isSameTypeRef(new TypeRef(update.application, update.type), MailTypeRef)
-				) {
+			) {
-					createUpdatesForLETs.push(update)
+				createUpdatesForLETs.push(update)
-				} else {
+			} else {
-					regularUpdates.push(update)
+				regularUpdates.push(update)
-				}
 			}
 		}
 
@@ -722,15 +737,18 @@ export class DefaultEntityRestCache implements EntityRestCache {
 		if (instanceListId != null) {
 			const deleteEvent = getEventOfType(batch, OperationType.DELETE, instanceId)
 
-			const element = deleteEvent && isSameTypeRef(MailTypeRef, typeRef) ? await this.storage.get(typeRef, deleteEvent.instanceListId, instanceId) : null
+			const mail = deleteEvent && isSameTypeRef(MailTypeRef, typeRef) ? await this.storage.get(MailTypeRef, deleteEvent.instanceListId, instanceId) : null
-			if (deleteEvent != null && element != null) {
+			// avoid downloading new mail element for non-mailSet user.
+			// can be removed once all mailbox have been migrated to mailSet (once lastNonOutdatedClientVersion is >= v242)
+			if (deleteEvent != null && mail != null && isEmpty(mail.sets)) {
 				// It is a move event for cached mail
 				await this.storage.deleteIfExists(typeRef, deleteEvent.instanceListId, instanceId)
-				await this.updateListIdOfMailAndUpdateCache(element as Mail, instanceListId, instanceId)
+				await this.updateListIdOfMailAndUpdateCache(mail, instanceListId, instanceId)
 				return update
 			} else if (await this.storage.isElementIdInCacheRange(typeRef, instanceListId, instanceId)) {
 				// No need to try to download something that's not there anymore
 				// We do not consult custom handlers here because they are only needed for list elements.
+				console.log("downloading create event for", getTypeId(typeRef), instanceListId, instanceId)
 				return this.entityRestClient
 					.load(typeRef, [instanceListId, instanceId])
 					.then((entity) => this.storage.put(entity))
@@ -880,16 +898,16 @@ export function getUpdateInstanceId(update: EntityUpdate): { instanceListId: Id
 /**
  * Check if a range request begins inside an existing range
  */
-function isStartIdWithinRange(range: Range, startId: Id): boolean {
+function isStartIdWithinRange(range: Range, startId: Id, typeModel: TypeModel): boolean {
-	return !firstBiggerThanSecond(startId, range.upper) && !firstBiggerThanSecond(range.lower, startId)
+	return !firstBiggerThanSecond(startId, range.upper, typeModel) && !firstBiggerThanSecond(range.lower, startId, typeModel)
 }
 
 /**
  * Check if a range request is going away from an existing range
  * Assumes that the range request doesn't start inside the range
  */
-function isRangeRequestAwayFromExistingRange(range: Range, reverse: boolean, start: string) {
+function isRangeRequestAwayFromExistingRange(range: Range, reverse: boolean, start: string, typeModel: TypeModel) {
-	return reverse ? firstBiggerThanSecond(range.lower, start) : firstBiggerThanSecond(start, range.upper)
+	return reverse ? firstBiggerThanSecond(range.lower, start, typeModel) : firstBiggerThanSecond(start, range.upper, typeModel)
 }
 
 /**
@@ -909,5 +927,9 @@ function isIgnoredType(typeRef: TypeRef<unknown>): boolean {
  * isIgnoredType(ref) -/-> !isCachedType(ref)
  */
 function isCachedType(typeModel: TypeModel, typeRef: TypeRef<unknown>): boolean {
-	return !isIgnoredType(typeRef) && typeModel.values._id.type === ValueType.GeneratedId
+	return (!isIgnoredType(typeRef) && isGeneratedIdType(typeModel)) || isSameTypeRef(typeRef, MailSetEntryTypeRef)
+}
+
+function isGeneratedIdType(typeModel: TypeModel): boolean {
+	return typeModel.values._id.type === ValueType.GeneratedId
 }

src/common/api/worker/rest/EphemeralCacheStorage.ts

@@ -111,6 +111,7 @@ export class EphemeralCacheStorage implements CacheStorage {
 		const entity = clone(originalEntity)
 		const typeRef = entity._type
 		const typeModel = await resolveTypeReference(typeRef)
+		// fixme: maybe we can fix the sortability of the custom id types here, directly before inserting?
 		switch (typeModel.type) {
 			case TypeId.Element:
 				const elementEntity = entity as ElementEntity

src/common/misc/ListModel.ts

@@ -1,4 +1,4 @@
-import { elementIdPart, getElementId, isSameId, ListElement } from "../api/common/utils/EntityUtils.js"
+import { elementIdPart, GENERATED_MAX_ID, getElementId, isSameId, ListElement } from "../api/common/utils/EntityUtils.js"
 import { ListLoadingState, ListState } from "../gui/base/List.js"
 
 import { OperationType } from "../api/common/TutanotaConstants.js"
@@ -25,14 +25,19 @@ import { ListFetchResult, PageSize } from "../gui/base/ListUtils.js"
 import { isOfflineError } from "../api/common/utils/ErrorUtils.js"
 import { ListAutoSelectBehavior } from "./DeviceConfig.js"
 
-export interface ListModelConfig<ListElementType> {
+export type ListModelConfig<ListElementType> = {
-	topId: Id
-
 	/**
 	 * Get the given number of entities starting after the given id. May return more elements than requested, e.g. if all elements are available on first fetch.
 	 */
 	fetch(startId: Id, count: number): Promise<ListFetchResult<ListElementType>>
 
+	/**
+	 * some lists load elements via an index indirection,
+	 * so they use a different list to load than is actually displayed.
+	 * in fact, the displayed entities might not even be stored in the same list
+	 */
+	getLoadIdForElement?: (element: ListElementType | null | undefined) => Id
+
 	/**
 	 * Returns null if the given element could not be loaded
 	 */
@@ -53,7 +58,14 @@ type PrivateListState<ElementType> = Omit<ListState<ElementType>, "items" | "act
 
 /** ListModel that does the state upkeep for the List, including loading state, loaded items, selection and filters*/
 export class ListModel<ElementType extends ListElement> {
-	constructor(private readonly config: ListModelConfig<ElementType>) {}
+	private readonly config: Required<ListModelConfig<ElementType>>
+
+	constructor(config: ListModelConfig<ElementType>) {
+		if (config.getLoadIdForElement == null) {
+			config.getLoadIdForElement = (element: ElementType) => (element != null ? getElementId(element) : GENERATED_MAX_ID)
+		}
+		this.config = config as Required<ListModelConfig<ElementType>>
+	}
 
 	private loadState: "created" | "initialized" = "created"
 	private loading: Promise<unknown> = Promise.resolve()
@@ -153,7 +165,8 @@ export class ListModel<ElementType extends ListElement> {
 		this.loading = Promise.resolve().then(async () => {
 			const lastItem = last(this.rawState.unfilteredItems)
 			try {
-				const { items: newItems, complete } = await this.config.fetch(lastItem ? getElementId(lastItem) : this.config.topId, PageSize)
+				const idToLoadFrom = this.config.getLoadIdForElement(lastItem)
+				const { items: newItems, complete } = await this.config.fetch(idToLoadFrom, PageSize)
 				// if the loading was cancelled in the meantime, don't insert anything so that it's not confusing
 				if (this.state.loadingStatus === ListLoadingState.ConnectionLost) {
 					return

src/common/settings/UserListView.ts

@@ -216,7 +216,6 @@ export class UserListView implements UpdatableSettingsViewer {
 
 	private makeListModel(): ListModel<GroupInfo> {
 		const listModel = new ListModel<GroupInfo>({
-			topId: GENERATED_MAX_ID,
 			sortCompare: compareGroupInfos,
 			fetch: async (startId) => {
 				if (startId !== GENERATED_MAX_ID) {

src/mail-app/contacts/view/ContactListViewModel.ts

@@ -8,7 +8,7 @@ import {
 	ContactTypeRef,
 	createContactListEntry,
 } from "../../../common/api/entities/tutanota/TypeRefs.js"
-import { GENERATED_MAX_ID, getEtId, isSameId } from "../../../common/api/common/utils/EntityUtils.js"
+import { getEtId, isSameId } from "../../../common/api/common/utils/EntityUtils.js"
 import { EntityClient } from "../../../common/api/common/EntityClient.js"
 import { GroupManagementFacade } from "../../../common/api/worker/facades/lazy/GroupManagementFacade.js"
 import { LoginController } from "../../../common/api/main/LoginController.js"
@@ -83,7 +83,6 @@ export class ContactListViewModel {
 
 	private readonly _listModel = memoized((listId: Id) => {
 		const newListModel = new ListModel<ContactListEntry>({
-			topId: GENERATED_MAX_ID,
 			fetch: async () => {
 				const items = await this.getRecipientsForList(listId)
 				return { items, complete: true }

src/mail-app/contacts/view/ContactViewModel.ts

@@ -6,7 +6,7 @@ import { Contact, ContactTypeRef } from "../../../common/api/entities/tutanota/T
 import { compareContacts } from "./ContactGuiUtils.js"
 import { ListState } from "../../../common/gui/base/List.js"
 import { assertNotNull, lazyMemoized } from "@tutao/tutanota-utils"
-import { GENERATED_MAX_ID, getElementId } from "../../../common/api/common/utils/EntityUtils.js"
+import { getElementId } from "../../../common/api/common/utils/EntityUtils.js"
 import Stream from "mithril/stream"
 import { Router } from "../../../common/gui/ScopedRouter.js"
 import { isUpdateForTypeRef } from "../../../common/api/common/utils/EntityUpdateUtils.js"
@@ -29,7 +29,6 @@ export class ContactViewModel {
 	) {}
 
 	readonly listModel: ListModel<Contact> = new ListModel<Contact>({
-		topId: GENERATED_MAX_ID,
 		fetch: async () => {
 			const items = await this.entityClient.loadAll(ContactTypeRef, this.contactListId)
 			return { items, complete: true }

src/mail-app/mail/view/MailViewModel.ts

@@ -1,11 +1,13 @@
-import { ListModel } from "../../../common/misc/ListModel.js"
+import { ListModel, ListModelConfig } from "../../../common/misc/ListModel.js"
 import { MailboxDetail, MailModel } from "../../../common/mailFunctionality/MailModel.js"
 import { EntityClient } from "../../../common/api/common/EntityClient.js"
 import { Mail, MailFolder, MailSetEntry, MailSetEntryTypeRef, MailTypeRef } from "../../../common/api/entities/tutanota/TypeRefs.js"
 import {
+	constructMailSetEntryId,
+	CUSTOM_MAX_ID,
+	customIdToUint8array,
 	elementIdPart,
 	firstBiggerThanSecond,
-	GENERATED_MAX_ID,
 	getElementId,
 	isSameId,
 	listIdPart,
@@ -13,6 +15,7 @@ import {
 } from "../../../common/api/common/utils/EntityUtils.js"
 import {
 	assertNotNull,
+	compare,
 	count,
 	debounce,
 	groupByAndMap,
@@ -48,6 +51,13 @@ export interface MailOpenedListener {
 	onEmailOpened(mail: Mail): unknown
 }
 
+/** sort mail set mails in descending order according to their receivedDate, not their element id */
+function sortCompareMailSetMails(firstMail: Mail, secondMail: Mail): number {
+	const firstMailEntryId = constructMailSetEntryId(firstMail.receivedDate, getElementId(firstMail))
+	const secondMailEntryId = constructMailSetEntryId(secondMail.receivedDate, getElementId(secondMail))
+	return compare(customIdToUint8array(secondMailEntryId), customIdToUint8array(firstMailEntryId))
+}
+
 /** ViewModel for the overall mail view. */
 export class MailViewModel {
 	private _folder: MailFolder | null = null
@@ -215,8 +225,11 @@ export class MailViewModel {
 	}
 
 	private listModelForFolder = memoized((folderId: Id) => {
+		const customGetLoadIdForElement: ListModelConfig<Mail>["getLoadIdForElement"] = (mail) =>
+			mail == null ? CUSTOM_MAX_ID : constructMailSetEntryId(mail.receivedDate, getElementId(mail))
+
 		return new ListModel<Mail>({
-			topId: GENERATED_MAX_ID,
+			getLoadIdForElement: this._folder?.isMailSet ? customGetLoadIdForElement : undefined,
 			fetch: async (startId, count) => {
 				const folder = assertNotNull(this._folder)
 				const { complete, items } = await this.loadMailRange(folder, startId, count)
@@ -225,10 +238,12 @@ export class MailViewModel {
 				}
 				return { complete, items }
 			},
-			loadSingle: (listId: Id, elementId: Id): Promise<Mail | null> => {
+			loadSingle: async (listId: Id, elementId: Id): Promise<Mail | null> => {
+				// await this.entityClient.load(MailSetEntryTypeRef, )
 				return this.entityClient.load(MailTypeRef, [listId, elementId])
 			},
-			sortCompare: sortCompareByReverseId,
+			sortCompare: (firstMail, secondMail): number =>
+				assertNotNull(this._folder).isMailSet ? sortCompareMailSetMails(firstMail, secondMail) : sortCompareByReverseId(firstMail, secondMail),
 			autoSelectBehavior: () => this.conversationPrefProvider.getMailAutoSelectBehavior(),
 		})
 	})
@@ -354,9 +369,10 @@ export class MailViewModel {
 		}
 	}
 
-	private async loadMailSetMailRange(folder: MailFolder, start: string, count: number) {
+	private async loadMailSetMailRange(folder: MailFolder, startId: string, count: number): Promise<ListFetchResult<Mail>> {
+		console.log("range request for ", folder.entries, startId, count)
 		try {
-			const loadMailSetEntries = () => this.entityClient.loadRange(MailSetEntryTypeRef, folder.entries, start, count, true)
+			const loadMailSetEntries = () => this.entityClient.loadRange(MailSetEntryTypeRef, folder.entries, startId, count, true)
 			const loadMails = (listId: Id, mailIds: Array<Id>) => this.entityClient.loadMultiple(MailTypeRef, listId, mailIds)
 
 			const mails = await this.acquireMails(loadMailSetEntries, loadMails)
@@ -382,7 +398,7 @@ export class MailViewModel {
 			// give it to the list and let list make another request (and almost certainly fail that request) to show a retry button. This way we both show
 			// the items we have and also show that we couldn't load everything.
 			if (isOfflineError(e)) {
-				const loadMailSetEntries = () => this.cacheStorage.provideFromRange(MailSetEntryTypeRef, folder.entries, start, count, true)
+				const loadMailSetEntries = () => this.cacheStorage.provideFromRange(MailSetEntryTypeRef, folder.entries, startId, count, true)
 				const loadMails = (listId: Id, mailIds: Array<Id>) => this.cacheStorage.provideMultiple(MailTypeRef, listId, mailIds)
 				const items = await this.acquireMails(loadMailSetEntries, loadMails)
 				if (items.length === 0) throw e
@@ -396,8 +412,12 @@ export class MailViewModel {
 	/**
 	 * Load mails either from remote or from offline storage. Loader functions must be implemented for each use case.
 	 */
-	private async acquireMails(loadMailSetEntries: () => Promise<MailSetEntry[]>, loadMails: (listId: Id, mailIds: Array<Id>) => Promise<Mail[]>) {
+	private async acquireMails(
+		loadMailSetEntries: () => Promise<MailSetEntry[]>,
+		loadMails: (listId: Id, mailIds: Array<Id>) => Promise<Mail[]>,
+	): Promise<Array<Mail>> {
 		const mailSetEntries = await loadMailSetEntries()
+		console.log("entries", mailSetEntries)
 		const mailListIdToMailIds = groupByAndMap(
 			mailSetEntries,
 			(mse) => listIdPart(mse.mail),
@@ -407,11 +427,11 @@ export class MailViewModel {
 		for (const [listId, mailIds] of mailListIdToMailIds) {
 			mails.push(...(await loadMails(listId, mailIds)))
 		}
-		mails.sort((a, b) => b.receivedDate.getTime() - a.receivedDate.getTime())
+		console.log("mails", mails)
 		return mails
 	}
 
-	private async loadLegacyMailRange(folder: MailFolder, start: string, count: number) {
+	private async loadLegacyMailRange(folder: MailFolder, start: string, count: number): Promise<ListFetchResult<Mail>> {
 		const listId = folder.mails
 		try {
 			const items = await this.entityClient.loadRange(MailTypeRef, listId, start, count, true)

src/mail-app/search/view/SearchViewModel.ts

@@ -685,7 +685,6 @@ export class SearchViewModel {
 		// note in case of refactor: the fact that the list updates the URL every time it changes
 		// its state is a major source of complexity and makes everything very order-dependent
 		return new ListModel<SearchResultListEntry>({
-			topId: GENERATED_MAX_ID,
 			fetch: async (startId: Id, count: number) => {
 				const lastResult = this._searchResult
 				if (lastResult !== this._searchResult) {

src/mail-app/settings/KnowledgeBaseListView.ts

@@ -82,7 +82,6 @@ export class KnowledgeBaseListView implements UpdatableSettingsViewer {
 
 	private makeListModel() {
 		const listModel = new ListModel<KnowledgeBaseEntry>({
-			topId: GENERATED_MAX_ID,
 			sortCompare: (a: KnowledgeBaseEntry, b: KnowledgeBaseEntry) => {
 				var titleA = a.title.toUpperCase()
 				var titleB = b.title.toUpperCase()

src/mail-app/settings/TemplateListView.ts

@@ -81,7 +81,6 @@ export class TemplateListView implements UpdatableSettingsViewer {
 
 	private makeListModel() {
 		const listModel = new ListModel<EmailTemplate>({
-			topId: GENERATED_MAX_ID,
 			sortCompare: (a: EmailTemplate, b: EmailTemplate) => {
 				const titleA = a.title.toUpperCase()
 				const titleB = b.title.toUpperCase()

src/mail-app/settings/groups/GroupListView.ts

@@ -159,7 +159,6 @@ export class GroupListView implements UpdatableSettingsViewer {
 
 	private makeListModel(): ListModel<GroupInfo> {
 		const listModel = new ListModel<GroupInfo>({
-			topId: GENERATED_MAX_ID,
 			sortCompare: compareGroupInfos,
 			fetch: async (startId) => {
 				if (startId === GENERATED_MAX_ID) {

test/tests/api/common/utils/EntityUtilsTest.ts

@@ -1,5 +1,6 @@
 import o from "@tutao/otest"
 import {
+	constructMailSetEntryId,
 	create,
 	GENERATED_MIN_ID,
 	generatedIdToTimestamp,
@@ -31,6 +32,16 @@ o.spec("EntityUtils", function () {
 		o(generatedIdToTimestamp("IwQvgF------")).equals(1370563200000)
 	})
 
+	o("test constructcustomId for mailSetEntry", function () {
+		const mailId: Id = "-----------0"
+
+		const expected = "V7iDsQAAAAAAAAAAAQ"
+		const receiveDate = new Date("2017-10-03 13:46:13")
+
+		const calculatedId = constructMailSetEntryId(receiveDate, mailId)
+		o(expected).equals(calculatedId)
+	})
+
 	o("create new entity without error object ", function () {
 		const mailEntity = create(typeModels.Mail, MailTypeRef)
 		o(mailEntity._errors).equals(undefined)

test/tests/api/worker/offline/OfflineStorageMigratorTest.ts

@@ -1,13 +1,14 @@
 import o from "@tutao/otest"
 import { OfflineMigration, OfflineStorageMigrator } from "../../../../../src/common/api/worker/offline/OfflineStorageMigrator.js"
-import { OfflineStorage } from "../../../../../src/common/api/worker/offline/OfflineStorage.js"
+import { ensureBase64Ext, OfflineStorage } from "../../../../../src/common/api/worker/offline/OfflineStorage.js"
 import { func, instance, matchers, object, when } from "testdouble"
 import { assertThrows, verify } from "@tutao/tutanota-test-utils"
-import { ModelInfos } from "../../../../../src/common/api/common/EntityFunctions.js"
+import { ModelInfos, resolveTypeReference } from "../../../../../src/common/api/common/EntityFunctions.js"
-import { typedEntries } from "@tutao/tutanota-utils"
+import { repeat, typedEntries } from "@tutao/tutanota-utils"
 import { ProgrammingError } from "../../../../../src/common/api/common/error/ProgrammingError.js"
 import { SqlCipherFacade } from "../../../../../src/common/native/common/generatedipc/SqlCipherFacade.js"
 import { OutOfSyncError } from "../../../../../src/common/api/common/error/OutOfSyncError.js"
+import { CalendarEventTypeRef, MailSetEntryTypeRef } from "../../../../../src/common/api/entities/tutanota/TypeRefs.js"
 
 o.spec("OfflineStorageMigrator", function () {
 	const modelInfos: ModelInfos = {
@@ -114,4 +115,17 @@ o.spec("OfflineStorageMigrator", function () {
 		verify(migration.migrate(storage, sqlCipherFacade), { times: 0 })
 		verify(storage.setStoredModelVersion("tutanota", matchers.anything()), { times: 0 })
 	})
+
+	o("sanity check for customids", async function () {
+		const mailSetEntryCustomId = "ZFvk7GRb5OwzgABABQ"
+		const typeModel = await resolveTypeReference(MailSetEntryTypeRef)
+		const convertedId = ensureBase64Ext(typeModel, mailSetEntryCustomId)
+		console.log(convertedId)
+	})
+
+	o("sanity check for max custom id", async function () {
+		const typeModel = await resolveTypeReference(CalendarEventTypeRef)
+		const convertedId = ensureBase64Ext(typeModel, repeat("z", 340))
+		console.log(convertedId)
+	})
 })

test/tests/misc/ListModelTest.ts

@@ -1,6 +1,6 @@
 import o from "@tutao/otest"
 import { ListModel, ListModelConfig } from "../../../src/common/misc/ListModel.js"
-import { GENERATED_MAX_ID, getElementId, getListId, sortCompareById, timestampToGeneratedId } from "../../../src/common/api/common/utils/EntityUtils.js"
+import { getElementId, getListId, sortCompareById, timestampToGeneratedId } from "../../../src/common/api/common/utils/EntityUtils.js"
 import { defer, DeferredObject } from "@tutao/tutanota-utils"
 import { KnowledgeBaseEntry, KnowledgeBaseEntryTypeRef } from "../../../src/common/api/entities/tutanota/TypeRefs.js"
 import { ListFetchResult } from "../../../src/common/gui/base/ListUtils.js"
@@ -15,7 +15,6 @@ o.spec("ListModel", function () {
 	let fetchDefer: DeferredObject<ListFetchResult<KnowledgeBaseEntry>>
 	let listModel: ListModel<KnowledgeBaseEntry>
 	const defaultListConfig = {
-		topId: GENERATED_MAX_ID,
 		fetch: () => fetchDefer.promise,
 		sortCompare: sortCompareById,
 		loadSingle: () => {