v6.2.2

Prepare changelog for v6.2.2
Merge pull request #1407 from matrix-org/jryans/1403-release
2020-06-16 11:00:41 +01:00 · 2020-06-16 11:00:41 +01:00 · 2020-06-16 10:56:00 +01:00 · 2020-06-16 10:38:41 +01:00 · 2020-06-16 10:38:41 +01:00 · 2020-06-16 10:38:41 +01:00
67 changed files with 6296 additions and 2805 deletions
@@ -1,58 +0,0 @@
-steps:
-  - label: ":eslint: JS Lint"
-    command:
-      - "yarn install"
-      - "yarn lint:js"
-    plugins:
-      - docker#v3.0.1:
-          image: "node:12"
-
-  - label: ":tslint: TS Lint"
-    command:
-      - "yarn install"
-      - "yarn lint:ts"
-    plugins:
-      - docker#v3.0.1:
-          image: "node:12"
-
-  - label: ":typescript: Types Lint"
-    command:
-      - "yarn install"
-      - "yarn lint:types"
-    plugins:
-      - docker#v3.0.1:
-          image: "node:12"
-
-  - label: "🛠 Build"
-    command:
-      - "yarn install"
-      - "yarn build"
-    plugins:
-      - docker#v3.0.1:
-          image: "node:12"
-
-  - label: ":jest: Tests"
-    command:
-      - "yarn install"
-      - "yarn test"
-    plugins:
-      - docker#v3.0.1:
-          image: "node:12"
-
-  - label: "📃 Docs"
-    command:
-      - "yarn install"
-      - "yarn gendoc"
-    plugins:
-      - docker#v3.0.1:
-          image: "node:12"
-
-  - wait
-
-  - label: "🐴 Trigger matrix-react-sdk"
-    trigger: "matrix-react-sdk"
-    branches: "develop"
-    build:
-        branch: "develop"
-        message: "[js-sdk] ${BUILDKITE_MESSAGE}"
-    async: true
@@ -1,3 +1,426 @@
+Changes in [6.2.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.2.2) (2020-06-16)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.2.1...v6.2.2)
+
+ * Use existing session id for fetching flows as to not get a new session
+   [\#1407](https://github.com/matrix-org/matrix-js-sdk/pull/1407)
+
+Changes in [6.2.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.2.1) (2020-06-05)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.2.0...v6.2.1)
+
+ * Bring back backup key format migration
+   [\#1399](https://github.com/matrix-org/matrix-js-sdk/pull/1399)
+
+Changes in [6.2.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.2.0) (2020-06-04)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.2.0-rc.1...v6.2.0)
+
+ * No changes since rc.1
+
+Changes in [6.2.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.2.0-rc.1) (2020-06-02)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.1.0...v6.2.0-rc.1)
+
+ * Make auth argument in the register request compliant with r0.6.0
+   [\#1304](https://github.com/matrix-org/matrix-js-sdk/pull/1304)
+ * Send the wrong auth params with the right auth params
+   [\#1393](https://github.com/matrix-org/matrix-js-sdk/pull/1393)
+ * encrypt cached keys with pickle key
+   [\#1387](https://github.com/matrix-org/matrix-js-sdk/pull/1387)
+ * Fix replying to key share requests
+   [\#1385](https://github.com/matrix-org/matrix-js-sdk/pull/1385)
+ * Add dist to package.json files so CDNs can serve it
+   [\#1384](https://github.com/matrix-org/matrix-js-sdk/pull/1384)
+ * Fix getVersion warning saying undefined room
+   [\#1382](https://github.com/matrix-org/matrix-js-sdk/pull/1382)
+ * Combine the two places we processed client-level default push rules
+   [\#1379](https://github.com/matrix-org/matrix-js-sdk/pull/1379)
+ * make MAC check robust against unpadded vs padded base64 differences
+   [\#1378](https://github.com/matrix-org/matrix-js-sdk/pull/1378)
+ * Remove key backup format migration
+   [\#1375](https://github.com/matrix-org/matrix-js-sdk/pull/1375)
+ * Add simple browserify browser-matrix.js tests
+   [\#1241](https://github.com/matrix-org/matrix-js-sdk/pull/1241)
+ * support new key agreement method for SAS
+   [\#1376](https://github.com/matrix-org/matrix-js-sdk/pull/1376)
+
+Changes in [6.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.1.0) (2020-05-19)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.1.0-rc.1...v6.1.0)
+
+ * No changes since rc.1
+
+Changes in [6.1.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.1.0-rc.1) (2020-05-14)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.0.0...v6.1.0-rc.1)
+
+ * Remove support for asymmetric 4S encryption
+   [\#1373](https://github.com/matrix-org/matrix-js-sdk/pull/1373)
+ * Increase timeout for 2nd phase of Olm session creation
+   [\#1367](https://github.com/matrix-org/matrix-js-sdk/pull/1367)
+ * Add logging on decryption retries
+   [\#1366](https://github.com/matrix-org/matrix-js-sdk/pull/1366)
+ * Emit event when a trusted self-key is stored
+   [\#1364](https://github.com/matrix-org/matrix-js-sdk/pull/1364)
+ * Customize error payload for oversized messages
+   [\#1352](https://github.com/matrix-org/matrix-js-sdk/pull/1352)
+ * Return null for key backup state when we haven't checked yet
+   [\#1363](https://github.com/matrix-org/matrix-js-sdk/pull/1363)
+ * Added a progressCallback for backup key loading
+   [\#1351](https://github.com/matrix-org/matrix-js-sdk/pull/1351)
+ * Add initialFetch param to willUpdateDevices / devicesUpdated
+   [\#1360](https://github.com/matrix-org/matrix-js-sdk/pull/1360)
+ * Fix race between sending .request and receiving .ready over to_device
+   [\#1359](https://github.com/matrix-org/matrix-js-sdk/pull/1359)
+ * Handle race between sending and await next event from other party
+   [\#1357](https://github.com/matrix-org/matrix-js-sdk/pull/1357)
+ * Add crypto.willUpdateDevices event and make
+   getStoredDevices/getStoredDevicesForUser synchronous
+   [\#1354](https://github.com/matrix-org/matrix-js-sdk/pull/1354)
+ * Fix sender of local echo events in unsigned redactions
+   [\#1350](https://github.com/matrix-org/matrix-js-sdk/pull/1350)
+ * Remove redundant key backup setup path
+   [\#1353](https://github.com/matrix-org/matrix-js-sdk/pull/1353)
+ * Remove some dead code from _retryDecryption
+   [\#1349](https://github.com/matrix-org/matrix-js-sdk/pull/1349)
+ * Don't send key requests until after sync processing is finished
+   [\#1348](https://github.com/matrix-org/matrix-js-sdk/pull/1348)
+ * Prevent attempts to send olm messages to ourselves
+   [\#1346](https://github.com/matrix-org/matrix-js-sdk/pull/1346)
+ * Retry account data upload requests
+   [\#1345](https://github.com/matrix-org/matrix-js-sdk/pull/1345)
+ * Log first known index with megolm session updates
+   [\#1344](https://github.com/matrix-org/matrix-js-sdk/pull/1344)
+ * Prune to_device messages to avoid sending empty messages
+   [\#1343](https://github.com/matrix-org/matrix-js-sdk/pull/1343)
+ * Convert bunch of things to TypeScript
+   [\#1335](https://github.com/matrix-org/matrix-js-sdk/pull/1335)
+ * Add logging when making new Olm sessions
+   [\#1342](https://github.com/matrix-org/matrix-js-sdk/pull/1342)
+ * Fix: handle filter not found
+   [\#1340](https://github.com/matrix-org/matrix-js-sdk/pull/1340)
+ * Make getAccountDataFromServer return null if not found
+   [\#1338](https://github.com/matrix-org/matrix-js-sdk/pull/1338)
+ * Fix setDefaultKeyId to fail if the request fails
+   [\#1336](https://github.com/matrix-org/matrix-js-sdk/pull/1336)
+ * Document setRoomEncryption not modifying room state
+   [\#1328](https://github.com/matrix-org/matrix-js-sdk/pull/1328)
+ * Fix: don't do extra /filter request when enabling lazy loading of members
+   [\#1332](https://github.com/matrix-org/matrix-js-sdk/pull/1332)
+ * Reject attemptAuth promise if no auth flow found
+   [\#1329](https://github.com/matrix-org/matrix-js-sdk/pull/1329)
+ * Fix FilterComponent allowed_values check
+   [\#1327](https://github.com/matrix-org/matrix-js-sdk/pull/1327)
+ * Serialise Olm prekey decryptions
+   [\#1326](https://github.com/matrix-org/matrix-js-sdk/pull/1326)
+ * Fix: crash when backup key needs fixing from corruption issue
+   [\#1324](https://github.com/matrix-org/matrix-js-sdk/pull/1324)
+ * Fix cross-signing/SSSS reset
+   [\#1322](https://github.com/matrix-org/matrix-js-sdk/pull/1322)
+ * Implement QR code reciprocate for self-verification with untrusted MSK
+   [\#1320](https://github.com/matrix-org/matrix-js-sdk/pull/1320)
+
+Changes in [6.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.0.0) (2020-05-05)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.0.0-rc.2...v6.0.0)
+
+ * Add progress callback for key backups
+   [\#1368](https://github.com/matrix-org/matrix-js-sdk/pull/1368)
+
+Changes in [6.0.0-rc.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.0.0-rc.2) (2020-05-01)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.0.0-rc.1...v6.0.0-rc.2)
+
+ * Emit event when a trusted self-key is stored
+   [\#1365](https://github.com/matrix-org/matrix-js-sdk/pull/1365)
+
+Changes in [6.0.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.0.0-rc.1) (2020-04-30)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.3.1-rc.4...v6.0.0-rc.1)
+
+BREAKING CHANGES
+---
+
+ * client.getStoredDevicesForUser and client.getStoredDevices are no longer async
+
+All Changes
+---
+
+ * Add initialFetch param to willUpdateDevices / devicesUpdated
+   [\#1362](https://github.com/matrix-org/matrix-js-sdk/pull/1362)
+ * Fix race between sending .request and receiving .ready over to_device
+   [\#1361](https://github.com/matrix-org/matrix-js-sdk/pull/1361)
+ * Handle race between sending and await next event from other party
+   [\#1358](https://github.com/matrix-org/matrix-js-sdk/pull/1358)
+ * Add crypto.willUpdateDevices event and make
+   getStoredDevices/getStoredDevicesForUser synchronous
+   [\#1356](https://github.com/matrix-org/matrix-js-sdk/pull/1356)
+ * Remove redundant key backup setup path
+   [\#1355](https://github.com/matrix-org/matrix-js-sdk/pull/1355)
+
+Changes in [5.3.1-rc.4](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.3.1-rc.4) (2020-04-23)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.3.1-rc.3...v5.3.1-rc.4)
+
+ * Retry account data upload requests
+   [\#1347](https://github.com/matrix-org/matrix-js-sdk/pull/1347)
+ * Fix: handle filter not found
+   [\#1341](https://github.com/matrix-org/matrix-js-sdk/pull/1341)
+ * Make getAccountDataFromServer return null if not found
+   [\#1339](https://github.com/matrix-org/matrix-js-sdk/pull/1339)
+ * Fix setDefaultKeyId to fail if the request fails
+   [\#1337](https://github.com/matrix-org/matrix-js-sdk/pull/1337)
+ * Fix: don't do extra /filter request when enabling lazy loading of members
+   [\#1333](https://github.com/matrix-org/matrix-js-sdk/pull/1333)
+ * Reject attemptAuth promise if no auth flow found
+   [\#1331](https://github.com/matrix-org/matrix-js-sdk/pull/1331)
+ * Serialise Olm prekey decryptions
+   [\#1330](https://github.com/matrix-org/matrix-js-sdk/pull/1330)
+
+Changes in [5.3.1-rc.3](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.3.1-rc.3) (2020-04-17)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.3.1-rc.2...v5.3.1-rc.3)
+
+ * Fix cross-signing/SSSS reset
+   [\#1323](https://github.com/matrix-org/matrix-js-sdk/pull/1323)
+ * Fix: crash when backup key needs fixing from corruption issue
+   [\#1325](https://github.com/matrix-org/matrix-js-sdk/pull/1325)
+
+Changes in [5.3.1-rc.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.3.1-rc.2) (2020-04-16)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.3.1-rc.1...v5.3.1-rc.2)
+
+ * Implement QR code reciprocate for self-verification with untrusted MSK
+   [\#1321](https://github.com/matrix-org/matrix-js-sdk/pull/1321)
+
+Changes in [5.3.1-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.3.1-rc.1) (2020-04-15)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.3.0-rc.1...v5.3.1-rc.1)
+
+ * Adapt release script for riot-desktop
+   [\#1319](https://github.com/matrix-org/matrix-js-sdk/pull/1319)
+ * Fix: prevent spurious notifications from indexer
+   [\#1318](https://github.com/matrix-org/matrix-js-sdk/pull/1318)
+ * Always create our own user object
+   [\#1317](https://github.com/matrix-org/matrix-js-sdk/pull/1317)
+ * Fix incorrect backup key format in SSSS
+   [\#1311](https://github.com/matrix-org/matrix-js-sdk/pull/1311)
+ * Fix e2ee crash after refreshing after having received a cross-singing key
+   reset
+   [\#1315](https://github.com/matrix-org/matrix-js-sdk/pull/1315)
+ * Fix: catch send errors in SAS verifier
+   [\#1314](https://github.com/matrix-org/matrix-js-sdk/pull/1314)
+ * Clear cross-signing keys when detecting the keys have changed
+   [\#1312](https://github.com/matrix-org/matrix-js-sdk/pull/1312)
+ * Upgrade deps
+   [\#1310](https://github.com/matrix-org/matrix-js-sdk/pull/1310)
+
+Changes in [5.3.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.3.0-rc.1) (2020-04-08)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.2.0...v5.3.0-rc.1)
+
+ * Store key backup key in cache as Uint8Array
+   [\#1308](https://github.com/matrix-org/matrix-js-sdk/pull/1308)
+ * Use the correct request body for the /keys/query endpoint.
+   [\#1307](https://github.com/matrix-org/matrix-js-sdk/pull/1307)
+ * Avoid creating two devices on registration
+   [\#1305](https://github.com/matrix-org/matrix-js-sdk/pull/1305)
+ * Lower max-warnings to 81
+   [\#1306](https://github.com/matrix-org/matrix-js-sdk/pull/1306)
+ * Move key backup key creation before caching
+   [\#1303](https://github.com/matrix-org/matrix-js-sdk/pull/1303)
+ * Expose function to force-reset outgoing room key requests
+   [\#1298](https://github.com/matrix-org/matrix-js-sdk/pull/1298)
+ * Add isSelfVerification property to VerificationRequest
+   [\#1302](https://github.com/matrix-org/matrix-js-sdk/pull/1302)
+ * QR code reciprocation
+   [\#1297](https://github.com/matrix-org/matrix-js-sdk/pull/1297)
+ * Add ability to check symmetric SSSS key before we try to use it
+   [\#1294](https://github.com/matrix-org/matrix-js-sdk/pull/1294)
+ * Add some debug logging for events stuck to bottom of timeline
+   [\#1296](https://github.com/matrix-org/matrix-js-sdk/pull/1296)
+ * Fix: spontanous verification request cancellation under some circumstances
+   [\#1295](https://github.com/matrix-org/matrix-js-sdk/pull/1295)
+ * Receive private key for caching from the app layer
+   [\#1293](https://github.com/matrix-org/matrix-js-sdk/pull/1293)
+ * Track whether we have verified a user before
+   [\#1292](https://github.com/matrix-org/matrix-js-sdk/pull/1292)
+ * Fix: error during tests
+   [\#1222](https://github.com/matrix-org/matrix-js-sdk/pull/1222)
+ * Send .done event for to_device verification
+   [\#1288](https://github.com/matrix-org/matrix-js-sdk/pull/1288)
+ * Request the key backup key & restore backup
+   [\#1291](https://github.com/matrix-org/matrix-js-sdk/pull/1291)
+ * Make screen sharing works on Chrome using getDisplayMedia()
+   [\#1276](https://github.com/matrix-org/matrix-js-sdk/pull/1276)
+ * Fix isVerified returning false
+   [\#1289](https://github.com/matrix-org/matrix-js-sdk/pull/1289)
+ * Fix: verification gets cancelled when event gets duplicated
+   [\#1286](https://github.com/matrix-org/matrix-js-sdk/pull/1286)
+ * Use requestSecret on the client to request secrets
+   [\#1287](https://github.com/matrix-org/matrix-js-sdk/pull/1287)
+ * Allow guests to fetch TURN servers
+   [\#1277](https://github.com/matrix-org/matrix-js-sdk/pull/1277)
+
+Changes in [5.2.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.2.0) (2020-03-30)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.2.0-rc.1...v5.2.0)
+
+ * Fix isVerified returning false
+   [\#1290](https://github.com/matrix-org/matrix-js-sdk/pull/1290)
+
+Changes in [5.2.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.2.0-rc.1) (2020-03-26)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.1.1...v5.2.0-rc.1)
+
+ * Add a flag for whether cross signing signatures are trusted
+   [\#1285](https://github.com/matrix-org/matrix-js-sdk/pull/1285)
+ * Cache user and self signing keys during bootstrap
+   [\#1282](https://github.com/matrix-org/matrix-js-sdk/pull/1282)
+ * remove unnecessary promise
+   [\#1283](https://github.com/matrix-org/matrix-js-sdk/pull/1283)
+ * Functions to cache session backups key automatically
+   [\#1281](https://github.com/matrix-org/matrix-js-sdk/pull/1281)
+ * Add function for checking cross-signing is ready
+   [\#1279](https://github.com/matrix-org/matrix-js-sdk/pull/1279)
+ * Use symmetric encryption for SSSS
+   [\#1228](https://github.com/matrix-org/matrix-js-sdk/pull/1228)
+ * Migrate SSSS to use symmetric algorithm
+   [\#1238](https://github.com/matrix-org/matrix-js-sdk/pull/1238)
+ * Migration to symmetric SSSS
+   [\#1272](https://github.com/matrix-org/matrix-js-sdk/pull/1272)
+ * Reduce number of one-time-key requests
+   [\#1280](https://github.com/matrix-org/matrix-js-sdk/pull/1280)
+ * Fix: assume the requested method is supported by other party with to_device
+   [\#1275](https://github.com/matrix-org/matrix-js-sdk/pull/1275)
+ * Use checkDeviceTrust when computing untrusted devices
+   [\#1278](https://github.com/matrix-org/matrix-js-sdk/pull/1278)
+ * Add a store for backup keys
+   [\#1271](https://github.com/matrix-org/matrix-js-sdk/pull/1271)
+ * Upload only new device signature of master key
+   [\#1268](https://github.com/matrix-org/matrix-js-sdk/pull/1268)
+ * Expose prepareToEncrypt in the client API
+   [\#1270](https://github.com/matrix-org/matrix-js-sdk/pull/1270)
+ * Don't kill the whole device download if one device gives an error
+   [\#1269](https://github.com/matrix-org/matrix-js-sdk/pull/1269)
+ * Handle racing .start event during self verification
+   [\#1267](https://github.com/matrix-org/matrix-js-sdk/pull/1267)
+ * A crypto.keySignatureUploadFailure event reported the wrong source
+   [\#1266](https://github.com/matrix-org/matrix-js-sdk/pull/1266)
+ * Fix editing of unsent messages by waiting for actual event id
+   [\#1263](https://github.com/matrix-org/matrix-js-sdk/pull/1263)
+ * Fix: ensureOlmSessionsForDevices parameter format
+   [\#1264](https://github.com/matrix-org/matrix-js-sdk/pull/1264)
+ * Remove stuff that yarn install doesn't think we need
+   [\#1261](https://github.com/matrix-org/matrix-js-sdk/pull/1261)
+ * Fix: prevent error being thrown during sync in some cases
+   [\#1258](https://github.com/matrix-org/matrix-js-sdk/pull/1258)
+ * Force `is_verified` for key backups to bool and fix computation
+   [\#1259](https://github.com/matrix-org/matrix-js-sdk/pull/1259)
+ * Add a method for legacy single device verification, returning a verification
+   request
+   [\#1257](https://github.com/matrix-org/matrix-js-sdk/pull/1257)
+ * yarn upgrade
+   [\#1256](https://github.com/matrix-org/matrix-js-sdk/pull/1256)
+
+Changes in [5.1.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.1.1) (2020-03-17)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.1.1-rc.1...v5.1.1)
+
+ * Fix: ensureOlmSessionsForDevices parameter format
+   [\#1265](https://github.com/matrix-org/matrix-js-sdk/pull/1265)
+ * Fix: prevent error being thrown during sync in some cases
+   [\#1262](https://github.com/matrix-org/matrix-js-sdk/pull/1262)
+ * Force `is_verified` for key backups to bool and fix computation
+   [\#1260](https://github.com/matrix-org/matrix-js-sdk/pull/1260)
+
+Changes in [5.1.1-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.1.1-rc.1) (2020-03-11)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.1.0...v5.1.1-rc.1)
+
+ * refactor megolm encryption to improve perceived speed
+   [\#1252](https://github.com/matrix-org/matrix-js-sdk/pull/1252)
+ * Remove v1 identity server fallbacks
+   [\#1253](https://github.com/matrix-org/matrix-js-sdk/pull/1253)
+ * Use alt_aliases instead of local ones for room names
+   [\#1251](https://github.com/matrix-org/matrix-js-sdk/pull/1251)
+ * Upload cross-signing key signatures in the background
+   [\#1250](https://github.com/matrix-org/matrix-js-sdk/pull/1250)
+ * Fix secret sharing names to match spec
+   [\#1249](https://github.com/matrix-org/matrix-js-sdk/pull/1249)
+ * Cleanup: remove crypto.verification.start event
+   [\#1248](https://github.com/matrix-org/matrix-js-sdk/pull/1248)
+ * Fix regression in key backup request params
+   [\#1246](https://github.com/matrix-org/matrix-js-sdk/pull/1246)
+ * Use cross-signing trust to mark backups verified
+   [\#1244](https://github.com/matrix-org/matrix-js-sdk/pull/1244)
+ * Check both cross-signing and local trust for key sharing
+   [\#1243](https://github.com/matrix-org/matrix-js-sdk/pull/1243)
+ * Fixed up tests to match new way that crypto stores are created
+   [\#1242](https://github.com/matrix-org/matrix-js-sdk/pull/1242)
+ * Store USK and SSK locally
+   [\#1235](https://github.com/matrix-org/matrix-js-sdk/pull/1235)
+ * Use unpadded base64 for QR code secrets
+   [\#1236](https://github.com/matrix-org/matrix-js-sdk/pull/1236)
+ * Don't require .done event for finishing self-verification
+   [\#1239](https://github.com/matrix-org/matrix-js-sdk/pull/1239)
+ * Don't cancel as 3rd party in verification request
+   [\#1237](https://github.com/matrix-org/matrix-js-sdk/pull/1237)
+ * Verification: log when switching start event
+   [\#1234](https://github.com/matrix-org/matrix-js-sdk/pull/1234)
+ * Perform crypto store operations directly after transaction
+   [\#1233](https://github.com/matrix-org/matrix-js-sdk/pull/1233)
+ * More verification request logging
+   [\#1232](https://github.com/matrix-org/matrix-js-sdk/pull/1232)
+ * Upgrade deps
+   [\#1231](https://github.com/matrix-org/matrix-js-sdk/pull/1231)
+
+Changes in [5.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.1.0) (2020-03-02)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.1.0-rc.1...v5.1.0)
+
+ * No changes since rc.1
+
+Changes in [5.1.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.1.0-rc.1) (2020-02-26)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.0.1...v5.1.0-rc.1)
+
+ * Add latest dist-tag for releases
+   [\#1230](https://github.com/matrix-org/matrix-js-sdk/pull/1230)
+ * Add room method for alt_aliases
+   [\#1225](https://github.com/matrix-org/matrix-js-sdk/pull/1225)
+ * Remove buildkite pipeline
+   [\#1227](https://github.com/matrix-org/matrix-js-sdk/pull/1227)
+ * don't assume verify has been called when receiving a cancellation in
+   verifier
+   [\#1226](https://github.com/matrix-org/matrix-js-sdk/pull/1226)
+ * Reduce secret size for new binary packing
+   [\#1221](https://github.com/matrix-org/matrix-js-sdk/pull/1221)
+ * misc rageshake fixes
+   [\#1223](https://github.com/matrix-org/matrix-js-sdk/pull/1223)
+ * Fix cancelled historical requests not appearing as cancelled
+   [\#1220](https://github.com/matrix-org/matrix-js-sdk/pull/1220)
+ * Fix renaming error that broke QR code verification
+   [\#1217](https://github.com/matrix-org/matrix-js-sdk/pull/1217)
+
+Changes in [5.0.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.0.1) (2020-02-19)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.0.0...v5.0.1)
+
+ * add method for new /aliases endpoint
+   [\#1219](https://github.com/matrix-org/matrix-js-sdk/pull/1219)
+ * method for checking if other party supports verification method
+   [\#1213](https://github.com/matrix-org/matrix-js-sdk/pull/1213)
+ * add local echo state for accepting or declining a verif req
+   [\#1210](https://github.com/matrix-org/matrix-js-sdk/pull/1210)
+ * make logging compatible with rageshakes
+   [\#1214](https://github.com/matrix-org/matrix-js-sdk/pull/1214)
+ * Find existing requests when starting a new verification request
+   [\#1209](https://github.com/matrix-org/matrix-js-sdk/pull/1209)
+ * log MAC calculation during SAS
+   [\#1211](https://github.com/matrix-org/matrix-js-sdk/pull/1211)
+
 Changes in [5.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.0.0) (2020-02-17)
 ================================================================================================
 [Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.0.0-rc.1...v5.0.0)
@@ -1,6 +1,6 @@
 {
  "name": "matrix-js-sdk",
-  "version": "5.0.0",
+  "version": "6.2.2",
  "description": "Matrix Client-Server SDK for Javascript",
  "scripts": {
    "prepare": "yarn build",
@@ -14,7 +14,7 @@
    "build:minify-browser": "terser dist/browser-matrix.js --compress --mangle --source-map --output dist/browser-matrix.min.js",
    "gendoc": "jsdoc -c jsdoc.json -P package.json",
    "lint": "yarn lint:types && yarn lint:ts && yarn lint:js",
-    "lint:js": "eslint --max-warnings 93 src spec",
+    "lint:js": "eslint --max-warnings 81 src spec",
    "lint:types": "tsc --noEmit",
    "lint:ts": "tslint --project ./tsconfig.json -t stylish",
    "test": "jest spec/ --coverage --testEnvironment node",
@@ -35,6 +35,7 @@
  "author": "matrix.org",
  "license": "Apache-2.0",
  "files": [
+    "dist",
    "lib",
    "src",
    "git-revision.txt",
@@ -68,6 +69,7 @@
    "@babel/preset-typescript": "^7.7.4",
    "@babel/register": "^7.7.4",
    "@types/node": "12",
+    "@types/request": "^2.48.4",
    "babel-eslint": "^10.0.3",
    "babel-jest": "^24.9.0",
    "babelify": "^10.0.0",
@@ -78,7 +80,9 @@
    "eslint-plugin-babel": "^5.3.0",
    "eslint-plugin-jest": "^23.0.4",
    "exorcist": "^1.0.1",
+    "fake-indexeddb": "^3.0.0",
    "jest": "^24.9.0",
+    "jest-localstorage-mock": "^2.4.0",
    "jsdoc": "^3.5.5",
    "matrix-mock-request": "^1.2.3",
    "olm": "https://packages.matrix.org/npm/olm/olm-3.1.4.tgz",
@@ -38,6 +38,7 @@ $USAGE
    -c changelog_file:  specify name of file containing changelog
    -x:                 skip updating the changelog
    -z:                 skip generating the jsdoc
+    -n:                 skip publish to NPM
 EOF
 }

@@ -60,9 +61,10 @@ fi

 skip_changelog=
 skip_jsdoc=
+skip_npm=
 changelog_file="CHANGELOG.md"
 expected_npm_user="matrixdotorg"
-while getopts hc:u:xz f; do
+while getopts hc:u:xzn f; do
    case $f in
        h)
            help
@@ -77,6 +79,9 @@ while getopts hc:u:xz f; do
        z)
            skip_jsdoc=1
            ;;
+        n)
+            skip_npm=1
+            ;;
        u)
            expected_npm_user="$OPTARG"
            ;;
@@ -96,10 +101,12 @@ fi

 # Login and publish continues to use `npm`, as it seems to have more clearly
 # defined options and semantics than `yarn` for writing to the registry.
-actual_npm_user=`npm whoami`;
-if [ $expected_npm_user != $actual_npm_user ]; then
-    echo "you need to be logged into npm as $expected_npm_user, but you are logged in as $actual_npm_user" >&2
-    exit 1
+if [ -z "$skip_npm" ]; then
+    actual_npm_user=`npm whoami`;
+    if [ $expected_npm_user != $actual_npm_user ]; then
+        echo "you need to be logged into npm as $expected_npm_user, but you are logged in as $actual_npm_user" >&2
+        exit 1
+    fi
 fi

 # ignore leading v on release
@@ -294,14 +301,18 @@ fi
 rm "${release_text}"
 rm "${latest_changes}"

-npm_publish_flags=''
-if [ $prerelease -eq 1 ]; then
-    # Tag prereleases as `next` so the last stable release remains the default
-    npm_publish_flags='--tag next'
-fi
 # Login and publish continues to use `npm`, as it seems to have more clearly
 # defined options and semantics than `yarn` for writing to the registry.
-npm publish $npm_publish_flags
+# Tag both releases and prereleases as `next` so the last stable release remains
+# the default.
+if [ -z "$skip_npm" ]; then
+    npm publish --tag next
+    if [ $prerelease -eq 0 ]; then
+        # For a release, also add the default `latest` tag.
+        package=$(cat package.json | jq -er .name)
+        npm dist-tag add "$package@$release" latest
+    fi
+fi

 if [ -z "$skip_jsdoc" ]; then
    echo "generating jsdocs"
@@ -336,8 +347,10 @@ if [ -z "$skip_jsdoc" ]; then
    git push origin gh-pages
 fi

-# finally, merge master back onto develop
-git checkout develop
-git pull
-git merge master
-git push origin develop
+# finally, merge master back onto develop (if it exists)
+if [ $(git branch -lr | grep origin/develop -c) -ge 1 ]; then
+    git checkout develop
+    git pull
+    git merge master
+    git push origin develop
+fi
@@ -185,7 +185,7 @@ TestClient.prototype.expectKeyQuery = function(response) {
        200, (path, content) => {
            Object.keys(response.device_keys).forEach((userId) => {
                expect(content.device_keys[userId]).toEqual(
-                    {},
+                    [],
                    "Expected key query for " + userId + ", got " +
                    Object.keys(content.device_keys),
                );
@@ -0,0 +1,23 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// stub for browser-matrix browserify tests
+global.XMLHttpRequest = jest.fn();
+
+afterAll(() => {
+    // clean up XMLHttpRequest mock
+    global.XMLHttpRequest = undefined;
+});
@@ -0,0 +1,103 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// load XmlHttpRequest mock
+import "./setupTests";
+import "../../dist/browser-matrix"; // uses browser-matrix instead of the src
+import {MockStorageApi} from "../MockStorageApi";
+import {WebStorageSessionStore} from "../../src/store/session/webstorage";
+import MockHttpBackend from "matrix-mock-request";
+import {LocalStorageCryptoStore} from "../../src/crypto/store/localStorage-crypto-store";
+import * as utils from "../test-utils";
+
+const USER_ID = "@user:test.server";
+const DEVICE_ID = "device_id";
+const ACCESS_TOKEN = "access_token";
+const ROOM_ID = "!room_id:server.test";
+
+/* global matrixcs */
+
+describe("Browserify Test", function() {
+    let client;
+    let httpBackend;
+
+    async function createTestClient() {
+        const sessionStoreBackend = new MockStorageApi();
+        const sessionStore = new WebStorageSessionStore(sessionStoreBackend);
+        const httpBackend = new MockHttpBackend();
+
+        const options = {
+            baseUrl: "http://" + USER_ID + ".test.server",
+            userId: USER_ID,
+            accessToken: ACCESS_TOKEN,
+            deviceId: DEVICE_ID,
+            sessionStore: sessionStore,
+            request: httpBackend.requestFn,
+            cryptoStore: new LocalStorageCryptoStore(sessionStoreBackend),
+        };
+
+        const client = matrixcs.createClient(options);
+
+        httpBackend.when("GET", "/pushrules").respond(200, {});
+        httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
+
+        return { client, httpBackend };
+    }
+
+    beforeEach(async () => {
+        ({client, httpBackend} = await createTestClient());
+        await client.startClient();
+    });
+
+    afterEach(async () => {
+        client.stopClient();
+        await httpBackend.stop();
+    });
+
+    it("Sync", async function() {
+        const event = utils.mkMembership({
+            room: ROOM_ID,
+            mship: "join",
+            user: "@other_user:server.test",
+            name: "Displayname",
+        });
+
+        const syncData = {
+            next_batch: "batch1",
+            rooms: {
+                join: {},
+            },
+        };
+        syncData.rooms.join[ROOM_ID] = {
+            timeline: {
+                events: [
+                    event,
+                ],
+                limited: false,
+            },
+        };
+
+        httpBackend.when("GET", "/sync").respond(200, syncData);
+        await Promise.race([
+            Promise.all([
+                httpBackend.flushAllExpected(),
+            ]),
+            new Promise((_, reject) => {
+                client.once("sync.unexpectedError", reject);
+            }),
+        ]);
+    }, 10000);
+});
@@ -70,7 +70,7 @@ function expectAliQueryKeys() {
    aliTestClient.httpBackend.when("POST", "/keys/query")
            .respond(200, function(path, content) {
        expect(content.device_keys[bobUserId]).toEqual(
-            {},
+            [],
            "Expected Alice to key query for " + bobUserId + ", got " +
            Object.keys(content.device_keys),
        );
@@ -98,7 +98,7 @@ function expectBobQueryKeys() {
        "POST", "/keys/query",
    ).respond(200, function(path, content) {
        expect(content.device_keys[aliUserId]).toEqual(
-            {},
+            [],
            "Expected Bob to key query for " + aliUserId + ", got " +
            Object.keys(content.device_keys),
        );
@@ -347,8 +347,8 @@ describe("MatrixClient", function() {

            httpBackend.when("POST", "/keys/query").check(function(req) {
                expect(req.data).toEqual({device_keys: {
-                    'boris': {},
-                    'chaz': {},
+                    'boris': [],
+                    'chaz': [],
                }});
            }).respond(200, {
                device_keys: {
@@ -3,6 +3,7 @@ import HttpBackend from "matrix-mock-request";
 import {MatrixClient} from "../../src/matrix";
 import {MatrixScheduler} from "../../src/scheduler";
 import {MemoryStore} from "../../src/store/memory";
+import {MatrixError} from "../../src/http-api";

 describe("MatrixClient opts", function() {
    const baseUrl = "http://localhost.or.something";
@@ -132,10 +133,10 @@ describe("MatrixClient opts", function() {
        });

        it("shouldn't retry sending events", function(done) {
-            httpBackend.when("PUT", "/txn1").fail(500, {
+            httpBackend.when("PUT", "/txn1").fail(500, new MatrixError({
                errcode: "M_SOMETHING",
                error: "Ruh roh",
-            });
+            }));
            client.sendTextMessage("!foo:bar", "a body", "txn1").then(function(res) {
                expect(false).toBe(true, "sendTextMessage resolved but shouldn't");
            }, function(err) {
@@ -16,6 +16,7 @@ limitations under the License.
 */

 import {logger} from '../src/logger';
+import * as utils from "../src/utils";

 // try to load the olm library.
 try {
@@ -24,3 +25,11 @@ try {
 } catch (e) {
    logger.warn("unable to run crypto tests: libolm not available");
 }
+
+// also try to set node crypto
+try {
+    const crypto = require('crypto');
+    utils.setCrypto(crypto);
+} catch (err) {
+    console.log('nodejs was compiled without crypto support: some tests will fail');
+}
@@ -313,6 +313,10 @@ describe("Crypto", function() {
            // make a room key request, and record the transaction ID for the
            // sendToDevice call
            await aliceClient.cancelAndResendEventRoomKeyRequest(event);
+            // key requests get queued until the sync has finished, but we don't
+            // let the client set up enough for that to happen, so gut-wrench a bit
+            // to force it to send now.
+            aliceClient._crypto._outgoingRoomKeyRequestManager.sendQueuedRequests();
            jest.runAllTimers();
            await Promise.resolve();
            expect(aliceClient.sendToDevice).toBeCalledTimes(1);
@@ -0,0 +1,250 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import '../../olm-loader';
+import {
+    CrossSigningInfo,
+    createCryptoStoreCacheCallbacks,
+} from '../../../src/crypto/CrossSigning';
+import {
+    IndexedDBCryptoStore,
+} from '../../../src/crypto/store/indexeddb-crypto-store';
+import {MemoryCryptoStore} from '../../../src/crypto/store/memory-crypto-store';
+import 'fake-indexeddb/auto';
+import 'jest-localstorage-mock';
+import {OlmDevice} from "../../../src/crypto/OlmDevice";
+
+const userId = "@alice:example.com";
+
+// Private key for tests only
+const testKey = new Uint8Array([
+    0xda, 0x5a, 0x27, 0x60, 0xe3, 0x3a, 0xc5, 0x82,
+    0x9d, 0x12, 0xc3, 0xbe, 0xe8, 0xaa, 0xc2, 0xef,
+    0xae, 0xb1, 0x05, 0xc1, 0xe7, 0x62, 0x78, 0xa6,
+    0xd7, 0x1f, 0xf8, 0x2c, 0x51, 0x85, 0xf0, 0x1d,
+]);
+
+const types = [
+    { type: "master", shouldCache: false },
+    { type: "self_signing", shouldCache: true },
+    { type: "user_signing", shouldCache: true },
+    { type: "invalid", shouldCache: false },
+];
+
+const badKey = Uint8Array.from(testKey);
+badKey[0] ^= 1;
+
+const masterKeyPub = "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk";
+
+describe("CrossSigningInfo.getCrossSigningKey", function() {
+    if (!global.Olm) {
+        console.warn('Not running megolm backup unit tests: libolm not present');
+        return;
+    }
+
+    beforeAll(function() {
+        return global.Olm.init();
+    });
+
+    it("should throw if no callback is provided", async () => {
+        const info = new CrossSigningInfo(userId);
+        await expect(info.getCrossSigningKey("master")).rejects.toThrow();
+    });
+
+    it.each(types)("should throw if the callback returns falsey",
+                   async ({type, shouldCache}) => {
+        const info = new CrossSigningInfo(userId, {
+            getCrossSigningKey: () => false,
+        });
+        await expect(info.getCrossSigningKey(type)).rejects.toThrow("falsey");
+    });
+
+    it("should throw if the expected key doesn't come back", async () => {
+        const info = new CrossSigningInfo(userId, {
+            getCrossSigningKey: () => masterKeyPub,
+        });
+        await expect(info.getCrossSigningKey("master", "")).rejects.toThrow();
+    });
+
+    it("should return a key from its callback", async () => {
+        const info = new CrossSigningInfo(userId, {
+            getCrossSigningKey: () => testKey,
+        });
+        const [pubKey, ab] = await info.getCrossSigningKey("master", masterKeyPub);
+        expect(pubKey).toEqual(masterKeyPub);
+        expect(ab).toEqual({a: 106712, b: 106712});
+    });
+
+    it.each(types)("should request a key from the cache callback (if set)" +
+                   " and does not call app if one is found" +
+                   " %o",
+                   async ({ type, shouldCache }) => {
+        const getCrossSigningKey = jest.fn().mockImplementation(() => {
+            if (shouldCache) {
+                return Promise.reject(new Error("Regular callback called"));
+            } else {
+                return Promise.resolve(testKey);
+            }
+        });
+        const getCrossSigningKeyCache = jest.fn().mockResolvedValue(testKey);
+        const info = new CrossSigningInfo(
+            userId,
+            { getCrossSigningKey },
+            { getCrossSigningKeyCache },
+        );
+        const [pubKey] = await info.getCrossSigningKey(type, masterKeyPub);
+        expect(pubKey).toEqual(masterKeyPub);
+        expect(getCrossSigningKeyCache.mock.calls.length).toBe(shouldCache ? 1 : 0);
+        if (shouldCache) {
+            expect(getCrossSigningKeyCache.mock.calls[0][0]).toBe(type);
+        }
+    });
+
+    it.each(types)("should store a key with the cache callback (if set)",
+                   async ({ type, shouldCache }) => {
+        const getCrossSigningKey = jest.fn().mockResolvedValue(testKey);
+        const storeCrossSigningKeyCache = jest.fn().mockResolvedValue(undefined);
+        const info = new CrossSigningInfo(
+            userId,
+            { getCrossSigningKey },
+            { storeCrossSigningKeyCache },
+        );
+        const [pubKey] = await info.getCrossSigningKey(type, masterKeyPub);
+        expect(pubKey).toEqual(masterKeyPub);
+        expect(storeCrossSigningKeyCache.mock.calls.length).toEqual(shouldCache ? 1 : 0);
+        if (shouldCache) {
+            expect(storeCrossSigningKeyCache.mock.calls[0][0]).toBe(type);
+            expect(storeCrossSigningKeyCache.mock.calls[0][1]).toBe(testKey);
+        }
+    });
+
+    it.each(types)("does not store a bad key to the cache",
+                   async ({ type, shouldCache }) => {
+        const getCrossSigningKey = jest.fn().mockResolvedValue(badKey);
+        const storeCrossSigningKeyCache = jest.fn().mockResolvedValue(undefined);
+        const info = new CrossSigningInfo(
+            userId,
+            { getCrossSigningKey },
+            { storeCrossSigningKeyCache },
+        );
+        await expect(info.getCrossSigningKey(type, masterKeyPub)).rejects.toThrow();
+        expect(storeCrossSigningKeyCache.mock.calls.length).toEqual(0);
+    });
+
+    it.each(types)("does not store a value to the cache if it came from the cache",
+        async ({ type, shouldCache }) => {
+        const getCrossSigningKey = jest.fn().mockImplementation(() => {
+            if (shouldCache) {
+                return Promise.reject(new Error("Regular callback called"));
+            } else {
+                return Promise.resolve(testKey);
+            }
+        });
+        const getCrossSigningKeyCache = jest.fn().mockResolvedValue(testKey);
+        const storeCrossSigningKeyCache = jest.fn().mockRejectedValue(
+            new Error("Tried to store a value from cache"),
+        );
+        const info = new CrossSigningInfo(
+            userId,
+            { getCrossSigningKey },
+            { getCrossSigningKeyCache, storeCrossSigningKeyCache },
+        );
+        expect(storeCrossSigningKeyCache.mock.calls.length).toBe(0);
+        const [pubKey] = await info.getCrossSigningKey(type, masterKeyPub);
+        expect(pubKey).toEqual(masterKeyPub);
+    });
+
+    it.each(types)("requests a key from the cache callback (if set) and then calls app" +
+        " if one is not found", async ({ type, shouldCache }) => {
+        const getCrossSigningKey = jest.fn().mockResolvedValue(testKey);
+        const getCrossSigningKeyCache = jest.fn().mockResolvedValue(undefined);
+        const storeCrossSigningKeyCache = jest.fn();
+        const info = new CrossSigningInfo(
+            userId,
+            { getCrossSigningKey },
+            { getCrossSigningKeyCache, storeCrossSigningKeyCache },
+        );
+        const [pubKey] = await info.getCrossSigningKey(type, masterKeyPub);
+        expect(pubKey).toEqual(masterKeyPub);
+        expect(getCrossSigningKey.mock.calls.length).toBe(1);
+        expect(getCrossSigningKeyCache.mock.calls.length).toBe(shouldCache ? 1 : 0);
+
+        /* Also expect that the cache gets updated */
+        expect(storeCrossSigningKeyCache.mock.calls.length).toBe(shouldCache ? 1 : 0);
+    });
+
+    it.each(types)("requests a key from the cache callback (if set) and then" +
+        " calls app if that key doesn't match", async ({ type, shouldCache }) => {
+        const getCrossSigningKey = jest.fn().mockResolvedValue(testKey);
+        const getCrossSigningKeyCache = jest.fn().mockResolvedValue(badKey);
+        const storeCrossSigningKeyCache = jest.fn();
+        const info = new CrossSigningInfo(
+            userId,
+            { getCrossSigningKey },
+            { getCrossSigningKeyCache, storeCrossSigningKeyCache },
+        );
+        const [pubKey] = await info.getCrossSigningKey(type, masterKeyPub);
+        expect(pubKey).toEqual(masterKeyPub);
+        expect(getCrossSigningKey.mock.calls.length).toBe(1);
+        expect(getCrossSigningKeyCache.mock.calls.length).toBe(shouldCache ? 1 : 0);
+
+        /* Also expect that the cache gets updated */
+        expect(storeCrossSigningKeyCache.mock.calls.length).toBe(shouldCache ? 1 : 0);
+    });
+});
+
+/*
+ * Note that MemoryStore is weird.  It's only used for testing - as far as I can tell,
+ * it's not possible to get one in normal execution unless you hack as we do here.
+ */
+describe.each([
+    ["IndexedDBCryptoStore",
+     () => new IndexedDBCryptoStore(global.indexedDB, "tests")],
+    ["LocalStorageCryptoStore",
+     () => new IndexedDBCryptoStore(undefined, "tests")],
+    ["MemoryCryptoStore", () => {
+        const store = new IndexedDBCryptoStore(undefined, "tests");
+        store._backend = new MemoryCryptoStore();
+        store._backendPromise = Promise.resolve(store._backend);
+        return store;
+    }],
+])("CrossSigning > createCryptoStoreCacheCallbacks [%s]", function(name, dbFactory) {
+    let store;
+
+    beforeAll(() => {
+        store = dbFactory();
+    });
+
+    beforeEach(async () => {
+        await store.deleteAllData();
+    });
+
+    it("should cache data to the store and retrieve it", async () => {
+        await store.startup();
+        const olmDevice = new OlmDevice(store);
+        const { getCrossSigningKeyCache, storeCrossSigningKeyCache } =
+              createCryptoStoreCacheCallbacks(store, olmDevice);
+        await storeCrossSigningKeyCache("self_signing", testKey);
+
+        // If we've not saved anything, don't expect anything
+        // Definitely don't accidentally return the wrong key for the type
+        const nokey = await getCrossSigningKeyCache("self", "");
+        expect(nokey).toBeNull();
+
+        const key = await getCrossSigningKeyCache("self_signing", "");
+        expect(new Uint8Array(key)).toEqual(testKey);
+    });
+});
@@ -297,6 +297,10 @@ describe("MegolmDecryption", function() {
                },
            }));

+            mockCrypto.checkDeviceTrust.mockReturnValue({
+                isVerified: () => false,
+            });
+
            const megolmEncryption = new MegolmEncryption({
                userId: '@user:id',
                crypto: mockCrypto,
@@ -320,14 +324,14 @@ describe("MegolmDecryption", function() {

            // this should have claimed a key for alice as it's starting a new session
            expect(mockBaseApis.claimOneTimeKeys).toHaveBeenCalledWith(
-                [['@alice:home.server', 'aliceDevice']], 'signed_curve25519',
+                [['@alice:home.server', 'aliceDevice']], 'signed_curve25519', 2000,
            );
            expect(mockCrypto.downloadKeys).toHaveBeenCalledWith(
                ['@alice:home.server'], false,
            );
            expect(mockBaseApis.sendToDevice).toHaveBeenCalled();
            expect(mockBaseApis.claimOneTimeKeys).toHaveBeenCalledWith(
-                [['@alice:home.server', 'aliceDevice']], 'signed_curve25519',
+                [['@alice:home.server', 'aliceDevice']], 'signed_curve25519', 2000,
            );

            mockBaseApis.claimOneTimeKeys.mockReset();
@@ -516,21 +520,22 @@ describe("MegolmDecryption", function() {
            };
        };

-        let run = false;
-        aliceClient.sendToDevice = async (msgtype, contentMap) => {
-            run = true;
-            expect(msgtype).toBe("org.matrix.room_key.withheld");
-            expect(contentMap).toStrictEqual({
-                '@bob:example.com': {
-                    bobdevice: {
-                        algorithm: "m.megolm.v1.aes-sha2",
-                        code: 'm.no_olm',
-                        reason: 'Unable to establish a secure channel.',
-                        sender_key: aliceDevice.deviceCurve25519Key,
+        const sendPromise = new Promise((resolve, reject) => {
+            aliceClient.sendToDevice = async (msgtype, contentMap) => {
+                expect(msgtype).toBe("org.matrix.room_key.withheld");
+                expect(contentMap).toStrictEqual({
+                    '@bob:example.com': {
+                        bobdevice: {
+                            algorithm: "m.megolm.v1.aes-sha2",
+                            code: 'm.no_olm',
+                            reason: 'Unable to establish a secure channel.',
+                            sender_key: aliceDevice.deviceCurve25519Key,
+                        },
                    },
-                },
-            });
-        };
+                });
+                resolve();
+            };
+        });

        const event = new MatrixEvent({
            type: "m.room.message",
@@ -540,8 +545,7 @@ describe("MegolmDecryption", function() {
            content: {},
        });
        await aliceClient._crypto.encryptEvent(event, aliceRoom);
-
-        expect(run).toBe(true);
+        await sendPromise;
    });

    it("throws an error describing why it doesn't have a key", async function() {
@@ -598,6 +602,7 @@ describe("MegolmDecryption", function() {
            aliceClient.initCrypto(),
            bobClient.initCrypto(),
        ]);
+        aliceClient._crypto.downloadKeys = async () => {};
        const bobDevice = bobClient._crypto._olmDevice;

        const roomId = "!someroom";
@@ -649,6 +654,7 @@ describe("MegolmDecryption", function() {
            bobClient.initCrypto(),
        ]);
        const bobDevice = bobClient._crypto._olmDevice;
+        aliceClient._crypto.downloadKeys = async () => {};

        const roomId = "!someroom";

@@ -541,5 +541,31 @@ describe("MegolmBackup", function() {
                expect(res.clearEvent.content).toEqual('testytest');
            });
        });
+
+        it('has working cache functions', async function() {
+            const key = Uint8Array.from([1, 2, 3, 4, 5, 6, 7, 8]);
+            await client._crypto.storeSessionBackupPrivateKey(key);
+            const result = await client._crypto.getSessionBackupPrivateKey();
+            expect(new Uint8Array(result)).toEqual(key);
+        });
+
+        it('caches session backup keys as it encounters them', async function() {
+            const cachedNull = await client._crypto.getSessionBackupPrivateKey();
+            expect(cachedNull).toBeNull();
+            client._http.authedRequest = function() {
+                return Promise.resolve(KEY_BACKUP_DATA);
+            };
+            await new Promise((resolve) => {
+                client.restoreKeyBackupWithRecoveryKey(
+                    "EsTc LW2K PGiF wKEA 3As5 g5c4 BXwk qeeJ ZJV8 Q9fu gUMN UE4d",
+                    ROOM_ID,
+                    SESSION_ID,
+                    BACKUP_INFO,
+                    { cacheCompleteCallback: resolve },
+                );
+            });
+            const cachedKey = await client._crypto.getSessionBackupPrivateKey();
+            expect(cachedKey).not.toBeNull();
+        });
    });
 });
@@ -0,0 +1,87 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {
+    IndexedDBCryptoStore,
+} from '../../../src/crypto/store/indexeddb-crypto-store';
+import {MemoryCryptoStore} from '../../../src/crypto/store/memory-crypto-store';
+import 'fake-indexeddb/auto';
+import 'jest-localstorage-mock';
+
+import {
+    ROOM_KEY_REQUEST_STATES,
+} from '../../../src/crypto/OutgoingRoomKeyRequestManager';
+
+const requests = [
+    {
+        requestId: "A",
+        requestBody: { session_id: "A", room_id: "A" },
+        state: ROOM_KEY_REQUEST_STATES.SENT,
+    },
+    {
+        requestId: "B",
+        requestBody: { session_id: "B", room_id: "B" },
+        state: ROOM_KEY_REQUEST_STATES.SENT,
+    },
+    {
+        requestId: "C",
+        requestBody: { session_id: "C", room_id: "C" },
+        state: ROOM_KEY_REQUEST_STATES.UNSENT,
+    },
+];
+
+describe.each([
+    ["IndexedDBCryptoStore",
+     () => new IndexedDBCryptoStore(global.indexedDB, "tests")],
+    ["LocalStorageCryptoStore",
+     () => new IndexedDBCryptoStore(undefined, "tests")],
+    ["MemoryCryptoStore", () => {
+        const store = new IndexedDBCryptoStore(undefined, "tests");
+        store._backend = new MemoryCryptoStore();
+        store._backendPromise = Promise.resolve(store._backend);
+        return store;
+    }],
+])("Outgoing room key requests [%s]", function(name, dbFactory) {
+    let store;
+
+    beforeAll(async () => {
+        store = dbFactory();
+        await store.startup();
+        await Promise.all(requests.map((request) =>
+            store.getOrAddOutgoingRoomKeyRequest(request),
+        ));
+    });
+
+    it("getAllOutgoingRoomKeyRequestsByState retrieves all entries in a given state",
+    async () => {
+        const r = await
+            store.getAllOutgoingRoomKeyRequestsByState(ROOM_KEY_REQUEST_STATES.SENT);
+        expect(r).toHaveLength(2);
+        requests.filter((e) => e.state == ROOM_KEY_REQUEST_STATES.SENT).forEach((e) => {
+            expect(r).toContainEqual(e);
+        });
+    });
+
+    test("getOutgoingRoomKeyRequestByState retrieves any entry in a given state",
+    async () => {
+        const r =
+            await store.getOutgoingRoomKeyRequestByState([ROOM_KEY_REQUEST_STATES.SENT]);
+        expect(r).not.toBeNull();
+        expect(r).not.toBeUndefined();
+        expect(r.state).toEqual(ROOM_KEY_REQUEST_STATES.SENT);
+        expect(requests).toContainEqual(r);
+    });
+});
@@ -16,10 +16,20 @@ limitations under the License.

 import '../../olm-loader';
 import * as olmlib from "../../../src/crypto/olmlib";
-import {SECRET_STORAGE_ALGORITHM_V1} from "../../../src/crypto/SecretStorage";
+import {SECRET_STORAGE_ALGORITHM_V1_AES} from "../../../src/crypto/SecretStorage";
 import {MatrixEvent} from "../../../src/models/event";
 import {TestClient} from '../../TestClient';
 import {makeTestClients} from './verification/util';
+import {encryptAES} from "../../../src/crypto/aes";
+
+import * as utils from "../../../src/utils";
+
+try {
+    const crypto = require('crypto');
+    utils.setCrypto(crypto);
+} catch (err) {
+    console.log('nodejs was compiled without crypto support');
+}

 async function makeTestClient(userInfo, options) {
    const client = (new TestClient(
@@ -40,6 +50,13 @@ async function makeTestClient(userInfo, options) {
    return client;
 }

+// Wrapper around pkSign to return a signed object. pkSign returns the
+// signature, rather than the signed object.
+function sign(obj, key, userId) {
+    olmlib.pkSign(obj, key, userId);
+    return obj;
+}
+
 describe("Secrets", function() {
    if (!global.Olm) {
        console.warn('Not running megolm backup unit tests: libolm not present');
@@ -51,9 +68,8 @@ describe("Secrets", function() {
    });

    it("should store and retrieve a secret", async function() {
-        const decryption = new global.Olm.PkDecryption();
-        const pubkey = decryption.generate_key();
-        const privkey = decryption.get_private_key();
+        const key = new Uint8Array(16);
+        for (let i = 0; i < 16; i++) key[i] = i;

        const signing = new global.Olm.PkSigning();
        const signingKey = signing.generate_seed();
@@ -69,7 +85,7 @@ describe("Secrets", function() {

        const getKey = jest.fn(e => {
            expect(Object.keys(e.keys)).toEqual(["abc"]);
-            return ['abc', privkey];
+            return ['abc', key];
        });

        const alice = await makeTestClient(
@@ -100,8 +116,7 @@ describe("Secrets", function() {
        };

        const keyAccountData = {
-            algorithm: SECRET_STORAGE_ALGORITHM_V1,
-            pubkey: pubkey,
+            algorithm: SECRET_STORAGE_ALGORITHM_V1_AES,
        };
        await alice._crypto._crossSigningInfo.signObject(keyAccountData, 'master');

@@ -112,11 +127,11 @@ describe("Secrets", function() {
            }),
        ]);

-        expect(await secretStorage.isStored("foo")).toBe(false);
+        expect(await secretStorage.isStored("foo")).toBeFalsy();

        await secretStorage.store("foo", "bar", ["abc"]);

-        expect(await secretStorage.isStored("foo")).toBe(true);
+        expect(await secretStorage.isStored("foo")).toBeTruthy();
        expect(await secretStorage.get("foo")).toBe("bar");

        expect(getKey).toHaveBeenCalled();
@@ -149,6 +164,13 @@ describe("Secrets", function() {
    });

    it("should encrypt with default key if keys is null", async function() {
+        const key = new Uint8Array(16);
+        for (let i = 0; i < 16; i++) key[i] = i;
+        const getKey = jest.fn(e => {
+            expect(Object.keys(e.keys)).toEqual([newKeyId]);
+            return [newKeyId, key];
+        });
+
        let keys = {};
        const alice = await makeTestClient(
            {userId: "@alice:example.com", deviceId: "Osborne2"},
@@ -156,6 +178,7 @@ describe("Secrets", function() {
                cryptoCallbacks: {
                    getCrossSigningKey: t => keys[t],
                    saveCrossSigningKeys: k => keys = k,
+                    getSecretStorageKey: getKey,
                },
            },
        );
@@ -170,7 +193,7 @@ describe("Secrets", function() {
        alice.resetCrossSigningKeys();

        const newKeyId = await alice.addSecretStorageKey(
-            SECRET_STORAGE_ALGORITHM_V1,
+            SECRET_STORAGE_ALGORITHM_V1_AES,
        );
        // we don't await on this because it waits for the event to come down the sync
        // which won't happen in the test setup
@@ -251,89 +274,389 @@ describe("Secrets", function() {
        expect(secret).toBe("bar");
    });

-    it("bootstraps when no storage or cross-signing keys locally", async function() {
-        const bob = await makeTestClient(
-            {
-                userId: "@bob:example.com",
-                deviceId: "bob1",
-            },
+    describe("bootstrap", function() {
+        // keys used in some of the tests
+        const XSK = new Uint8Array(
+            olmlib.decodeBase64("3lo2YdJugHjfE+Or7KJ47NuKbhE7AAGLgQ/dc19913Q="),
        );
-        bob.uploadDeviceSigningKeys = async () => {};
-        bob.uploadKeySignatures = async () => {};
-        bob.setAccountData = async function(eventType, contents, callback) {
-            const event = new MatrixEvent({
-                type: eventType,
-                content: contents,
+        const XSPubKey = "DRb8pFVJyEJ9OWvXeUoM0jq/C2Wt+NxzBZVuk2nRb+0";
+        const USK = new Uint8Array(
+            olmlib.decodeBase64("lKWi3hJGUie5xxHgySoz8PHFnZv6wvNaud/p2shN9VU="),
+        );
+        const USPubKey = "CUpoiTtHiyXpUmd+3ohb7JVxAlUaOG1NYs9Jlx8soQU";
+        const SSK = new Uint8Array(
+            olmlib.decodeBase64("1R6JVlXX99UcfUZzKuCDGQgJTw8ur1/ofgPD8pp+96M="),
+        );
+        const SSPubKey = "0DfNsRDzEvkCLA0gD3m7VAGJ5VClhjEsewI35xq873Q";
+        const SSSSKey = new Uint8Array(
+            olmlib.decodeBase64(
+                "XrmITOOdBhw6yY5Bh7trb/bgp1FRdIGyCUxxMP873R0=",
+            ),
+        );
+
+        it("bootstraps when no storage or cross-signing keys locally", async function() {
+            const key = new Uint8Array(16);
+            for (let i = 0; i < 16; i++) key[i] = i;
+            const getKey = jest.fn(e => {
+                return [Object.keys(e.keys)[0], key];
            });
-            this.store.storeAccountDataEvents([
-                event,
-            ]);
-            this.emit("accountData", event);
-        };

-        await bob.bootstrapSecretStorage();
-
-        const crossSigning = bob._crypto._crossSigningInfo;
-        const secretStorage = bob._crypto._secretStorage;
-
-        expect(crossSigning.getId()).toBeTruthy();
-        expect(await crossSigning.isStoredInSecretStorage(secretStorage)).toBeTruthy();
-        expect(await secretStorage.hasKey()).toBeTruthy();
-    });
-
-    it("bootstraps when cross-signing keys in secret storage", async function() {
-        const decryption = new global.Olm.PkDecryption();
-        const storagePublicKey = decryption.generate_key();
-        const storagePrivateKey = decryption.get_private_key();
-
-        const bob = await makeTestClient(
-            {
-                userId: "@bob:example.com",
-                deviceId: "bob1",
-            },
-            {
-                cryptoCallbacks: {
-                    getSecretStorageKey: async request => {
-                        const defaultKeyId = await bob.getDefaultSecretStorageKeyId();
-                        expect(Object.keys(request.keys)).toEqual([defaultKeyId]);
-                        return [defaultKeyId, storagePrivateKey];
+            const bob = await makeTestClient(
+                {
+                    userId: "@bob:example.com",
+                    deviceId: "bob1",
+                },
+                {
+                    cryptoCallbacks: {
+                        getSecretStorageKey: getKey,
                    },
                },
-            },
-        );
+            );
+            bob.uploadDeviceSigningKeys = async () => {};
+            bob.uploadKeySignatures = async () => {};
+            bob.setAccountData = async function(eventType, contents, callback) {
+                const event = new MatrixEvent({
+                    type: eventType,
+                    content: contents,
+                });
+                this.store.storeAccountDataEvents([
+                    event,
+                ]);
+                this.emit("accountData", event);
+            };

-        bob.uploadDeviceSigningKeys = async () => {};
-        bob.uploadKeySignatures = async () => {};
-        bob.setAccountData = async function(eventType, contents, callback) {
-            const event = new MatrixEvent({
-                type: eventType,
-                content: contents,
-            });
-            this.store.storeAccountDataEvents([
-                event,
-            ]);
-            this.emit("accountData", event);
-        };
-        bob._crypto.checkKeyBackup = async () => {};
+            await bob.bootstrapSecretStorage();

-        const crossSigning = bob._crypto._crossSigningInfo;
-        const secretStorage = bob._crypto._secretStorage;
+            const crossSigning = bob._crypto._crossSigningInfo;
+            const secretStorage = bob._crypto._secretStorage;

-        // Set up cross-signing keys from scratch with specific storage key
-        await bob.bootstrapSecretStorage({
-            createSecretStorageKey: async () => ({ pubkey: storagePublicKey }),
+            expect(crossSigning.getId()).toBeTruthy();
+            expect(await crossSigning.isStoredInSecretStorage(secretStorage))
+                .toBeTruthy();
+            expect(await secretStorage.hasKey()).toBeTruthy();
        });

-        // Clear local cross-signing keys and read from secret storage
-        bob._crypto._deviceList.storeCrossSigningForUser(
-            "@bob:example.com",
-            crossSigning.toStorage(),
-        );
-        crossSigning.keys = {};
-        await bob.bootstrapSecretStorage();
+        it("bootstraps when cross-signing keys in secret storage", async function() {
+            const decryption = new global.Olm.PkDecryption();
+            const storagePublicKey = decryption.generate_key();
+            const storagePrivateKey = decryption.get_private_key();

-        expect(crossSigning.getId()).toBeTruthy();
-        expect(await crossSigning.isStoredInSecretStorage(secretStorage)).toBeTruthy();
-        expect(await secretStorage.hasKey()).toBeTruthy();
+            const bob = await makeTestClient(
+                {
+                    userId: "@bob:example.com",
+                    deviceId: "bob1",
+                },
+                {
+                    cryptoCallbacks: {
+                        getSecretStorageKey: async request => {
+                            const defaultKeyId = await bob.getDefaultSecretStorageKeyId();
+                            expect(Object.keys(request.keys)).toEqual([defaultKeyId]);
+                            return [defaultKeyId, storagePrivateKey];
+                        },
+                    },
+                },
+            );
+
+            bob.uploadDeviceSigningKeys = async () => {};
+            bob.uploadKeySignatures = async () => {};
+            bob.setAccountData = async function(eventType, contents, callback) {
+                const event = new MatrixEvent({
+                    type: eventType,
+                    content: contents,
+                });
+                this.store.storeAccountDataEvents([
+                    event,
+                ]);
+                this.emit("accountData", event);
+            };
+            bob._crypto.checkKeyBackup = async () => {};
+
+            const crossSigning = bob._crypto._crossSigningInfo;
+            const secretStorage = bob._crypto._secretStorage;
+
+            // Set up cross-signing keys from scratch with specific storage key
+            await bob.bootstrapSecretStorage({
+                createSecretStorageKey: async () => ({
+                    // `pubkey` not used anymore with symmetric 4S
+                    keyInfo: { pubkey: storagePublicKey },
+                    privateKey: storagePrivateKey,
+                }),
+            });
+
+            // Clear local cross-signing keys and read from secret storage
+            bob._crypto._deviceList.storeCrossSigningForUser(
+                "@bob:example.com",
+                crossSigning.toStorage(),
+            );
+            crossSigning.keys = {};
+            await bob.bootstrapSecretStorage();
+
+            expect(crossSigning.getId()).toBeTruthy();
+            expect(await crossSigning.isStoredInSecretStorage(secretStorage))
+                .toBeTruthy();
+            expect(await secretStorage.hasKey()).toBeTruthy();
+        });
+
+        it("adds passphrase checking if it's lacking", async function() {
+            let crossSigningKeys = {
+                master: XSK,
+                user_signing: USK,
+                self_signing: SSK,
+            };
+            const secretStorageKeys = {
+                key_id: SSSSKey,
+            };
+            const alice = await makeTestClient(
+                {userId: "@alice:example.com", deviceId: "Osborne2"},
+                {
+                    cryptoCallbacks: {
+                        getCrossSigningKey: t => crossSigningKeys[t],
+                        saveCrossSigningKeys: k => crossSigningKeys = k,
+                        getSecretStorageKey: ({keys}, name) => {
+                            for (const keyId of Object.keys(keys)) {
+                                if (secretStorageKeys[keyId]) {
+                                    return [keyId, secretStorageKeys[keyId]];
+                                }
+                            }
+                        },
+                    },
+                },
+            );
+            alice.store.storeAccountDataEvents([
+                new MatrixEvent({
+                    type: "m.secret_storage.default_key",
+                    content: {
+                        key: "key_id",
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.secret_storage.key.key_id",
+                    content: {
+                        algorithm: "m.secret_storage.v1.aes-hmac-sha2",
+                        passphrase: {
+                            algorithm: "m.pbkdf2",
+                            iterations: 500000,
+                            salt: "GbkvwKHVMveo1zGVSb2GMMdCinG2npJK",
+                        },
+                    },
+                }),
+                // we never use these values, other than checking that they
+                // exist, so just use dummy values
+                new MatrixEvent({
+                    type: "m.cross_signing.master",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.self_signing",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.user_signing",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+            ]);
+            alice._crypto._deviceList.storeCrossSigningForUser("@alice:example.com", {
+                keys: {
+                    master: {
+                        user_id: "@alice:example.com",
+                        usage: ["master"],
+                        keys: {
+                            [`ed25519:${XSPubKey}`]: XSPubKey,
+                        },
+                    },
+                    self_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["self_signing"],
+                        keys: {
+                            [`ed25519:${SSPubKey}`]: SSPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                    user_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["user_signing"],
+                        keys: {
+                            [`ed25519:${USPubKey}`]: USPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                },
+            });
+            alice.getKeyBackupVersion = async () => {
+                return {
+                    version: "1",
+                    algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
+                    auth_data: sign({
+                        public_key: "pxEXhg+4vdMf/kFwP4bVawFWdb0EmytL3eFJx++zQ0A",
+                    }, XSK, "@alice:example.com"),
+                };
+            };
+            alice.setAccountData = async function(name, data) {
+                const event = new MatrixEvent({
+                    type: name,
+                    content: data,
+                });
+                alice.store.storeAccountDataEvents([event]);
+                this.emit("accountData", event);
+            };
+
+            await alice.bootstrapSecretStorage();
+
+            expect(alice.getAccountData("m.secret_storage.default_key").getContent())
+                .toEqual({key: "key_id"});
+            const keyInfo = alice.getAccountData("m.secret_storage.key.key_id")
+                .getContent();
+            expect(keyInfo.algorithm)
+                .toEqual("m.secret_storage.v1.aes-hmac-sha2");
+            expect(keyInfo.passphrase).toEqual({
+                algorithm: "m.pbkdf2",
+                iterations: 500000,
+                salt: "GbkvwKHVMveo1zGVSb2GMMdCinG2npJK",
+            });
+            expect(keyInfo).toHaveProperty("iv");
+            expect(keyInfo).toHaveProperty("mac");
+            expect(alice.checkSecretStorageKey(secretStorageKeys.key_id, keyInfo))
+                .toBeTruthy();
+        });
+        it("fixes backup keys in the wrong format", async function() {
+            let crossSigningKeys = {
+                master: XSK,
+                user_signing: USK,
+                self_signing: SSK,
+            };
+            const secretStorageKeys = {
+                key_id: SSSSKey,
+            };
+            const alice = await makeTestClient(
+                {userId: "@alice:example.com", deviceId: "Osborne2"},
+                {
+                    cryptoCallbacks: {
+                        getCrossSigningKey: t => crossSigningKeys[t],
+                        saveCrossSigningKeys: k => crossSigningKeys = k,
+                        getSecretStorageKey: ({keys}, name) => {
+                            for (const keyId of Object.keys(keys)) {
+                                if (secretStorageKeys[keyId]) {
+                                    return [keyId, secretStorageKeys[keyId]];
+                                }
+                            }
+                        },
+                    },
+                },
+            );
+            alice.store.storeAccountDataEvents([
+                new MatrixEvent({
+                    type: "m.secret_storage.default_key",
+                    content: {
+                        key: "key_id",
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.secret_storage.key.key_id",
+                    content: {
+                        algorithm: "m.secret_storage.v1.aes-hmac-sha2",
+                        passphrase: {
+                            algorithm: "m.pbkdf2",
+                            iterations: 500000,
+                            salt: "GbkvwKHVMveo1zGVSb2GMMdCinG2npJK",
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.master",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.self_signing",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.user_signing",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.megolm_backup.v1",
+                    content: {
+                        encrypted: {
+                            key_id: await encryptAES(
+                                "123,45,6,7,89,1,234,56,78,90,12,34,5,67,8,90",
+                                secretStorageKeys.key_id, "m.megolm_backup.v1",
+                            ),
+                        },
+                    },
+                }),
+            ]);
+            alice._crypto._deviceList.storeCrossSigningForUser("@alice:example.com", {
+                keys: {
+                    master: {
+                        user_id: "@alice:example.com",
+                        usage: ["master"],
+                        keys: {
+                            [`ed25519:${XSPubKey}`]: XSPubKey,
+                        },
+                    },
+                    self_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["self_signing"],
+                        keys: {
+                            [`ed25519:${SSPubKey}`]: SSPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                    user_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["user_signing"],
+                        keys: {
+                            [`ed25519:${USPubKey}`]: USPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                },
+            });
+            alice.getKeyBackupVersion = async () => {
+                return {
+                    version: "1",
+                    algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
+                    auth_data: sign({
+                        public_key: "pxEXhg+4vdMf/kFwP4bVawFWdb0EmytL3eFJx++zQ0A",
+                    }, XSK, "@alice:example.com"),
+                };
+            };
+            alice.setAccountData = async function(name, data) {
+                const event = new MatrixEvent({
+                    type: name,
+                    content: data,
+                });
+                alice.store.storeAccountDataEvents([event]);
+                this.emit("accountData", event);
+            };
+
+            await alice.bootstrapSecretStorage();
+
+            const backupKey = alice.getAccountData("m.megolm_backup.v1")
+                .getContent();
+            expect(backupKey.encrypted).toHaveProperty("key_id");
+            expect(await alice.getSecret("m.megolm_backup.v1"))
+                .toEqual("ey0GB1kB6jhOWgwiBUMIWg==");
+        });
    });
 });
@@ -44,7 +44,16 @@ describe("SAS verification", function() {
    });

    it("should error on an unexpected event", async function() {
-        const sas = new SAS({}, "@alice:example.com", "ABCDEFG");
+        //channel, baseApis, userId, deviceId, startEvent, request
+        const request = {
+            onVerifierCancelled: function() {},
+        };
+        const channel = {
+            send: function() {
+                return Promise.resolve();
+            },
+        };
+        const sas = new SAS(channel, {}, "@alice:example.com", "ABCDEFG", null, request);
        sas.handleEvent(new MatrixEvent({
            sender: "@alice:example.com",
            type: "es.inquisition",
@@ -122,8 +131,8 @@ describe("SAS verification", function() {
            bobSasEvent = null;

            bobPromise = new Promise((resolve, reject) => {
-                bob.client.on("crypto.verification.start", (verifier) => {
-                    verifier.on("show_sas", (e) => {
+                bob.client.on("crypto.verification.request", request => {
+                    request.verifier.on("show_sas", (e) => {
                        if (!e.sas.emoji || !e.sas.decimal) {
                            e.cancel();
                        } else if (!aliceSasEvent) {
@@ -139,7 +148,7 @@ describe("SAS verification", function() {
                            }
                        }
                    });
-                    resolve(verifier);
+                    resolve(request.verifier);
                });
            });

@@ -172,11 +181,14 @@ describe("SAS verification", function() {

        it("should verify a key", async () => {
            let macMethod;
+            let keyAgreement;
            const origSendToDevice = bob.client.sendToDevice.bind(bob.client);
            bob.client.sendToDevice = function(type, map) {
                if (type === "m.key.verification.accept") {
                    macMethod = map[alice.client.getUserId()][alice.client.deviceId]
                        .message_authentication_code;
+                    keyAgreement = map[alice.client.getUserId()][alice.client.deviceId]
+                        .key_agreement_protocol;
                }
                return origSendToDevice(type, map);
            };
@@ -203,6 +215,7 @@ describe("SAS verification", function() {

            // make sure that it uses the preferred method
            expect(macMethod).toBe("hkdf-hmac-sha256");
+            expect(keyAgreement).toBe("curve25519-hkdf-sha256");

            // make sure Alice and Bob verified each other
            const bobDevice
@@ -339,11 +352,11 @@ describe("SAS verification", function() {
        };

        const bobPromise = new Promise((resolve, reject) => {
-            bob.client.on("crypto.verification.start", (verifier) => {
-                verifier.on("show_sas", (e) => {
+            bob.client.on("crypto.verification.request", request => {
+                request.verifier.on("show_sas", (e) => {
                    e.mismatch();
                });
-                resolve(verifier);
+                resolve(request.verifier);
            });
        });

@@ -0,0 +1,117 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {VerificationBase} from '../../../../src/crypto/verification/Base';
+import {CrossSigningInfo} from '../../../../src/crypto/CrossSigning';
+import {encodeBase64} from "../../../../src/crypto/olmlib";
+import {setupWebcrypto, teardownWebcrypto} from './util';
+
+jest.useFakeTimers();
+
+// Private key for tests only
+const testKey = new Uint8Array([
+    0xda, 0x5a, 0x27, 0x60, 0xe3, 0x3a, 0xc5, 0x82,
+    0x9d, 0x12, 0xc3, 0xbe, 0xe8, 0xaa, 0xc2, 0xef,
+    0xae, 0xb1, 0x05, 0xc1, 0xe7, 0x62, 0x78, 0xa6,
+    0xd7, 0x1f, 0xf8, 0x2c, 0x51, 0x85, 0xf0, 0x1d,
+]);
+const testKeyPub = "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk";
+
+describe("self-verifications", () => {
+    beforeAll(function() {
+        setupWebcrypto();
+        return global.Olm.init();
+    });
+
+    afterAll(() => {
+        teardownWebcrypto();
+    });
+
+    it("triggers a request for key sharing upon completion", async () => {
+        const userId = "@test:localhost";
+
+        const cacheCallbacks = {
+            getCrossSigningKeyCache: jest.fn().mockReturnValue(null),
+            storeCrossSigningKeyCache: jest.fn(),
+        };
+
+        const _crossSigningInfo = new CrossSigningInfo(
+            userId,
+            {},
+            cacheCallbacks,
+        );
+        _crossSigningInfo.keys = {
+            self_signing: { keys: { X: testKeyPub } },
+            user_signing: { keys: { X: testKeyPub } },
+        };
+
+        const _secretStorage = {
+            request: jest.fn().mockReturnValue({
+                promise: Promise.resolve(encodeBase64(testKey)),
+            }),
+        };
+
+        const storeSessionBackupPrivateKey = jest.fn();
+        const restoreKeyBackupWithCache = jest.fn(() => Promise.resolve());
+
+        const client = {
+            _crypto: {
+                _crossSigningInfo,
+                _secretStorage,
+                storeSessionBackupPrivateKey,
+                getSessionBackupPrivateKey: () => null,
+            },
+            requestSecret: _secretStorage.request.bind(_secretStorage),
+            getUserId: () => userId,
+            getKeyBackupVersion: () => Promise.resolve({}),
+            restoreKeyBackupWithCache,
+        };
+
+        const request = {
+            onVerifierFinished: () => undefined,
+        };
+
+        const verification = new VerificationBase(
+            undefined, // channel
+            client, // baseApis
+            userId,
+            "ABC", // deviceId
+            undefined, // startEvent
+            request,
+        );
+        verification._resolve = () => undefined;
+
+        const result = await verification.done();
+
+        /* We should request, and store, two cross signing key and the key backup key */
+        expect(cacheCallbacks.storeCrossSigningKeyCache.mock.calls.length).toBe(2);
+        expect(_secretStorage.request.mock.calls.length).toBe(3);
+
+        expect(cacheCallbacks.storeCrossSigningKeyCache.mock.calls[0][1])
+          .toEqual(testKey);
+        expect(cacheCallbacks.storeCrossSigningKeyCache.mock.calls[1][1])
+          .toEqual(testKey);
+
+        expect(storeSessionBackupPrivateKey.mock.calls[0][0])
+          .toEqual(testKey);
+
+        expect(restoreKeyBackupWithCache).toHaveBeenCalled();
+
+        expect(result).toBeInstanceOf(Array);
+        expect(result[0][0]).toBe(testKeyPub);
+        expect(result[1][0]).toBe(testKeyPub);
+    });
+});
@@ -120,7 +120,6 @@ async function distributeEvent(ownRequest, theirRequest, event) {
 }

 describe("verification request unit tests", function() {
-
    beforeAll(function() {
        setupWebcrypto();
    });
@@ -0,0 +1,34 @@
+import {FilterComponent} from "../../src/filter-component";
+import {mkEvent} from '../test-utils';
+
+describe("Filter Component", function() {
+    describe("types", function() {
+        it("should filter out events with other types", function() {
+            const filter = new FilterComponent({ types: ['m.room.message'] });
+            const event = mkEvent({
+                type: 'm.room.member',
+                content: { },
+                room: 'roomId',
+                event: true,
+            });
+
+            const checkResult = filter.check(event);
+
+            expect(checkResult).toBe(false);
+        });
+
+        it("should validate events with the same type", function() {
+            const filter = new FilterComponent({ types: ['m.room.message'] });
+            const event = mkEvent({
+                type: 'm.room.message',
+                content: { },
+                room: 'roomId',
+                event: true,
+            });
+
+            const checkResult = filter.check(event);
+
+            expect(checkResult).toBe(true);
+        });
+    });
+});
@@ -97,7 +97,7 @@ describe("InteractiveAuth", function() {
        // first we expect a call to doRequest
        doRequest.mockImplementation(function(authData) {
            logger.log("request1", authData);
-            expect(authData).toEqual({});
+            expect(authData).toEqual(null); // first request should be null
            const err = new MatrixError({
                session: "sessionId",
                flows: [
@@ -143,4 +143,33 @@ describe("InteractiveAuth", function() {
            expect(stateUpdated).toBeCalledTimes(1);
        });
    });
+
+    it("should start an auth stage and reject if no auth flow", function() {
+        const doRequest = jest.fn();
+        const stateUpdated = jest.fn();
+
+        const ia = new InteractiveAuth({
+            matrixClient: new FakeClient(),
+            doRequest: doRequest,
+            stateUpdated: stateUpdated,
+        });
+
+        doRequest.mockImplementation(function(authData) {
+            logger.log("request1", authData);
+            expect(authData).toEqual(null); // first request should be null
+            const err = new MatrixError({
+                session: "sessionId",
+                flows: [],
+                params: {
+                    "logintype": { param: "aa" },
+                },
+            });
+            err.httpStatus = 401;
+            throw err;
+        });
+
+        return ia.attemptAuth().catch(function(error) {
+            expect(error.message).toBe('No appropriate authentication flow found');
+        });
+    });
 });
@@ -617,15 +617,10 @@ describe("Room", function() {
                }, event: true,
            })]);
        };
-        const setAliases = function(aliases, stateKey) {
-            if (!stateKey) {
-                stateKey = aliases.length
-                    ? aliases[0].split(':').splice(1).join(':') // domain+port
-                    : 'fibble';
-            }
+        const setAltAliases = function(aliases) {
            room.addLiveEvents([utils.mkEvent({
-                type: "m.room.aliases", room: roomId, skey: stateKey, content: {
-                    aliases: aliases,
+                type: "m.room.canonical_alias", room: roomId, skey: "", content: {
+                    alt_aliases: aliases,
                }, event: true,
            })]);
        };
@@ -862,7 +857,7 @@ describe("Room", function() {
            "(invite join_rules) rooms if a room name doesn't exist.", function() {
                const alias = "#room_alias:here";
                setJoinRule("invite");
-                setAliases([alias, "#another:here"]);
+                setAltAliases([alias, "#another:here"]);
                room.recalculate();
                const name = room.name;
                expect(name).toEqual(alias);
@@ -872,7 +867,7 @@ describe("Room", function() {
            "(public join_rules) rooms if a room name doesn't exist.", function() {
                const alias = "#room_alias:here";
                setJoinRule("public");
-                setAliases([alias, "#another:here"]);
+                setAltAliases([alias, "#another:here"]);
                room.recalculate();
                const name = room.name;
                expect(name).toEqual(alias);
@@ -0,0 +1,25 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export {};
+
+declare global {
+    namespace NodeJS {
+        interface Global {
+            localStorage: Storage;
+        }
+    }
+}
@@ -502,10 +502,12 @@ export class AutoDiscovery {
            request(
                { method: "GET", uri: url, timeout: 5000 },
                (err, response, body) => {
-                    if (err || response.statusCode < 200 || response.statusCode >= 300) {
+                    if (err || response &&
+                        (response.statusCode < 200 || response.statusCode >= 300)
+                    ) {
                        let action = "FAIL_PROMPT";
                        let reason = (err ? err.message : null) || "General failure";
-                        if (response.statusCode === 404) {
+                        if (response && response.statusCode === 404) {
                            action = "IGNORE";
                            reason = AutoDiscovery.ERROR_MISSING_WELLKNOWN;
                        }
@@ -28,13 +28,7 @@ import {SERVICE_TYPES} from './service-types';
 import {logger} from './logger';
 import {PushProcessor} from "./pushprocessor";
 import * as utils from "./utils";
-import {
-    MatrixHttpApi,
-    PREFIX_IDENTITY_V1,
-    PREFIX_IDENTITY_V2,
-    PREFIX_R0,
-    PREFIX_UNSTABLE,
-} from "./http-api";
+import {MatrixHttpApi, PREFIX_IDENTITY_V2, PREFIX_R0, PREFIX_UNSTABLE} from "./http-api";

 function termsUrlForService(serviceType, baseUrl) {
    switch (serviceType) {
@@ -212,9 +206,6 @@ MatrixBaseApis.prototype.register = function(
        inhibitLogin = undefined;
    }

-    if (auth === undefined || auth === null) {
-        auth = {};
-    }
    if (sessionId) {
        auth.session = sessionId;
    }
@@ -1119,6 +1110,21 @@ MatrixBaseApis.prototype.deleteAlias = function(alias, callback) {
    );
 };

+/**
+ * @param {string} roomId
+ * @param {module:client.callback} callback Optional.
+ * @return {Promise} Resolves: an object with an `aliases` property, containing an array of local aliases
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.unstableGetLocalAliases =
+function(roomId, callback) {
+    const path = utils.encodeUri("/rooms/$roomId/aliases",
+        {$roomId: roomId});
+    const prefix = PREFIX_UNSTABLE + "/org.matrix.msc2432";
+    return this._http.authedRequest(callback, "GET", path,
+        null, null, { prefix });
+};
+
 /**
 * Get room info for the given alias.
 * @param {string} alias The room alias to resolve.
@@ -1748,7 +1754,7 @@ MatrixBaseApis.prototype.downloadKeysForUsers = function(userIds, opts) {
        content.token = opts.token;
    }
    userIds.forEach((u) => {
-        content.device_keys[u] = {};
+        content.device_keys[u] = [];
    });

    return this._http.authedRequest(undefined, "POST", "/keys/query", undefined, content);
@@ -1761,10 +1767,13 @@ MatrixBaseApis.prototype.downloadKeysForUsers = function(userIds, opts) {
 *
 * @param {string} [key_algorithm = signed_curve25519]  desired key type
 *
+ * @param {number} [timeout] the time (in milliseconds) to wait for keys from remote
+ *     servers
+ *
 * @return {Promise} Resolves: result object. Rejects: with
 *     an error response ({@link module:http-api.MatrixError}).
 */
-MatrixBaseApis.prototype.claimOneTimeKeys = function(devices, key_algorithm) {
+MatrixBaseApis.prototype.claimOneTimeKeys = function(devices, key_algorithm, timeout) {
    const queries = {};

    if (key_algorithm === undefined) {
@@ -1779,6 +1788,9 @@ MatrixBaseApis.prototype.claimOneTimeKeys = function(devices, key_algorithm) {
        query[deviceId] = key_algorithm;
    }
    const content = {one_time_keys: queries};
+    if (timeout) {
+        content.timeout = timeout;
+    }
    const path = "/keys/claim";
    return this._http.authedRequest(undefined, "POST", path, undefined, content);
 };
@@ -1880,28 +1892,10 @@ MatrixBaseApis.prototype.requestEmailToken = async function(
        next_link: nextLink,
    };

-    try {
-        const response = await this._http.idServerRequest(
-            undefined, "POST", "/validate/email/requestToken",
-            params, PREFIX_IDENTITY_V2, identityAccessToken,
-        );
-        // TODO: Fold callback into above call once v1 path below is removed
-        if (callback) callback(null, response);
-        return response;
-    } catch (err) {
-        if (err.cors === "rejected" || err.httpStatus === 404) {
-            // Fall back to deprecated v1 API for now
-            // TODO: Remove this path once v2 is only supported version
-            // See https://github.com/vector-im/riot-web/issues/10443
-            logger.warn("IS doesn't support v2, falling back to deprecated v1");
-            return await this._http.idServerRequest(
-                callback, "POST", "/validate/email/requestToken",
-                params, PREFIX_IDENTITY_V1,
-            );
-        }
-        if (callback) callback(err);
-        throw err;
-    }
+    return await this._http.idServerRequest(
+        callback, "POST", "/validate/email/requestToken",
+        params, PREFIX_IDENTITY_V2, identityAccessToken,
+    );
 };

 /**
@@ -1948,28 +1942,10 @@ MatrixBaseApis.prototype.requestMsisdnToken = async function(
        next_link: nextLink,
    };

-    try {
-        const response = await this._http.idServerRequest(
-            undefined, "POST", "/validate/msisdn/requestToken",
-            params, PREFIX_IDENTITY_V2, identityAccessToken,
-        );
-        // TODO: Fold callback into above call once v1 path below is removed
-        if (callback) callback(null, response);
-        return response;
-    } catch (err) {
-        if (err.cors === "rejected" || err.httpStatus === 404) {
-            // Fall back to deprecated v1 API for now
-            // TODO: Remove this path once v2 is only supported version
-            // See https://github.com/vector-im/riot-web/issues/10443
-            logger.warn("IS doesn't support v2, falling back to deprecated v1");
-            return await this._http.idServerRequest(
-                callback, "POST", "/validate/msisdn/requestToken",
-                params, PREFIX_IDENTITY_V1,
-            );
-        }
-        if (callback) callback(err);
-        throw err;
-    }
+    return await this._http.idServerRequest(
+        callback, "POST", "/validate/msisdn/requestToken",
+        params, PREFIX_IDENTITY_V2, identityAccessToken,
+    );
 };

 /**
@@ -2003,24 +1979,10 @@ MatrixBaseApis.prototype.submitMsisdnToken = async function(
        token: msisdnToken,
    };

-    try {
-        return await this._http.idServerRequest(
-            undefined, "POST", "/validate/msisdn/submitToken",
-            params, PREFIX_IDENTITY_V2, identityAccessToken,
-        );
-    } catch (err) {
-        if (err.cors === "rejected" || err.httpStatus === 404) {
-            // Fall back to deprecated v1 API for now
-            // TODO: Remove this path once v2 is only supported version
-            // See https://github.com/vector-im/riot-web/issues/10443
-            logger.warn("IS doesn't support v2, falling back to deprecated v1");
-            return await this._http.idServerRequest(
-                undefined, "POST", "/validate/msisdn/submitToken",
-                params, PREFIX_IDENTITY_V1,
-            );
-        }
-        throw err;
-    }
+    return await this._http.idServerRequest(
+        undefined, "POST", "/validate/msisdn/submitToken",
+        params, PREFIX_IDENTITY_V2, identityAccessToken,
+    );
 };

 /**
@@ -2175,53 +2137,32 @@ MatrixBaseApis.prototype.lookupThreePid = async function(
    callback,
    identityAccessToken,
 ) {
-    try {
-        // Note: we're using the V2 API by calling this function, but our
-        // function contract requires a V1 response. We therefore have to
-        // convert it manually.
-        const response = await this.identityHashedLookup(
-            [[address, medium]], identityAccessToken,
-        );
-        const result = response.find(p => p.address === address);
-        if (!result) {
-            // TODO: Fold callback into above call once v1 path below is removed
-            if (callback) callback(null, {});
-            return {};
-        }
-
-        const mapping = {
-            address,
-            medium,
-            mxid: result.mxid,
-
-            // We can't reasonably fill these parameters:
-            // not_before
-            // not_after
-            // ts
-            // signatures
-        };
-
-        // TODO: Fold callback into above call once v1 path below is removed
-        if (callback) callback(null, mapping);
-        return mapping;
-    } catch (err) {
-        if (err.cors === "rejected" || err.httpStatus === 404) {
-            // Fall back to deprecated v1 API for now
-            // TODO: Remove this path once v2 is only supported version
-            // See https://github.com/vector-im/riot-web/issues/10443
-            const params = {
-                medium: medium,
-                address: address,
-            };
-            logger.warn("IS doesn't support v2, falling back to deprecated v1");
-            return await this._http.idServerRequest(
-                callback, "GET", "/lookup",
-                params, PREFIX_IDENTITY_V1,
-            );
-        }
-        if (callback) callback(err, undefined);
-        throw err;
+    // Note: we're using the V2 API by calling this function, but our
+    // function contract requires a V1 response. We therefore have to
+    // convert it manually.
+    const response = await this.identityHashedLookup(
+        [[address, medium]], identityAccessToken,
+    );
+    const result = response.find(p => p.address === address);
+    if (!result) {
+        if (callback) callback(null, {});
+        return {};
    }
+
+    const mapping = {
+        address,
+        medium,
+        mxid: result.mxid,
+
+        // We can't reasonably fill these parameters:
+        // not_before
+        // not_after
+        // ts
+        // signatures
+    };
+
+    if (callback) callback(null, mapping);
+    return mapping;
 };

 /**
@@ -2239,46 +2180,29 @@ MatrixBaseApis.prototype.bulkLookupThreePids = async function(
    query,
    identityAccessToken,
 ) {
-    try {
-        // Note: we're using the V2 API by calling this function, but our
-        // function contract requires a V1 response. We therefore have to
-        // convert it manually.
-        const response = await this.identityHashedLookup(
-            // We have to reverse the query order to get [address, medium] pairs
-            query.map(p => [p[1], p[0]]), identityAccessToken,
-        );
+    // Note: we're using the V2 API by calling this function, but our
+    // function contract requires a V1 response. We therefore have to
+    // convert it manually.
+    const response = await this.identityHashedLookup(
+        // We have to reverse the query order to get [address, medium] pairs
+        query.map(p => [p[1], p[0]]), identityAccessToken,
+    );

-        const v1results = [];
-        for (const mapping of response) {
-            const originalQuery = query.find(p => p[1] === mapping.address);
-            if (!originalQuery) {
-                throw new Error("Identity sever returned unexpected results");
-            }
-
-            v1results.push([
-                originalQuery[0], // medium
-                mapping.address,
-                mapping.mxid,
-            ]);
+    const v1results = [];
+    for (const mapping of response) {
+        const originalQuery = query.find(p => p[1] === mapping.address);
+        if (!originalQuery) {
+            throw new Error("Identity sever returned unexpected results");
        }

-        return {threepids: v1results};
-    } catch (err) {
-        if (err.cors === "rejected" || err.httpStatus === 404) {
-            // Fall back to deprecated v1 API for now
-            // TODO: Remove this path once v2 is only supported version
-            // See https://github.com/vector-im/riot-web/issues/10443
-            const params = {
-                threepids: query,
-            };
-            logger.warn("IS doesn't support v2, falling back to deprecated v1");
-            return await this._http.idServerRequest(
-                undefined, "POST", "/bulk_lookup", params,
-                PREFIX_IDENTITY_V1, identityAccessToken,
-            );
-        }
-        throw err;
+        v1results.push([
+            originalQuery[0], // medium
+            mapping.address,
+            mapping.mxid,
+        ]);
    }
+
+    return {threepids: v1results};
 };

 /**
@@ -35,19 +35,25 @@ import {StubStore} from "./store/stub";
 import {createNewMatrixCall} from "./webrtc/call";
 import * as utils from './utils';
 import {sleep} from './utils';
-import {MatrixError, PREFIX_MEDIA_R0, PREFIX_UNSTABLE} from "./http-api";
+import {
+    MatrixError,
+    PREFIX_MEDIA_R0,
+    PREFIX_UNSTABLE,
+    retryNetworkOperation,
+} from "./http-api";
 import {getHttpUriForMxc} from "./content-repo";
 import * as ContentHelpers from "./content-helpers";
 import * as olmlib from "./crypto/olmlib";
 import {ReEmitter} from './ReEmitter';
 import {RoomList} from './crypto/RoomList';
 import {logger} from './logger';
-import {Crypto, isCryptoAvailable} from './crypto';
+import {Crypto, isCryptoAvailable, fixBackupKey} from './crypto';
 import {decodeRecoveryKey} from './crypto/recoverykey';
 import {keyFromAuthData} from './crypto/key_passphrase';
 import {randomString} from './randomstring';
 import {PushProcessor} from "./pushprocessor";
 import {encodeBase64, decodeBase64} from "./crypto/olmlib";
+import { User } from "./models/user";

 const SCROLLBACK_DELAY_MS = 3000;
 export const CRYPTO_ENABLED = isCryptoAvailable();
@@ -104,6 +110,9 @@ function keyFromRecoverySession(session, decryptionKey) {
 *     If provided, opts.deviceId and opts.userId should **NOT** be provided
 *     (they are present in the exported data).
 *
+ * @param {string} opts.pickleKey Key used to pickle olm objects or other
+ *     sensitive data.
+ *
 * @param {IdentityServerProvider} [opts.identityServer]
 * Optional. A provider object with one function `getAccessToken`, which is a
 * callback that returns a Promise<String> of an identity access token to supply
@@ -217,8 +226,10 @@ function keyFromRecoverySession(session, decryptionKey) {
 * Args:
 *   {object} keys Information about the keys:
 *       {
- *           <key name>: {
- *               pubkey: {UInt8Array}
+ *           keys: {
+ *               <key name>: {
+ *                   pubkey: {UInt8Array}
+ *               }, ...
 *           }
 *       }
 *   {string} name the name of the value we want to read out of SSSS, for UI purposes.
@@ -276,18 +287,26 @@ export function MatrixClient(opts) {
            // will be used during async initialization of the crypto
            this._exportedOlmDeviceToImport = opts.deviceToImport.olmDevice;
        }
+    } else if (opts.pickleKey) {
+        this.pickleKey = opts.pickleKey;
    }

    this.scheduler = opts.scheduler;
    if (this.scheduler) {
        const self = this;
-        this.scheduler.setProcessFunction(function(eventToSend) {
+        this.scheduler.setProcessFunction(async function(eventToSend) {
            const room = self.getRoom(eventToSend.getRoomId());
            if (eventToSend.status !== EventStatus.SENDING) {
                _updatePendingEventStatus(room, eventToSend,
                                          EventStatus.SENDING);
            }
-            return _sendEventHttpRequest(self, eventToSend);
+            const res = await _sendEventHttpRequest(self, eventToSend);
+            if (room) {
+                // ensure we update pending event before the next scheduler run so that any listeners to event id
+                // updates on the synchronous event emitter get a chance to run first.
+                room.updatePendingEvent(eventToSend, EventStatus.SENT, res.event_id);
+            }
+            return res;
        });
    }
    this.clientRunning = false;
@@ -407,9 +426,9 @@ export function MatrixClient(opts) {
                    break;
                }

-                highlightCount += this.getPushActionsForEvent(
-                    event,
-                ).tweaks.highlight ? 1 : 0;
+                const pushActions = this.getPushActionsForEvent(event);
+                highlightCount += pushActions.tweaks &&
+                    pushActions.tweaks.highlight ? 1 : 0;
            }

            // Note: we don't need to handle 'total' notifications because the counts
@@ -685,6 +704,9 @@ MatrixClient.prototype.initCrypto = async function() {
        throw new Error(`Cannot enable encryption: no cryptoStore provided`);
    }

+    logger.log("Crypto: Starting up crypto store...");
+    await this._cryptoStore.startup();
+
    // initialise the list of encrypted rooms (whether or not crypto is enabled)
    logger.log("Crypto: initialising roomlist...");
    await this._roomList.init();
@@ -720,6 +742,7 @@ MatrixClient.prototype.initCrypto = async function() {
        "crypto.roomKeyRequestCancellation",
        "crypto.warning",
        "crypto.devicesUpdated",
+        "crypto.willUpdateDevices",
        "deviceVerificationChanged",
        "userTrustStatusChanged",
        "crossSigning.keysChanged",
@@ -728,6 +751,7 @@ MatrixClient.prototype.initCrypto = async function() {
    logger.log("Crypto: initialising crypto object...");
    await crypto.init({
        exportedOlmDevice: this._exportedOlmDeviceToImport,
+        pickleKey: this.pickleKey,
    });
    delete this._exportedOlmDeviceToImport;

@@ -749,7 +773,6 @@ MatrixClient.prototype.isCryptoEnabled = function() {
    return this._crypto !== null;
 };

-
 /**
 * Get the Ed25519 key for this device
 *
@@ -763,6 +786,19 @@ MatrixClient.prototype.getDeviceEd25519Key = function() {
    return this._crypto.getDeviceEd25519Key();
 };

+/**
+ * Get the Curve25519 key for this device
+ *
+ * @return {?string} base64-encoded curve25519 key. Null if crypto is
+ *    disabled.
+ */
+MatrixClient.prototype.getDeviceCurve25519Key = function() {
+    if (!this._crypto) {
+        return null;
+    }
+    return this._crypto.getDeviceCurve25519Key();
+};
+
 /**
 * Upload the device keys to the homeserver.
 * @return {object} A promise that will resolve when the keys are uploaded.
@@ -796,9 +832,9 @@ MatrixClient.prototype.downloadKeys = function(userIds, forceDownload) {
 *
 * @param {string} userId the user to list keys for.
 *
- * @return {Promise<module:crypto/deviceinfo[]>} list of devices
+ * @return {module:crypto/deviceinfo[]} list of devices
 */
-MatrixClient.prototype.getStoredDevicesForUser = async function(userId) {
+MatrixClient.prototype.getStoredDevicesForUser = function(userId) {
    if (this._crypto === null) {
        throw new Error("End-to-end encryption disabled");
    }
@@ -811,9 +847,9 @@ MatrixClient.prototype.getStoredDevicesForUser = async function(userId) {
 * @param {string} userId the user to list keys for.
 * @param {string} deviceId unique identifier for the device
 *
- * @return {Promise<?module:crypto/deviceinfo>} device or null
+ * @return {module:crypto/deviceinfo} device or null
 */
-MatrixClient.prototype.getStoredDevice = async function(userId, deviceId) {
+MatrixClient.prototype.getStoredDevice = function(userId, deviceId) {
    if (this._crypto === null) {
        throw new Error("End-to-end encryption disabled");
    }
@@ -918,6 +954,20 @@ MatrixClient.prototype.requestVerificationDM = function(userId, roomId) {
    return this._crypto.requestVerificationDM(userId, roomId);
 };

+/**
+ * Finds a DM verification request that is already in progress for the given room id
+ *
+ * @param {string} roomId the room to use for verification
+ *
+ * @returns {module:crypto/verification/request/VerificationRequest?} the VerificationRequest that is in progress, if any
+ */
+MatrixClient.prototype.findVerificationRequestDMInProgress = function(roomId) {
+    if (this._crypto === null) {
+        throw new Error("End-to-end encryption disabled");
+    }
+    return this._crypto.findVerificationRequestDMInProgress(roomId);
+};
+
 /**
 * Request a key verification from another user.
 *
@@ -1097,6 +1147,12 @@ function wrapCryptoFuncs(MatrixClient, names) {
 * @returns {boolean} true if the key matches, otherwise false
 */

+/**
+ * Perform any background tasks that can be done before a message is ready to
+ * send, in order to speed up sending of the message.
+ *
+ * @param {module:models/room} room the room the event is in
+ */
 wrapCryptoFuncs(MatrixClient, [
    "resetCrossSigningKeys",
    "getCrossSigningId",
@@ -1105,6 +1161,11 @@ wrapCryptoFuncs(MatrixClient, [
    "checkDeviceTrust",
    "checkOwnCrossSigningTrust",
    "checkCrossSigningPrivateKey",
+    "legacyDeviceVerification",
+    "prepareToEncrypt",
+    "isCrossSigningReady",
+    "getCryptoTrustCrossSignedDevices",
+    "setCryptoTrustCrossSignedDevices",
 ]);

 /**
@@ -1131,7 +1192,9 @@ MatrixClient.prototype.checkEventSenderTrust = async function(event) {
 * @param {string} password Passphrase string that can be entered by the user
 *     when restoring the backup as an alternative to entering the recovery key.
 *     Optional.
- * @returns {Promise<String>} The user-facing recovery key string.
+ * @returns {Promise<Object>} Object with public key metadata, encoded private
+ *     recovery key which should be disposed of after displaying to the user,
+ *     and raw private key to avoid round tripping if needed.
 */

 /**
@@ -1203,7 +1266,9 @@ MatrixClient.prototype.checkEventSenderTrust = async function(event) {
 * @param {boolean} checkKey check if the secret is encrypted by a trusted
 *     key
 *
- * @return {boolean} whether or not the secret is stored
+ * @return {object?} map of key name to key info the secret is encrypted
+ *     with, or null if it is not present or not encrypted with a trusted
+ *     key
 */

 /**
@@ -1257,6 +1322,7 @@ wrapCryptoFuncs(MatrixClient, [
    "requestSecret",
    "getDefaultSecretStorageKeyId",
    "setDefaultSecretStorageKeyId",
+    "checkSecretStorageKey",
    "checkSecretStoragePrivateKey",
 ]);

@@ -1303,7 +1369,8 @@ MatrixClient.prototype.cancelAndResendEventRoomKeyRequest = function(event) {
 };

 /**
- * Enable end-to-end encryption for a room.
+ * Enable end-to-end encryption for a room. This does not modify room state.
+ * Any messages sent before the returned promise resolves will be sent unencrypted.
 * @param {string} roomId The room ID to enable encryption in.
 * @param {object} config The encryption config for the room.
 * @return {Promise} A promise that will resolve when encryption is set up.
@@ -1375,15 +1442,17 @@ MatrixClient.prototype.exportRoomKeys = function() {
 * Import a list of room keys previously exported by exportRoomKeys
 *
 * @param {Object[]} keys a list of session export objects
+ * @param {Object} opts
+ * @param {Function} opts.progressCallback called with an object that has a "stage" param
 *
 * @return {Promise} a promise which resolves when the keys
 *    have been imported
 */
-MatrixClient.prototype.importRoomKeys = function(keys) {
+MatrixClient.prototype.importRoomKeys = function(keys, opts) {
    if (!this._crypto) {
        throw new Error("End-to-end encryption disabled");
    }
-    return this._crypto.importRoomKeys(keys);
+    return this._crypto.importRoomKeys(keys, opts);
 };

 /**
@@ -1443,12 +1512,16 @@ MatrixClient.prototype.isKeyBackupTrusted = function(info) {

 /**
 * @returns {bool} true if the client is configured to back up keys to
- *     the server, otherwise false.
+ *     the server, otherwise false. If we haven't completed a successful check
+ *     of key backup status yet, returns null.
 */
 MatrixClient.prototype.getKeyBackupEnabled = function() {
    if (this._crypto === null) {
        throw new Error("End-to-end encryption disabled");
    }
+    if (!this._crypto._checkedForBackup) {
+        return null;
+    }
    return Boolean(this._crypto.backupKey);
 };

@@ -1512,7 +1585,7 @@ MatrixClient.prototype.prepareKeyBackupVersion = async function(
        throw new Error("End-to-end encryption disabled");
    }

-    const [keyInfo, encodedPrivateKey, privateKey] =
+    const { keyInfo, encodedPrivateKey, privateKey } =
        await this.createRecoveryKeyFromPassphrase(password);

    if (secureSecretStorage) {
@@ -1537,7 +1610,9 @@ MatrixClient.prototype.prepareKeyBackupVersion = async function(

 /**
 * Check whether the key backup private key is stored in secret storage.
- * @return {Promise<boolean>} Whether the backup key is stored.
+ * @return {Promise<object?>} map of key name to key info the secret is
+ *     encrypted with, or null if it is not present or not encrypted with a
+ *     trusted key
 */
 MatrixClient.prototype.isKeyBackupKeyStored = async function() {
    return this.isSecretStored("m.megolm_backup.v1", false /* checkKey */);
@@ -1690,6 +1765,35 @@ MatrixClient.prototype.isValidRecoveryKey = function(recoveryKey) {
    }
 };

+/**
+ * Get the raw key for a key backup from the password
+ * Used when migrating key backups into SSSS
+ *
+ * The cross-signing API is currently UNSTABLE and may change without notice.
+ *
+ * @param {string} password Passphrase
+ * @param {object} backupInfo Backup metadata from `checkKeyBackup`
+ * @return {Promise<Buffer>} key backup key
+ */
+MatrixClient.prototype.keyBackupKeyFromPassword = function(
+    password, backupInfo,
+) {
+    return keyFromAuthData(backupInfo.auth_data, password);
+};
+
+/**
+ * Get the raw key for a key backup from the recovery key
+ * Used when migrating key backups into SSSS
+ *
+ * The cross-signing API is currently UNSTABLE and may change without notice.
+ *
+ * @param {string} recoveryKey The recovery key
+ * @return {Buffer} key backup key
+ */
+MatrixClient.prototype.keyBackupKeyFromRecoveryKey = function(recoveryKey) {
+    return decodeRecoveryKey(recoveryKey);
+};
+
 MatrixClient.RESTORE_BACKUP_ERROR_BAD_KEY = 'RESTORE_BACKUP_ERROR_BAD_KEY';

 /**
@@ -1701,15 +1805,16 @@ MatrixClient.RESTORE_BACKUP_ERROR_BAD_KEY = 'RESTORE_BACKUP_ERROR_BAD_KEY';
 * @param {string} [targetSessionId] Session ID to target a specific session.
 * Restores all sessions if omitted.
 * @param {object} backupInfo Backup metadata from `checkKeyBackup`
+ * @param {object} opts Optional params such as callbacks
 * @return {Promise<object>} Status of restoration with `total` and `imported`
 * key counts.
 */
 MatrixClient.prototype.restoreKeyBackupWithPassword = async function(
-    password, targetRoomId, targetSessionId, backupInfo,
+    password, targetRoomId, targetSessionId, backupInfo, opts,
 ) {
    const privKey = await keyFromAuthData(backupInfo.auth_data, password);
    return this._restoreKeyBackup(
-        privKey, targetRoomId, targetSessionId, backupInfo,
+        privKey, targetRoomId, targetSessionId, backupInfo, opts,
    );
 };

@@ -1722,15 +1827,26 @@ MatrixClient.prototype.restoreKeyBackupWithPassword = async function(
 * Restores all rooms if omitted.
 * @param {string} [targetSessionId] Session ID to target a specific session.
 * Restores all sessions if omitted.
+ * @param {object} opts Optional params such as callbacks
 * @return {Promise<object>} Status of restoration with `total` and `imported`
 * key counts.
 */
 MatrixClient.prototype.restoreKeyBackupWithSecretStorage = async function(
-    backupInfo, targetRoomId, targetSessionId,
+    backupInfo, targetRoomId, targetSessionId, opts,
 ) {
-    const privKey = decodeBase64(await this.getSecret("m.megolm_backup.v1"));
+    const storedKey = await this.getSecret("m.megolm_backup.v1");
+
+    // ensure that the key is in the right format.  If not, fix the key and
+    // store the fixed version
+    const fixedKey = fixBackupKey(storedKey);
+    if (fixedKey) {
+        const [keyId] = await this._crypto.getSecretStorageKey();
+        await this.storeSecret("m.megolm_backup.v1", fixedKey, [keyId]);
+    }
+
+    const privKey = decodeBase64(fixedKey || storedKey);
    return this._restoreKeyBackup(
-        privKey, targetRoomId, targetSessionId, backupInfo,
+        privKey, targetRoomId, targetSessionId, backupInfo, opts,
    );
 };

@@ -1743,20 +1859,50 @@ MatrixClient.prototype.restoreKeyBackupWithSecretStorage = async function(
 * @param {string} [targetSessionId] Session ID to target a specific session.
 * Restores all sessions if omitted.
 * @param {object} backupInfo Backup metadata from `checkKeyBackup`
+ * @param {object} opts Optional params such as callbacks
+
 * @return {Promise<object>} Status of restoration with `total` and `imported`
 * key counts.
 */
 MatrixClient.prototype.restoreKeyBackupWithRecoveryKey = function(
-    recoveryKey, targetRoomId, targetSessionId, backupInfo,
+    recoveryKey, targetRoomId, targetSessionId, backupInfo, opts,
 ) {
    const privKey = decodeRecoveryKey(recoveryKey);
    return this._restoreKeyBackup(
-        privKey, targetRoomId, targetSessionId, backupInfo,
+        privKey, targetRoomId, targetSessionId, backupInfo, opts,
+    );
+};
+
+/**
+ * Restore from an existing key backup using a cached key, or fail
+ *
+ * @param {string} [targetRoomId] Room ID to target a specific room.
+ * Restores all rooms if omitted.
+ * @param {string} [targetSessionId] Session ID to target a specific session.
+ * Restores all sessions if omitted.
+ * @param {object} backupInfo Backup metadata from `checkKeyBackup`
+ * @param {object} opts Optional params such as callbacks
+ * @return {Promise<object>} Status of restoration with `total` and `imported`
+ * key counts.
+ */
+MatrixClient.prototype.restoreKeyBackupWithCache = async function(
+    targetRoomId, targetSessionId, backupInfo, opts,
+) {
+    const privKey = await this._crypto.getSessionBackupPrivateKey();
+    if (!privKey) {
+        throw new Error("Couldn't get key");
+    }
+    return this._restoreKeyBackup(
+        privKey, targetRoomId, targetSessionId, backupInfo, opts,
    );
 };

 MatrixClient.prototype._restoreKeyBackup = function(
    privKey, targetRoomId, targetSessionId, backupInfo,
+    {
+        cacheCompleteCallback, // For sequencing during tests
+        progressCallback,
+    }={},
 ) {
    if (this._crypto === null) {
        throw new Error("End-to-end encryption disabled");
@@ -1784,6 +1930,19 @@ MatrixClient.prototype._restoreKeyBackup = function(
        return Promise.reject({errcode: MatrixClient.RESTORE_BACKUP_ERROR_BAD_KEY});
    }

+    // Cache the key, if possible.
+    // This is async.
+    this._crypto.storeSessionBackupPrivateKey(privKey)
+    .catch((e) => {
+        console.warn("Error caching session backup key:", e);
+    }).then(cacheCompleteCallback);
+
+    if (progressCallback) {
+        progressCallback({
+            stage: "fetch",
+        });
+    }
+
    return this._http.authedRequest(
        undefined, "GET", path.path, path.queryData, undefined,
        {prefix: PREFIX_UNSTABLE},
@@ -1818,7 +1977,7 @@ MatrixClient.prototype._restoreKeyBackup = function(
            }
        }

-        return this.importRoomKeys(keys);
+        return this.importRoomKeys(keys, { progressCallback });
    }).then(() => {
        return this._crypto.setTrustedBackupPubKey(backupPubKey);
    }).then(() => {
@@ -1954,6 +2113,7 @@ MatrixClient.prototype.getUsers = function() {

 /**
 * Set account data event for the current user.
+ * It will retry the request up to 5 times.
 * @param {string} eventType The event type
 * @param {Object} contents the contents object for the event
 * @param {module:client.callback} callback Optional.
@@ -1965,9 +2125,13 @@ MatrixClient.prototype.setAccountData = function(eventType, contents, callback)
        $userId: this.credentials.userId,
        $type: eventType,
    });
-    return this._http.authedRequest(
-        callback, "PUT", path, undefined, contents,
-    );
+    const promise = retryNetworkOperation(5, () => {
+        return this._http.authedRequest(undefined, "PUT", path, undefined, contents);
+    });
+    if (callback) {
+        promise.then(result => callback(null, result), callback);
+    }
+    return promise;
 };

 /**
@@ -2002,9 +2166,17 @@ MatrixClient.prototype.getAccountDataFromServer = async function(eventType) {
        $userId: this.credentials.userId,
        $type: eventType,
    });
-    return this._http.authedRequest(
-        undefined, "GET", path, undefined,
-    );
+    try {
+        const result = await this._http.authedRequest(
+            undefined, "GET", path, undefined,
+        );
+        return result;
+    } catch (e) {
+        if (e.data && e.data.errcode === 'M_NOT_FOUND') {
+            return null;
+        }
+        throw e;
+    }
 };

 /**
@@ -2318,6 +2490,7 @@ MatrixClient.prototype._sendCompleteEvent = function(roomId, eventObject, txnId,
    const localEvent = new MatrixEvent(Object.assign(eventObject, {
        event_id: "~" + roomId + ":" + txnId,
        user_id: this.credentials.userId,
+        sender: this.credentials.userId,
        room_id: roomId,
        origin_server_ts: new Date().getTime(),
    }));
@@ -2381,7 +2554,7 @@ function _sendEvent(client, room, event, callback) {
        let promise;
        // this event may be queued
        if (client.scheduler) {
-            // if this returns a promsie then the scheduler has control now and will
+            // if this returns a promise then the scheduler has control now and will
            // resolve/reject when it is done. Internally, the scheduler will invoke
            // processFn which is set to this._sendEventHttpRequest so the same code
            // path is executed regardless.
@@ -2395,12 +2568,15 @@ function _sendEvent(client, room, event, callback) {

        if (!promise) {
            promise = _sendEventHttpRequest(client, event);
+            if (room) {
+                promise = promise.then(res => {
+                    room.updatePendingEvent(event, EventStatus.SENT, res.event_id);
+                    return res;
+                });
+            }
        }
        return promise;
    }).then(function(res) {  // the request was sent OK
-        if (room) {
-            room.updatePendingEvent(event, EventStatus.SENT, res.event_id);
-        }
        if (callback) {
            callback(null, res);
        }
@@ -4375,64 +4551,54 @@ MatrixClient.prototype.getFilter = function(userId, filterId, allowCached) {
 * @param {Filter} filter
 * @return {Promise<String>} Filter ID
 */
-MatrixClient.prototype.getOrCreateFilter = function(filterName, filter) {
+MatrixClient.prototype.getOrCreateFilter = async function(filterName, filter) {
    const filterId = this.store.getFilterIdByName(filterName);
-    let promise = Promise.resolve();
-    const self = this;
+    let existingId = undefined;

    if (filterId) {
        // check that the existing filter matches our expectations
-        promise = self.getFilter(self.credentials.userId,
-                         filterId, true,
-        ).then(function(existingFilter) {
-            const oldDef = existingFilter.getDefinition();
-            const newDef = filter.getDefinition();
+        try {
+            const existingFilter =
+                await this.getFilter(this.credentials.userId, filterId, true);
+            if (existingFilter) {
+                const oldDef = existingFilter.getDefinition();
+                const newDef = filter.getDefinition();

-            if (utils.deepCompare(oldDef, newDef)) {
-                // super, just use that.
-                // debuglog("Using existing filter ID %s: %s", filterId,
-                //          JSON.stringify(oldDef));
-                return Promise.resolve(filterId);
+                if (utils.deepCompare(oldDef, newDef)) {
+                    // super, just use that.
+                    // debuglog("Using existing filter ID %s: %s", filterId,
+                    //          JSON.stringify(oldDef));
+                    existingId = filterId;
+                }
            }
-            // debuglog("Existing filter ID %s: %s; new filter: %s",
-            //          filterId, JSON.stringify(oldDef), JSON.stringify(newDef));
-            self.store.setFilterIdByName(filterName, undefined);
-            return undefined;
-        }, function(error) {
+        } catch (error) {
            // Synapse currently returns the following when the filter cannot be found:
            // {
            //     errcode: "M_UNKNOWN",
            //     name: "M_UNKNOWN",
            //     message: "No row found",
-            //     data: Object, httpStatus: 404
            // }
-            if (error.httpStatus === 404 &&
-                (error.errcode === "M_UNKNOWN" || error.errcode === "M_NOT_FOUND")) {
-                // Clear existing filterId from localStorage
-                // if it no longer exists on the server
-                self.store.setFilterIdByName(filterName, undefined);
-                // Return a undefined value for existingId further down the promise chain
-                return undefined;
-            } else {
+            if (error.errcode !== "M_UNKNOWN" && error.errcode !== "M_NOT_FOUND") {
                throw error;
            }
-        });
+        }
+        // if the filter doesn't exist anymore on the server, remove from store
+        if (!existingId) {
+            this.store.setFilterIdByName(filterName, undefined);
+        }
    }

-    return promise.then(function(existingId) {
-        if (existingId) {
-            return existingId;
-        }
+    if (existingId) {
+        return existingId;
+    }

-        // create a new filter
-        return self.createFilter(filter.getDefinition(),
-        ).then(function(createdFilter) {
-            // debuglog("Created new filter ID %s: %s", createdFilter.filterId,
-            //          JSON.stringify(createdFilter.getDefinition()));
-            self.store.setFilterIdByName(filterName, createdFilter.filterId);
-            return createdFilter.filterId;
-        });
-    });
+    // create a new filter
+    const createdFilter = await this.createFilter(filter.getDefinition());
+
+    // debuglog("Created new filter ID %s: %s", createdFilter.filterId,
+    //          JSON.stringify(createdFilter.getDefinition()));
+    this.store.setFilterIdByName(filterName, createdFilter.filterId);
+    return createdFilter.filterId;
 };


@@ -4610,6 +4776,13 @@ MatrixClient.prototype.startClient = async function(opts) {
        };
    }

+    // Create our own user object artificially (instead of waiting for sync)
+    // so it's always available, even if the user is not in any rooms etc.
+    const userId = this.getUserId();
+    if (userId) {
+        this.store.storeUser(new User(userId));
+    }
+
    if (this._crypto) {
        this._crypto.uploadDeviceKeys();
        this._crypto.start();
@@ -4783,6 +4956,19 @@ MatrixClient.prototype.doesServerSupportSeparateAddAndBind = async function() {
        || (unstableFeatures && unstableFeatures["m.separate_add_and_bind"]);
 };

+/**
+ * Query the server to see if it lists support for an unstable feature
+ * in the /versions response
+ * @param {string} feature the feature name
+ * @return {Promise<boolean>} true if the feature is supported
+ */
+MatrixClient.prototype.doesServerSupportUnstableFeature = async function(feature) {
+    const response = await this.getVersions();
+    if (!response) return false;
+    const unstableFeatures = response["unstable_features"];
+    return unstableFeatures && !!unstableFeatures[feature];
+};
+
 /**
 * Get if lazy loading members is being used.
 * @return {boolean} Whether or not members are lazy loaded by this client
@@ -5070,9 +5256,6 @@ function checkTurnServers(client) {
    if (!client._supportsVoip) {
        return;
    }
-    if (client.isGuest()) {
-        return; // guests can't access TURN servers
-    }

    client.turnServer().then(function(res) {
        if (res.uris) {
@@ -5113,17 +5296,20 @@ function _resolve(callback, resolve, res) {
    resolve(res);
 }

-function _PojoToMatrixEventMapper(client) {
+function _PojoToMatrixEventMapper(client, options) {
+    const preventReEmit = Boolean(options && options.preventReEmit);
    function mapper(plainOldJsObject) {
        const event = new MatrixEvent(plainOldJsObject);
        if (event.isEncrypted()) {
-            client.reEmitter.reEmit(event, [
-                "Event.decrypted",
-            ]);
+            if (!preventReEmit) {
+                client.reEmitter.reEmit(event, [
+                    "Event.decrypted",
+                ]);
+            }
            event.attemptDecryption(client._crypto);
        }
        const room = client.getRoom(event.getRoomId());
-        if (room) {
+        if (room && !preventReEmit) {
            room.reEmitter.reEmit(event, ["Event.replaced"]);
        }
        return event;
@@ -5132,10 +5318,21 @@ function _PojoToMatrixEventMapper(client) {
 }

 /**
+ * @param {object} [options]
+ * @param {bool} options.preventReEmit don't reemit events emitted on an event mapped by this mapper on the client
 * @return {Function}
 */
-MatrixClient.prototype.getEventMapper = function() {
-    return _PojoToMatrixEventMapper(this);
+MatrixClient.prototype.getEventMapper = function(options = undefined) {
+    return _PojoToMatrixEventMapper(this, options);
+};
+
+/**
+ * The app may wish to see if we have a key cached without
+ * triggering a user interaction.
+ * @return {object}
+ */
+MatrixClient.prototype.getCrossSigningCacheCallbacks = function() {
+    return this._crypto && this._crypto._crossSigningInfo.getCacheCallbacks();
 };

 // Identity Server Operations
@@ -5412,6 +5609,22 @@ MatrixClient.prototype.generateClientSecret = function() {
 * });
 */

+/**
+ * Fires whenever the stored devices for a user have changed
+ * @event module:client~MatrixClient#"crypto.devicesUpdated"
+ * @param {String[]} users A list of user IDs that were updated
+ * @param {bool} initialFetch If true, the store was empty (apart
+ *     from our own device) and has been seeded.
+ */
+
+/**
+ * Fires whenever the stored devices for a user will be updated
+ * @event module:client~MatrixClient#"crypto.willUpdateDevices"
+ * @param {String[]} users A list of user IDs that will be updated
+ * @param {bool} initialFetch If true, the store is empty (apart
+ *     from our own device) and is being seeded.
+ */
+
 /**
 * Fires whenever the status of e2e key backup changes, as returned by getKeyBackupEnabled()
 * @event module:client~MatrixClient#"crypto.keyBackupStatus"
@@ -5454,13 +5667,6 @@ MatrixClient.prototype.generateClientSecret = function() {
 *     reject the key verification.
 */

-/**
- * Fires when a key verification started message is received.
- * @event module:client~MatrixClient#"crypto.verification.start"
- * @param {module:crypto/verification/Base} verifier a verifier object to
- *     perform the key verification
- */
-
 /**
 * Fires when a secret request has been cancelled.  If the client is prompting
 * the user to ask whether they want to share a secret, the prompt can be
@@ -23,6 +23,8 @@ limitations under the License.
 import {decodeBase64, encodeBase64, pkSign, pkVerify} from './olmlib';
 import {EventEmitter} from 'events';
 import {logger} from '../logger';
+import {IndexedDBCryptoStore} from '../crypto/store/indexeddb-crypto-store';
+import {decryptAES, encryptAES} from './aes';

 function publicKeyFromKeyInfo(keyInfo) {
    // `keys` is an object with { [`ed25519:${pubKey}`]: pubKey }
@@ -40,8 +42,9 @@ export class CrossSigningInfo extends EventEmitter {
     * @param {string} userId the user that the information is about
     * @param {object} callbacks Callbacks used to interact with the app
     *     Requires getCrossSigningKey and saveCrossSigningKeys
+     * @param {object} cacheCallbacks Callbacks used to interact with the cache
     */
-    constructor(userId, callbacks) {
+    constructor(userId, callbacks, cacheCallbacks) {
        super();

        // you can't change the userId
@@ -50,8 +53,15 @@ export class CrossSigningInfo extends EventEmitter {
            value: userId,
        });
        this._callbacks = callbacks || {};
+        this._cacheCallbacks = cacheCallbacks || {};
        this.keys = {};
        this.firstUse = true;
+        // This tracks whether we've ever verified this user with any identity.
+        // When you verify a user, any devices online at the time that receive
+        // the verifying signature via the homeserver will latch this to true
+        // and can use it in the future to detect cases where the user has
+        // become unverifed later for any reason.
+        this.crossSigningVerifiedBefore = false;
    }

    /**
@@ -62,6 +72,8 @@ export class CrossSigningInfo extends EventEmitter {
     * @returns {Array} An array with [ public key, Olm.PkSigning ]
     */
    async getCrossSigningKey(type, expectedPubkey) {
+        const shouldCache = ["self_signing", "user_signing"].indexOf(type) >= 0;
+
        if (!this._callbacks.getCrossSigningKey) {
            throw new Error("No getCrossSigningKey callback supplied");
        }
@@ -70,22 +82,47 @@ export class CrossSigningInfo extends EventEmitter {
            expectedPubkey = this.getId(type);
        }

-        const privkey = await this._callbacks.getCrossSigningKey(type, expectedPubkey);
+        function validateKey(key) {
+            if (!key) return;
+            const signing = new global.Olm.PkSigning();
+            const gotPubkey = signing.init_with_seed(key);
+            if (gotPubkey === expectedPubkey) {
+                return [gotPubkey, signing];
+            }
+            signing.free();
+        }
+
+        let privkey;
+        if (this._cacheCallbacks.getCrossSigningKeyCache && shouldCache) {
+            privkey = await this._cacheCallbacks
+              .getCrossSigningKeyCache(type, expectedPubkey);
+        }
+
+        const cacheresult = validateKey(privkey);
+        if (cacheresult) {
+            return cacheresult;
+        }
+
+        privkey = await this._callbacks.getCrossSigningKey(type, expectedPubkey);
+        const result = validateKey(privkey);
+        if (result) {
+            if (this._cacheCallbacks.storeCrossSigningKeyCache && shouldCache) {
+                await this._cacheCallbacks.storeCrossSigningKeyCache(type, privkey);
+            }
+            return result;
+        }
+
+        /* No keysource even returned a key */
        if (!privkey) {
            throw new Error(
                "getCrossSigningKey callback for " + type + " returned falsey",
            );
        }
-        const signing = new global.Olm.PkSigning();
-        const gotPubkey = signing.init_with_seed(privkey);
-        if (gotPubkey !== expectedPubkey) {
-            signing.free();
-            throw new Error(
-                "Key type " + type + " from getCrossSigningKey callback did not match",
-            );
-        } else {
-            return [gotPubkey, signing];
-        }
+
+        /* We got some keys from the keysource, but none of them were valid */
+        throw new Error(
+            "Key type " + type + " from getCrossSigningKey callback did not match",
+        );
    }

    static fromStorage(obj, userId) {
@@ -102,6 +139,7 @@ export class CrossSigningInfo extends EventEmitter {
        return {
            keys: this.keys,
            firstUse: this.firstUse,
+            crossSigningVerifiedBefore: this.crossSigningVerifiedBefore,
        };
    }

@@ -111,14 +149,28 @@ export class CrossSigningInfo extends EventEmitter {
     * want to know this anyway...
     *
     * @param {SecretStorage} secretStorage The secret store using account data
-     * @returns {boolean} Whether all private keys were found in storage
+     * @returns {object} map of key name to key info the secret is encrypted
+     *     with, or null if it is not present or not encrypted with a trusted
+     *     key
     */
    async isStoredInSecretStorage(secretStorage) {
-        let stored = true;
-        for (const type of ["master", "self_signing", "user_signing"]) {
-            stored &= await secretStorage.isStored(`m.cross_signing.${type}`, false);
+        // check what SSSS keys have encrypted the master key (if any)
+        const stored =
+              await secretStorage.isStored("m.cross_signing.master", false) || {};
+        // then check which of those SSSS keys have also encrypted the SSK and USK
+        function intersect(s) {
+            for (const k of Object.keys(stored)) {
+                if (!s[k]) {
+                    delete stored[k];
+                }
+            }
        }
-        return stored;
+        for (const type of ["self_signing", "user_signing"]) {
+            intersect(
+                await secretStorage.isStored(`m.cross_signing.${type}`, false) || {},
+            );
+        }
+        return Object.keys(stored).length ? stored : null;
    }

    /**
@@ -259,6 +311,13 @@ export class CrossSigningInfo extends EventEmitter {
        }
    }

+    /**
+     * unsets the keys, used when another session has reset the keys, to disable cross-signing
+     */
+    clearKeys() {
+        this.keys = {};
+    }
+
    setKeys(keys) {
        const signingKeys = {};
        if (keys.master) {
@@ -331,6 +390,14 @@ export class CrossSigningInfo extends EventEmitter {
        }
    }

+    updateCrossSigningVerifiedBefore(isCrossSigningVerified) {
+        // It is critical that this value latches forward from false to true but
+        // never back to false to avoid a downgrade attack.
+        if (!this.crossSigningVerifiedBefore && isCrossSigningVerified) {
+            this.crossSigningVerifiedBefore = true;
+        }
+    }
+
    async signObject(data, type) {
        if (!this.keys[type]) {
            throw new Error(
@@ -389,13 +456,13 @@ export class CrossSigningInfo extends EventEmitter {
            && this.getId("self_signing")
            && this.getId("self_signing") === userCrossSigning.getId("self_signing")
        ) {
-            return new UserTrustLevel(true, this.firstUse);
+            return new UserTrustLevel(true, true, this.firstUse);
        }

        if (!this.keys.user_signing) {
            // If there's no user signing key, they can't possibly be verified.
            // They may be TOFU trusted though.
-            return new UserTrustLevel(false, userCrossSigning.firstUse);
+            return new UserTrustLevel(false, false, userCrossSigning.firstUse);
        }

        let userTrusted;
@@ -407,7 +474,11 @@ export class CrossSigningInfo extends EventEmitter {
        } catch (e) {
            userTrusted = false;
        }
-        return new UserTrustLevel(userTrusted, userCrossSigning.firstUse);
+        return new UserTrustLevel(
+            userTrusted,
+            userCrossSigning.crossSigningVerifiedBefore,
+            userCrossSigning.firstUse,
+        );
    }

    /**
@@ -416,17 +487,20 @@ export class CrossSigningInfo extends EventEmitter {
     * @param {CrossSigningInfo} userCrossSigning Cross signing info for user
     * @param {module:crypto/deviceinfo} device The device to check
     * @param {bool} localTrust Whether the device is trusted locally
+     * @param {bool} trustCrossSignedDevices Whether we trust cross signed devices
     *
     * @returns {DeviceTrustLevel}
     */
-    checkDeviceTrust(userCrossSigning, device, localTrust) {
+    checkDeviceTrust(userCrossSigning, device, localTrust, trustCrossSignedDevices) {
        const userTrust = this.checkUserTrust(userCrossSigning);

        const userSSK = userCrossSigning.keys.self_signing;
        if (!userSSK) {
            // if the user has no self-signing key then we cannot make any
            // trust assertions about this device from cross-signing
-            return new DeviceTrustLevel(false, false, localTrust);
+            return new DeviceTrustLevel(
+                false, false, localTrust, trustCrossSignedDevices,
+            );
        }

        const deviceObj = deviceToObject(device, userCrossSigning.userId);
@@ -438,11 +512,22 @@ export class CrossSigningInfo extends EventEmitter {
                deviceObj, publicKeyFromKeyInfo(userSSK), userCrossSigning.userId,
            );
            // ...then we trust this device as much as far as we trust the user
-            return DeviceTrustLevel.fromUserTrustLevel(userTrust, localTrust);
+            return DeviceTrustLevel.fromUserTrustLevel(
+                userTrust, localTrust, trustCrossSignedDevices,
+            );
        } catch (e) {
-            return new DeviceTrustLevel(false, false, localTrust);
+            return new DeviceTrustLevel(
+                false, false, localTrust, trustCrossSignedDevices,
+            );
        }
    }
+
+    /**
+     * @returns {object} Cache callbacks
+     */
+    getCacheCallbacks() {
+        return this._cacheCallbacks;
+    }
 }

 function deviceToObject(device, userId) {
@@ -465,8 +550,9 @@ export const CrossSigningLevel = {
 * Represents the ways in which we trust a user
 */
 export class UserTrustLevel {
-    constructor(crossSigningVerified, tofu) {
+    constructor(crossSigningVerified, crossSigningVerifiedBefore, tofu) {
        this._crossSigningVerified = crossSigningVerified;
+        this._crossSigningVerifiedBefore = crossSigningVerifiedBefore;
        this._tofu = tofu;
    }

@@ -484,6 +570,14 @@ export class UserTrustLevel {
        return this._crossSigningVerified;
    }

+    /**
+     * @returns {bool} true if we ever verified this user before (at least for
+     * the history of verifications observed by this device).
+     */
+    wasCrossSigningVerified() {
+        return this._crossSigningVerifiedBefore;
+    }
+
    /**
     * @returns {bool} true if this user's key is trusted on first use
     */
@@ -496,17 +590,19 @@ export class UserTrustLevel {
 * Represents the ways in which we trust a device
 */
 export class DeviceTrustLevel {
-    constructor(crossSigningVerified, tofu, localVerified) {
+    constructor(crossSigningVerified, tofu, localVerified, trustCrossSignedDevices) {
        this._crossSigningVerified = crossSigningVerified;
        this._tofu = tofu;
        this._localVerified = localVerified;
+        this._trustCrossSignedDevices = trustCrossSignedDevices;
    }

-    static fromUserTrustLevel(userTrustLevel, localVerified) {
+    static fromUserTrustLevel(userTrustLevel, localVerified, trustCrossSignedDevices) {
        return new DeviceTrustLevel(
            userTrustLevel._crossSigningVerified,
            userTrustLevel._tofu,
            localVerified,
+            trustCrossSignedDevices,
        );
    }

@@ -514,7 +610,9 @@ export class DeviceTrustLevel {
     * @returns {bool} true if this device is verified via any means
     */
    isVerified() {
-        return this.isCrossSigningVerified() || this.isLocallyVerified();
+        return Boolean(this.isLocallyVerified() || (
+            this._trustCrossSignedDevices && this.isCrossSigningVerified()
+        ));
    }

    /**
@@ -539,3 +637,43 @@ export class DeviceTrustLevel {
        return this._tofu;
    }
 }
+
+export function createCryptoStoreCacheCallbacks(store, olmdevice) {
+    return {
+        getCrossSigningKeyCache: async function(type, _expectedPublicKey) {
+            const key = await new Promise((resolve) => {
+                return store.doTxn(
+                    'readonly',
+                    [IndexedDBCryptoStore.STORE_ACCOUNT],
+                    (txn) => {
+                        store.getSecretStorePrivateKey(txn, resolve, type);
+                    },
+                );
+            });
+
+            if (key && key.ciphertext) {
+                const pickleKey = Buffer.from(olmdevice._pickleKey);
+                const decrypted = await decryptAES(key, pickleKey, type);
+                return decodeBase64(decrypted);
+            } else {
+                return key;
+            }
+        },
+        storeCrossSigningKeyCache: async function(type, key) {
+            if (!(key instanceof Uint8Array)) {
+                throw new Error(
+                    `storeCrossSigningKeyCache expects Uint8Array, got ${key}`,
+                );
+            }
+            const pickleKey = Buffer.from(olmdevice._pickleKey);
+            key = await encryptAES(encodeBase64(key), pickleKey, type);
+            return store.doTxn(
+                'readwrite',
+                [IndexedDBCryptoStore.STORE_ACCOUNT],
+                (txn) => {
+                    store.storeSecretStorePrivateKey(txn, type, key);
+                },
+            );
+        },
+    };
+}
@@ -109,6 +109,9 @@ export class DeviceList extends EventEmitter {
        this._savePromiseTime = null;
        // The timer used to delay the save
        this._saveTimer = null;
+        // True if we have fetched data from the server or loaded a non-empty
+        // set of device data from the store
+        this._hasFetched = null;
    }

    /**
@@ -118,6 +121,7 @@ export class DeviceList extends EventEmitter {
        await this._cryptoStore.doTxn(
            'readonly', [IndexedDBCryptoStore.STORE_DEVICE_DATA], (txn) => {
                this._cryptoStore.getEndToEndDeviceData(txn, (deviceData) => {
+                    this._hasFetched = Boolean(deviceData && deviceData.devices);
                    this._devices = deviceData ? deviceData.devices : {},
                    this._crossSigningInfo = deviceData ?
                        deviceData.crossSigningInfo || {} : {};
@@ -197,16 +201,16 @@ export class DeviceList extends EventEmitter {
            const resolveSavePromise = this._resolveSavePromise;
            this._savePromiseTime = targetTime;
            this._saveTimer = setTimeout(() => {
-                logger.log('Saving device tracking data at token ' + this._syncToken);
+                logger.log('Saving device tracking data', this._syncToken);
+
                // null out savePromise now (after the delay but before the write),
                // otherwise we could return the existing promise when the save has
-                // actually already happened. Likewise for the dirty flag.
+                // actually already happened.
                this._savePromiseTime = null;
                this._saveTimer = null;
                this._savePromise = null;
                this._resolveSavePromise = null;

-                this._dirty = false;
                this._cryptoStore.doTxn(
                    'readwrite', [IndexedDBCryptoStore.STORE_DEVICE_DATA], (txn) => {
                        this._cryptoStore.storeEndToEndDeviceData({
@@ -217,7 +221,13 @@ export class DeviceList extends EventEmitter {
                        }, txn);
                    },
                ).then(() => {
+                    // The device list is considered dirty until the write
+                    // completes.
+                    this._dirty = false;
                    resolveSavePromise();
+                }, err => {
+                    logger.error('Failed to save device tracking data', this._syncToken);
+                    logger.error(err);
                });
            }, delay);
        }
@@ -311,6 +321,15 @@ export class DeviceList extends EventEmitter {
        return stored;
    }

+    /**
+     * Returns a list of all user IDs the DeviceList knows about
+     *
+     * @return {array} All known user IDs
+     */
+    getKnownUserIds() {
+        return Object.keys(this._devices);
+    }
+
    /**
     * Get the stored device keys for a user id
     *
@@ -373,6 +392,26 @@ export class DeviceList extends EventEmitter {
        return DeviceInfo.fromStorage(devs[deviceId], deviceId);
    }

+    /**
+     * Get a user ID by one of their device's curve25519 identity key
+     *
+     * @param {string} algorithm  encryption algorithm
+     * @param {string} senderKey  curve25519 key to match
+     *
+     * @return {string} user ID
+     */
+    getUserByIdentityKey(algorithm, senderKey) {
+        if (
+            algorithm !== olmlib.OLM_ALGORITHM &&
+            algorithm !== olmlib.MEGOLM_ALGORITHM
+        ) {
+            // we only deal in olm keys
+            return null;
+        }
+
+        return this._userByIdentityKey[senderKey];
+    }
+
    /**
     * Find a device by curve25519 identity key
     *
@@ -382,19 +421,11 @@ export class DeviceList extends EventEmitter {
     * @return {module:crypto/deviceinfo?}
     */
    getDeviceByIdentityKey(algorithm, senderKey) {
-        const userId = this._userByIdentityKey[senderKey];
+        const userId = this.getUserByIdentityKey(algorithm, senderKey);
        if (!userId) {
            return null;
        }

-        if (
-            algorithm !== olmlib.OLM_ALGORITHM &&
-            algorithm !== olmlib.MEGOLM_ALGORITHM
-        ) {
-            // we only deal in olm keys
-            return null;
-        }
-
        const devices = this._devices[userId];
        if (!devices) {
            return null;
@@ -625,6 +656,7 @@ export class DeviceList extends EventEmitter {
        });

        const finished = (success) => {
+            this.emit("crypto.willUpdateDevices", users, !this._hasFetched);
            users.forEach((u) => {
                this._dirty = true;

@@ -650,7 +682,8 @@ export class DeviceList extends EventEmitter {
                }
            });
            this.saveIfDirty();
-            this.emit("crypto.devicesUpdated", users);
+            this.emit("crypto.devicesUpdated", users, !this._hasFetched);
+            this._hasFetched = true;
        };

        return prom;
@@ -749,31 +782,34 @@ class DeviceListUpdateSerialiser {

        this._baseApis.downloadKeysForUsers(
            downloadUsers, opts,
-        ).then((res) => {
+        ).then(async (res) => {
            const dk = res.device_keys || {};
            const masterKeys = res.master_keys || {};
            const ssks = res.self_signing_keys || {};
            const usks = res.user_signing_keys || {};

-            // do each user in a separate promise, to avoid wedging the CPU
+            // yield to other things that want to execute in between users, to
+            // avoid wedging the CPU
            // (https://github.com/vector-im/riot-web/issues/3158)
            //
            // of course we ought to do this in a web worker or similar, but
            // this serves as an easy solution for now.
-            let prom = Promise.resolve();
            for (const userId of downloadUsers) {
-                prom = prom.then(sleep(5)).then(() => {
-                    return this._processQueryResponseForUser(
+                await sleep(5);
+                try {
+                    await this._processQueryResponseForUser(
                        userId, dk[userId], {
                            master: masterKeys[userId],
                            self_signing: ssks[userId],
                            user_signing: usks[userId],
                        },
                    );
-                });
+                } catch (e) {
+                    // log the error but continue, so that one bad key
+                    // doesn't kill the whole process
+                    logger.error(`Error processing keys for ${userId}:`, e);
+                }
            }
-
-            return prom;
        }).then(() => {
            logger.log('Completed key download for ' + downloadUsers);

@@ -794,7 +830,7 @@ class DeviceListUpdateSerialiser {
    }

    async _processQueryResponseForUser(
-        userId, dkResponse, crossSigningResponse, sskResponse,
+        userId, dkResponse, crossSigningResponse,
    ) {
        logger.log('got device keys for ' + userId + ':', dkResponse);
        logger.log('got cross-signing keys for ' + userId + ':', crossSigningResponse);
@@ -36,9 +36,15 @@ function checkPayloadLength(payloadString) {
        // Note that even if we manage to do the encryption, the message send may fail,
        // because by the time we've wrapped the ciphertext in the event object, it may
        // exceed 65K. But at least we won't just fail with "abort()" in that case.
-        throw new Error("Message too long (" + payloadString.length + " bytes). " +
+        const err = new Error("Message too long (" + payloadString.length + " bytes). " +
                        "The maximum for an encrypted message is " +
                        MAX_PLAINTEXT_LENGTH + " bytes.");
+        // TODO: [TypeScript] We should have our own error types
+        err.data = {
+            errcode: "M_TOO_LARGE",
+            error: "Payload too large for encrypted message",
+        };
+        throw err;
    }
 }

@@ -105,6 +111,9 @@ export function OlmDevice(cryptoStore) {
    // Keep track of sessions that we're starting, so that we don't start
    // multiple sessions for the same device at the same time.
    this._sessionsInProgress = {};
+
+    // Used by olm to serialise prekey message decryptions
+    this._olmPrekeyPromise = Promise.resolve();
 }

 /**
@@ -1029,6 +1038,11 @@ OlmDevice.prototype.addInboundGroupSession = async function(
                            }
                        }

+                        logger.info(
+                            "Storing megolm session " + senderKey + "/" + sessionId +
+                            " with first index " + session.first_known_index(),
+                        );
+
                        const sessionData = {
                            room_id: roomId,
                            session: session.pickle(this._pickleKey),
@@ -58,7 +58,7 @@ const SEND_KEY_REQUESTS_DELAY_MS = 500;
 *
 * @enum {number}
 */
-const ROOM_KEY_REQUEST_STATES = {
+export const ROOM_KEY_REQUEST_STATES = {
    /** request not yet sent */
    UNSENT: 0,

@@ -97,10 +97,6 @@ export class OutgoingRoomKeyRequestManager {
     */
    start() {
        this._clientRunning = true;
-
-        // set the timer going, to handle any requests which didn't get sent
-        // on the previous run of the client.
-        this._startTimer();
    }

    /**
@@ -113,7 +109,14 @@ export class OutgoingRoomKeyRequestManager {
    }

    /**
-     * Send off a room key request, if we haven't already done so.
+     * Send any requests that have been queued
+     */
+    sendQueuedRequests() {
+        this._startTimer();
+    }
+
+    /**
+     * Queue up a room key request, if we haven't already queued or sent one.
     *
     * The `requestBody` is compared (with a deep-equality check) against
     * previous queued or sent requests and if it matches, no change is made.
@@ -129,7 +132,7 @@ export class OutgoingRoomKeyRequestManager {
     *    pending list (or we have established that a similar request already
     *    exists)
     */
-    async sendRoomKeyRequest(requestBody, recipients, resend=false) {
+    async queueRoomKeyRequest(requestBody, recipients, resend=false) {
        const req = await this._cryptoStore.getOutgoingRoomKeyRequest(
            requestBody,
        );
@@ -184,7 +187,7 @@ export class OutgoingRoomKeyRequestManager {
                        // in state ROOM_KEY_REQUEST_STATES.SENT, so we must have
                        // raced with another tab to mark the request cancelled.
                        // Try again, to make sure the request is resent.
-                        return await this.sendRoomKeyRequest(
+                        return await this.queueRoomKeyRequest(
                            requestBody, recipients, resend,
                        );
                    }
@@ -220,9 +223,6 @@ export class OutgoingRoomKeyRequestManager {
                throw new Error('unhandled state: ' + req.state);
            }
        }
-        // some of the branches require the timer to be started.  Just start it
-        // all the time, because it doesn't hurt to start it.
-        this._startTimer();
    }

    /**
@@ -327,6 +327,21 @@ export class OutgoingRoomKeyRequestManager {
        );
    }

+    /**
+     * Find anything in `sent` state, and kick it around the loop again.
+     * This is intended for situations where something substantial has changed, and we
+     * don't really expect the other end to even care about the cancellation.
+     * For example, after initialization or self-verification.
+     * @return {Promise} An array of `queueRoomKeyRequest` outputs.
+     */
+    async cancelAndResendAllOutgoingRequests() {
+        const outgoings = await this._cryptoStore.getAllOutgoingRoomKeyRequestsByState(
+            ROOM_KEY_REQUEST_STATES.SENT,
+        );
+        return Promise.all(outgoings.map(({ requestBody, recipients }) =>
+            this.queueRoomKeyRequest(requestBody, recipients, true)));
+    }
+
    // start the background timer to send queued requests, if the timer isn't
    // already running
    _startTimer() {
@@ -366,15 +381,12 @@ export class OutgoingRoomKeyRequestManager {
            return Promise.resolve();
        }

-        logger.log("Looking for queued outgoing room key requests");
-
        return this._cryptoStore.getOutgoingRoomKeyRequestByState([
            ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING,
            ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING_AND_WILL_RESEND,
            ROOM_KEY_REQUEST_STATES.UNSENT,
        ]).then((req) => {
            if (!req) {
-                logger.log("No more outgoing room key requests");
                this._sendOutgoingRoomKeyRequestsTimer = null;
                return;
            }
@@ -398,7 +410,6 @@ export class OutgoingRoomKeyRequestManager {
            }).catch((e) => {
                logger.error("Error sending room key request; will retry later.", e);
                this._sendOutgoingRoomKeyRequestsTimer = null;
-                this._startTimer();
            });
        });
    }
@@ -1,5 +1,5 @@
 /*
-Copyright 2019 The Matrix.org Foundation C.I.C.
+Copyright 2019, 2020 The Matrix.org Foundation C.I.C.

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -17,21 +17,24 @@ limitations under the License.
 import {EventEmitter} from 'events';
 import {logger} from '../logger';
 import * as olmlib from './olmlib';
-import {pkVerify} from './olmlib';
 import {randomString} from '../randomstring';
+import {encryptAES, decryptAES} from './aes';
+import {encodeBase64} from "./olmlib";

-export const SECRET_STORAGE_ALGORITHM_V1 = "m.secret_storage.v1.curve25519-aes-sha2";
+export const SECRET_STORAGE_ALGORITHM_V1_AES
+    = "m.secret_storage.v1.aes-hmac-sha2";
+
+const ZERO_STR = "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0";

 /**
 * Implements Secure Secret Storage and Sharing (MSC1946)
 * @module crypto/SecretStorage
 */
 export class SecretStorage extends EventEmitter {
-    constructor(baseApis, cryptoCallbacks, crossSigningInfo) {
+    constructor(baseApis, cryptoCallbacks) {
        super();
        this._baseApis = baseApis;
        this._cryptoCallbacks = cryptoCallbacks;
-        this._crossSigningInfo = crossSigningInfo;
        this._requests = {};
        this._incomingRequests = {};
    }
@@ -45,7 +48,7 @@ export class SecretStorage extends EventEmitter {
    }

    setDefaultKeyId(keyId) {
-        return new Promise((resolve) => {
+        return new Promise(async (resolve, reject) => {
            const listener = (ev) => {
                if (
                    ev.getType() === 'm.secret_storage.default_key' &&
@@ -57,10 +60,15 @@ export class SecretStorage extends EventEmitter {
            };
            this._baseApis.on('accountData', listener);

-            this._baseApis.setAccountData(
-                'm.secret_storage.default_key',
-                { key: keyId },
-            );
+            try {
+                await this._baseApis.setAccountData(
+                    'm.secret_storage.default_key',
+                    { key: keyId },
+                );
+            } catch (e) {
+                this._baseApis.removeListener('accountData', listener);
+                reject(e);
+            }
        });
    }

@@ -84,28 +92,16 @@ export class SecretStorage extends EventEmitter {
            keyData.name = opts.name;
        }

-        switch (algorithm) {
-        case SECRET_STORAGE_ALGORITHM_V1:
-        {
-            const decryption = new global.Olm.PkDecryption();
-            try {
-                const { passphrase, pubkey } = opts;
-                // Copies in public key details of the form generated by
-                // the Crypto module's `createRecoveryKeyFromPassphrase`.
-                if (passphrase && pubkey) {
-                    keyData.passphrase = passphrase;
-                    keyData.pubkey = pubkey;
-                } else if (pubkey) {
-                    keyData.pubkey = pubkey;
-                } else {
-                    keyData.pubkey = decryption.generate_key();
-                }
-            } finally {
-                decryption.free();
+        if (algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+            if (opts.passphrase) {
+                keyData.passphrase = opts.passphrase;
            }
-            break;
-        }
-        default:
+            if (opts.key) {
+                const {iv, mac} = await SecretStorage._calculateKeyCheck(opts.key);
+                keyData.iv = iv;
+                keyData.mac = mac;
+            }
+        } else {
            throw new Error(`Unknown key algorithm ${opts.algorithm}`);
        }

@@ -119,8 +115,6 @@ export class SecretStorage extends EventEmitter {
            );
        }

-        await this._crossSigningInfo.signObject(keyData, 'master');
-
        await this._baseApis.setAccountData(
            `m.secret_storage.key.${keyId}`, keyData,
        );
@@ -129,30 +123,25 @@ export class SecretStorage extends EventEmitter {
    }

    /**
-     * Signs a given secret storage key with the cross-signing master key.
+     * Get the key information for a given ID.
     *
-     * @param {string} [keyId = default key's ID] The ID of the key to sign.
-     *     Defaults to the default key ID if not provided.
+     * @param {string} [keyId = default key's ID] The ID of the key to check
+     *     for. Defaults to the default key ID if not provided.
+     * @returns {Array?} If the key was found, the return value is an array of
+     *     the form [keyId, keyInfo].  Otherwise, null is returned.
     */
-    async signKey(keyId) {
+    async getKey(keyId) {
        if (!keyId) {
            keyId = await this.getDefaultKeyId();
        }
        if (!keyId) {
-            throw new Error("signKey requires a key ID");
+            return null;
        }

        const keyInfo = await this._baseApis.getAccountDataFromServer(
-            `m.secret_storage.key.${keyId}`,
-        );
-        if (!keyInfo) {
-            throw new Error(`Key ${keyId} does not exist in account data`);
-        }
-
-        await this._crossSigningInfo.signObject(keyInfo, 'master');
-        await this._baseApis.setAccountData(
-            `m.secret_storage.key.${keyId}`, keyInfo,
+            "m.secret_storage.key." + keyId,
        );
+        return keyInfo ? [keyId, keyInfo] : null;
    }

    /**
@@ -163,16 +152,33 @@ export class SecretStorage extends EventEmitter {
     * @return {boolean} Whether we have the key.
     */
    async hasKey(keyId) {
-        if (!keyId) {
-            keyId = await this.getDefaultKeyId();
-        }
-        if (!keyId) {
-            return false;
-        }
+        return !!(await this.getKey(keyId));
+    }

-        return !!this._baseApis.getAccountDataFromServer(
-            "m.secret_storage.key." + keyId,
-        );
+    /**
+     * Check whether a key matches what we expect based on the key info
+     *
+     * @param {Uint8Array} key the key to check
+     * @param {object} info the key info
+     *
+     * @return {boolean} whether or not the key matches
+     */
+    async checkKey(key, info) {
+        if (info.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+            if (info.mac) {
+                const {mac} = await SecretStorage._calculateKeyCheck(key, info.iv);
+                return info.mac.replace(/=+$/g, '') === mac.replace(/=+$/g, '');
+            } else {
+                // if we have no information, we have to assume the key is right
+                return true;
+            }
+        } else {
+            throw new Error("Unknown algorithm");
+        }
+    }
+
+    static async _calculateKeyCheck(key, iv) {
+        return await encryptAES(ZERO_STR, key, "", iv);
    }

    /**
@@ -207,27 +213,12 @@ export class SecretStorage extends EventEmitter {
                throw new Error("Unknown key: " + keyId);
            }

-            // check signature of key info
-            pkVerify(
-                keyInfo,
-                this._crossSigningInfo.getId('master'),
-                this._crossSigningInfo.userId,
-            );
-
            // encrypt secret, based on the algorithm
-            switch (keyInfo.algorithm) {
-            case SECRET_STORAGE_ALGORITHM_V1:
-            {
-                const encryption = new global.Olm.PkEncryption();
-                try {
-                    encryption.set_recipient_key(keyInfo.pubkey);
-                    encrypted[keyId] = encryption.encrypt(secret);
-                } finally {
-                    encryption.free();
-                }
-                break;
-            }
-            default:
+            if (keyInfo.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+                const keys = {[keyId]: keyInfo};
+                const [, encryption] = await this._getSecretStorageKey(keys, name);
+                encrypted[keyId] = await encryption.encrypt(secret);
+            } else {
                logger.warn("unknown algorithm for secret storage key " + keyId
                            + ": " + keyInfo.algorithm);
                // do nothing if we don't understand the encryption algorithm
@@ -238,29 +229,6 @@ export class SecretStorage extends EventEmitter {
        await this._baseApis.setAccountData(name, {encrypted});
    }

-    /**
-     * Store a secret defined to be the same as the given key.
-     * No secret information will be stored, instead the secret will
-     * be stored with a marker to say that the contents of the secret is
-     * the value of the given key.
-     * This is useful for migration from systems that predate SSSS such as
-     * key backup.
-     *
-     * @param {string} name The name of the secret
-     * @param {string} keyId The ID of the key whose value will be the
-     *     value of the secret
-     * @returns {Promise} resolved when account data is saved
-     */
-    storePassthrough(name, keyId) {
-        return this._baseApis.setAccountData(name, {
-            encrypted: {
-                [keyId]: {
-                    passthrough: true,
-                },
-            },
-        });
-    }
-
    /**
     * Temporary method to fix up existing accounts where secrets
     * are incorrectly stored without the 'encrypted' level
@@ -316,19 +284,11 @@ export class SecretStorage extends EventEmitter {
                "m.secret_storage.key." + keyId,
            );
            const encInfo = secretInfo.encrypted[keyId];
-            switch (keyInfo.algorithm) {
-            case SECRET_STORAGE_ALGORITHM_V1:
-                if (
-                    keyInfo.pubkey && (
-                        (encInfo.ciphertext && encInfo.mac && encInfo.ephemeral) ||
-                        encInfo.passthrough
-                    )
-                ) {
+            // only use keys we understand the encryption algorithm of
+            if (keyInfo.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+                if (encInfo.iv && encInfo.ciphertext && encInfo.mac) {
                    keys[keyId] = keyInfo;
                }
-                break;
-            default:
-                // do nothing if we don't understand the encryption algorithm
            }
        }

@@ -341,18 +301,13 @@ export class SecretStorage extends EventEmitter {
            const encInfo = secretInfo.encrypted[keyId];

            // We don't actually need the decryption object if it's a passthrough
-            // since we just want to return the key itself.
-            if (encInfo.passthrough) return decryption.get_private_key();
+            // since we just want to return the key itself. It must be base64
+            // encoded, since this is how a key would normally be stored.
+            if (encInfo.passthrough) return encodeBase64(decryption.get_private_key());

-            // decrypt secret
-            switch (keys[keyId].algorithm) {
-            case SECRET_STORAGE_ALGORITHM_V1:
-                return decryption.decrypt(
-                    encInfo.ephemeral, encInfo.mac, encInfo.ciphertext,
-                );
-            }
+            return await decryption.decrypt(encInfo);
        } finally {
-            if (decryption) decryption.free();
+            if (decryption && decryption.free) decryption.free();
        }
    }

@@ -362,55 +317,43 @@ export class SecretStorage extends EventEmitter {
     * @param {string} name the name of the secret
     * @param {boolean} checkKey check if the secret is encrypted by a trusted key
     *
-     * @return {boolean} whether or not the secret is stored
+     * @return {object?} map of key name to key info the secret is encrypted
+     *     with, or null if it is not present or not encrypted with a trusted
+     *     key
     */
    async isStored(name, checkKey) {
        // check if secret exists
        let secretInfo = await this._baseApis.getAccountDataFromServer(name);
-        if (!secretInfo) return false;
+        if (!secretInfo) return null;
        if (!secretInfo.encrypted) {
            // try to fix it up
            secretInfo = await this._fixupStoredSecret(name, secretInfo);
            if (!secretInfo || !secretInfo.encrypted) {
-                return false;
+                return null;
            }
        }

        if (checkKey === undefined) checkKey = true;

-        // check if secret is encrypted by a known/trusted secret and
-        // encryption looks sane
+        const ret = {};
+
+        // filter secret encryption keys with supported algorithm
        for (const keyId of Object.keys(secretInfo.encrypted)) {
            // get key information from key storage
            const keyInfo = await this._baseApis.getAccountDataFromServer(
                "m.secret_storage.key." + keyId,
            );
-            if (!keyInfo) return false;
+            if (!keyInfo) continue;
            const encInfo = secretInfo.encrypted[keyId];
-            if (checkKey) {
-                pkVerify(
-                    keyInfo,
-                    this._crossSigningInfo.getId('master'),
-                    this._crossSigningInfo.userId,
-                );
-            }

-            // We don't actually need the decryption object if it's a passthrough
-            // since we just want to return the key itself.
-            if (encInfo.passthrough) return true;
-
-            switch (keyInfo.algorithm) {
-            case SECRET_STORAGE_ALGORITHM_V1:
-                if (keyInfo.pubkey && encInfo.ciphertext && encInfo.mac
-                    && encInfo.ephemeral) {
-                    return true;
+            // only use keys we understand the encryption algorithm of
+            if (keyInfo.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+                if (encInfo.iv && encInfo.ciphertext && encInfo.mac) {
+                    ret[keyId] = keyInfo;
                }
-                break;
-            default:
-                // do nothing if we don't understand the encryption algorithm
            }
        }
-        return false;
+        return Object.keys(ret).length ? ret : null;
    }

    /**
@@ -462,6 +405,7 @@ export class SecretStorage extends EventEmitter {
        for (const device of devices) {
            toDevice[device] = requestData;
        }
+        logger.info(`Request secret ${name} from ${devices}, id ${requestId}`);
        this._baseApis.sendToDevice("m.secret.request", {
            [this._baseApis.getUserId()]: toDevice,
        });
@@ -515,6 +459,7 @@ export class SecretStorage extends EventEmitter {
                device_trust: this._baseApis.checkDeviceTrust(sender, deviceId),
            });
            if (secret) {
+                logger.info(`Preparing ${content.name} secret for ${deviceId}`);
                const payload = {
                    type: "m.secret.send",
                    content: {
@@ -532,7 +477,7 @@ export class SecretStorage extends EventEmitter {
                    this._baseApis,
                    {
                        [sender]: [
-                            await this._baseApis.getStoredDevice(sender, deviceId),
+                            this._baseApis.getStoredDevice(sender, deviceId),
                        ],
                    },
                );
@@ -542,7 +487,7 @@ export class SecretStorage extends EventEmitter {
                    this._baseApis.deviceId,
                    this._baseApis._crypto._olmDevice,
                    sender,
-                    this._baseApis._crypto.getStoredDevice(sender, deviceId),
+                    this._baseApis.getStoredDevice(sender, deviceId),
                    payload,
                );
                const contentMap = {
@@ -551,7 +496,10 @@ export class SecretStorage extends EventEmitter {
                    },
                };

+                logger.info(`Sending ${content.name} secret for ${deviceId}`);
                this._baseApis.sendToDevice("m.room.encrypted", contentMap);
+            } else {
+                logger.info(`Request denied for ${content.name} secret for ${deviceId}`);
            }
        }
    }
@@ -563,7 +511,7 @@ export class SecretStorage extends EventEmitter {
            return;
        }
        const content = event.getContent();
-        logger.log("got secret share for request ", content.request_id);
+        logger.log("got secret share for request", content.request_id);
        const requestControl = this._requests[content.request_id];
        if (requestControl) {
            // make sure that the device that sent it is one of the devices that
@@ -606,27 +554,18 @@ export class SecretStorage extends EventEmitter {
            throw new Error("App returned unknown key from getSecretStorageKey!");
        }

-        switch (keys[keyId].algorithm) {
-            case SECRET_STORAGE_ALGORITHM_V1:
-            {
-                const decryption = new global.Olm.PkDecryption();
-                let pubkey;
-                try {
-                    pubkey = decryption.init_with_private_key(privateKey);
-                } catch (e) {
-                    decryption.free();
-                    throw new Error("getSecretStorageKey callback returned invalid key");
-                }
-                if (pubkey !== keys[keyId].pubkey) {
-                    decryption.free();
-                    throw new Error(
-                        "getSecretStorageKey callback returned incorrect key",
-                    );
-                }
-                return [keyId, decryption];
-            }
-            default:
-                throw new Error("Unknown key type: " + keys[keyId].algorithm);
+        if (keys[keyId].algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+            const decryption = {
+                encrypt: async function(secret) {
+                    return await encryptAES(secret, privateKey, name);
+                },
+                decrypt: async function(encInfo) {
+                    return await decryptAES(encInfo, privateKey, name);
+                },
+            };
+            return [keyId, decryption];
+        } else {
+            throw new Error("Unknown key type: " + keys[keyId].algorithm);
        }
    }
 }
@@ -0,0 +1,251 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {getCrypto} from '../utils';
+import {decodeBase64, encodeBase64} from './olmlib';
+
+const subtleCrypto = (typeof window !== "undefined" && window.crypto) ?
+    (window.crypto.subtle || window.crypto.webkitSubtle) : null;
+
+// salt for HKDF, with 8 bytes of zeros
+const zerosalt = new Uint8Array(8);
+
+/**
+ * encrypt a string in Node.js
+ *
+ * @param {string} data the plaintext to encrypt
+ * @param {Uint8Array} key the encryption key to use
+ * @param {string} name the name of the secret
+ * @param {string} ivStr the initialization vector to use
+ */
+async function encryptNode(data, key, name, ivStr) {
+    const crypto = getCrypto();
+    if (!crypto) {
+        throw new Error("No usable crypto implementation");
+    }
+
+    let iv;
+    if (ivStr) {
+        iv = decodeBase64(ivStr);
+    } else {
+        iv = crypto.randomBytes(16);
+    }
+
+    // clear bit 63 of the IV to stop us hitting the 64-bit counter boundary
+    // (which would mean we wouldn't be able to decrypt on Android). The loss
+    // of a single bit of iv is a price we have to pay.
+    iv[8] &= 0x7f;
+
+    const [aesKey, hmacKey] = deriveKeysNode(key, name);
+
+    const cipher = crypto.createCipheriv("aes-256-ctr", aesKey, iv);
+    const ciphertext = cipher.update(data, "utf-8", "base64")
+          + cipher.final("base64");
+
+    const hmac = crypto.createHmac("sha256", hmacKey)
+        .update(ciphertext, "base64").digest("base64");
+
+    return {
+        iv: encodeBase64(iv),
+        ciphertext: ciphertext,
+        mac: hmac,
+    };
+}
+
+/**
+ * decrypt a string in Node.js
+ *
+ * @param {object} data the encrypted data
+ * @param {string} data.ciphertext the ciphertext in base64
+ * @param {string} data.iv the initialization vector in base64
+ * @param {string} data.mac the HMAC in base64
+ * @param {Uint8Array} key the encryption key to use
+ * @param {string} name the name of the secret
+ */
+async function decryptNode(data, key, name) {
+    const crypto = getCrypto();
+    if (!crypto) {
+        throw new Error("No usable crypto implementation");
+    }
+
+    const [aesKey, hmacKey] = deriveKeysNode(key, name);
+
+    const hmac = crypto.createHmac("sha256", hmacKey)
+        .update(data.ciphertext, "base64").digest("base64").replace(/=+$/g, '');
+
+    if (hmac !== data.mac.replace(/=+$/g, '')) {
+        throw new Error(`Error decrypting secret ${name}: bad MAC`);
+    }
+
+    const decipher = crypto.createDecipheriv(
+        "aes-256-ctr", aesKey, decodeBase64(data.iv),
+    );
+    return decipher.update(data.ciphertext, "base64", "utf-8")
+          + decipher.final("utf-8");
+}
+
+function deriveKeysNode(key, name) {
+    const crypto = getCrypto();
+    const prk = crypto.createHmac("sha256", zerosalt)
+        .update(key).digest();
+
+    const b = Buffer.alloc(1, 1);
+    const aesKey = crypto.createHmac("sha256", prk)
+        .update(name, "utf-8").update(b).digest();
+    b[0] = 2;
+    const hmacKey = crypto.createHmac("sha256", prk)
+        .update(aesKey).update(name, "utf-8").update(b).digest();
+
+    return [aesKey, hmacKey];
+}
+
+/**
+ * encrypt a string in Node.js
+ *
+ * @param {string} data the plaintext to encrypt
+ * @param {Uint8Array} key the encryption key to use
+ * @param {string} name the name of the secret
+ * @param {string} ivStr the initialization vector to use
+ */
+async function encryptBrowser(data, key, name, ivStr) {
+    let iv;
+    if (ivStr) {
+        iv = decodeBase64(ivStr);
+    } else {
+        iv = new Uint8Array(16);
+        window.crypto.getRandomValues(iv);
+    }
+
+    // clear bit 63 of the IV to stop us hitting the 64-bit counter boundary
+    // (which would mean we wouldn't be able to decrypt on Android). The loss
+    // of a single bit of iv is a price we have to pay.
+    iv[8] &= 0x7f;
+
+    const [aesKey, hmacKey] = await deriveKeysBrowser(key, name);
+    const encodedData = new TextEncoder().encode(data);
+
+    const ciphertext = await subtleCrypto.encrypt(
+        {
+            name: "AES-CTR",
+            counter: iv,
+            length: 64,
+        },
+        aesKey,
+        encodedData,
+    );
+
+    const hmac = await subtleCrypto.sign(
+        {name: 'HMAC'},
+        hmacKey,
+        ciphertext,
+    );
+
+    return {
+        iv: encodeBase64(iv),
+        ciphertext: encodeBase64(ciphertext),
+        mac: encodeBase64(hmac),
+    };
+}
+
+/**
+ * decrypt a string in the browser
+ *
+ * @param {object} data the encrypted data
+ * @param {string} data.ciphertext the ciphertext in base64
+ * @param {string} data.iv the initialization vector in base64
+ * @param {string} data.mac the HMAC in base64
+ * @param {Uint8Array} key the encryption key to use
+ * @param {string} name the name of the secret
+ */
+async function decryptBrowser(data, key, name) {
+    const [aesKey, hmacKey] = await deriveKeysBrowser(key, name);
+
+    const ciphertext = decodeBase64(data.ciphertext);
+
+    if (!await subtleCrypto.verify(
+        {name: "HMAC"},
+        hmacKey,
+        decodeBase64(data.mac),
+        ciphertext,
+    )) {
+        throw new Error(`Error decrypting secret ${name}: bad MAC`);
+    }
+
+    const plaintext = await subtleCrypto.decrypt(
+        {
+            name: "AES-CTR",
+            counter: decodeBase64(data.iv),
+            length: 64,
+        },
+        aesKey,
+        ciphertext,
+    );
+
+    return new TextDecoder().decode(new Uint8Array(plaintext));
+}
+
+async function deriveKeysBrowser(key, name) {
+    const hkdfkey = await subtleCrypto.importKey(
+        'raw',
+        key,
+        {name: "HKDF"},
+        false,
+        ["deriveBits"],
+    );
+    const keybits = await subtleCrypto.deriveBits(
+        {
+            name: "HKDF",
+            salt: zerosalt,
+            info: (new TextEncoder().encode(name)),
+            hash: "SHA-256",
+        },
+        hkdfkey,
+        512,
+    );
+
+    const aesKey = keybits.slice(0, 32);
+    const hmacKey = keybits.slice(32);
+
+    const aesProm = subtleCrypto.importKey(
+        'raw',
+        aesKey,
+        {name: 'AES-CTR'},
+        false,
+        ['encrypt', 'decrypt'],
+    );
+
+    const hmacProm = subtleCrypto.importKey(
+        'raw',
+        hmacKey,
+        {
+            name: 'HMAC',
+            hash: {name: 'SHA-256'},
+        },
+        false,
+        ['sign', 'verify'],
+    );
+
+    return await Promise.all([aesProm, hmacProm]);
+}
+
+export function encryptAES(...args) {
+    return subtleCrypto ? encryptBrowser(...args) : encryptNode(...args);
+}
+
+export function decryptAES(...args) {
+    return subtleCrypto ? decryptBrowser(...args) : decryptNode(...args);
+}
+
@@ -60,6 +60,15 @@ export class EncryptionAlgorithm {
        this._roomId = params.roomId;
    }

+    /**
+     * Perform any background tasks that can be done before a message is ready to
+     * send, in order to speed up sending of the message.
+     *
+     * @param {module:models/room} room the room the event is in
+     */
+    prepareToEncrypt(room) {
+    }
+
    /**
     * Encrypt a message event
     *
@@ -185,15 +185,15 @@ utils.inherits(MegolmEncryption, EncryptionAlgorithm);
 *
 * @param {Object} devicesInRoom The devices in this room, indexed by user ID
 * @param {Object} blocked The devices that are blocked, indexed by user ID
+ * @param {boolean} [singleOlmCreationPhase] Only perform one round of olm
+ *     session creation
 *
 * @return {Promise} Promise which resolves to the
 *    OutboundSessionInfo when setup is complete.
 */
 MegolmEncryption.prototype._ensureOutboundSession = async function(
-    devicesInRoom, blocked,
+    devicesInRoom, blocked, singleOlmCreationPhase,
 ) {
-    const self = this;
-
    let session;

    // takes the previous OutboundSessionInfo, and considers whether to create
@@ -201,12 +201,12 @@ MegolmEncryption.prototype._ensureOutboundSession = async function(
    // Updates `session` to hold the final OutboundSessionInfo.
    //
    // returns a promise which resolves once the keyshare is successful.
-    async function prepareSession(oldSession) {
+    const prepareSession = async (oldSession) => {
        session = oldSession;

        // need to make a brand new session?
-        if (session && session.needsRotation(self._sessionRotationPeriodMsgs,
-                                             self._sessionRotationPeriodMs)
+        if (session && session.needsRotation(this._sessionRotationPeriodMsgs,
+                                             this._sessionRotationPeriodMs)
           ) {
            logger.log("Starting new megolm session because we need to rotate.");
            session = null;
@@ -218,32 +218,20 @@ MegolmEncryption.prototype._ensureOutboundSession = async function(
        }

        if (!session) {
-            logger.log(`Starting new megolm session for room ${self._roomId}`);
-            session = await self._prepareNewSession();
+            logger.log(`Starting new megolm session for room ${this._roomId}`);
+            session = await this._prepareNewSession();
            logger.log(`Started new megolm session ${session.sessionId} ` +
-                       `for room ${self._roomId}`);
-            self._outboundSessions[session.sessionId] = session;
+                       `for room ${this._roomId}`);
+            this._outboundSessions[session.sessionId] = session;
        }

        // now check if we need to share with any devices
        const shareMap = {};

-        for (const userId in devicesInRoom) {
-            if (!devicesInRoom.hasOwnProperty(userId)) {
-                continue;
-            }
-
-            const userDevices = devicesInRoom[userId];
-
-            for (const deviceId in userDevices) {
-                if (!userDevices.hasOwnProperty(deviceId)) {
-                    continue;
-                }
-
-                const deviceInfo = userDevices[deviceId];
-
+        for (const [userId, userDevices] of Object.entries(devicesInRoom)) {
+            for (const [deviceId, deviceInfo] of Object.entries(userDevices)) {
                const key = deviceInfo.getIdentityKey();
-                if (key == self._olmDevice.deviceCurve25519Key) {
+                if (key == this._olmDevice.deviceCurve25519Key) {
                    // don't bother sending to ourself
                    continue;
                }
@@ -258,51 +246,99 @@ MegolmEncryption.prototype._ensureOutboundSession = async function(
            }
        }

-        const errorDevices = [];
-
-        await self._shareKeyWithDevices(
-            session, shareMap, errorDevices,
+        const key = this._olmDevice.getOutboundGroupSessionKey(session.sessionId);
+        const payload = {
+            type: "m.room_key",
+            content: {
+                algorithm: olmlib.MEGOLM_ALGORITHM,
+                room_id: this._roomId,
+                session_id: session.sessionId,
+                session_key: key.key,
+                chain_index: key.chain_index,
+            },
+        };
+        const [devicesWithoutSession, olmSessions] = await olmlib.getExistingOlmSessions(
+            this._olmDevice, this._baseApis, shareMap,
        );

-        // are there any new blocked devices that we need to notify?
-        const blockedMap = {};
-        for (const userId in blocked) {
-            if (!blocked.hasOwnProperty(userId)) {
-                continue;
-            }
+        await Promise.all([
+            (async () => {
+                // share keys with devices that we already have a session for
+                await this._shareKeyWithOlmSessions(
+                    session, key, payload, olmSessions,
+                );
+            })(),
+            (async () => {
+                const errorDevices = [];

-            const userBlockedDevices = blocked[userId];
+                // meanwhile, establish olm sessions for devices that we don't
+                // already have a session for, and share keys with them.  If
+                // we're doing two phases of olm session creation, use a
+                // shorter timeout when fetching one-time keys for the first
+                // phase.
+                const start = Date.now();
+                const failedServers = [];
+                await this._shareKeyWithDevices(
+                    session, key, payload, devicesWithoutSession, errorDevices,
+                    singleOlmCreationPhase ? 10000 : 2000, failedServers,
+                );

-            for (const deviceId in userBlockedDevices) {
-                if (!userBlockedDevices.hasOwnProperty(deviceId)) {
-                    continue;
+                if (!singleOlmCreationPhase && (Date.now() - start < 10000)) {
+                    // perform the second phase of olm session creation if requested,
+                    // and if the first phase didn't take too long
+                    (async () => {
+                        // Retry sending keys to devices that we were unable to establish
+                        // an olm session for.  This time, we use a longer timeout, but we
+                        // do this in the background and don't block anything else while we
+                        // do this.  We only need to retry users from servers that didn't
+                        // respond the first time.
+                        const retryDevices = {};
+                        const failedServerMap = new Set;
+                        for (const server of failedServers) {
+                            failedServerMap.add(server);
+                        }
+                        const failedDevices = [];
+                        for (const {userId, deviceInfo} of errorDevices) {
+                            const userHS = userId.slice(userId.indexOf(":") + 1);
+                            if (failedServerMap.has(userHS)) {
+                                retryDevices[userId] = retryDevices[userId] || [];
+                                retryDevices[userId].push(deviceInfo);
+                            } else {
+                                // if we aren't going to retry, then handle it
+                                // as a failed device
+                                failedDevices.push({userId, deviceInfo});
+                            }
+                        }
+
+                        await this._shareKeyWithDevices(
+                            session, key, payload, retryDevices, failedDevices, 30000,
+                        );
+
+                        await this._notifyFailedOlmDevices(session, key, failedDevices);
+                    })();
+                } else {
+                    await this._notifyFailedOlmDevices(session, key, errorDevices);
+                }
+            })(),
+            (async () => {
+                // also, notify blocked devices that they're blocked
+                const blockedMap = {};
+                for (const [userId, userBlockedDevices] of Object.entries(blocked)) {
+                    for (const [deviceId, device] of Object.entries(userBlockedDevices)) {
+                        if (
+                            !session.blockedDevicesNotified[userId] ||
+                            session.blockedDevicesNotified[userId][deviceId] === undefined
+                        ) {
+                            blockedMap[userId] = blockedMap[userId] || {};
+                            blockedMap[userId][deviceId] = {device};
+                        }
+                    }
                }

-                if (
-                    !session.blockedDevicesNotified[userId] ||
-                        session.blockedDevicesNotified[userId][deviceId] === undefined
-                ) {
-                    blockedMap[userId] = blockedMap[userId] || [];
-                    blockedMap[userId].push(userBlockedDevices[deviceId]);
-                }
-            }
-        }
-
-
-        const filteredErrorDevices =
-              await self._olmDevice.filterOutNotifiedErrorDevices(errorDevices);
-        for (const {userId, deviceInfo} of filteredErrorDevices) {
-            blockedMap[userId] = blockedMap[userId] || [];
-            blockedMap[userId].push({
-                code: "m.no_olm",
-                reason: WITHHELD_MESSAGES["m.no_olm"],
-                deviceInfo,
-            });
-        }
-
-        // notify blocked devices that they're blocked
-        await self._notifyBlockedDevices(session, blockedMap);
-    }
+                await this._notifyBlockedDevices(session, blockedMap);
+            })(),
+        ]);
+    };

    // helper which returns the session prepared by prepareSession
    function returnSession() {
@@ -349,96 +385,47 @@ MegolmEncryption.prototype._prepareNewSession = async function() {
 };

 /**
- * Splits the user device map into multiple chunks to reduce the number of
- * devices we encrypt to per API call. Also filters out devices we don't have
- * a session with.
+ * Determines what devices in devicesByUser don't have an olm session as given
+ * in devicemap.
 *
 * @private
 *
- * @param {module:crypto/algorithms/megolm.OutboundSessionInfo} session
+ * @param {object} devicemap the devices that have olm sessions, as returned by
+ *     olmlib.ensureOlmSessionsForDevices.
+ * @param {object} devicesByUser a map of user IDs to array of deviceInfo
+ * @param {array} [noOlmDevices] an array to fill with devices that don't have
+ *     olm sessions
 *
- * @param {number} chainIndex current chain index
- *
- * @param {object<userId, deviceId>} devicemap
- *   mapping from userId to deviceId to {@link module:crypto~OlmSessionResult}
- *
- * @param {object<string, module:crypto/deviceinfo[]>} devicesByUser
- *    map from userid to list of devices
- *
- * @param {array<object>} errorDevices
- *    array that will be populated with the devices that can't get an
- *    olm session for
- *
- * @return {array<object<userid, deviceInfo>>}
+ * @return {array} an array of devices that don't have olm sessions.  If
+ *     noOlmDevices is specified, then noOlmDevices will be returned.
 */
-MegolmEncryption.prototype._splitUserDeviceMap = function(
-    session, chainIndex, devicemap, devicesByUser, errorDevices,
+MegolmEncryption.prototype._getDevicesWithoutSessions = function(
+    devicemap, devicesByUser, noOlmDevices,
 ) {
-    const maxUsersPerRequest = 20;
+    noOlmDevices = noOlmDevices || [];

-    // use an array where the slices of a content map gets stored
-    const mapSlices = [];
-    let currentSliceId = 0; // start inserting in the first slice
-    let entriesInCurrentSlice = 0;
-
-    for (const userId of Object.keys(devicesByUser)) {
-        const devicesToShareWith = devicesByUser[userId];
+    for (const [userId, devicesToShareWith] of Object.entries(devicesByUser)) {
        const sessionResults = devicemap[userId];

-        for (let i = 0; i < devicesToShareWith.length; i++) {
-            const deviceInfo = devicesToShareWith[i];
+        for (const deviceInfo of devicesToShareWith) {
            const deviceId = deviceInfo.deviceId;

            const sessionResult = sessionResults[deviceId];
            if (!sessionResult.sessionId) {
                // no session with this device, probably because there
                // were no one-time keys.
-                //
-                // we could send them a to_device message anyway, as a
-                // signal that they have missed out on the key sharing
-                // message because of the lack of keys, but there's not
-                // much point in that really; it will mostly serve to clog
-                // up to_device inboxes.

-                // mark this device as "handled" because we don't want to try
-                // to claim a one-time-key for dead devices on every message.
-                session.markSharedWithDevice(userId, deviceId, chainIndex);
-
-                errorDevices.push({userId, deviceInfo});
+                noOlmDevices.push({userId, deviceInfo});
+                delete sessionResults[deviceId];

                // ensureOlmSessionsForUsers has already done the logging,
                // so just skip it.
                continue;
            }
-
-            logger.log(
-                "share keys with device " + userId + ":" + deviceId,
-            );
-
-            if (!mapSlices[currentSliceId]) {
-                mapSlices[currentSliceId] = [];
-            }
-
-            mapSlices[currentSliceId].push({
-                userId: userId,
-                deviceInfo: deviceInfo,
-            });
-
-            entriesInCurrentSlice++;
-        }
-
-        // We do this in the per-user loop as we prefer that all messages to the
-        // same user end up in the same API call to make it easier for the
-        // server (e.g. only have to send one EDU if a remote user, etc). This
-        // does mean that if a user has many devices we may go over the desired
-        // limit, but its not a hard limit so that is fine.
-        if (entriesInCurrentSlice > maxUsersPerRequest) {
-            // the current slice is filled up. Start inserting into the next slice
-            entriesInCurrentSlice = 0;
-            currentSliceId++;
        }
    }
-    return mapSlices;
+
+    return noOlmDevices;
 };

 /**
@@ -451,20 +438,18 @@ MegolmEncryption.prototype._splitUserDeviceMap = function(
 *
 * @return {array<array<object>>} the blocked devices, split into chunks
 */
-MegolmEncryption.prototype._splitBlockedDevices = function(devicesByUser) {
-    const maxUsersPerRequest = 20;
+MegolmEncryption.prototype._splitDevices = function(devicesByUser) {
+    const maxDevicesPerRequest = 20;

    // use an array where the slices of a content map gets stored
    let currentSlice = [];
    const mapSlices = [currentSlice];

-    for (const userId of Object.keys(devicesByUser)) {
-        const userBlockedDevicesToShareWith = devicesByUser[userId];
-
-        for (const blockedInfo of userBlockedDevicesToShareWith) {
+    for (const [userId, userDevices] of Object.entries(devicesByUser)) {
+        for (const deviceInfo of Object.values(userDevices)) {
            currentSlice.push({
                userId: userId,
-                blockedInfo: blockedInfo,
+                deviceInfo: deviceInfo.device,
            });
        }

@@ -473,7 +458,7 @@ MegolmEncryption.prototype._splitBlockedDevices = function(devicesByUser) {
        // server (e.g. only have to send one EDU if a remote user, etc). This
        // does mean that if a user has many devices we may go over the desired
        // limit, but its not a hard limit so that is fine.
-        if (currentSlice.length > maxUsersPerRequest) {
+        if (currentSlice.length > maxDevicesPerRequest) {
            // the current slice is filled up. Start inserting into the next slice
            currentSlice = [];
            mapSlices.push(currentSlice);
@@ -536,6 +521,33 @@ MegolmEncryption.prototype._encryptAndSendKeysToDevices = function(
    }

    return Promise.all(promises).then(() => {
+        // prune out any devices that encryptMessageForDevice could not encrypt for,
+        // in which case it will have just not added anything to the ciphertext object.
+        // There's no point sending messages to devices if we couldn't encrypt to them,
+        // since that's effectively a blank message.
+        for (const userId of Object.keys(contentMap)) {
+            for (const deviceId of Object.keys(contentMap[userId])) {
+                if (Object.keys(contentMap[userId][deviceId].ciphertext).length === 0) {
+                    logger.log(
+                        "No ciphertext for device " +
+                        userId + ":" + deviceId + ": pruning",
+                    );
+                    delete contentMap[userId][deviceId];
+                }
+            }
+            // No devices left for that user? Strip that too.
+            if (Object.keys(contentMap[userId]).length === 0) {
+                logger.log("Pruned all devices for user " + userId);
+                delete contentMap[userId];
+            }
+        }
+
+        // Is there anything left?
+        if (Object.keys(contentMap).length === 0) {
+            logger.log("No users left to send to: aborting");
+            return;
+        }
+
        return this._baseApis.sendToDevice("m.room.encrypted", contentMap).then(() => {
            // store that we successfully uploaded the keys of the current slice
            for (const userId of Object.keys(contentMap)) {
@@ -568,7 +580,7 @@ MegolmEncryption.prototype._sendBlockedNotificationsToDevices = async function(

    for (const val of userDeviceMap) {
        const userId = val.userId;
-        const blockedInfo = val.blockedInfo;
+        const blockedInfo = val.deviceInfo;
        const deviceInfo = blockedInfo.deviceInfo;
        const deviceId = deviceInfo.deviceId;

@@ -643,9 +655,7 @@ MegolmEncryption.prototype.reshareKeyWithDevice = async function(

    await olmlib.ensureOlmSessionsForDevices(
        this._olmDevice, this._baseApis, {
-            [userId]: {
-                [device.deviceId]: device,
-            },
+            [userId]: [device],
        },
    );

@@ -688,37 +698,44 @@ MegolmEncryption.prototype.reshareKeyWithDevice = async function(
 };

 /**
+ * @private
+ *
 * @param {module:crypto/algorithms/megolm.OutboundSessionInfo} session
 *
+ * @param {object} key the session key as returned by
+ *    OlmDevice.getOutboundGroupSessionKey
+ *
+ * @param {object} payload the base to-device message payload for sharing keys
+ *
 * @param {object<string, module:crypto/deviceinfo[]>} devicesByUser
 *    map from userid to list of devices
 *
 * @param {array<object>} errorDevices
 *    array that will be populated with the devices that we can't get an
 *    olm session for
+ *
+ * @param {Number} [otkTimeout] The timeout in milliseconds when requesting
+ *     one-time keys for establishing new olm sessions.
+ *
+ * @param {Array} [failedServers] An array to fill with remote servers that
+ *     failed to respond to one-time-key requests.
 */
 MegolmEncryption.prototype._shareKeyWithDevices = async function(
-    session, devicesByUser, errorDevices,
+    session, key, payload, devicesByUser, errorDevices, otkTimeout, failedServers,
 ) {
-    const key = this._olmDevice.getOutboundGroupSessionKey(session.sessionId);
-    const payload = {
-        type: "m.room_key",
-        content: {
-            algorithm: olmlib.MEGOLM_ALGORITHM,
-            room_id: this._roomId,
-            session_id: session.sessionId,
-            session_key: key.key,
-            chain_index: key.chain_index,
-        },
-    };
-
    const devicemap = await olmlib.ensureOlmSessionsForDevices(
-        this._olmDevice, this._baseApis, devicesByUser,
+        this._olmDevice, this._baseApis, devicesByUser, otkTimeout, failedServers,
    );

-    const userDeviceMaps = this._splitUserDeviceMap(
-        session, key.chain_index, devicemap, devicesByUser, errorDevices,
-    );
+    this._getDevicesWithoutSessions(devicemap, devicesByUser, errorDevices);
+
+    await this._shareKeyWithOlmSessions(session, key, payload, devicemap);
+};
+
+MegolmEncryption.prototype._shareKeyWithOlmSessions = async function(
+    session, key, payload, devicemap,
+) {
+    const userDeviceMaps = this._splitDevices(devicemap);

    for (let i = 0; i < userDeviceMaps.length; i++) {
        try {
@@ -736,6 +753,52 @@ MegolmEncryption.prototype._shareKeyWithDevices = async function(
    }
 };

+/**
+ * Notify devices that we weren't able to create olm sessions.
+ *
+ * @param {module:crypto/algorithms/megolm.OutboundSessionInfo} session
+ *
+ * @param {object} key
+ *
+ * @param {Array<object>} failedDevices the devices that we were unable to
+ *     create olm sessions for, as returned by _shareKeyWithDevices
+ */
+MegolmEncryption.prototype._notifyFailedOlmDevices = async function(
+    session, key, failedDevices,
+) {
+    // mark the devices that failed as "handled" because we don't want to try
+    // to claim a one-time-key for dead devices on every message.
+    for (const {userId, deviceInfo} of failedDevices) {
+        const deviceId = deviceInfo.deviceId;
+
+        session.markSharedWithDevice(
+            userId, deviceId, key.chain_index,
+        );
+    }
+
+    const filteredFailedDevices =
+          await this._olmDevice.filterOutNotifiedErrorDevices(
+              failedDevices,
+          );
+    const blockedMap = {};
+    for (const {userId, deviceInfo} of filteredFailedDevices) {
+        blockedMap[userId] = blockedMap[userId] || {};
+        // we use a similar format to what
+        // olmlib.ensureOlmSessionsForDevices returns, so that
+        // we can use the same function to split
+        blockedMap[userId][deviceInfo.deviceId] = {
+            device: {
+                code: "m.no_olm",
+                reason: WITHHELD_MESSAGES["m.no_olm"],
+                deviceInfo,
+            },
+        };
+    }
+
+    // send the notifications
+    await this._notifyBlockedDevices(session, blockedMap);
+};
+
 /**
 * Notify blocked devices that they have been blocked.
 *
@@ -754,7 +817,7 @@ MegolmEncryption.prototype._notifyBlockedDevices = async function(
        sender_key: this._olmDevice.deviceCurve25519Key,
    };

-    const userDeviceMaps = this._splitBlockedDevices(devicesByUser);
+    const userDeviceMaps = this._splitDevices(devicesByUser);

    for (let i = 0; i < userDeviceMaps.length; i++) {
        try {
@@ -772,6 +835,38 @@ MegolmEncryption.prototype._notifyBlockedDevices = async function(
    }
 };

+/**
+ * Perform any background tasks that can be done before a message is ready to
+ * send, in order to speed up sending of the message.
+ *
+ * @param {module:models/room} room the room the event is in
+ */
+MegolmEncryption.prototype.prepareToEncrypt = function(room) {
+    if (this.encryptionPreparation) {
+        // We're already preparing something, so don't do anything else.
+        // FIXME: check if we need to restart
+        // (https://github.com/matrix-org/matrix-js-sdk/issues/1255)
+        return;
+    }
+
+    logger.debug(`Preparing to encrypt events for ${this._roomId}`);
+
+    this.encryptionPreparation = (async () => {
+        const [devicesInRoom, blocked] = await this._getDevicesInRoom(room);
+
+        if (this._crypto.getGlobalErrorOnUnknownDevices()) {
+            // Drop unknown devices for now.  When the message gets sent, we'll
+            // throw an error, but we'll still be prepared to send to the known
+            // devices.
+            this._removeUnknownDevices(devicesInRoom);
+        }
+
+        await this._ensureOutboundSession(devicesInRoom, blocked, true);
+
+        delete this.encryptionPreparation;
+    })();
+};
+
 /**
 * @inheritdoc
 *
@@ -782,38 +877,48 @@ MegolmEncryption.prototype._notifyBlockedDevices = async function(
 * @return {Promise} Promise which resolves to the new event body
 */
 MegolmEncryption.prototype.encryptMessage = async function(room, eventType, content) {
-    const self = this;
    logger.log(`Starting to encrypt event for ${this._roomId}`);

+    if (this.encryptionPreparation) {
+        // If we started sending keys, wait for it to be done.
+        // FIXME: check if we need to cancel
+        // (https://github.com/matrix-org/matrix-js-sdk/issues/1255)
+        try {
+            await this.encryptionPreparation;
+        } catch (e) {
+            // ignore any errors -- if the preparation failed, we'll just
+            // restart everything here
+        }
+    }
+
    const [devicesInRoom, blocked] = await this._getDevicesInRoom(room);

    // check if any of these devices are not yet known to the user.
    // if so, warn the user so they can verify or ignore.
    if (this._crypto.getGlobalErrorOnUnknownDevices()) {
-        self._checkForUnknownDevices(devicesInRoom);
+        this._checkForUnknownDevices(devicesInRoom);
    }

-    const session = await self._ensureOutboundSession(devicesInRoom, blocked);
+    const session = await this._ensureOutboundSession(devicesInRoom, blocked);
    const payloadJson = {
-        room_id: self._roomId,
+        room_id: this._roomId,
        type: eventType,
        content: content,
    };

-    const ciphertext = self._olmDevice.encryptGroupMessage(
+    const ciphertext = this._olmDevice.encryptGroupMessage(
        session.sessionId, JSON.stringify(payloadJson),
    );
-
    const encryptedContent = {
        algorithm: olmlib.MEGOLM_ALGORITHM,
-        sender_key: self._olmDevice.deviceCurve25519Key,
+        sender_key: this._olmDevice.deviceCurve25519Key,
        ciphertext: ciphertext,
        session_id: session.sessionId,
        // Include our device ID so that recipients can send us a
        // m.new_device message if they don't have our session key.
        // XXX: Do we still need this now that m.new_device messages
        // no longer exist since #483?
-        device_id: self._deviceId,
+        device_id: this._deviceId,
    };

    session.useCount++;
@@ -861,6 +966,27 @@ MegolmEncryption.prototype._checkForUnknownDevices = function(devicesInRoom) {
    }
 };

+/**
+ * Remove unknown devices from a set of devices.  The devicesInRoom parameter
+ * will be modified.
+ *
+ * @param {Object} devicesInRoom userId -> {deviceId -> object}
+ *   devices we should shared the session with.
+ */
+MegolmEncryption.prototype._removeUnknownDevices = function(devicesInRoom) {
+    for (const [userId, userDevices] of Object.entries(devicesInRoom)) {
+        for (const [deviceId, device] of Object.entries(userDevices)) {
+            if (device.isUnverified() && !device.isKnown()) {
+                delete userDevices[deviceId];
+            }
+        }
+
+        if (Object.keys(userDevices).length === 0) {
+            delete devicesInRoom[userId];
+        }
+    }
+};
+
 /**
 * Get the list of unblocked devices for all users in the room
 *
@@ -904,8 +1030,10 @@ MegolmEncryption.prototype._getDevicesInRoom = async function(room) {
                continue;
            }

+            const deviceTrust = this._crypto.checkDeviceTrust(userId, deviceId);
+
            if (userDevices[deviceId].isBlocked() ||
-                (userDevices[deviceId].isUnverified() && isBlacklisting)
+                (!deviceTrust.isVerified() && isBlacklisting)
            ) {
                if (!blocked[userId]) {
                    blocked[userId] = {};
@@ -1195,7 +1323,6 @@ MegolmDecryption.prototype.onRoomKeyEvent = function(event) {
        keysClaimed = event.getKeysClaimed();
    }

-    logger.log(`Received and adding key for megolm session ${senderKey}|${sessionId}`);
    return this._olmDevice.addInboundGroupSession(
        content.room_id, senderKey, forwardingKeyChain, sessionId,
        content.session_key, keysClaimed,
@@ -1247,6 +1374,9 @@ MegolmDecryption.prototype.onRoomKeyWithheldEvent = async function(event) {

    if (content.code === "m.no_olm") {
        const sender = event.getSender();
+        logger.warn(
+            `${sender}:${senderKey} was unable to establish an olm session with us`,
+        );
        // if the sender says that they haven't been able to establish an olm
        // session, let's proactively establish one

@@ -1258,21 +1388,30 @@ MegolmDecryption.prototype.onRoomKeyWithheldEvent = async function(event) {
        if (await this._olmDevice.getSessionIdForDevice(senderKey)) {
            // a session has already been established, so we don't need to
            // create a new one.
+            logger.debug("New session already created.  Not creating a new one.");
            await this._olmDevice.recordSessionProblem(senderKey, "no_olm", true);
            this.retryDecryptionFromSender(senderKey);
            return;
        }
-        const device = this._crypto._deviceList.getDeviceByIdentityKey(
+        let device = this._crypto._deviceList.getDeviceByIdentityKey(
            content.algorithm, senderKey,
        );
        if (!device) {
-            logger.info(
-                "Couldn't find device for identity key " + senderKey +
-                    ": not establishing session",
+            // if we don't know about the device, fetch the user's devices again
+            // and retry before giving up
+            await this._crypto.downloadKeys([sender], false);
+            device = this._crypto._deviceList.getDeviceByIdentityKey(
+                content.algorithm, senderKey,
            );
-            await this._olmDevice.recordSessionProblem(senderKey, "no_olm", false);
-            this.retryDecryptionFromSender(senderKey);
-            return;
+            if (!device) {
+                logger.info(
+                    "Couldn't find device for identity key " + senderKey +
+                        ": not establishing session",
+                );
+                await this._olmDevice.recordSessionProblem(senderKey, "no_olm", false);
+                this.retryDecryptionFromSender(senderKey);
+                return;
+            }
        }
        await olmlib.ensureOlmSessionsForDevices(
            this._olmDevice, this._baseApis, {[sender]: [device]}, false,
@@ -1442,7 +1581,8 @@ MegolmDecryption.prototype.importRoomKey = function(session) {
 };

 /**
- * Have another go at decrypting events after we receive a key
+ * Have another go at decrypting events after we receive a key. Resolves once
+ * decryption has been re-attempted on all events.
 *
 * @private
 * @param {String} senderKey
@@ -1461,27 +1601,22 @@ MegolmDecryption.prototype._retryDecryption = async function(senderKey, sessionI
        return true;
    }

-    pending.delete(sessionId);
-    if (pending.size === 0) {
-        this._pendingEvents[senderKey];
-    }
+    logger.debug("Retrying decryption on events", [...pending]);

    await Promise.all([...pending].map(async (ev) => {
        try {
-            await ev.attemptDecryption(this._crypto);
+            await ev.attemptDecryption(this._crypto, true);
        } catch (e) {
            // don't die if something goes wrong
        }
    }));

-    // ev.attemptDecryption will re-add to this._pendingEvents if an event
-    // couldn't be decrypted
+    // If decrypted successfully, they'll have been removed from _pendingEvents
    return !((this._pendingEvents[senderKey] || {})[sessionId]);
 };

 MegolmDecryption.prototype.retryDecryptionFromSender = async function(senderKey) {
    const senderPendingEvents = this._pendingEvents[senderKey];
-    logger.warn(senderPendingEvents);
    if (!senderPendingEvents) {
        return true;
    }
@@ -1491,7 +1626,6 @@ MegolmDecryption.prototype.retryDecryptionFromSender = async function(senderKey)
    await Promise.all([...senderPendingEvents].map(async ([_sessionId, pending]) => {
        await Promise.all([...pending].map(async (ev) => {
            try {
-                logger.warn(ev.getId());
                await ev.attemptDecryption(this._crypto);
            } catch (e) {
                // don't die if something goes wrong
@@ -264,6 +264,25 @@ OlmDecryption.prototype.decryptEvent = async function(event) {
 */
 OlmDecryption.prototype._decryptMessage = async function(
    theirDeviceIdentityKey, message,
+) {
+    // This is a wrapper that serialises decryptions of prekey messages, because
+    // otherwise we race between deciding we have no active sessions for the message
+    // and creating a new one, which we can only do once because it removes the OTK.
+    if (message.type !== 0) {
+        // not a prekey message: we can safely just try & decrypt it
+        return this._reallyDecryptMessage(theirDeviceIdentityKey, message);
+    } else {
+        const myPromise = this._olmDevice._olmPrekeyPromise.then(() => {
+            return this._reallyDecryptMessage(theirDeviceIdentityKey, message);
+        });
+        // we want the error, but don't propagate it to the next decryption
+        this._olmDevice._olmPrekeyPromise = myPromise.catch(() => {});
+        return await myPromise;
+    }
+};
+
+OlmDecryption.prototype._reallyDecryptMessage = async function(
+    theirDeviceIdentityKey, message,
 ) {
    const sessionIds = await this._olmDevice.getSessionIdsForDevice(
        theirDeviceIdentityKey,
@@ -19,6 +19,8 @@ import {randomString} from '../randomstring';

 const DEFAULT_ITERATIONS = 500000;

+const DEFAULT_BITSIZE = 256;
+
 export async function keyFromAuthData(authData, password) {
    if (!global.Olm) {
        throw new Error("Olm is not available");
@@ -34,6 +36,7 @@ export async function keyFromAuthData(authData, password) {
    return await deriveKey(
        password, authData.private_key_salt,
        authData.private_key_iterations,
+        authData.private_key_bits || DEFAULT_BITSIZE,
    );
 }

@@ -44,12 +47,12 @@ export async function keyFromPassphrase(password) {

    const salt = randomString(32);

-    const key = await deriveKey(password, salt, DEFAULT_ITERATIONS);
+    const key = await deriveKey(password, salt, DEFAULT_ITERATIONS, DEFAULT_BITSIZE);

    return { key, salt, iterations: DEFAULT_ITERATIONS };
 }

-export async function deriveKey(password, salt, iterations) {
+export async function deriveKey(password, salt, iterations, numBits = DEFAULT_BITSIZE) {
    const subtleCrypto = global.crypto.subtle;
    const TextEncoder = global.TextEncoder;
    if (!subtleCrypto || !TextEncoder) {
@@ -73,7 +76,7 @@ export async function deriveKey(password, salt, iterations) {
            hash: 'SHA-512',
        },
        key,
-        global.Olm.PRIVATE_KEY_LENGTH * 8,
+        numBits,
    );

    return new Uint8Array(keybits);
@@ -113,6 +113,57 @@ export async function encryptMessageForDevice(
    );
 }

+/**
+ * Get the existing olm sessions for the given devices, and the devices that
+ * don't have olm sessions.
+ *
+ * @param {module:crypto/OlmDevice} olmDevice
+ *
+ * @param {module:base-apis~MatrixBaseApis} baseApis
+ *
+ * @param {object<string, module:crypto/deviceinfo[]>} devicesByUser
+ *    map from userid to list of devices to ensure sessions for
+ *
+ * @return {Promise} resolves to an array.  The first element of the array is a
+ *    a map of user IDs to arrays of deviceInfo, representing the devices that
+ *    don't have established olm sessions.  The second element of the array is
+ *    a map from userId to deviceId to {@link module:crypto~OlmSessionResult}
+ */
+export async function getExistingOlmSessions(
+    olmDevice, baseApis, devicesByUser,
+) {
+    const devicesWithoutSession = {};
+    const sessions = {};
+
+    const promises = [];
+
+    for (const [userId, devices] of Object.entries(devicesByUser)) {
+        for (const deviceInfo of devices) {
+            const deviceId = deviceInfo.deviceId;
+            const key = deviceInfo.getIdentityKey();
+            promises.push((async () => {
+                const sessionId = await olmDevice.getSessionIdForDevice(
+                    key, true,
+                );
+                if (sessionId === null) {
+                    devicesWithoutSession[userId] = devicesWithoutSession[userId] || [];
+                    devicesWithoutSession[userId].push(deviceInfo);
+                } else {
+                    sessions[userId] = sessions[userId] || {};
+                    sessions[userId][deviceId] = {
+                        device: deviceInfo,
+                        sessionId: sessionId,
+                    };
+                }
+            })());
+        }
+    }
+
+    await Promise.all(promises);
+
+    return [devicesWithoutSession, sessions];
+}
+
 /**
 * Try to make sure we have established olm sessions for the given devices.
 *
@@ -123,32 +174,58 @@ export async function encryptMessageForDevice(
 * @param {object<string, module:crypto/deviceinfo[]>} devicesByUser
 *    map from userid to list of devices to ensure sessions for
 *
- * @param {boolean} force If true, establish a new session even if one already exists.
- *     Optional.
+ * @param {boolean} [force=false] If true, establish a new session even if one
+ *     already exists.
+ *
+ * @param {Number} [otkTimeout] The timeout in milliseconds when requesting
+ *     one-time keys for establishing new olm sessions.
+ *
+ * @param {Array} [failedServers] An array to fill with remote servers that
+ *     failed to respond to one-time-key requests.
 *
 * @return {Promise} resolves once the sessions are complete, to
 *    an Object mapping from userId to deviceId to
 *    {@link module:crypto~OlmSessionResult}
 */
 export async function ensureOlmSessionsForDevices(
-    olmDevice, baseApis, devicesByUser, force,
+    olmDevice, baseApis, devicesByUser, force, otkTimeout, failedServers,
 ) {
+    if (typeof force === "number") {
+        failedServers = otkTimeout;
+        otkTimeout = force;
+        force = false;
+    }
+
    const devicesWithoutSession = [
        // [userId, deviceId], ...
    ];
    const result = {};
    const resolveSession = {};

-    for (const userId in devicesByUser) {
-        if (!devicesByUser.hasOwnProperty(userId)) {
-            continue;
-        }
+    for (const [userId, devices] of Object.entries(devicesByUser)) {
        result[userId] = {};
-        const devices = devicesByUser[userId];
-        for (let j = 0; j < devices.length; j++) {
-            const deviceInfo = devices[j];
+        for (const deviceInfo of devices) {
            const deviceId = deviceInfo.deviceId;
            const key = deviceInfo.getIdentityKey();
+
+            if (key === olmDevice.deviceCurve25519Key) {
+                // We should never be trying to start a session with ourself.
+                // Apart from talking to yourself being the first sign of madness,
+                // olm sessions can't do this because they get confused when
+                // they get a message and see that the 'other side' has started a
+                // new chain when this side has an active sender chain.
+                // If you see this message being logged in the wild, we should find
+                // the thing that is trying to send Olm messages to itself and fix it.
+                logger.info("Attempted to start session with ourself! Ignoring");
+                // We must fill in the section in the return value though, as callers
+                // expect it to be there.
+                result[userId][deviceId] = {
+                    device: deviceInfo,
+                    sessionId: null,
+                };
+                continue;
+            }
+
            if (!olmDevice._sessionsInProgress[key]) {
                // pre-emptively mark the session as in-progress to avoid race
                // conditions.  If we find that we already have a session, then
@@ -180,6 +257,11 @@ export async function ensureOlmSessionsForDevices(
                delete resolveSession[key];
            }
            if (sessionId === null || force) {
+                if (force) {
+                    logger.info("Forcing new Olm session for " + userId + ":" + deviceId);
+                } else {
+                    logger.info("Making new Olm session for " + userId + ":" + deviceId);
+                }
                devicesWithoutSession.push([userId, deviceId]);
            }
            result[userId][deviceId] = {
@@ -197,7 +279,7 @@ export async function ensureOlmSessionsForDevices(
    let res;
    try {
        res = await baseApis.claimOneTimeKeys(
-            devicesWithoutSession, oneTimeKeyAlgorithm,
+            devicesWithoutSession, oneTimeKeyAlgorithm, otkTimeout,
        );
    } catch (e) {
        for (const resolver of Object.values(resolveSession)) {
@@ -207,18 +289,26 @@ export async function ensureOlmSessionsForDevices(
        throw e;
    }

+    if (failedServers && "failures" in res) {
+        failedServers.push(...Object.keys(res.failures));
+    }
+
    const otk_res = res.one_time_keys || {};
    const promises = [];
-    for (const userId in devicesByUser) {
-        if (!devicesByUser.hasOwnProperty(userId)) {
-            continue;
-        }
+    for (const [userId, devices] of Object.entries(devicesByUser)) {
        const userRes = otk_res[userId] || {};
-        const devices = devicesByUser[userId];
        for (let j = 0; j < devices.length; j++) {
            const deviceInfo = devices[j];
            const deviceId = deviceInfo.deviceId;
            const key = deviceInfo.getIdentityKey();
+
+            if (key === olmDevice.deviceCurve25519Key) {
+                // We've already logged about this above. Skip here too
+                // otherwise we'll log saying there are no one-time keys
+                // which will be confusing.
+                continue;
+            }
+
            if (result[userId][deviceId].sessionId && !force) {
                // we already have a result for this device
                continue;
@@ -407,6 +497,15 @@ export function encodeBase64(uint8Array) {
    return Buffer.from(uint8Array).toString("base64");
 }

+/**
+ * Encode a typed array of uint8 as unpadded base64.
+ * @param {Uint8Array} uint8Array The data to encode.
+ * @return {string} The unpadded base64.
+ */
+export function encodeUnpaddedBase64(uint8Array) {
+    return encodeBase64(uint8Array).replace(/=+$/g, '');
+}
+
 /**
 * Decode a base64 string to a typed array of uint8.
 * @param {string} base64 The base64 to decode.
@@ -59,8 +59,8 @@ export function decodeRecoveryKey(recoverykey) {
        throw new Error("Incorrect length");
    }

-    return result.slice(
+    return Uint8Array.from(result.slice(
        OLM_RECOVERY_KEY_PREFIX.length,
        OLM_RECOVERY_KEY_PREFIX.length + global.Olm.PRIVATE_KEY_LENGTH,
-    );
+    ));
 }
@@ -203,6 +203,23 @@ export class Backend {
        return promiseifyTxn(txn).then(() => result);
    }

+    /**
+     *
+     * @param {Number} wantedState
+     * @return {Promise<Array<*>>} All elements in a given state
+     */
+    getAllOutgoingRoomKeyRequestsByState(wantedState) {
+        return new Promise((resolve, reject) => {
+            const txn = this._db.transaction("outgoingRoomKeyRequests", "readonly");
+            const store = txn.objectStore("outgoingRoomKeyRequests");
+            const index = store.index("state");
+            const request = index.getAll(wantedState);
+
+            request.onsuccess = (ev) => resolve(ev.target.result);
+            request.onerror = (ev) => reject(ev.target.error);
+        });
+    }
+
    getOutgoingRoomKeyRequestsByTarget(userId, deviceId, wantedStates) {
        let stateIndex = 0;
        const results = [];
@@ -341,18 +358,39 @@ export class Backend {
        };
    }

+    getSecretStorePrivateKey(txn, func, type) {
+        const objectStore = txn.objectStore("account");
+        const getReq = objectStore.get(`ssss_cache:${type}`);
+        getReq.onsuccess = function() {
+            try {
+                func(getReq.result || null);
+            } catch (e) {
+                abortWithException(txn, e);
+            }
+        };
+    }
+
    storeCrossSigningKeys(txn, keys) {
        const objectStore = txn.objectStore("account");
        objectStore.put(keys, "crossSigningKeys");
    }

+    storeSecretStorePrivateKey(txn, type, key) {
+        const objectStore = txn.objectStore("account");
+        objectStore.put(key, `ssss_cache:${type}`);
+    }
+
    // Olm Sessions

    countEndToEndSessions(txn, func) {
        const objectStore = txn.objectStore("sessions");
        const countReq = objectStore.count();
        countReq.onsuccess = function() {
-            func(countReq.result);
+            try {
+                func(countReq.result);
+            } catch (e) {
+                abortWithException(txn, e);
+            }
        };
    }

@@ -402,16 +440,16 @@ export class Backend {
        const objectStore = txn.objectStore("sessions");
        const getReq = objectStore.openCursor();
        getReq.onsuccess = function() {
-            const cursor = getReq.result;
-            if (cursor) {
-                func(cursor.value);
-                cursor.continue();
-            } else {
-                try {
+            try {
+                const cursor = getReq.result;
+                if (cursor) {
+                    func(cursor.value);
+                    cursor.continue();
+                } else {
                    func(null);
-                } catch (e) {
-                    abortWithException(txn, e);
                }
+            } catch (e) {
+                abortWithException(txn, e);
            }
        };
    }
@@ -46,6 +46,7 @@ export class IndexedDBCryptoStore {
        this._indexedDB = indexedDB;
        this._dbName = dbName;
        this._backendPromise = null;
+        this._backend = null;
    }

    static exists(indexedDB, dbName) {
@@ -56,10 +57,12 @@ export class IndexedDBCryptoStore {
     * Ensure the database exists and is up-to-date, or fall back to
     * a local storage or in-memory store.
     *
+     * This must be called before the store can be used.
+     *
     * @return {Promise} resolves to either an IndexedDBCryptoStoreBackend.Backend,
     * or a MemoryCryptoStore
     */
-    _connect() {
+    startup() {
        if (this._backendPromise) {
            return this._backendPromise;
        }
@@ -135,6 +138,8 @@ export class IndexedDBCryptoStore {
                );
                return new MemoryCryptoStore();
            }
+        }).then(backend => {
+            this._backend = backend;
        });

        return this._backendPromise;
@@ -189,9 +194,7 @@ export class IndexedDBCryptoStore {
     *    same instance as passed in, or the existing one.
     */
    getOrAddOutgoingRoomKeyRequest(request) {
-        return this._connect().then((backend) => {
-            return backend.getOrAddOutgoingRoomKeyRequest(request);
-        });
+        return this._backend.getOrAddOutgoingRoomKeyRequest(request);
    }

    /**
@@ -205,9 +208,7 @@ export class IndexedDBCryptoStore {
     *    not found
     */
    getOutgoingRoomKeyRequest(requestBody) {
-        return this._connect().then((backend) => {
-            return backend.getOutgoingRoomKeyRequest(requestBody);
-        });
+        return this._backend.getOutgoingRoomKeyRequest(requestBody);
    }

    /**
@@ -221,9 +222,18 @@ export class IndexedDBCryptoStore {
     *    requests in those states, an arbitrary one is chosen.
     */
    getOutgoingRoomKeyRequestByState(wantedStates) {
-        return this._connect().then((backend) => {
-            return backend.getOutgoingRoomKeyRequestByState(wantedStates);
-        });
+        return this._backend.getOutgoingRoomKeyRequestByState(wantedStates);
+    }
+
+    /**
+     * Look for room key requests by state –
+     * unlike above, return a list of all entries in one state.
+     *
+     * @param {Number} wantedState
+     * @return {Promise<Array<*>>} Returns an array of requests in the given state
+     */
+    getAllOutgoingRoomKeyRequestsByState(wantedState) {
+        return this._backend.getAllOutgoingRoomKeyRequestsByState(wantedState);
    }

    /**
@@ -237,11 +247,9 @@ export class IndexedDBCryptoStore {
     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}
     */
    getOutgoingRoomKeyRequestsByTarget(userId, deviceId, wantedStates) {
-        return this._connect().then((backend) => {
-            return backend.getOutgoingRoomKeyRequestsByTarget(
-                userId, deviceId, wantedStates,
-            );
-        });
+        return this._backend.getOutgoingRoomKeyRequestsByTarget(
+            userId, deviceId, wantedStates,
+        );
    }

    /**
@@ -257,11 +265,9 @@ export class IndexedDBCryptoStore {
     *    updated request, or null if no matching row was found
     */
    updateOutgoingRoomKeyRequest(requestId, expectedState, updates) {
-        return this._connect().then((backend) => {
-            return backend.updateOutgoingRoomKeyRequest(
-                requestId, expectedState, updates,
-            );
-        });
+        return this._backend.updateOutgoingRoomKeyRequest(
+            requestId, expectedState, updates,
+        );
    }

    /**
@@ -274,9 +280,7 @@ export class IndexedDBCryptoStore {
     * @returns {Promise} resolves once the operation is completed
     */
    deleteOutgoingRoomKeyRequest(requestId, expectedState) {
-        return this._connect().then((backend) => {
-            return backend.deleteOutgoingRoomKeyRequest(requestId, expectedState);
-        });
+        return this._backend.deleteOutgoingRoomKeyRequest(requestId, expectedState);
    }

    // Olm Account
@@ -289,9 +293,7 @@ export class IndexedDBCryptoStore {
     * @param {function(string)} func Called with the account pickle
     */
    getAccount(txn, func) {
-        this._backendPromise.then(backend => {
-            backend.getAccount(txn, func);
-        });
+        this._backend.getAccount(txn, func);
    }

    /**
@@ -302,9 +304,7 @@ export class IndexedDBCryptoStore {
     * @param {string} newData The new account pickle to store.
     */
    storeAccount(txn, newData) {
-        this._backendPromise.then(backend => {
-            backend.storeAccount(txn, newData);
-        });
+        this._backend.storeAccount(txn, newData);
    }

    /**
@@ -316,9 +316,16 @@ export class IndexedDBCryptoStore {
     *        { key_type: base64 encoded seed } where key type = user_signing_key_seed or self_signing_key_seed
     */
    getCrossSigningKeys(txn, func) {
-        this._backendPromise.then(backend => {
-            backend.getCrossSigningKeys(txn, func);
-        });
+        this._backend.getCrossSigningKeys(txn, func);
+    }
+
+    /**
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(string)} func Called with the private key
+     * @param {string} type A key type
+     */
+    getSecretStorePrivateKey(txn, func, type) {
+        this._backend.getSecretStorePrivateKey(txn, func, type);
    }

    /**
@@ -328,9 +335,18 @@ export class IndexedDBCryptoStore {
     * @param {string} keys keys object as getCrossSigningKeys()
     */
    storeCrossSigningKeys(txn, keys) {
-        this._backendPromise.then(backend => {
-            backend.storeCrossSigningKeys(txn, keys);
-        });
+        this._backend.storeCrossSigningKeys(txn, keys);
+    }
+
+    /**
+     * Write the cross-signing private keys back to the store
+     *
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {string} type The type of cross-signing private key to store
+     * @param {string} key keys object as getCrossSigningKeys()
+     */
+    storeSecretStorePrivateKey(txn, type, key) {
+        this._backend.storeSecretStorePrivateKey(txn, type, key);
    }

    // Olm sessions
@@ -341,9 +357,7 @@ export class IndexedDBCryptoStore {
     * @param {function(int)} func Called with the count of sessions
     */
    countEndToEndSessions(txn, func) {
-        this._backendPromise.then(backend => {
-            backend.countEndToEndSessions(txn, func);
-        });
+        this._backend.countEndToEndSessions(txn, func);
    }

    /**
@@ -359,9 +373,7 @@ export class IndexedDBCryptoStore {
     *     a message.
     */
    getEndToEndSession(deviceKey, sessionId, txn, func) {
-        this._backendPromise.then(backend => {
-            backend.getEndToEndSession(deviceKey, sessionId, txn, func);
-        });
+        this._backend.getEndToEndSession(deviceKey, sessionId, txn, func);
    }

    /**
@@ -376,9 +388,7 @@ export class IndexedDBCryptoStore {
     *     a message.
     */
    getEndToEndSessions(deviceKey, txn, func) {
-        this._backendPromise.then(backend => {
-            backend.getEndToEndSessions(deviceKey, txn, func);
-        });
+        this._backend.getEndToEndSessions(deviceKey, txn, func);
    }

    /**
@@ -389,9 +399,7 @@ export class IndexedDBCryptoStore {
     *     and session keys.
     */
    getAllEndToEndSessions(txn, func) {
-        this._backendPromise.then(backend => {
-            backend.getAllEndToEndSessions(txn, func);
-        });
+        this._backend.getAllEndToEndSessions(txn, func);
    }

    /**
@@ -402,29 +410,21 @@ export class IndexedDBCryptoStore {
     * @param {*} txn An active transaction. See doTxn().
     */
    storeEndToEndSession(deviceKey, sessionId, sessionInfo, txn) {
-        this._backendPromise.then(backend => {
-            backend.storeEndToEndSession(
-                deviceKey, sessionId, sessionInfo, txn,
-            );
-        });
+        this._backend.storeEndToEndSession(
+            deviceKey, sessionId, sessionInfo, txn,
+        );
    }

    storeEndToEndSessionProblem(deviceKey, type, fixed) {
-        return this._backendPromise.then(async (backend) => {
-            await backend.storeEndToEndSessionProblem(deviceKey, type, fixed);
-        });
+        return this._backend.storeEndToEndSessionProblem(deviceKey, type, fixed);
    }

    getEndToEndSessionProblem(deviceKey, timestamp) {
-        return this._backendPromise.then(async (backend) => {
-            return await backend.getEndToEndSessionProblem(deviceKey, timestamp);
-        });
+        return this._backend.getEndToEndSessionProblem(deviceKey, timestamp);
    }

    filterOutNotifiedErrorDevices(devices) {
-        return this._backendPromise.then(async (backend) => {
-            return await backend.filterOutNotifiedErrorDevices(devices);
-        });
+        return this._backend.filterOutNotifiedErrorDevices(devices);
    }

    // Inbound group sessions
@@ -439,11 +439,9 @@ export class IndexedDBCryptoStore {
     *     to Base64 end-to-end session.
     */
    getEndToEndInboundGroupSession(senderCurve25519Key, sessionId, txn, func) {
-        this._backendPromise.then(backend => {
-            backend.getEndToEndInboundGroupSession(
-                senderCurve25519Key, sessionId, txn, func,
-            );
-        });
+        this._backend.getEndToEndInboundGroupSession(
+            senderCurve25519Key, sessionId, txn, func,
+        );
    }

    /**
@@ -454,9 +452,7 @@ export class IndexedDBCryptoStore {
     *     sessionData}, then once with null to indicate the end of the list.
     */
    getAllEndToEndInboundGroupSessions(txn, func) {
-        this._backendPromise.then(backend => {
-            backend.getAllEndToEndInboundGroupSessions(txn, func);
-        });
+        this._backend.getAllEndToEndInboundGroupSessions(txn, func);
    }

    /**
@@ -469,11 +465,9 @@ export class IndexedDBCryptoStore {
     * @param {*} txn An active transaction. See doTxn().
     */
    addEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
-        this._backendPromise.then(backend => {
-            backend.addEndToEndInboundGroupSession(
-                senderCurve25519Key, sessionId, sessionData, txn,
-            );
-        });
+        this._backend.addEndToEndInboundGroupSession(
+            senderCurve25519Key, sessionId, sessionData, txn,
+        );
    }

    /**
@@ -486,21 +480,17 @@ export class IndexedDBCryptoStore {
     * @param {*} txn An active transaction. See doTxn().
     */
    storeEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
-        this._backendPromise.then(backend => {
-            backend.storeEndToEndInboundGroupSession(
-                senderCurve25519Key, sessionId, sessionData, txn,
-            );
-        });
+        this._backend.storeEndToEndInboundGroupSession(
+            senderCurve25519Key, sessionId, sessionData, txn,
+        );
    }

    storeEndToEndInboundGroupSessionWithheld(
        senderCurve25519Key, sessionId, sessionData, txn,
    ) {
-        this._backendPromise.then(backend => {
-            backend.storeEndToEndInboundGroupSessionWithheld(
-                senderCurve25519Key, sessionId, sessionData, txn,
-            );
-        });
+        this._backend.storeEndToEndInboundGroupSessionWithheld(
+            senderCurve25519Key, sessionId, sessionData, txn,
+        );
    }

    // End-to-end device tracking
@@ -516,9 +506,7 @@ export class IndexedDBCryptoStore {
     * @param {*} txn An active transaction. See doTxn().
     */
    storeEndToEndDeviceData(deviceData, txn) {
-        this._backendPromise.then(backend => {
-            backend.storeEndToEndDeviceData(deviceData, txn);
-        });
+        this._backend.storeEndToEndDeviceData(deviceData, txn);
    }

    /**
@@ -529,9 +517,7 @@ export class IndexedDBCryptoStore {
     *     device data
     */
    getEndToEndDeviceData(txn, func) {
-        this._backendPromise.then(backend => {
-            backend.getEndToEndDeviceData(txn, func);
-        });
+        this._backend.getEndToEndDeviceData(txn, func);
    }

    // End to End Rooms
@@ -543,9 +529,7 @@ export class IndexedDBCryptoStore {
     * @param {*} txn An active transaction. See doTxn().
     */
    storeEndToEndRoom(roomId, roomInfo, txn) {
-        this._backendPromise.then(backend => {
-            backend.storeEndToEndRoom(roomId, roomInfo, txn);
-        });
+        this._backend.storeEndToEndRoom(roomId, roomInfo, txn);
    }

    /**
@@ -554,9 +538,7 @@ export class IndexedDBCryptoStore {
     * @param {function(Object)} func Function called with the end to end encrypted rooms
     */
    getEndToEndRooms(txn, func) {
-        this._backendPromise.then(backend => {
-            backend.getEndToEndRooms(txn, func);
-        });
+        this._backend.getEndToEndRooms(txn, func);
    }

    // session backups
@@ -568,9 +550,7 @@ export class IndexedDBCryptoStore {
     * @returns {Promise} resolves to an array of inbound group sessions
     */
    getSessionsNeedingBackup(limit) {
-        return this._connect().then((backend) => {
-            return backend.getSessionsNeedingBackup(limit);
-        });
+        return this._backend.getSessionsNeedingBackup(limit);
    }

    /**
@@ -579,9 +559,7 @@ export class IndexedDBCryptoStore {
     * @returns {Promise} resolves to the number of sessions
     */
    countSessionsNeedingBackup(txn) {
-        return this._connect().then((backend) => {
-            return backend.countSessionsNeedingBackup(txn);
-        });
+        return this._backend.countSessionsNeedingBackup(txn);
    }

    /**
@@ -591,9 +569,7 @@ export class IndexedDBCryptoStore {
     * @returns {Promise} resolves when the sessions are unmarked
     */
    unmarkSessionsNeedingBackup(sessions, txn) {
-        return this._connect().then((backend) => {
-            return backend.unmarkSessionsNeedingBackup(sessions, txn);
-        });
+        return this._backend.unmarkSessionsNeedingBackup(sessions, txn);
    }

    /**
@@ -603,9 +579,7 @@ export class IndexedDBCryptoStore {
     * @returns {Promise} resolves when the sessions are marked
     */
    markSessionsNeedingBackup(sessions, txn) {
-        return this._connect().then((backend) => {
-            return backend.markSessionsNeedingBackup(sessions, txn);
-        });
+        return this._backend.markSessionsNeedingBackup(sessions, txn);
    }

    /**
@@ -630,9 +604,7 @@ export class IndexedDBCryptoStore {
     *     exception will propagate to the caller of the getFoo method.
     */
    doTxn(mode, stores, func) {
-        return this._connect().then((backend) => {
-            return backend.doTxn(mode, stores, func);
-        });
+        return this._backend.doTxn(mode, stores, func);
    }
 }

@@ -367,12 +367,23 @@ export class LocalStorageCryptoStore extends MemoryCryptoStore {
        func(keys);
    }

+    getSecretStorePrivateKey(txn, func, type) {
+        const key = getJsonItem(this.store, E2E_PREFIX + `ssss_cache.${type}`);
+        func(key);
+    }
+
    storeCrossSigningKeys(txn, keys) {
        setJsonItem(
            this.store, KEY_CROSS_SIGNING_KEYS, keys,
        );
    }

+    storeSecretStorePrivateKey(txn, type, key) {
+        setJsonItem(
+            this.store, E2E_PREFIX + `ssss_cache.${type}`, key,
+        );
+    }
+
    doTxn(mode, stores, func) {
        return Promise.resolve(func(null));
    }
@@ -33,6 +33,8 @@ export class MemoryCryptoStore {
        this._outgoingRoomKeyRequests = [];
        this._account = null;
        this._crossSigningKeys = null;
+        this._privateKeys = {};
+        this._backupKeys = {};

        // Map of {devicekey -> {sessionId -> session pickle}}
        this._sessions = {};
@@ -51,6 +53,18 @@ export class MemoryCryptoStore {
        this._sessionsNeedingBackup = {};
    }

+    /**
+     * Ensure the database exists and is up-to-date.
+     *
+     * This must be called before the store can be used.
+     *
+     * @return {Promise} resolves to the store.
+     */
+    async startup() {
+        // No startup work to do for the memory store.
+        return this;
+    }
+
    /**
     * Delete all data from this store.
     *
@@ -152,6 +166,19 @@ export class MemoryCryptoStore {
        return Promise.resolve(null);
    }

+    /**
+     *
+     * @param {Number} wantedState
+     * @return {Promise<Array<*>>} All OutgoingRoomKeyRequests in state
+     */
+    getAllOutgoingRoomKeyRequestsByState(wantedState) {
+        return Promise.resolve(
+            this._outgoingRoomKeyRequests.filter(
+                (r) => r.state == wantedState,
+            ),
+        );
+    }
+
    getOutgoingRoomKeyRequestsByTarget(userId, deviceId, wantedStates) {
        const results = [];

@@ -243,10 +270,19 @@ export class MemoryCryptoStore {
        func(this._crossSigningKeys);
    }

+    getSecretStorePrivateKey(txn, func, type) {
+        const result = this._privateKeys[type];
+        return func(result || null);
+    }
+
    storeCrossSigningKeys(txn, keys) {
        this._crossSigningKeys = keys;
    }

+    storeSecretStorePrivateKey(txn, type, key) {
+        this._privateKeys[type] = key;
+    }
+
    // Olm Sessions

    countEndToEndSessions(txn, func) {
@@ -1,5 +1,6 @@
 /*
 Copyright 2018 New Vector Ltd
+Copyright 2020 The Matrix.org Foundation C.I.C.

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -24,6 +25,8 @@ import {EventEmitter} from 'events';
 import {logger} from '../../logger';
 import {DeviceInfo} from '../deviceinfo';
 import {newTimeoutError} from "./Error";
+import {CrossSigningInfo} from "../CrossSigning";
+import {decodeBase64} from "../olmlib";

 const timeoutException = new Error("Verification timed out");

@@ -76,6 +79,8 @@ export class VerificationBase extends EventEmitter {
        this._transactionTimeoutTimer = null;
    }

+    static keyRequestTimeoutMs = 1000 * 60;
+
    get initiatedByMe() {
        // if there is no start event yet,
        // we probably want to send it,
@@ -117,6 +122,11 @@ export class VerificationBase extends EventEmitter {
        if (this._done) {
            return Promise.reject(new Error("Verification is already done"));
        }
+        const existingEvent = this.request.getEventFromOtherParty(type);
+        if (existingEvent) {
+            return Promise.resolve(existingEvent);
+        }
+
        this._expectedEvent = type;
        return new Promise((resolve, reject) => {
            this._resolveEvent = resolve;
@@ -130,6 +140,8 @@ export class VerificationBase extends EventEmitter {

    switchStartEvent(event) {
        if (this.canSwitchStartEvent(event)) {
+            logger.log("Verification Base: switching verification start event",
+                {restartingFlow: !!this._rejectEvent});
            if (this._rejectEvent) {
                const reject = this._rejectEvent;
                this._rejectEvent = undefined;
@@ -155,10 +167,13 @@ export class VerificationBase extends EventEmitter {
        } else if (e.getType() === "m.key.verification.cancel") {
            const reject = this._reject;
            this._reject = undefined;
-            const content = e.getContent();
-            const {reason, code} = content;
-            reject(new Error(`Other side cancelled verification ` +
-                `because ${reason} (${code})`));
+            // there is only promise to reject if verify has been called
+            if (reject) {
+                const content = e.getContent();
+                const {reason, code} = content;
+                reject(new Error(`Other side cancelled verification ` +
+                    `because ${reason} (${code})`));
+            }
        } else if (this._expectedEvent) {
            // only cancel if there is an event expected.
            // if there is no event expected, it means verify() wasn't called
@@ -181,11 +196,95 @@ export class VerificationBase extends EventEmitter {
    done() {
        this._endTimer(); // always kill the activity timer
        if (!this._done) {
-            if (this._channel.needsDoneMessage) {
-                // verification in DM requires a done message
-                this._send("m.key.verification.done", {});
-            }
+            this.request.onVerifierFinished();
            this._resolve();
+
+            //#region Cross-signing keys request
+            // If this is a self-verification, ask the other party for keys
+            if (this._baseApis.getUserId() !== this.userId) {
+                return;
+            }
+            // FIXME: This is a lot of logic that isn't anything to do with verification
+            // and probably ought to be somewhere else.
+            console.log("VerificationBase.done: Self-verification done; requesting keys");
+            /* This happens asynchronously, and we're not concerned about
+             * waiting for it.  We return here in order to test. */
+            return new Promise((resolve, reject) => {
+                const client = this._baseApis;
+                const original = client._crypto._crossSigningInfo;
+
+                /* We already have all of the infrastructure we need to validate and
+                 * cache cross-signing keys, so instead of replicating that, here we
+                 * set up callbacks that request them from the other device and call
+                 * CrossSigningInfo.getCrossSigningKey() to validate/cache */
+                const crossSigning = new CrossSigningInfo(
+                    original.userId,
+                    { getCrossSigningKey: async (type) => {
+                        console.debug("VerificationBase.done: requesting secret",
+                                      type, this.deviceId);
+                        const { promise } = client.requestSecret(
+                            `m.cross_signing.${type}`, [this.deviceId],
+                        );
+                        const result = await promise;
+                        const decoded = decodeBase64(result);
+                        return Uint8Array.from(decoded);
+                    } },
+                    original._cacheCallbacks,
+                );
+                crossSigning.keys = original.keys;
+
+                // XXX: get all keys out if we get one key out
+                // https://github.com/vector-im/riot-web/issues/12604
+                // then change here to reject on the timeout
+                /* Requests can be ignored, so don't wait around forever */
+                const timeout = new Promise((resolve, reject) => {
+                    setTimeout(
+                        resolve,
+                        VerificationBase.keyRequestTimeoutMs,
+                        new Error("Timeout"),
+                    );
+                });
+
+                // also request and cache the key backup key
+                const backupKeyPromise = new Promise(async resolve => {
+                    const cachedKey = await client._crypto.getSessionBackupPrivateKey();
+                    if (!cachedKey) {
+                        logger.info("No cached backup key found. Requesting...");
+                        const secretReq = client.requestSecret(
+                            'm.megolm_backup.v1', [this.deviceId],
+                        );
+                        const base64Key = await secretReq.promise;
+                        logger.info("Got key backup key, decoding...");
+                        const decodedKey = decodeBase64(base64Key);
+                        logger.info("Decoded backup key, storing...");
+                        client._crypto.storeSessionBackupPrivateKey(
+                            Uint8Array.from(decodedKey),
+                        );
+                        logger.info("Backup key stored. Starting backup restore...");
+                        const backupInfo = await client.getKeyBackupVersion();
+                        // no need to await for this - just let it go in the bg
+                        client.restoreKeyBackupWithCache(
+                            undefined, undefined, backupInfo,
+                        ).then(() => {
+                            logger.info("Backup restored.");
+                        });
+                    }
+                    resolve();
+                });
+
+                /* We call getCrossSigningKey() for its side-effects */
+                return Promise.race([
+                    Promise.all([
+                        crossSigning.getCrossSigningKey("self_signing"),
+                        crossSigning.getCrossSigningKey("user_signing"),
+                        backupKeyPromise,
+                    ]),
+                    timeout,
+                ]).then(resolve, reject);
+            }).catch((e) => {
+                console.warn("VerificationBase: failure while requesting keys:", e);
+            });
+            //#endregion
        }
    }

@@ -193,6 +292,7 @@ export class VerificationBase extends EventEmitter {
        this._endTimer(); // always kill the activity timer
        if (!this._done) {
            this.cancelled = true;
+            this.request.onVerifierCancelled();
            if (this.userId && this.deviceId) {
                // send a cancellation to the other user (if it wasn't
                // cancelled by the other user)
@@ -275,7 +375,7 @@ export class VerificationBase extends EventEmitter {

        for (const [keyId, keyInfo] of Object.entries(keys)) {
            const deviceId = keyId.split(':', 2)[1];
-            const device = await this._baseApis.getStoredDevice(userId, deviceId);
+            const device = this._baseApis.getStoredDevice(userId, deviceId);
            if (device) {
                await verifier(keyId, device, keyInfo);
                verifiedDevices.push(deviceId);
@@ -23,8 +23,9 @@ limitations under the License.
 import {VerificationBase as Base} from "./Base";
 import {
    newKeyMismatchError,
-    newUserMismatchError,
+    newUserCancelledError,
 } from './Error';
+import {encodeUnpaddedBase64, decodeBase64} from "../olmlib";

 export const SHOW_QR_CODE_METHOD = "m.qr_code.show.v1";
 export const SCAN_QR_CODE_METHOD = "m.qr_code.scan.v1";
@@ -49,47 +50,46 @@ export class ReciprocateQRCode extends Base {
                "with this method yet.");
        }

-        const targetUserId = this.startEvent.getSender();
-        if (!this.userId) {
-            console.log("Asking to confirm user ID");
-            this.userId = await new Promise((resolve, reject) => {
-                this.emit("confirm_user_id", {
-                    userId: targetUserId,
-                    confirm: resolve, // takes a userId
-                    cancel: () => reject(newUserMismatchError()),
-                });
-            });
-        } else if (targetUserId !== this.userId) {
-            throw newUserMismatchError({
-                expected: this.userId,
-                actual: targetUserId,
-            });
-        }
-
-        if (this.startEvent.getContent()['secret'] !== this.request.encodedSharedSecret) {
+        const {qrCodeData} = this.request;
+        // 1. check the secret
+        if (this.startEvent.getContent()['secret'] !== qrCodeData.encodedSharedSecret) {
            throw newKeyMismatchError();
        }

-        // If we've gotten this far, verify the user's master cross signing key
-        const xsignInfo = this._baseApis.getStoredCrossSigningForUser(this.userId);
-        if (!xsignInfo) throw new Error("Missing cross signing info");
-
-        const masterKey = xsignInfo.getId("master");
-        const masterKeyId = `ed25519:${masterKey}`;
-        const keys = {[masterKeyId]: masterKey};
-
-        const devices = (await this._baseApis.getStoredDevicesForUser(this.userId)) || [];
-        const targetDevice = devices.find(d => {
-            return d.deviceId === this.request.estimatedTargetDevice.deviceId;
+        // 2. ask if other user shows shield as well
+        await new Promise((resolve, reject) => {
+            this.reciprocateQREvent = {
+                confirm: resolve,
+                cancel: () => reject(newUserCancelledError()),
+            };
+            this.emit("show_reciprocate_qr", this.reciprocateQREvent);
        });
-        if (!targetDevice) throw new Error("Device not found, somehow");
-        keys[`ed25519:${targetDevice.deviceId}`] = targetDevice.getFingerprint();

-        if (this.request.requestingUserId === this.request.receivingUserId) {
-            delete keys[masterKeyId];
+        // 3. determine key to sign / mark as trusted
+        const keys = {};
+
+        switch (qrCodeData.mode) {
+            case MODE_VERIFY_OTHER_USER: {
+                // add master key to keys to be signed, only if we're not doing self-verification
+                const masterKey = qrCodeData.otherUserMasterKey;
+                keys[`ed25519:${masterKey}`] = masterKey;
+                break;
+            }
+            case MODE_VERIFY_SELF_TRUSTED: {
+                const deviceId = this.request.targetDevice.deviceId;
+                keys[`ed25519:${deviceId}`] = qrCodeData.otherDeviceKey;
+                break;
+            }
+            case MODE_VERIFY_SELF_UNTRUSTED: {
+                const masterKey = qrCodeData.myMasterKey;
+                keys[`ed25519:${masterKey}`] = masterKey;
+                break;
+            }
        }

+        // 4. sign the key (or mark own MSK as verified in case of MODE_VERIFY_SELF_TRUSTED)
        await this._verifyKeys(this.userId, keys, (keyId, device, keyInfo) => {
+            // make sure the device has the expected keys
            const targetKey = keys[keyId];
            if (!targetKey) throw newKeyMismatchError();

@@ -106,8 +106,200 @@ export class ReciprocateQRCode extends Base {
                    throw newKeyMismatchError();
                }
            }
-
-            // Otherwise it is probably fine
        });
    }
 }
+
+const CODE_VERSION = 0x02; // the version of binary QR codes we support
+const BINARY_PREFIX = "MATRIX"; // ASCII, used to prefix the binary format
+const MODE_VERIFY_OTHER_USER = 0x00; // Verifying someone who isn't us
+const MODE_VERIFY_SELF_TRUSTED = 0x01; // We trust the master key
+const MODE_VERIFY_SELF_UNTRUSTED = 0x02; // We do not trust the master key
+
+export class QRCodeData {
+    constructor(
+        mode, sharedSecret, otherUserMasterKey,
+        otherDeviceKey, myMasterKey, buffer,
+    ) {
+        this._sharedSecret = sharedSecret;
+        this._mode = mode;
+        this._otherUserMasterKey = otherUserMasterKey;
+        this._otherDeviceKey = otherDeviceKey;
+        this._myMasterKey = myMasterKey;
+        this._buffer = buffer;
+    }
+
+    static async create(request, client) {
+        const sharedSecret = QRCodeData._generateSharedSecret();
+        const mode = QRCodeData._determineMode(request, client);
+        let otherUserMasterKey = null;
+        let otherDeviceKey = null;
+        let myMasterKey = null;
+        if (mode === MODE_VERIFY_OTHER_USER) {
+            const otherUserCrossSigningInfo =
+                client.getStoredCrossSigningForUser(request.otherUserId);
+            otherUserMasterKey = otherUserCrossSigningInfo.getId("master");
+        } else if (mode === MODE_VERIFY_SELF_TRUSTED) {
+            otherDeviceKey = await QRCodeData._getOtherDeviceKey(request, client);
+        } else if (mode === MODE_VERIFY_SELF_UNTRUSTED) {
+            const myUserId = client.getUserId();
+            const myCrossSigningInfo = client.getStoredCrossSigningForUser(myUserId);
+            myMasterKey = myCrossSigningInfo.getId("master");
+        }
+        const qrData = QRCodeData._generateQrData(
+            request, client, mode,
+            sharedSecret,
+            otherUserMasterKey,
+            otherDeviceKey,
+            myMasterKey,
+        );
+        const buffer = QRCodeData._generateBuffer(qrData);
+        return new QRCodeData(mode, sharedSecret,
+            otherUserMasterKey, otherDeviceKey, myMasterKey, buffer);
+    }
+
+    get buffer() {
+        return this._buffer;
+    }
+
+    get mode() {
+        return this._mode;
+    }
+
+    /**
+     * only set when mode is MODE_VERIFY_SELF_TRUSTED
+     * @return {string} device key of other party at time of generating QR code
+     */
+    get otherDeviceKey() {
+        return this._otherDeviceKey;
+    }
+
+    /**
+     * only set when mode is MODE_VERIFY_OTHER_USER
+     * @return {string} master key of other party at time of generating QR code
+     */
+    get otherUserMasterKey() {
+        return this._otherUserMasterKey;
+    }
+
+    /**
+     * only set when mode is MODE_VERIFY_SELF_UNTRUSTED
+     * @return {string} own master key at time of generating QR code
+     */
+    get myMasterKey() {
+        return this._myMasterKey;
+    }
+
+    /**
+     * The unpadded base64 encoded shared secret.
+     */
+    get encodedSharedSecret() {
+        return this._sharedSecret;
+    }
+
+    static _generateSharedSecret() {
+        const secretBytes = new Uint8Array(11);
+        global.crypto.getRandomValues(secretBytes);
+        return encodeUnpaddedBase64(secretBytes);
+    }
+
+    static async _getOtherDeviceKey(request, client) {
+        const myUserId = client.getUserId();
+        const otherDevice = request.targetDevice;
+        const otherDeviceId = otherDevice ? otherDevice.deviceId : null;
+        const device = client.getStoredDevice(myUserId, otherDeviceId);
+        if (!device) {
+            throw new Error("could not find device " + otherDeviceId);
+        }
+        const key = device.getFingerprint();
+        return key;
+    }
+
+    static _determineMode(request, client) {
+        const myUserId = client.getUserId();
+        const otherUserId = request.otherUserId;
+
+        let mode = MODE_VERIFY_OTHER_USER;
+        if (myUserId === otherUserId) {
+            // Mode changes depending on whether or not we trust the master cross signing key
+            const myTrust = client.checkUserTrust(myUserId);
+            if (myTrust.isCrossSigningVerified()) {
+                mode = MODE_VERIFY_SELF_TRUSTED;
+            } else {
+                mode = MODE_VERIFY_SELF_UNTRUSTED;
+            }
+        }
+        return mode;
+    }
+
+    static _generateQrData(request, client, mode,
+        encodedSharedSecret, otherUserMasterKey,
+        otherDeviceKey, myMasterKey,
+    ) {
+        const myUserId = client.getUserId();
+        const transactionId = request.channel.transactionId;
+        const qrData = {
+            prefix: BINARY_PREFIX,
+            version: CODE_VERSION,
+            mode,
+            transactionId,
+            firstKeyB64: '', // worked out shortly
+            secondKeyB64: '', // worked out shortly
+            secretB64: encodedSharedSecret,
+        };
+
+        const myCrossSigningInfo = client.getStoredCrossSigningForUser(myUserId);
+
+        if (mode === MODE_VERIFY_OTHER_USER) {
+            // First key is our master cross signing key
+            qrData.firstKeyB64 = myCrossSigningInfo.getId("master");
+            // Second key is the other user's master cross signing key
+            qrData.secondKeyB64 = otherUserMasterKey;
+        } else if (mode === MODE_VERIFY_SELF_TRUSTED) {
+            // First key is our master cross signing key
+            qrData.firstKeyB64 = myCrossSigningInfo.getId("master");
+            qrData.secondKeyB64 = otherDeviceKey;
+        } else if (mode === MODE_VERIFY_SELF_UNTRUSTED) {
+            // First key is our device's key
+            qrData.firstKeyB64 = client.getDeviceEd25519Key();
+            // Second key is what we think our master cross signing key is
+            qrData.secondKeyB64 = myMasterKey;
+        }
+        return qrData;
+    }
+
+    static _generateBuffer(qrData) {
+        let buf = Buffer.alloc(0); // we'll concat our way through life
+
+        const appendByte = (b) => {
+            const tmpBuf = Buffer.from([b]);
+            buf = Buffer.concat([buf, tmpBuf]);
+        };
+        const appendInt = (i) => {
+            const tmpBuf = Buffer.alloc(2);
+            tmpBuf.writeInt16BE(i, 0);
+            buf = Buffer.concat([buf, tmpBuf]);
+        };
+        const appendStr = (s, enc, withLengthPrefix = true) => {
+            const tmpBuf = Buffer.from(s, enc);
+            if (withLengthPrefix) appendInt(tmpBuf.byteLength);
+            buf = Buffer.concat([buf, tmpBuf]);
+        };
+        const appendEncBase64 = (b64) => {
+            const b = decodeBase64(b64);
+            const tmpBuf = Buffer.from(b);
+            buf = Buffer.concat([buf, tmpBuf]);
+        };
+
+        // Actually build the buffer for the QR code
+        appendStr(qrData.prefix, "ascii", false);
+        appendByte(qrData.version);
+        appendByte(qrData.mode);
+        appendStr(qrData.transactionId, "utf-8");
+        appendEncBase64(qrData.firstKeyB64);
+        appendEncBase64(qrData.secondKeyB64);
+        appendEncBase64(qrData.secretB64);
+
+        return buf;
+    }
+}
@@ -28,6 +28,7 @@ import {
    newUnknownMethodError,
    newUserCancelledError,
 } from './Error';
+import {logger} from '../../logger';

 const START_TYPE = "m.key.verification.start";

@@ -165,11 +166,42 @@ const macMethods = {
    "hmac-sha256": "calculate_mac_long_kdf",
 };

+function calculateMAC(olmSAS, method) {
+    return function(...args) {
+        const macFunction = olmSAS[macMethods[method]];
+        const mac = macFunction.apply(olmSAS, args);
+        logger.log("SAS calculateMAC:", method, args, mac);
+        return mac;
+    };
+}
+
+const calculateKeyAgreement = {
+    "curve25519-hkdf-sha256": function(sas, olmSAS, bytes) {
+        const ourInfo = `${sas._baseApis.getUserId()}|${sas._baseApis.deviceId}|`
+              + `${sas.ourSASPubKey}|`;
+        const theirInfo = `${sas.userId}|${sas.deviceId}|${sas.theirSASPubKey}|`;
+        const sasInfo =
+            "MATRIX_KEY_VERIFICATION_SAS|"
+              + (sas.initiatedByMe ? ourInfo + theirInfo : theirInfo + ourInfo)
+              + sas._channel.transactionId;
+        return olmSAS.generate_bytes(sasInfo, bytes);
+    },
+    "curve25519": function(sas, olmSAS, bytes) {
+        const ourInfo = `${sas._baseApis.getUserId()}${sas._baseApis.deviceId}`;
+        const theirInfo = `${sas.userId}${sas.deviceId}`;
+        const sasInfo =
+            "MATRIX_KEY_VERIFICATION_SAS"
+              + (sas.initiatedByMe ? ourInfo + theirInfo : theirInfo + ourInfo)
+              + sas._channel.transactionId;
+        return olmSAS.generate_bytes(sasInfo, bytes);
+    },
+};
+
 /* lists of algorithms/methods that are supported.  The key agreement, hashes,
 * and MAC lists should be sorted in order of preference (most preferred
 * first).
 */
-const KEY_AGREEMENT_LIST = ["curve25519"];
+const KEY_AGREEMENT_LIST = ["curve25519-hkdf-sha256", "curve25519"];
 const HASHES_LIST = ["sha256"];
 const MAC_LIST = ["hkdf-hmac-sha256", "hmac-sha256"];
 const SAS_LIST = Object.keys(sasGenerators);
@@ -281,12 +313,14 @@ export class SAS extends Base {
        if (typeof content.commitment !== "string") {
            throw newInvalidMessageError();
        }
+        const keyAgreement = content.key_agreement_protocol;
        const macMethod = content.message_authentication_code;
        const hashCommitment = content.commitment;
        const olmSAS = new global.Olm.SAS();
        try {
-            this._send("m.key.verification.key", {
-                key: olmSAS.get_pubkey(),
+            this.ourSASPubKey = olmSAS.get_pubkey();
+            await this._send("m.key.verification.key", {
+                key: this.ourSASPubKey,
            });


@@ -298,23 +332,25 @@ export class SAS extends Base {
            if (olmutil.sha256(commitmentStr) !== hashCommitment) {
                throw newMismatchedCommitmentError();
            }
+            this.theirSASPubKey = content.key;
            olmSAS.set_their_key(content.key);

-            const sasInfo = "MATRIX_KEY_VERIFICATION_SAS"
-                  + this._baseApis.getUserId() + this._baseApis.deviceId
-                  + this.userId + this.deviceId
-                  + this._channel.transactionId;
-            const sasBytes = olmSAS.generate_bytes(sasInfo, 6);
+            const sasBytes = calculateKeyAgreement[keyAgreement](this, olmSAS, 6);
            const verifySAS = new Promise((resolve, reject) => {
-                this.emit("show_sas", {
+                this.sasEvent = {
                    sas: generateSas(sasBytes, sasMethods),
-                    confirm: () => {
-                        this._sendMAC(olmSAS, macMethod);
-                        resolve();
+                    confirm: async () => {
+                        try {
+                            await this._sendMAC(olmSAS, macMethod);
+                            resolve();
+                        } catch (err) {
+                            reject(err);
+                        }
                    },
                    cancel: () => reject(newUserCancelledError()),
                    mismatch: () => reject(newMismatchedSASError()),
-                });
+                };
+                this.emit("show_sas", this.sasEvent);
            });


@@ -366,7 +402,7 @@ export class SAS extends Base {
        const olmSAS = new global.Olm.SAS();
        try {
            const commitmentStr = olmSAS.get_pubkey() + anotherjson.stringify(content);
-            this._send("m.key.verification.accept", {
+            await this._send("m.key.verification.accept", {
                key_agreement_protocol: keyAgreement,
                hash: hashMethod,
                message_authentication_code: macMethod,
@@ -379,26 +415,29 @@ export class SAS extends Base {
            let e = await this._waitForEvent("m.key.verification.key");
            // FIXME: make sure event is properly formed
            content = e.getContent();
+            this.theirSASPubKey = content.key;
            olmSAS.set_their_key(content.key);
-            this._send("m.key.verification.key", {
-                key: olmSAS.get_pubkey(),
+            this.ourSASPubKey = olmSAS.get_pubkey();
+            await this._send("m.key.verification.key", {
+                key: this.ourSASPubKey,
            });

-            const sasInfo = "MATRIX_KEY_VERIFICATION_SAS"
-                  + this.userId + this.deviceId
-                  + this._baseApis.getUserId() + this._baseApis.deviceId
-                  + this._channel.transactionId;
-            const sasBytes = olmSAS.generate_bytes(sasInfo, 6);
+            const sasBytes = calculateKeyAgreement[keyAgreement](this, olmSAS, 6);
            const verifySAS = new Promise((resolve, reject) => {
-                this.emit("show_sas", {
+                this.sasEvent = {
                    sas: generateSas(sasBytes, sasMethods),
-                    confirm: () => {
-                        this._sendMAC(olmSAS, macMethod);
-                        resolve();
+                    confirm: async () => {
+                        try {
+                            await this._sendMAC(olmSAS, macMethod);
+                            resolve();
+                        } catch(err) {
+                            reject(err);
+                        }
                    },
                    cancel: () => reject(newUserCancelledError()),
                    mismatch: () => reject(newMismatchedSASError()),
-                });
+                };
+                this.emit("show_sas", this.sasEvent);
            });


@@ -429,7 +468,7 @@ export class SAS extends Base {
              + this._channel.transactionId;

        const deviceKeyId = `ed25519:${this._baseApis.deviceId}`;
-        mac[deviceKeyId] = olmSAS[macMethods[method]](
+        mac[deviceKeyId] = calculateMAC(olmSAS, method)(
            this._baseApis.getDeviceEd25519Key(),
            baseInfo + deviceKeyId,
        );
@@ -438,18 +477,18 @@ export class SAS extends Base {
        const crossSigningId = this._baseApis.getCrossSigningId();
        if (crossSigningId) {
            const crossSigningKeyId = `ed25519:${crossSigningId}`;
-            mac[crossSigningKeyId] = olmSAS[macMethods[method]](
+            mac[crossSigningKeyId] = calculateMAC(olmSAS, method)(
                crossSigningId,
                baseInfo + crossSigningKeyId,
            );
            keyList.push(crossSigningKeyId);
        }

-        const keys = olmSAS[macMethods[method]](
+        const keys = calculateMAC(olmSAS, method)(
            keyList.sort().join(","),
            baseInfo + "KEY_IDS",
        );
-        this._send("m.key.verification.mac", { mac, keys });
+        return this._send("m.key.verification.mac", { mac, keys });
    }

    async _checkMAC(olmSAS, content, method) {
@@ -458,7 +497,7 @@ export class SAS extends Base {
              + this._baseApis.getUserId() + this._baseApis.deviceId
              + this._channel.transactionId;

-        if (content.keys !== olmSAS[macMethods[method]](
+        if (content.keys !== calculateMAC(olmSAS, method)(
            Object.keys(content.mac).sort().join(","),
            baseInfo + "KEY_IDS",
        )) {
@@ -466,7 +505,7 @@ export class SAS extends Base {
        }

        await this._verifyKeys(this.userId, content.mac, (keyId, device, keyInfo) => {
-            if (keyInfo !== olmSAS[macMethods[method]](
+            if (keyInfo !== calculateMAC(olmSAS, method)(
                device.keys[keyId],
                baseInfo + keyId,
            )) {
@@ -44,11 +44,6 @@ export class InRoomChannel {
        this._requestEventId = null;
    }

-    /** Whether this channel needs m.key.verification.done messages to be sent after a successful verification */
-    get needsDoneMessage() {
-        return true;
-    }
-
    get receiveStartFromOtherDevices() {
        return true;
    }
@@ -183,6 +178,11 @@ export class InRoomChannel {
     * @returns {Promise} a promise that resolves when any requests as an anwser to the passed-in event are sent.
     */
    async handleEvent(event, request, isLiveEvent) {
+        // prevent processing the same event multiple times, as under
+        // some circumstances Room.timeline can get emitted twice for the same event
+        if (request.hasEventId(event.getId())) {
+            return;
+        }
        const type = InRoomChannel.getEventType(event);
        // do validations that need state (roomId, userId),
        // ignore if invalid
@@ -305,7 +305,6 @@ export class InRoomRequests {
    getRequest(event) {
        const roomId = event.getRoomId();
        const txnId = InRoomChannel.getTransactionId(event);
-//        console.log(`looking for request in room ${roomId} with txnId ${txnId} for an ${event.getType()} from ${event.getSender()}...`);
        return this._getRequestByTxnId(roomId, txnId);
    }

@@ -351,4 +350,15 @@ export class InRoomRequests {
            }
        }
    }
+
+    findRequestInProgress(roomId) {
+        const requestsByTxnId = this._requestsByRoomId.get(roomId);
+        if (requestsByTxnId) {
+            for (const request of requestsByTxnId.values()) {
+                if (request.pending) {
+                    return request;
+                }
+            }
+        }
+    }
 }
@@ -43,12 +43,22 @@ export class ToDeviceChannel {
        this._deviceId = deviceId;
    }

-    get deviceId() {
-        return this._deviceId;
+    isToDevices(devices) {
+        if (devices.length === this._devices.length) {
+            for (const device of devices) {
+                const d = this._devices.find(d => d.deviceId === device.deviceId);
+                if (!d) {
+                    return false;
+                }
+            }
+            return true;
+        } else {
+            return false;
+        }
    }

-    get needsDoneMessage() {
-        return true;
+    get deviceId() {
+        return this._deviceId;
    }

    static getEventType(event) {
@@ -335,4 +345,15 @@ export class ToDeviceRequests {
            }
        }
    }
+
+    findRequestInProgress(userId, devices) {
+        const requestsByTxnId = this._requestsByUserId.get(userId);
+        if (requestsByTxnId) {
+            for (const request of requestsByTxnId.values()) {
+                if (request.pending && request.channel.isToDevices(devices)) {
+                    return request;
+                }
+            }
+        }
+    }
 }
@@ -23,7 +23,7 @@ import {
    newUnexpectedMessageError,
    newUnknownMethodError,
 } from "../Error";
-import * as olmlib from "../../olmlib";
+import {QRCodeData, SCAN_QR_CODE_METHOD} from "../QRCode";

 // the recommended amount of time before a verification request
 // should be (automatically) cancelled without user interaction
@@ -70,7 +70,16 @@ export class VerificationRequest extends EventEmitter {
        this._eventsByThem = new Map();
        this._observeOnly = false;
        this._timeoutTimer = null;
-        this._sharedSecret = null; // used for QR codes
+        this._accepting = false;
+        this._declining = false;
+        this._verifierHasFinished = false;
+        this._cancelled = false;
+        this._chosenMethod = null;
+        // we keep a copy of the QR Code data (including other user master key) around
+        // for QR reciprocate verification, to protect against
+        // cross-signing identity reset between the .ready and .start event
+        // and signing the wrong key after .start
+        this._qrCodeData = null;
    }

    /**
@@ -151,6 +160,11 @@ export class VerificationRequest extends EventEmitter {
        return this._commonMethods;
    }

+    /** the method picked in the .start event */
+    get chosenMethod() {
+        return this._chosenMethod;
+    }
+
    /** The current remaining amount of ms before the request should be automatically cancelled */
    get timeout() {
        const requestEvent = this._getEventByEither(REQUEST_TYPE);
@@ -179,10 +193,65 @@ export class VerificationRequest extends EventEmitter {
        return this._verifier;
    }

+    get canAccept() {
+        return this.phase < PHASE_READY && !this._accepting && !this._declining;
+    }
+
+    get accepting() {
+        return this._accepting;
+    }
+
+    get declining() {
+        return this._declining;
+    }
+
    /** whether this request has sent it's initial event and needs more events to complete */
    get pending() {
-        return this._phase !== PHASE_DONE
-            && this._phase !== PHASE_CANCELLED;
+        return !this.observeOnly &&
+            this._phase !== PHASE_DONE &&
+            this._phase !== PHASE_CANCELLED;
+    }
+
+    /** Only set after a .ready if the other party can scan a QR code */
+    get qrCodeData() {
+        return this._qrCodeData;
+    }
+
+    /** Checks whether the other party supports a given verification method.
+     *  This is useful when setting up the QR code UI, as it is somewhat asymmetrical:
+     *  if the other party supports SCAN_QR, we should show a QR code in the UI, and vice versa.
+     *  For methods that need to be supported by both ends, use the `methods` property.
+     *  @param {string} method the method to check
+     *  @param {boolean} force to check even if the phase is not ready or started yet, internal usage
+     *  @return {bool} whether or not the other party said the supported the method */
+    otherPartySupportsMethod(method, force = false) {
+        if (!force && !this.ready && !this.started) {
+            return false;
+        }
+        const theirMethodEvent = this._eventsByThem.get(REQUEST_TYPE) ||
+            this._eventsByThem.get(READY_TYPE);
+        if (!theirMethodEvent) {
+            // if we started straight away with .start event,
+            // we are assuming that the other side will support the
+            // chosen method, so return true for that.
+            if (this.started && this.initiatedByMe) {
+                const myStartEvent = this._eventsByUs.get(START_TYPE);
+                const content = myStartEvent && myStartEvent.getContent();
+                const myStartMethod = content && content.method;
+                return method == myStartMethod;
+            }
+            return false;
+        }
+        const content = theirMethodEvent.getContent();
+        if (!content) {
+            return false;
+        }
+        const {methods} = content;
+        if (!Array.isArray(methods)) {
+            return false;
+        }
+
+        return methods.includes(method);
    }

    /** Whether this request was initiated by the syncing user.
@@ -234,6 +303,10 @@ export class VerificationRequest extends EventEmitter {
        return this.channel.userId;
    }

+    get isSelfVerification() {
+        return this._client.getUserId() === this.otherUserId;
+    }
+
    /**
     * The id of the user that cancelled the request,
     * only defined when phase is PHASE_CANCELLED
@@ -263,14 +336,6 @@ export class VerificationRequest extends EventEmitter {
        return this._observeOnly;
    }

-    /**
-     * The unpadded base64 encoded shared secret. Primarily used for QR code
-     * verification.
-     */
-    get encodedSharedSecret() {
-        if (!this._sharedSecret) this._generateSharedSecret();
-        return this._sharedSecret;
-    }

    /**
     * Gets which device the verification should be started with
@@ -317,6 +382,7 @@ export class VerificationRequest extends EventEmitter {
                if (!this._verifier) {
                    throw newUnknownMethodError();
                }
+                this._chosenMethod = method;
            }
        }
        return this._verifier;
@@ -330,7 +396,6 @@ export class VerificationRequest extends EventEmitter {
        if (!this.observeOnly && this._phase === PHASE_UNSENT) {
            const methods = [...this._verificationMethods.keys()];
            await this.channel.send(REQUEST_TYPE, {methods});
-            this._generateSharedSecret();
        }
    }

@@ -342,6 +407,8 @@ export class VerificationRequest extends EventEmitter {
     */
    async cancel({reason = "User declined", code = "m.user"} = {}) {
        if (!this.observeOnly && this._phase !== PHASE_CANCELLED) {
+            this._declining = true;
+            this.emit("change");
            if (this._verifier) {
                return this._verifier.cancel(errorFactory(code, reason)());
            } else {
@@ -358,17 +425,12 @@ export class VerificationRequest extends EventEmitter {
    async accept() {
        if (!this.observeOnly && this.phase === PHASE_REQUESTED && !this.initiatedByMe) {
            const methods = [...this._verificationMethods.keys()];
+            this._accepting = true;
+            this.emit("change");
            await this.channel.send(READY_TYPE, {methods});
-            this._generateSharedSecret();
        }
    }

-    _generateSharedSecret() {
-        const secretBytes = new Uint8Array(32); // 256bits
-        global.crypto.getRandomValues(secretBytes);
-        this._sharedSecret = olmlib.encodeBase64(secretBytes);
-    }
-
    /**
     * Can be used to listen for state changes until the callback returns true.
     * @param {Function} fn callback to evaluate whether the request is in the desired state.
@@ -459,12 +521,12 @@ export class VerificationRequest extends EventEmitter {
        }

        const ourDoneEvent = this._eventsByUs.get(DONE_TYPE);
-        if (ourDoneEvent && phase() === PHASE_STARTED) {
+        if (this._verifierHasFinished || (ourDoneEvent && phase() === PHASE_STARTED)) {
            transitions.push({phase: PHASE_DONE});
        }

        const cancelEvent = this._getEventByEither(CANCEL_TYPE);
-        if (cancelEvent && phase() !== PHASE_DONE) {
+        if ((this._cancelled || cancelEvent) && phase() !== PHASE_DONE) {
            transitions.push({phase: PHASE_CANCELLED, event: cancelEvent});
            return transitions;
        }
@@ -503,10 +565,80 @@ export class VerificationRequest extends EventEmitter {
            const {method} = event.getContent();
            if (!this._verifier && !this.observeOnly) {
                this._verifier = this._createVerifier(method, event);
+                if (!this._verifier) {
+                    this.cancel({
+                        code: "m.unknown_method",
+                        reason: `Unknown method: ${method}`,
+                    });
+                } else {
+                    this._chosenMethod = method;
+                }
            }
        }
    }

+    _applyPhaseTransitions() {
+        const transitions = this._calculatePhaseTransitions();
+        const existingIdx = transitions.findIndex(t => t.phase === this.phase);
+        // trim off phases we already went through, if any
+        const newTransitions = transitions.slice(existingIdx + 1);
+        // transition to all new phases
+        for (const transition of newTransitions) {
+            this._transitionToPhase(transition);
+        }
+        return newTransitions;
+    }
+
+    _isWinningStartRace(newEvent) {
+        if (newEvent.getType() !== START_TYPE) {
+            return false;
+        }
+        const oldEvent = this._verifier.startEvent;
+
+        let oldRaceIdentifier;
+        if (this.isSelfVerification) {
+            // if the verifier does not have a startEvent,
+            // it is because it's still sending and we are on the initator side
+            // we know we are sending a .start event because we already
+            // have a verifier (checked in calling method)
+            if (oldEvent) {
+                const oldContent = oldEvent.getContent();
+                oldRaceIdentifier = oldContent && oldContent.from_device;
+            } else {
+                oldRaceIdentifier = this._client.getDeviceId();
+            }
+        } else {
+            if (oldEvent) {
+                oldRaceIdentifier = oldEvent.getSender();
+            } else {
+                oldRaceIdentifier = this._client.getUserId();
+            }
+        }
+
+        let newRaceIdentifier;
+        if (this.isSelfVerification) {
+            const newContent = newEvent.getContent();
+            newRaceIdentifier = newContent && newContent.from_device;
+        } else {
+            newRaceIdentifier = newEvent.getSender();
+        }
+        return newRaceIdentifier < oldRaceIdentifier;
+    }
+
+    hasEventId(eventId) {
+        for (const event of this._eventsByUs.values()) {
+            if (event.getId() === eventId) {
+                return true;
+            }
+        }
+        for (const event of this._eventsByThem.values()) {
+            if (event.getId() === eventId) {
+                return true;
+            }
+        }
+        return false;
+    }
+
    /**
     * Changes the state of the request and verifier in response to a key verification event.
     * @param {string} type the "symbolic" event type, as returned by the `getEventType` function on the channel.
@@ -519,7 +651,7 @@ export class VerificationRequest extends EventEmitter {
     */
    async handleEvent(type, event, isLiveEvent, isRemoteEcho, isSentByUs) {
        // if reached phase cancelled or done, ignore anything else that comes
-        if (!this.pending) {
+        if (this.done || this.cancelled) {
            return;
        }
        const wasObserveOnly = this._observeOnly;
@@ -532,31 +664,32 @@ export class VerificationRequest extends EventEmitter {
            }
        }

+        // This assumes verification won't need to send an event with
+        // the same type for the same party twice.
+        // This is true for QR and SAS verification, and was
+        // added here to prevent verification getting cancelled
+        // when the server duplicates an event (https://github.com/matrix-org/synapse/issues/3365)
+        const isDuplicateEvent = isSentByUs ?
+            this._eventsByUs.has(type) :
+            this._eventsByThem.has(type);
+        if (isDuplicateEvent) {
+            return;
+        }
+
        const oldPhase = this.phase;
        this._addEvent(type, event, isSentByUs);

-        const transitions = this._calculatePhaseTransitions();
-        const existingIdx = transitions.findIndex(t => t.phase === this.phase);
-        // trim off phases we already went through, if any
-        const newTransitions = transitions.slice(existingIdx + 1);
-        // transition to all new phases
-        for (const transition of newTransitions) {
-            this._transitionToPhase(transition);
-        }
+        // this will create if needed the verifier so needs to happen before calling it
+        const newTransitions = this._applyPhaseTransitions();
        try {
            // only pass events from the other side to the verifier,
            // no remote echos of our own events
            if (this._verifier && !this.observeOnly) {
-                const oldEvent = this._verifier.startEvent;
-                // if the verifier does not have a startEvent, it is because it's still sending and we are on the initator side
-                const oldSender = oldEvent ?
-                    oldEvent.getSender() :
-                    this._client.getUserId();
-                const newEventWinsRace = event.getSender() < oldSender;
+                const newEventWinsRace = this._isWinningStartRace(event);
                if (this._verifier.canSwitchStartEvent(event) && newEventWinsRace) {
                    this._verifier.switchStartEvent(event);
                } else if (!isRemoteEcho) {
-                     if (type === CANCEL_TYPE || (this._verifier.events
+                    if (type === CANCEL_TYPE || (this._verifier.events
                        && this._verifier.events.includes(type))) {
                        this._verifier.handleEvent(event);
                    }
@@ -564,6 +697,20 @@ export class VerificationRequest extends EventEmitter {
            }

            if (newTransitions.length) {
+                // create QRCodeData if the other side can scan
+                // important this happens before emitting a phase change,
+                // so listeners can rely on it being there already
+                // We only do this for live events because it is important that
+                // we sign the keys that were in the QR code, and not the keys
+                // we happen to have at some later point in time.
+                if (isLiveEvent && newTransitions.some(t => t.phase === PHASE_READY)) {
+                    const shouldGenerateQrCode =
+                        this.otherPartySupportsMethod(SCAN_QR_CODE_METHOD, true);
+                    if (shouldGenerateQrCode) {
+                        this._qrCodeData = await QRCodeData.create(this, this._client);
+                    }
+                }
+
                const lastTransition = newTransitions[newTransitions.length - 1];
                const {phase} = lastTransition;

@@ -576,9 +723,11 @@ export class VerificationRequest extends EventEmitter {
        } finally {
            // log events we processed so we can see from rageshakes what events were added to a request
            logger.log(`Verification request ${this.channel.transactionId}: ` +
-                `${type} event with ${JSON.stringify(event.getContent())} ` +
-                `deviceId:${this.channel.deviceId} ` +
-                `sender:${event.getSender()}, isSentByUs:${isSentByUs} ` +
+                `${type} event with id:${event.getId()}, ` +
+                `content:${JSON.stringify(event.getContent())} ` +
+                `deviceId:${this.channel.deviceId}, ` +
+                `sender:${event.getSender()}, isSentByUs:${isSentByUs}, ` +
+                `isLiveEvent:${isLiveEvent}, isRemoteEcho:${isRemoteEcho}, ` +
                `phase:${oldPhase}=>${this.phase}, ` +
                `observeOnly:${wasObserveOnly}=>${this._observeOnly}`);
        }
@@ -622,7 +771,12 @@ export class VerificationRequest extends EventEmitter {

        const isUnexpectedRequest = type === REQUEST_TYPE && this.phase !== PHASE_UNSENT;
        const isUnexpectedReady = type === READY_TYPE && this.phase !== PHASE_REQUESTED;
-        if (isUnexpectedRequest || isUnexpectedReady) {
+        // only if phase has passed from PHASE_UNSENT should we cancel, because events
+        // are allowed to come in in any order (at least with InRoomChannel). So we only know
+        // we're dealing with a valid request we should participate in once we've moved to PHASE_REQUESTED
+        // before that, we could be looking at somebody elses verification request and we just
+        // happen to be in the room
+        if (this.phase !== PHASE_UNSENT && (isUnexpectedRequest || isUnexpectedReady)) {
            logger.warn(`Cancelling, unexpected ${type} verification ` +
                `event from ${event.getSender()}`);
            const reason = `Unexpected ${type} event in phase ${this.phase}`;
@@ -704,4 +858,27 @@ export class VerificationRequest extends EventEmitter {
        }
        return true;
    }
+
+    onVerifierCancelled() {
+        this._cancelled = true;
+        // move to cancelled phase
+        const newTransitions = this._applyPhaseTransitions();
+        if (newTransitions.length) {
+            this._setPhase(newTransitions[newTransitions.length - 1].phase);
+        }
+    }
+
+    onVerifierFinished() {
+        this.channel.send("m.key.verification.done", {});
+        this._verifierHasFinished = true;
+        // move to .done phase
+        const newTransitions = this._applyPhaseTransitions();
+        if (newTransitions.length) {
+            this._setPhase(newTransitions[newTransitions.length - 1].phase);
+        }
+    }
+
+    getEventFromOtherParty(type) {
+        return this._eventsByThem.get(type);
+    }
 }
@@ -44,3 +44,10 @@ InvalidCryptoStoreError.prototype = Object.create(Error.prototype, {
  },
 });
 Reflect.setPrototypeOf(InvalidCryptoStoreError, Error);
+
+export class KeySignatureUploadError extends Error {
+  constructor(message, value) {
+    super(message);
+    this.value = value;
+  }
+}
@@ -108,8 +108,9 @@ FilterComponent.prototype._checkFields =
        }

        const allowed_values = self[name];
-        if (allowed_values) {
-            if (!allowed_values.map(match_func)) {
+        if (allowed_values && allowed_values.length > 0) {
+            const anyMatch = allowed_values.some(match_func);
+            if (!anyMatch) {
                return false;
            }
        }
@@ -56,13 +56,6 @@ Filter.LAZY_LOADING_MESSAGES_FILTER = {
    lazy_load_members: true,
 };

-Filter.LAZY_LOADING_SYNC_FILTER = {
-    room: {
-        state: Filter.LAZY_LOADING_MESSAGES_FILTER,
-    },
-};
-
-
 /**
 * Get the ID of this filter on your homeserver (if known)
 * @return {?Number} The filter ID
@@ -96,6 +89,7 @@ Filter.prototype.setDefinition = function(definition) {
    //     "state": {
    //       "types": ["m.room.*"],
    //       "not_rooms": ["!726s6s6q:example.com"],
+    //       "lazy_load_members": true,
    //     },
    //     "timeline": {
    //       "limit": 10,
@@ -177,6 +171,10 @@ Filter.prototype.setTimelineLimit = function(limit) {
    setProp(this.definition, "room.timeline.limit", limit);
 };

+Filter.prototype.setLazyLoadMembers = function(enabled) {
+    setProp(this.definition, "room.state.lazy_load_members", !!enabled);
+};
+
 /**
 * Control whether left rooms should be included in responses.
 * @param {boolean} includeLeave True to make rooms the user has left appear
@@ -47,6 +47,7 @@ export const PREFIX_UNSTABLE = "/_matrix/client/unstable";

 /**
 * URI path for v1 of the the identity API
+ * @deprecated Use v2.
 */
 export const PREFIX_IDENTITY_V1 = "/_matrix/identity/api/v1";

@@ -275,6 +276,9 @@ MatrixHttpApi.prototype = {
                        callbacks.clearTimeout(xhr.timeout_timer);
                        var resp;
                        try {
+                            if (xhr.status === 0) {
+                                throw new AbortError();
+                            }
                            if (!xhr.responseText) {
                                throw new Error('No response body.');
                            }
@@ -788,6 +792,17 @@ const requestCallback = function(
    userDefinedCallback = userDefinedCallback || function() {};

    return function(err, response, body) {
+        if (err) {
+            // the unit tests use matrix-mock-request, which throw the string "aborted" when aborting a request.
+            // See https://github.com/matrix-org/matrix-mock-request/blob/3276d0263a561b5b8326b47bae720578a2c7473a/src/index.js#L48
+            const aborted = err.name === "AbortError" || err === "aborted";
+            if (!aborted && !(err instanceof MatrixError)) {
+                // browser-request just throws normal Error objects,
+                // not `TypeError`s like fetch does. So just assume any
+                // error is due to the connection.
+                err = new ConnectionError("request failed", err);
+            }
+        }
        if (!err) {
            try {
                if (response.statusCode >= 400) {
@@ -891,12 +906,76 @@ function getResponseContentType(response) {
 * @prop {Object} data The raw Matrix error JSON used to construct this object.
 * @prop {integer} httpStatus The numeric HTTP status code given
 */
-export function MatrixError(errorJson) {
-    errorJson = errorJson || {};
-    this.errcode = errorJson.errcode;
-    this.name = errorJson.errcode || "Unknown error code";
-    this.message = errorJson.error || "Unknown message";
-    this.data = errorJson;
+export class MatrixError extends Error {
+    constructor(errorJson) {
+        errorJson = errorJson || {};
+        super(`MatrixError: ${errorJson.errcode}`);
+        this.errcode = errorJson.errcode;
+        this.name = errorJson.errcode || "Unknown error code";
+        this.message = errorJson.error || "Unknown message";
+        this.data = errorJson;
+    }
+}
+
+/**
+ * Construct a ConnectionError. This is a JavaScript Error indicating
+ * that a request failed because of some error with the connection, either
+ * CORS was not correctly configured on the server, the server didn't response,
+ * the request timed out, or the internet connection on the client side went down.
+ * @constructor
+ */
+export class ConnectionError extends Error {
+    constructor(message, cause = undefined) {
+        super(message + (cause ? `: ${cause.message}` : ""));
+        this._cause = cause;
+    }
+
+    get name() {
+        return "ConnectionError";
+    }
+
+    get cause() {
+        return this._cause;
+    }
+}
+
+export class AbortError extends Error {
+    constructor() {
+        super("Operation aborted");
+    }
+
+    get name() {
+        return "AbortError";
+    }
+}
+
+/**
+ * Retries a network operation run in a callback.
+ * @param  {number}   maxAttempts maximum attempts to try
+ * @param  {Function} callback    callback that returns a promise of the network operation. If rejected with ConnectionError, it will be retried by calling the callback again.
+ * @return {any} the result of the network operation
+ * @throws {ConnectionError} If after maxAttempts the callback still throws ConnectionError
+ */
+export async function retryNetworkOperation(maxAttempts, callback) {
+    let attempts = 0;
+    let lastConnectionError = null;
+    while (attempts < maxAttempts) {
+        try {
+            if (attempts > 0) {
+                const timeout = 1000 * Math.pow(2, attempts);
+                console.log(`network operation failed ${attempts} times,` +
+                    ` retrying in ${timeout}ms...`);
+                await new Promise(r => setTimeout(r, timeout));
+            }
+            return await callback();
+        } catch (err) {
+            if (err instanceof ConnectionError) {
+                attempts += 1;
+                lastConnectionError = err;
+            } else {
+                throw err;
+            }
+        }
+    }
+    throw lastConnectionError;
 }
-MatrixError.prototype = Object.create(Error.prototype);
-MatrixError.prototype.constructor = MatrixError;
@@ -21,5 +21,12 @@ import request from "request";
 matrixcs.request(request);
 utils.runPolyfills();

+try {
+    const crypto = require('crypto');
+    utils.setCrypto(crypto);
+} catch (err) {
+    console.log('nodejs was compiled without crypto support');
+}
+
 export * from "./matrix";
 export default matrixcs;
@@ -143,11 +143,19 @@ InteractiveAuth.prototype = {
            this._resolveFunc = resolve;
            this._rejectFunc = reject;

-            // if we have no flows, try a request (we'll have
-            // just a session ID in _data if resuming)
-            if (!this._data.flows) {
+            const hasFlows = this._data && this._data.flows;
+
+            // if we have no flows, try a request to acquire the flows
+            if (!hasFlows) {
                if (this._busyChangedCallback) this._busyChangedCallback(true);
-                this._doRequest(this._data).finally(() => {
+                // use the existing sessionid, if one is present.
+                let auth = null;
+                if (this._data.session) {
+                    auth = {
+                        session: this._data.session,
+                    };
+                }
+                this._doRequest(auth).finally(() => {
                    if (this._busyChangedCallback) this._busyChangedCallback(false);
                });
            } else {
@@ -163,6 +171,8 @@ InteractiveAuth.prototype = {
     */
    poll: async function() {
        if (!this._data.session) return;
+        // likewise don't poll if there is no auth session in progress
+        if (!this._resolveFunc) return;
        // if we currently have a request in flight, there's no point making
        // another just to check what the status is
        if (this._submitPromise) return;
@@ -184,7 +194,11 @@ InteractiveAuth.prototype = {
                }
                authDict = {
                    type: EMAIL_STAGE_TYPE,
+                    // TODO: Remove `threepid_creds` once servers support proper UIA
+                    // See https://github.com/matrix-org/synapse/issues/5665
+                    // See https://github.com/matrix-org/matrix-doc/issues/2220
                    threepid_creds: creds,
+                    threepidCreds: creds,
                };
            }
        }
@@ -262,11 +276,16 @@ InteractiveAuth.prototype = {
            }
        }

-        // use the sessionid from the last request.
-        const auth = {
-            session: this._data.session,
-        };
-        utils.extend(auth, authData);
+        // use the sessionid from the last request, if one is present.
+        let auth;
+        if (this._data.session) {
+            auth = {
+                session: this._data.session,
+            };
+            utils.extend(auth, authData);
+        } else {
+            auth = authData;
+        }

        try {
            // NB. the 'background' flag is deprecated by the busyChanged
@@ -318,10 +337,12 @@ InteractiveAuth.prototype = {
        try {
            const result = await this._requestCallback(auth, background);
            this._resolveFunc(result);
+            this._resolveFunc = null;
+            this._rejectFunc = null;
        } catch (error) {
            // sometimes UI auth errors don't come with flows
            const errorFlows = error.data ? error.data.flows : null;
-            const haveFlows = Boolean(this._data.flows) || Boolean(errorFlows);
+            const haveFlows = this._data.flows || Boolean(errorFlows);
            if (error.httpStatus !== 401 || !error.data || !haveFlows) {
                // doesn't look like an interactive-auth failure.
                if (!background) {
@@ -347,7 +368,13 @@ InteractiveAuth.prototype = {
                error.data.session = this._data.session;
            }
            this._data = error.data;
-            this._startNextAuthStage();
+            try {
+                this._startNextAuthStage();
+            } catch (e) {
+                this._rejectFunc(e);
+                this._resolveFunc = null;
+                this._rejectFunc = null;
+            }

            if (
                !this._emailSid &&
@@ -379,8 +406,10 @@ InteractiveAuth.prototype = {
                    // (or not being registered, depending on what we're trying
                    // to do) or it could be a network failure. Either way, pass
                    // the failure up as the user can't complete auth if we can't
-                    // send the email, foe whatever reason.
+                    // send the email, for whatever reason.
                    this._rejectFunc(e);
+                    this._resolveFunc = null;
+                    this._rejectFunc = null;
                } finally {
                    this._requestingEmailToken = false;
                }
@@ -408,7 +437,7 @@ InteractiveAuth.prototype = {
            return;
        }

-        if (this._data.errcode || this._data.error) {
+        if (this._data && this._data.errcode || this._data.error) {
            this._stateUpdatedCallback(nextStage, {
                errcode: this._data.errcode || "",
                error: this._data.error || "",
@@ -29,6 +29,25 @@ import log from "loglevel";
 // Part of #332 is introducing a logging library in the first place.
 const DEFAULT_NAMESPACE = "matrix";

+// because rageshakes in react-sdk hijack the console log, also at module load time,
+// initializing the logger here races with the initialization of rageshakes.
+// to avoid the issue, we override the methodFactory of loglevel that binds to the
+// console methods at initialization time by a factory that looks up the console methods
+// when logging so we always get the current value of console methods.
+log.methodFactory = function(methodName, logLevel, loggerName) {
+    return function(...args) {
+        const supportedByConsole = methodName === "error" ||
+            methodName === "warn" ||
+            methodName === "trace" ||
+            methodName === "info";
+        if (supportedByConsole) {
+            return console[methodName](...args);
+        } else {
+            return console.log(...args);
+        }
+    };
+};
+
 /**
 * Drop-in replacement for <code>console</code> using {@link https://www.npmjs.com/package/loglevel|loglevel}.
 * Can be tailored down to specific use cases if needed.
@@ -16,8 +16,15 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */

+import type Request from "request";
+
 import {MemoryCryptoStore} from "./crypto/store/memory-crypto-store";
+import {LocalStorageCryptoStore} from "./crypto/store/localStorage-crypto-store";
+import {IndexedDBCryptoStore} from "./crypto/store/indexeddb-crypto-store";
 import {MemoryStore} from "./store/memory";
+import {StubStore} from "./store/stub";
+import {LocalIndexedDBStoreBackend} from "./store/indexeddb-local-backend";
+import {RemoteIndexedDBStoreBackend} from "./store/indexeddb-remote-backend";
 import {MatrixScheduler} from "./scheduler";
 import {MatrixClient} from "./client";

@@ -89,6 +96,10 @@ export function wrapRequest(wrapper) {
    };
 }

+type Store =
+    StubStore | MemoryStore | LocalIndexedDBStoreBackend | RemoteIndexedDBStoreBackend;
+
+type CryptoStore = MemoryCryptoStore | LocalStorageCryptoStore | IndexedDBCryptoStore;

 let cryptoStoreFactory = () => new MemoryCryptoStore;

@@ -102,6 +113,15 @@ export function setCryptoStoreFactory(fac) {
    cryptoStoreFactory = fac;
 }

+
+interface ICreateClientOpts {
+    baseUrl: string;
+    store?: Store;
+    cryptoStore?: CryptoStore;
+    scheduler?: MatrixScheduler;
+    request?: Request;
+}
+
 /**
 * Construct a Matrix Client. Similar to {@link module:client.MatrixClient}
 * except that the 'request', 'store' and 'scheduler' dependencies are satisfied.
@@ -125,10 +145,10 @@ export function setCryptoStoreFactory(fac) {
 * @see {@link module:client.MatrixClient} for the full list of options for
 * <code>opts</code>.
 */
-export function createClient(opts) {
+export function createClient(opts: ICreateClientOpts | string) {
    if (typeof opts === "string") {
        opts = {
-            "baseUrl": opts,
+            "baseUrl": opts as string,
        };
    }
    opts.request = opts.request || requestInstance;
@@ -390,11 +390,12 @@ utils.extend(MatrixEvent.prototype, {
     * @internal
     *
     * @param {module:crypto} crypto crypto module
+     * @param {bool} isRetry True if this is a retry (enables more logging)
     *
     * @returns {Promise} promise which resolves (to undefined) when the decryption
     * attempt is completed.
     */
-    attemptDecryption: async function(crypto) {
+    attemptDecryption: async function(crypto, isRetry) {
        // start with a couple of sanity checks.
        if (!this.isEncrypted()) {
            throw new Error("Attempt to decrypt event which isn't encrypted");
@@ -406,7 +407,7 @@ utils.extend(MatrixEvent.prototype, {
        ) {
            // we may want to just ignore this? let's start with rejecting it.
            throw new Error(
-                "Attempt to decrypt event which has already been encrypted",
+                "Attempt to decrypt event which has already been decrypted",
            );
        }

@@ -424,7 +425,7 @@ utils.extend(MatrixEvent.prototype, {
            return this._decryptionPromise;
        }

-        this._decryptionPromise = this._decryptionLoop(crypto);
+        this._decryptionPromise = this._decryptionLoop(crypto, isRetry);
        return this._decryptionPromise;
    },

@@ -469,7 +470,7 @@ utils.extend(MatrixEvent.prototype, {
        return recipients;
    },

-    _decryptionLoop: async function(crypto) {
+    _decryptionLoop: async function(crypto, isRetry) {
        // make sure that this method never runs completely synchronously.
        // (doing so would mean that we would clear _decryptionPromise *before*
        // it is set in attemptDecryption - and hence end up with a stuck
@@ -486,13 +487,18 @@ utils.extend(MatrixEvent.prototype, {
                    res = this._badEncryptedMessage("Encryption not enabled");
                } else {
                    res = await crypto.decryptEvent(this);
+                    if (isRetry) {
+                        logger.info(`Decrypted event on retry (id=${this.getId()})`);
+                    }
                }
            } catch (e) {
                if (e.name !== "DecryptionError") {
                    // not a decryption error: log the whole exception as an error
                    // (and don't bother with a retry)
+                    const re = isRetry ? 're' : '';
                    logger.error(
-                        `Error decrypting event (id=${this.getId()}): ${e.stack || e}`,
+                        `Error ${re}decrypting event ` +
+                        `(id=${this.getId()}): ${e.stack || e}`,
                    );
                    this._decryptionPromise = null;
                    this._retryDecryption = false;
@@ -209,7 +209,7 @@ utils.inherits(Room, EventEmitter);
 Room.prototype.getVersion = function() {
    const createEvent = this.currentState.getStateEvents("m.room.create", "");
    if (!createEvent) {
-        logger.warn("Room " + this.room_id + " does not have an m.room.create event");
+        logger.warn("Room " + this.roomId + " does not have an m.room.create event");
        return '1';
    }
    const ver = createEvent.getContent()['room_version'];
@@ -675,7 +675,7 @@ Room.prototype.hasUnverifiedDevices = async function() {
    }
    const e2eMembers = await this.getEncryptionTargetMembers();
    for (const member of e2eMembers) {
-        const devices = await this._client.getStoredDevicesForUser(member.userId);
+        const devices = this._client.getStoredDevicesForUser(member.userId);
        if (devices.some((device) => device.isUnverified())) {
            return true;
        }
@@ -861,11 +861,23 @@ Room.prototype.getAliases = function() {
 Room.prototype.getCanonicalAlias = function() {
    const canonicalAlias = this.currentState.getStateEvents("m.room.canonical_alias", "");
    if (canonicalAlias) {
-        return canonicalAlias.getContent().alias;
+        return canonicalAlias.getContent().alias || null;
    }
    return null;
 };

+/**
+ * Get this room's alternative aliases
+ * @return {array} The room's alternative aliases, or an empty array
+ */
+Room.prototype.getAltAliases = function() {
+    const canonicalAlias = this.currentState.getStateEvents("m.room.canonical_alias", "");
+    if (canonicalAlias) {
+        return canonicalAlias.getContent().alt_aliases || [];
+    }
+    return [];
+};
+
 /**
 * Add events to a timeline
 *
@@ -1266,6 +1278,11 @@ Room.prototype._handleRemoteEcho = function(remoteEvent, localEvent) {
    const newEventId = remoteEvent.getId();
    const oldStatus = localEvent.status;

+    logger.debug(
+        `Got remote echo for event ${oldEventId} -> ${newEventId} ` +
+        `old status ${oldStatus}`,
+    );
+
    // no longer pending
    delete this._txnToEvent[remoteEvent.getUnsigned().transaction_id];

@@ -1335,7 +1352,10 @@ ALLOWED_TRANSITIONS[EventStatus.CANCELLED] =
 * @fires module:client~MatrixClient#event:"Room.localEchoUpdated"
 */
 Room.prototype.updatePendingEvent = function(event, newStatus, newEventId) {
-    logger.log(`setting pendingEvent status to ${newStatus} in ${event.getRoomId()}`);
+    logger.log(
+        `setting pendingEvent status to ${newStatus} in ${event.getRoomId()} ` +
+        `event ID ${event.getId()} -> ${newEventId}`,
+    );

    // if the message was sent, we expect an event id
    if (newStatus == EventStatus.SENT && !newEventId) {
@@ -1821,7 +1841,7 @@ function calculateRoomName(room, userId, ignoreRoomNameEvent) {
    let alias = room.getCanonicalAlias();

    if (!alias) {
-        const aliases = room.getAliases();
+        const aliases = room.getAltAliases();

        if (aliases.length) {
            alias = aliases[0];
@@ -23,9 +23,8 @@ import {escapeRegExp, globToRegexp, isNullOrUndefined} from "./utils";

 const RULEKINDS_IN_ORDER = ['override', 'content', 'room', 'sender', 'underride'];

-// The default override rules to apply when calculating actions for an event. These
-// defaults apply under no other circumstances to avoid confusing the client with server
-// state. We do this for two reasons:
+// The default override rules to apply to the push rules that arrive from the server.
+// We do this for two reasons:
 //   1. Synapse is unlikely to send us the push rule in an incremental sync - see
 //      https://github.com/matrix-org/synapse/pull/4867#issuecomment-481446072 for
 //      more details.
@@ -364,33 +363,6 @@ export function PushProcessor(client) {
        return actionObj;
    };

-    const applyRuleDefaults = function(clientRuleset) {
-        // Deep clone the object before we mutate it
-        const ruleset = JSON.parse(JSON.stringify(clientRuleset));
-
-        if (!clientRuleset['global']) {
-            clientRuleset['global'] = {};
-        }
-        if (!clientRuleset['global']['override']) {
-            clientRuleset['global']['override'] = [];
-        }
-
-        // Apply default overrides
-        const globalOverrides = clientRuleset['global']['override'];
-        for (const override of DEFAULT_OVERRIDE_RULES) {
-            const existingRule = globalOverrides
-                .find((r) => r.rule_id === override.rule_id);
-
-            if (!existingRule) {
-                const ruleId = override.rule_id;
-                console.warn(`Adding default global override for ${ruleId}`);
-                globalOverrides.push(override);
-            }
-        }
-
-        return ruleset;
-    };
-
    this.ruleMatchesEvent = function(rule, ev) {
        let ret = true;
        for (let i = 0; i < rule.conditions.length; ++i) {
@@ -410,8 +382,7 @@ export function PushProcessor(client) {
     * @return {PushAction}
     */
    this.actionsForEvent = function(ev) {
-        const rules = applyRuleDefaults(client.pushRules);
-        return pushActionsForEventAndRulesets(ev, rules);
+        return pushActionsForEventAndRulesets(ev, client.pushRules);
    };

    /**
@@ -476,18 +447,25 @@ PushProcessor.rewriteDefaultRules = function(incomingRules) {
    if (!newRules.global) newRules.global = {};
    if (!newRules.global.override) newRules.global.override = [];

-    // Fix default override rules
-    newRules.global.override = newRules.global.override.map(r => {
-        const defaultRule = DEFAULT_OVERRIDE_RULES.find(d => d.rule_id === r.rule_id);
-        if (!defaultRule) return r;
+    // Merge the client-level defaults with the ones from the server
+    const globalOverrides = newRules.global.override;
+    for (const override of DEFAULT_OVERRIDE_RULES) {
+        const existingRule = globalOverrides
+            .find((r) => r.rule_id === override.rule_id);

-        // Copy over the actions, default, and conditions. Don't touch the user's
-        // preference.
-        r.default = defaultRule.default;
-        r.conditions = defaultRule.conditions;
-        r.actions = defaultRule.actions;
-        return r;
-    });
+        if (existingRule) {
+            // Copy over the actions, default, and conditions. Don't touch the user's
+            // preference.
+            existingRule.default = override.default;
+            existingRule.conditions = override.conditions;
+            existingRule.actions = override.actions;
+        } else {
+            // Add the rule
+            const ruleId = override.rule_id;
+            console.warn(`Adding default global override for ${ruleId}`);
+            globalOverrides.push(override);
+        }
+    }

    return newRules;
 };
@@ -25,6 +25,15 @@ limitations under the License.
 import {User} from "../models/user";
 import * as utils from "../utils";

+function isValidFilterId(filterId) {
+    const isValidStr = typeof filterId === "string" &&
+        !!filterId &&
+        filterId !== "undefined" && // exclude these as we've serialized undefined in localStorage before
+        filterId !== "null";
+
+    return isValidStr || typeof filterId === "number";
+}
+
 /**
 * Construct a new in-memory data store for the Matrix Client.
 * @constructor
@@ -273,8 +282,17 @@ MemoryStore.prototype = {
        if (!this.localStorage) {
            return null;
        }
+        const key = "mxjssdk_memory_filter_" + filterName;
+        // XXX Storage.getItem doesn't throw ...
+        // or are we using something different
+        // than window.localStorage in some cases
+        // that does throw?
+        // that would be very naughty
        try {
-            return this.localStorage.getItem("mxjssdk_memory_filter_" + filterName);
+            const value = this.localStorage.getItem(key);
+            if (isValidFilterId(value)) {
+                return value;
+            }
        } catch (e) {}
        return null;
    },
@@ -288,8 +306,13 @@ MemoryStore.prototype = {
        if (!this.localStorage) {
            return;
        }
+        const key = "mxjssdk_memory_filter_" + filterName;
        try {
-            this.localStorage.setItem("mxjssdk_memory_filter_" + filterName, filterId);
+            if (isValidFilterId(filterId)) {
+                this.localStorage.setItem(key, filterId);
+            } else {
+                this.localStorage.removeItem(key);
+            }
        } catch (e) {}
    },

@@ -511,6 +511,12 @@ SyncApi.prototype.sync = function() {
        checkLazyLoadStatus(); // advance to the next stage
    }

+    function buildDefaultFilter() {
+        const filter = new Filter(client.credentials.userId);
+        filter.setTimelineLimit(self.opts.initialSyncLimit);
+        return filter;
+    }
+
    const checkLazyLoadStatus = async () => {
        debuglog("Checking lazy load status...");
        if (this.opts.lazyLoadMembers && client.isGuest()) {
@@ -520,19 +526,11 @@ SyncApi.prototype.sync = function() {
            debuglog("Checking server lazy load support...");
            const supported = await client.doesServerSupportLazyLoading();
            if (supported) {
-                try {
-                    debuglog("Creating and storing lazy load sync filter...");
-                    this.opts.filter = await client.createFilter(
-                        Filter.LAZY_LOADING_SYNC_FILTER,
-                    );
-                    debuglog("Created and stored lazy load sync filter");
-                } catch (err) {
-                    logger.error(
-                        "Creating and storing lazy load sync filter failed",
-                        err,
-                    );
-                    throw err;
+                debuglog("Enabling lazy load on sync filter...");
+                if (!this.opts.filter) {
+                    this.opts.filter = buildDefaultFilter();
                }
+                this.opts.filter.setLazyLoadMembers(true);
            } else {
                debuglog("LL: lazy loading requested but not supported " +
                    "by server, so disabling");
@@ -575,8 +573,7 @@ SyncApi.prototype.sync = function() {
        if (self.opts.filter) {
            filter = self.opts.filter;
        } else {
-            filter = new Filter(client.credentials.userId);
-            filter.setTimelineLimit(self.opts.initialSyncLimit);
+            filter = buildDefaultFilter();
        }

        let filterId;
@@ -21,6 +21,7 @@ limitations under the License.
 */

 import unhomoglyph from 'unhomoglyph';
+import {ConnectionError} from "./http-api";

 /**
 * Encode a dictionary of query parameters.
@@ -265,7 +266,7 @@ export function checkObjectHasNoAdditionalKeys(obj: object, allowedKeys: string[
 * @param {Object} obj The object to deep copy.
 * @return {Object} A copy of the object without any references to the original.
 */
-export function deepCopy(obj: object): object {
+export function deepCopy<T>(obj: T): T {
    return JSON.parse(JSON.stringify(obj));
 }

@@ -372,6 +373,7 @@ export function extend() {
    const target = arguments[0] || {};
    for (let i = 1; i < arguments.length; i++) {
        const source = arguments[i];
+        if (!source) continue;
        for (const propName in source) { // eslint-disable-line guard-for-in
            target[propName] = source[propName];
        }
@@ -734,3 +736,17 @@ export async function promiseMapSeries<T>(
 export function promiseTry<T>(fn: () => T): Promise<T> {
    return new Promise((resolve) => resolve(fn()));
 }
+
+// We need to be able to access the Node.js crypto library from within the
+// Matrix SDK without needing to `require("crypto")`, which will fail in
+// browsers.  So `index.ts` will call `setCrypto` to store it, and when we need
+// it, we can call `getCrypto`.
+let crypto: Object;
+
+export function setCrypto(c: Object) {
+    crypto = c;
+}
+
+export function getCrypto(): Object {
+    return crypto;
+}
@@ -165,29 +165,26 @@ MatrixCall.prototype.placeVideoCall = function(remoteVideoElement, localVideoEle
 * @throws If you have not specified a listener for 'error' events.
 */
 MatrixCall.prototype.placeScreenSharingCall =
-    function(remoteVideoElement, localVideoElement) {
+    async function(remoteVideoElement, localVideoElement) {
    debuglog("placeScreenSharingCall");
    checkForErrorListener(this);
-    const screenConstraints = _getScreenSharingConstraints(this);
-    if (!screenConstraints) {
-        return;
-    }
    this.localVideoElement = localVideoElement;
    this.remoteVideoElement = remoteVideoElement;
    const self = this;
-    this.webRtc.getUserMedia(screenConstraints, function(stream) {
-        self.screenSharingStream = stream;
+    try {
+        self.screenSharingStream = await this.webRtc.getDisplayMedia({'audio': false});
        debuglog("Got screen stream, requesting audio stream...");
        const audioConstraints = _getUserMediaVideoContraints('voice');
        _placeCallWithConstraints(self, audioConstraints);
-    }, function(err) {
+    } catch(err) {
        self.emit("error",
            callError(
                MatrixCall.ERR_NO_USER_MEDIA,
                "Failed to get screen-sharing stream: " + err,
            ),
        );
-    });
+    }
+
    this.type = 'video';
    _tryPlayRemoteStream(this);
 };
@@ -1231,31 +1228,6 @@ const _createPeerConnection = function(self) {
    return pc;
 };

-const _getScreenSharingConstraints = function(call) {
-    const screen = global.screen;
-    if (!screen) {
-        call.emit("error", callError(
-            MatrixCall.ERR_NO_USER_MEDIA,
-            "Couldn't determine screen sharing constaints.",
-        ));
-        return;
-    }
-
-    return {
-        video: {
-            mediaSource: 'screen',
-            mandatory: {
-                chromeMediaSource: "screen",
-                chromeMediaSourceId: "" + Date.now(),
-                maxWidth: screen.width,
-                maxHeight: screen.height,
-                minFrameRate: 1,
-                maxFrameRate: 10,
-            },
-        },
-    };
-};
-
 const _getUserMediaVideoContraints = function(callType) {
    const isWebkit = !!global.window.navigator.webkitGetUserMedia;

@@ -1371,6 +1343,14 @@ export function createNewMatrixCall(client, roomId, options) {
        };
    }

+    const getDisplayMedia = (
+        w.navigator.mediaDevices && w.navigator.mediaDevices.getDisplayMedia ||
+        w.navigator.getDisplayMedia
+    );
+    if (getDisplayMedia) {
+        webRtc.getDisplayMedia = getDisplayMedia.bind(w.navigator.mediaDevices);
+    }
+
    // Firefox throws on so little as accessing the RTCPeerConnection when operating in
    // a secure mode. There's some information at https://bugzilla.mozilla.org/show_bug.cgi?id=1542616
    // though the concern is that the browser throwing a SecurityError will brick the
@@ -2,6 +2,7 @@
  "compilerOptions": {
    "experimentalDecorators": true,
    "emitDecoratorMetadata": true,
+    "esModuleInterop": true,
    "module": "commonjs",
    "moduleResolution": "node",
    "target": "es2016",