v9.2.0

[Release] Fix dehydration method name

.eslintrc.js

@@ -1,71 +1,15 @@
 module.exports = {
-    parser: "babel-eslint", // now needed for class properties
-    parserOptions: {
-        sourceType: "module",
-        ecmaFeatures: {
-        }
-    },
+    extends: ["matrix-org"],
+    plugins: [
+        "babel",
+    ],
     env: {
         browser: true,
         node: true,
-
-        // babel's transform-runtime converts references to ES6 globals such as
-        // Promise and Map to core-js polyfills, so we can use ES6 globals.
-        es6: true,
-        jest: true,
     },
-    extends: ["eslint:recommended", "google"],
-    plugins: [
-        "babel",
-        "jest",
-    ],
+
     rules: {
-        // rules we've always adhered to or now do
-        "max-len": ["error", {
-            code: 90,
-            ignoreComments: true,
-        }],
-        curly: ["error", "multi-line"],
-        "prefer-const": ["error"],
-        "comma-dangle": ["error", {
-            arrays: "always-multiline",
-            objects: "always-multiline",
-            imports: "always-multiline",
-            exports: "always-multiline",
-            functions: "always-multiline",
-        }],
-
-        // loosen jsdoc requirements a little
-        "require-jsdoc": ["error", {
-            require: {
-                FunctionDeclaration: false,
-            }
-        }],
-        "valid-jsdoc": ["error", {
-            requireParamDescription: false,
-            requireReturn: false,
-            requireReturnDescription: false,
-        }],
-
-        // rules we do not want from eslint-recommended
-        "no-console": ["off"],
-        "no-constant-condition": ["off"],
-        "no-empty": ["error", { "allowEmptyCatch": true }],
-
-        // rules we do not want from the google styleguide
-        "object-curly-spacing": ["off"],
-        "spaced-comment": ["off"],
-        "guard-for-in": ["off"],
-
-        // in principle we prefer single quotes, but life is too short
-        quotes: ["off"],
-
-        // rules we'd ideally like to adhere to, but the current
-        // code does not (in most cases because it's still ES5)
-        // we set these to warnings, and assert that the number
-        // of warnings doesn't exceed a given threshold
         "no-var": ["warn"],
-        "brace-style": ["warn", "1tbs", {"allowSingleLine": true}],
         "prefer-rest-params": ["warn"],
         "prefer-spread": ["warn"],
         "one-var": ["warn"],
@@ -79,10 +23,21 @@ module.exports = {
             "asyncArrow": "always",
         }],
         "arrow-parens": "off",
-
-        // eslint's built in no-invalid-this rule breaks with class properties
-        "no-invalid-this": "off",
-        // so we replace it with a version that is class property aware
-        "babel/no-invalid-this": "error",
-    }
-}
+        "prefer-promise-reject-errors": "off",
+        "quotes": "off",
+        "indent": "off",
+        "no-constant-condition": "off",
+        "no-async-promise-executor": "off",
+        // We use a `logger` intermediary module
+        "no-console": "error",
+    },
+    overrides: [{
+        "files": ["src/**/*.ts"],
+        "extends": ["matrix-org/ts"],
+        "rules": {
+            // While we're converting to ts we make heavy use of this
+            "@typescript-eslint/no-explicit-any": "off",
+            "quotes": "off",
+        },
+    }],
+};

.gitignore

@@ -12,7 +12,6 @@ lib-cov
 out
 /dist
 /lib
-/specbuild
 
 # version file and tarball created by `npm pack` / `yarn pack`
 /git-revision.txt

CHANGELOG.md

@@ -1,3 +1,651 @@
+Changes in [9.2.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v9.2.0) (2020-11-23)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v9.2.0-rc.1...v9.2.0)
+
+ * [Release] Fix dehydration method name
+   [\#1545](https://github.com/matrix-org/matrix-js-sdk/pull/1545)
+
+Changes in [9.2.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v9.2.0-rc.1) (2020-11-18)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v9.1.0...v9.2.0-rc.1)
+
+ * Implement call holding functionality
+   [\#1532](https://github.com/matrix-org/matrix-js-sdk/pull/1532)
+ * Support awaitable one-time dehydration
+   [\#1537](https://github.com/matrix-org/matrix-js-sdk/pull/1537)
+ * Client set profile methods update own user
+   [\#1534](https://github.com/matrix-org/matrix-js-sdk/pull/1534)
+
+Changes in [9.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v9.1.0) (2020-11-09)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v9.1.0-rc.1...v9.1.0)
+
+* No changes since rc.1
+
+Changes in [9.1.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v9.1.0-rc.1) (2020-11-04)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v9.0.1...v9.1.0-rc.1)
+
+ * Fix spelling error in the server ACL event type
+   [\#1535](https://github.com/matrix-org/matrix-js-sdk/pull/1535)
+ * await idb operations from crypto store for dehydration
+   [\#1533](https://github.com/matrix-org/matrix-js-sdk/pull/1533)
+ * Fix stuck never-sending messages
+   [\#1531](https://github.com/matrix-org/matrix-js-sdk/pull/1531)
+ * Await key cache check to avoid prompts
+   [\#1529](https://github.com/matrix-org/matrix-js-sdk/pull/1529)
+ * Improve ICE candidate batching
+   [\#1524](https://github.com/matrix-org/matrix-js-sdk/pull/1524)
+ * Convert logger to typescript
+   [\#1527](https://github.com/matrix-org/matrix-js-sdk/pull/1527)
+ * Fix logger typo
+   [\#1525](https://github.com/matrix-org/matrix-js-sdk/pull/1525)
+ * bind online listener to window instead of document
+   [\#1523](https://github.com/matrix-org/matrix-js-sdk/pull/1523)
+ * Support m.call.select_answer
+   [\#1522](https://github.com/matrix-org/matrix-js-sdk/pull/1522)
+
+Changes in [9.0.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v9.0.1) (2020-10-28)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v9.0.0...v9.0.1)
+
+ * [Release] Await key cache check to avoid prompts
+   [\#1530](https://github.com/matrix-org/matrix-js-sdk/pull/1530)
+
+Changes in [9.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v9.0.0) (2020-10-26)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v9.0.0-rc.1...v9.0.0)
+
+ * Fix logger typo
+   [\#1528](https://github.com/matrix-org/matrix-js-sdk/pull/1528)
+
+Changes in [9.0.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v9.0.0-rc.1) (2020-10-21)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.5.0...v9.0.0-rc.1)
+
+BREAKING CHANGES
+---
+
+ * `hasPendingEvent` now returns false instead of throwing when pending ordering mode is not `detached`
+
+All changes
+---
+
+ * Don't cache failures when fetching /versions
+   [\#1521](https://github.com/matrix-org/matrix-js-sdk/pull/1521)
+ * Install deps first as part of release
+   [\#1518](https://github.com/matrix-org/matrix-js-sdk/pull/1518)
+ * [Breaking] Change hasPendingEvent to return false if pending ordering
+   !detached
+   [\#1517](https://github.com/matrix-org/matrix-js-sdk/pull/1517)
+ * Skip editor prompts for merges
+   [\#1519](https://github.com/matrix-org/matrix-js-sdk/pull/1519)
+ * Convert call test to TypeScript
+   [\#1516](https://github.com/matrix-org/matrix-js-sdk/pull/1516)
+ * Support party_id
+   [\#1512](https://github.com/matrix-org/matrix-js-sdk/pull/1512)
+ * Support m.call.reject
+   [\#1510](https://github.com/matrix-org/matrix-js-sdk/pull/1510)
+ * Remove specbuild from .gitignore
+   [\#1515](https://github.com/matrix-org/matrix-js-sdk/pull/1515)
+ * Log the error when we failed to send candidates
+   [\#1514](https://github.com/matrix-org/matrix-js-sdk/pull/1514)
+ * Fixes for call state machine
+   [\#1503](https://github.com/matrix-org/matrix-js-sdk/pull/1503)
+ * Fix call event handler listener removing
+   [\#1506](https://github.com/matrix-org/matrix-js-sdk/pull/1506)
+ * Set the type of the call based on the tracks
+   [\#1501](https://github.com/matrix-org/matrix-js-sdk/pull/1501)
+ * Use new local timestamp for calls
+   [\#1499](https://github.com/matrix-org/matrix-js-sdk/pull/1499)
+ * Adjust types and APIs to match React SDK
+   [\#1502](https://github.com/matrix-org/matrix-js-sdk/pull/1502)
+ * Make an accurate version of 'age' for events
+   [\#1495](https://github.com/matrix-org/matrix-js-sdk/pull/1495)
+ * Make 'options' parameter optional
+   [\#1498](https://github.com/matrix-org/matrix-js-sdk/pull/1498)
+ * Create a giant event type enum
+   [\#1497](https://github.com/matrix-org/matrix-js-sdk/pull/1497)
+ * Convert call.js to Typescript & update WebRTC APIs (re-apply)
+   [\#1494](https://github.com/matrix-org/matrix-js-sdk/pull/1494)
+
+Changes in [8.5.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.5.0) (2020-10-12)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.5.0-rc.1...v8.5.0)
+
+* No changes since rc.1
+
+Changes in [8.5.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.5.0-rc.1) (2020-10-07)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.4.1...v8.5.0-rc.1)
+
+ * Add support for olm fallback keys
+   [\#1467](https://github.com/matrix-org/matrix-js-sdk/pull/1467)
+ * Fix editing local echoes not updating them in real time
+   [\#1492](https://github.com/matrix-org/matrix-js-sdk/pull/1492)
+ * Fix re-emit of Event.replaced to be on client and not room
+   [\#1491](https://github.com/matrix-org/matrix-js-sdk/pull/1491)
+ * Add space to log line
+   [\#1496](https://github.com/matrix-org/matrix-js-sdk/pull/1496)
+ * Revert "Convert call.js to Typescript & update WebRTC APIs"
+   [\#1493](https://github.com/matrix-org/matrix-js-sdk/pull/1493)
+ * Convert call.js to Typescript & update WebRTC APIs
+   [\#1487](https://github.com/matrix-org/matrix-js-sdk/pull/1487)
+ * Dehydrate and rehydrate devices
+   [\#1436](https://github.com/matrix-org/matrix-js-sdk/pull/1436)
+ * Keep local device after processing device list sync
+   [\#1490](https://github.com/matrix-org/matrix-js-sdk/pull/1490)
+ * Enforce logger module via lint rules
+   [\#1489](https://github.com/matrix-org/matrix-js-sdk/pull/1489)
+ * Extend method redactEvent with reason
+   [\#1462](https://github.com/matrix-org/matrix-js-sdk/pull/1462)
+ * Catch exception from call event handler
+   [\#1484](https://github.com/matrix-org/matrix-js-sdk/pull/1484)
+ * Ignore invalid candidates
+   [\#1483](https://github.com/matrix-org/matrix-js-sdk/pull/1483)
+ * Always push docs if they are generated
+   [\#1478](https://github.com/matrix-org/matrix-js-sdk/pull/1478)
+ * Only sign key backup with cross-signing keys when available
+   [\#1481](https://github.com/matrix-org/matrix-js-sdk/pull/1481)
+ * Upgrade dependencies
+   [\#1479](https://github.com/matrix-org/matrix-js-sdk/pull/1479)
+
+Changes in [8.4.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.4.1) (2020-09-28)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.4.0...v8.4.1)
+
+ * Catch exception from call event handler
+   [\#1486](https://github.com/matrix-org/matrix-js-sdk/pull/1486)
+ * Ignore invalid candidates
+   [\#1485](https://github.com/matrix-org/matrix-js-sdk/pull/1485)
+
+Changes in [8.4.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.4.0) (2020-09-28)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.4.0-rc.1...v8.4.0)
+
+ * Only sign key backup with cross-signing keys when available
+   [\#1482](https://github.com/matrix-org/matrix-js-sdk/pull/1482)
+
+Changes in [8.4.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.4.0-rc.1) (2020-09-23)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.3.0...v8.4.0-rc.1)
+
+ * If there are extraParams set, ensure that queryParams is defined
+   [\#1477](https://github.com/matrix-org/matrix-js-sdk/pull/1477)
+ * Add diagnostics to security bootstrap paths
+   [\#1475](https://github.com/matrix-org/matrix-js-sdk/pull/1475)
+ * Switch to a combination of better-docs and docdash
+   [\#1459](https://github.com/matrix-org/matrix-js-sdk/pull/1459)
+ * Undo attempts to cache private keys aggressively
+   [\#1474](https://github.com/matrix-org/matrix-js-sdk/pull/1474)
+ * Repair secret storage reset, cache keys when missing
+   [\#1472](https://github.com/matrix-org/matrix-js-sdk/pull/1472)
+ * Prevent parallel getVersions calls
+   [\#1471](https://github.com/matrix-org/matrix-js-sdk/pull/1471)
+ * Send end-of-candidates
+   [\#1473](https://github.com/matrix-org/matrix-js-sdk/pull/1473)
+ * Add a function for checking the /versions flag for forced e2ee
+   [\#1470](https://github.com/matrix-org/matrix-js-sdk/pull/1470)
+ * Add option to allow users of pantialaimon to use the SDK
+   [\#1469](https://github.com/matrix-org/matrix-js-sdk/pull/1469)
+ * Fixed Yarn broken link
+   [\#1468](https://github.com/matrix-org/matrix-js-sdk/pull/1468)
+ * some TypeScript and doc fixes
+   [\#1466](https://github.com/matrix-org/matrix-js-sdk/pull/1466)
+ * Remove Travis CI reference
+   [\#1464](https://github.com/matrix-org/matrix-js-sdk/pull/1464)
+ * Inject identity server token for 3pid invites on createRoom
+   [\#1463](https://github.com/matrix-org/matrix-js-sdk/pull/1463)
+
+Changes in [8.3.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.3.0) (2020-09-14)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.3.0-rc.1...v8.3.0)
+
+* No changes since rc.1
+
+Changes in [8.3.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.3.0-rc.1) (2020-09-09)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.2.0...v8.3.0-rc.1)
+
+ * Add missing options in ICreateClientOpts
+   [\#1452](https://github.com/matrix-org/matrix-js-sdk/pull/1452)
+ * Ensure ready functions return boolean values
+   [\#1457](https://github.com/matrix-org/matrix-js-sdk/pull/1457)
+ * Handle missing cross-signing keys gracefully
+   [\#1456](https://github.com/matrix-org/matrix-js-sdk/pull/1456)
+ * Fix eslint ts override tsx matching
+   [\#1451](https://github.com/matrix-org/matrix-js-sdk/pull/1451)
+ * Untangle cross-signing and secret storage
+   [\#1450](https://github.com/matrix-org/matrix-js-sdk/pull/1450)
+
+Changes in [8.2.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.2.0) (2020-09-01)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.2.0-rc.1...v8.2.0)
+
+## Security notice
+
+JS SDK 8.2.0 fixes an issue where encrypted state events could break incoming call handling.
+Thanks to @awesome-michael from Awesome Technologies for responsibly disclosing this via Matrix's
+Security Disclosure Policy.
+
+## All changes
+
+* No changes since rc.1
+
+Changes in [8.2.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.2.0-rc.1) (2020-08-26)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.1.0...v8.2.0-rc.1)
+
+ * Add state event check
+   [\#1449](https://github.com/matrix-org/matrix-js-sdk/pull/1449)
+ * Add method to check whether client .well-known has been fetched
+   [\#1444](https://github.com/matrix-org/matrix-js-sdk/pull/1444)
+ * Handle auth errors during cross-signing key upload
+   [\#1443](https://github.com/matrix-org/matrix-js-sdk/pull/1443)
+ * Don't fail if the requested audio output isn't available
+   [\#1448](https://github.com/matrix-org/matrix-js-sdk/pull/1448)
+ * Fix logging failures
+   [\#1447](https://github.com/matrix-org/matrix-js-sdk/pull/1447)
+ * Log the constraints we pass to getUserMedia
+   [\#1446](https://github.com/matrix-org/matrix-js-sdk/pull/1446)
+ * Use SAS emoji data from matrix-doc
+   [\#1440](https://github.com/matrix-org/matrix-js-sdk/pull/1440)
+
+Changes in [8.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.1.0) (2020-08-17)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.1.0-rc.1...v8.1.0)
+
+* No changes since rc.1
+
+Changes in [8.1.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.1.0-rc.1) (2020-08-13)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.0.1...v8.1.0-rc.1)
+
+ * Update on Promises
+   [\#1438](https://github.com/matrix-org/matrix-js-sdk/pull/1438)
+ * Store and request master cross-signing key
+   [\#1437](https://github.com/matrix-org/matrix-js-sdk/pull/1437)
+ * Filter out non-string display names
+   [\#1433](https://github.com/matrix-org/matrix-js-sdk/pull/1433)
+ * Bump elliptic from 6.5.2 to 6.5.3
+   [\#1427](https://github.com/matrix-org/matrix-js-sdk/pull/1427)
+ * Replace Riot with Element in docs and comments
+   [\#1431](https://github.com/matrix-org/matrix-js-sdk/pull/1431)
+ * Remove leftover bits of TSLint
+   [\#1430](https://github.com/matrix-org/matrix-js-sdk/pull/1430)
+
+Changes in [8.0.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.0.1) (2020-08-05)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.0.1-rc.1...v8.0.1)
+
+ * Filter out non-string display names
+   [\#1434](https://github.com/matrix-org/matrix-js-sdk/pull/1434)
+
+Changes in [8.0.1-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.0.1-rc.1) (2020-07-31)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.0.0...v8.0.1-rc.1)
+
+ * Remove redundant lint dependencies
+   [\#1426](https://github.com/matrix-org/matrix-js-sdk/pull/1426)
+ * Upload all keys when we start using a new key backup version
+   [\#1428](https://github.com/matrix-org/matrix-js-sdk/pull/1428)
+ * Expose countSessionsNeedingBackup
+   [\#1429](https://github.com/matrix-org/matrix-js-sdk/pull/1429)
+ * Configure and use new eslint package
+   [\#1422](https://github.com/matrix-org/matrix-js-sdk/pull/1422)
+
+Changes in [8.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.0.0) (2020-07-27)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v7.1.0...v8.0.0)
+
+BREAKING CHANGES
+---
+
+* `RoomState` events changed to use a Map instead of an object, which changes the collection APIs available to access them.
+ 
+All Changes
+---
+
+ * Properly support txnId
+   [\#1424](https://github.com/matrix-org/matrix-js-sdk/pull/1424)
+ * [BREAKING] Remove deprecated getIdenticonUri
+   [\#1423](https://github.com/matrix-org/matrix-js-sdk/pull/1423)
+ * Bump lodash from 4.17.15 to 4.17.19
+   [\#1421](https://github.com/matrix-org/matrix-js-sdk/pull/1421)
+ * [BREAKING] Convert RoomState's stored state map to a real map
+   [\#1419](https://github.com/matrix-org/matrix-js-sdk/pull/1419)
+
+Changes in [7.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v7.1.0) (2020-07-03)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v7.1.0-rc.1...v7.1.0)
+
+* No changes since rc.1
+
+Changes in [7.1.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v7.1.0-rc.1) (2020-07-01)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v7.0.0...v7.1.0-rc.1)
+
+ * Ask general crypto callbacks for 4S privkey if operation adapter doesn't
+   have it yet
+   [\#1414](https://github.com/matrix-org/matrix-js-sdk/pull/1414)
+ * Fix ICreateClientOpts missing idBaseUrl
+   [\#1413](https://github.com/matrix-org/matrix-js-sdk/pull/1413)
+ * Increase max event listeners for rooms
+   [\#1411](https://github.com/matrix-org/matrix-js-sdk/pull/1411)
+ * Don't trust keys megolm received from backup for verifying the sender
+   [\#1406](https://github.com/matrix-org/matrix-js-sdk/pull/1406)
+ * Raise the last known account data / state event for an update
+   [\#1410](https://github.com/matrix-org/matrix-js-sdk/pull/1410)
+ * Isolate encryption bootstrap side-effects
+   [\#1380](https://github.com/matrix-org/matrix-js-sdk/pull/1380)
+ * Add method to get current in-flight to-device requests
+   [\#1405](https://github.com/matrix-org/matrix-js-sdk/pull/1405)
+
+Changes in [7.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v7.0.0) (2020-06-23)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v7.0.0-rc.1...v7.0.0)
+
+* No changes since rc.1
+
+Changes in [7.0.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v7.0.0-rc.1) (2020-06-17)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.2.2...v7.0.0-rc.1)
+
+BREAKING CHANGES
+---
+
+ * Presence lists were removed from the spec in r0.5.0, and the corresponding methods have now been removed here as well:
+   * `getPresenceList`
+   * `inviteToPresenceList`
+   * `dropFromPresenceList`
+
+All changes
+---
+
+ * Remove support for unspecced device-specific push rules
+   [\#1404](https://github.com/matrix-org/matrix-js-sdk/pull/1404)
+ * Use existing session id for fetching flows as to not get a new session
+   [\#1403](https://github.com/matrix-org/matrix-js-sdk/pull/1403)
+ * Upgrade deps
+   [\#1400](https://github.com/matrix-org/matrix-js-sdk/pull/1400)
+ * Bring back backup key format migration
+   [\#1398](https://github.com/matrix-org/matrix-js-sdk/pull/1398)
+ * Fix: more informative error message when we cant find a key to decrypt with
+   [\#1313](https://github.com/matrix-org/matrix-js-sdk/pull/1313)
+ * Add js-sdk mechanism for polling client well-known for config
+   [\#1394](https://github.com/matrix-org/matrix-js-sdk/pull/1394)
+ * Fix verification request timeouts to match spec
+   [\#1388](https://github.com/matrix-org/matrix-js-sdk/pull/1388)
+ * Drop presence list methods
+   [\#1391](https://github.com/matrix-org/matrix-js-sdk/pull/1391)
+ * Batch up URL previews to prevent excessive requests
+   [\#1395](https://github.com/matrix-org/matrix-js-sdk/pull/1395)
+
+Changes in [6.2.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.2.2) (2020-06-16)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.2.1...v6.2.2)
+
+ * Use existing session id for fetching flows as to not get a new session
+   [\#1407](https://github.com/matrix-org/matrix-js-sdk/pull/1407)
+
+Changes in [6.2.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.2.1) (2020-06-05)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.2.0...v6.2.1)
+
+ * Bring back backup key format migration
+   [\#1399](https://github.com/matrix-org/matrix-js-sdk/pull/1399)
+
+Changes in [6.2.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.2.0) (2020-06-04)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.2.0-rc.1...v6.2.0)
+
+ * No changes since rc.1
+
+Changes in [6.2.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.2.0-rc.1) (2020-06-02)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.1.0...v6.2.0-rc.1)
+
+ * Make auth argument in the register request compliant with r0.6.0
+   [\#1304](https://github.com/matrix-org/matrix-js-sdk/pull/1304)
+ * Send the wrong auth params with the right auth params
+   [\#1393](https://github.com/matrix-org/matrix-js-sdk/pull/1393)
+ * encrypt cached keys with pickle key
+   [\#1387](https://github.com/matrix-org/matrix-js-sdk/pull/1387)
+ * Fix replying to key share requests
+   [\#1385](https://github.com/matrix-org/matrix-js-sdk/pull/1385)
+ * Add dist to package.json files so CDNs can serve it
+   [\#1384](https://github.com/matrix-org/matrix-js-sdk/pull/1384)
+ * Fix getVersion warning saying undefined room
+   [\#1382](https://github.com/matrix-org/matrix-js-sdk/pull/1382)
+ * Combine the two places we processed client-level default push rules
+   [\#1379](https://github.com/matrix-org/matrix-js-sdk/pull/1379)
+ * make MAC check robust against unpadded vs padded base64 differences
+   [\#1378](https://github.com/matrix-org/matrix-js-sdk/pull/1378)
+ * Remove key backup format migration
+   [\#1375](https://github.com/matrix-org/matrix-js-sdk/pull/1375)
+ * Add simple browserify browser-matrix.js tests
+   [\#1241](https://github.com/matrix-org/matrix-js-sdk/pull/1241)
+ * support new key agreement method for SAS
+   [\#1376](https://github.com/matrix-org/matrix-js-sdk/pull/1376)
+
+Changes in [6.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.1.0) (2020-05-19)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.1.0-rc.1...v6.1.0)
+
+ * No changes since rc.1
+
+Changes in [6.1.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.1.0-rc.1) (2020-05-14)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.0.0...v6.1.0-rc.1)
+
+ * Remove support for asymmetric 4S encryption
+   [\#1373](https://github.com/matrix-org/matrix-js-sdk/pull/1373)
+ * Increase timeout for 2nd phase of Olm session creation
+   [\#1367](https://github.com/matrix-org/matrix-js-sdk/pull/1367)
+ * Add logging on decryption retries
+   [\#1366](https://github.com/matrix-org/matrix-js-sdk/pull/1366)
+ * Emit event when a trusted self-key is stored
+   [\#1364](https://github.com/matrix-org/matrix-js-sdk/pull/1364)
+ * Customize error payload for oversized messages
+   [\#1352](https://github.com/matrix-org/matrix-js-sdk/pull/1352)
+ * Return null for key backup state when we haven't checked yet
+   [\#1363](https://github.com/matrix-org/matrix-js-sdk/pull/1363)
+ * Added a progressCallback for backup key loading
+   [\#1351](https://github.com/matrix-org/matrix-js-sdk/pull/1351)
+ * Add initialFetch param to willUpdateDevices / devicesUpdated
+   [\#1360](https://github.com/matrix-org/matrix-js-sdk/pull/1360)
+ * Fix race between sending .request and receiving .ready over to_device
+   [\#1359](https://github.com/matrix-org/matrix-js-sdk/pull/1359)
+ * Handle race between sending and await next event from other party
+   [\#1357](https://github.com/matrix-org/matrix-js-sdk/pull/1357)
+ * Add crypto.willUpdateDevices event and make
+   getStoredDevices/getStoredDevicesForUser synchronous
+   [\#1354](https://github.com/matrix-org/matrix-js-sdk/pull/1354)
+ * Fix sender of local echo events in unsigned redactions
+   [\#1350](https://github.com/matrix-org/matrix-js-sdk/pull/1350)
+ * Remove redundant key backup setup path
+   [\#1353](https://github.com/matrix-org/matrix-js-sdk/pull/1353)
+ * Remove some dead code from _retryDecryption
+   [\#1349](https://github.com/matrix-org/matrix-js-sdk/pull/1349)
+ * Don't send key requests until after sync processing is finished
+   [\#1348](https://github.com/matrix-org/matrix-js-sdk/pull/1348)
+ * Prevent attempts to send olm messages to ourselves
+   [\#1346](https://github.com/matrix-org/matrix-js-sdk/pull/1346)
+ * Retry account data upload requests
+   [\#1345](https://github.com/matrix-org/matrix-js-sdk/pull/1345)
+ * Log first known index with megolm session updates
+   [\#1344](https://github.com/matrix-org/matrix-js-sdk/pull/1344)
+ * Prune to_device messages to avoid sending empty messages
+   [\#1343](https://github.com/matrix-org/matrix-js-sdk/pull/1343)
+ * Convert bunch of things to TypeScript
+   [\#1335](https://github.com/matrix-org/matrix-js-sdk/pull/1335)
+ * Add logging when making new Olm sessions
+   [\#1342](https://github.com/matrix-org/matrix-js-sdk/pull/1342)
+ * Fix: handle filter not found
+   [\#1340](https://github.com/matrix-org/matrix-js-sdk/pull/1340)
+ * Make getAccountDataFromServer return null if not found
+   [\#1338](https://github.com/matrix-org/matrix-js-sdk/pull/1338)
+ * Fix setDefaultKeyId to fail if the request fails
+   [\#1336](https://github.com/matrix-org/matrix-js-sdk/pull/1336)
+ * Document setRoomEncryption not modifying room state
+   [\#1328](https://github.com/matrix-org/matrix-js-sdk/pull/1328)
+ * Fix: don't do extra /filter request when enabling lazy loading of members
+   [\#1332](https://github.com/matrix-org/matrix-js-sdk/pull/1332)
+ * Reject attemptAuth promise if no auth flow found
+   [\#1329](https://github.com/matrix-org/matrix-js-sdk/pull/1329)
+ * Fix FilterComponent allowed_values check
+   [\#1327](https://github.com/matrix-org/matrix-js-sdk/pull/1327)
+ * Serialise Olm prekey decryptions
+   [\#1326](https://github.com/matrix-org/matrix-js-sdk/pull/1326)
+ * Fix: crash when backup key needs fixing from corruption issue
+   [\#1324](https://github.com/matrix-org/matrix-js-sdk/pull/1324)
+ * Fix cross-signing/SSSS reset
+   [\#1322](https://github.com/matrix-org/matrix-js-sdk/pull/1322)
+ * Implement QR code reciprocate for self-verification with untrusted MSK
+   [\#1320](https://github.com/matrix-org/matrix-js-sdk/pull/1320)
+
+Changes in [6.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.0.0) (2020-05-05)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.0.0-rc.2...v6.0.0)
+
+ * Add progress callback for key backups
+   [\#1368](https://github.com/matrix-org/matrix-js-sdk/pull/1368)
+
+Changes in [6.0.0-rc.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.0.0-rc.2) (2020-05-01)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.0.0-rc.1...v6.0.0-rc.2)
+
+ * Emit event when a trusted self-key is stored
+   [\#1365](https://github.com/matrix-org/matrix-js-sdk/pull/1365)
+
+Changes in [6.0.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.0.0-rc.1) (2020-04-30)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.3.1-rc.4...v6.0.0-rc.1)
+
+BREAKING CHANGES
+---
+
+ * client.getStoredDevicesForUser and client.getStoredDevices are no longer async
+
+All Changes
+---
+
+ * Add initialFetch param to willUpdateDevices / devicesUpdated
+   [\#1362](https://github.com/matrix-org/matrix-js-sdk/pull/1362)
+ * Fix race between sending .request and receiving .ready over to_device
+   [\#1361](https://github.com/matrix-org/matrix-js-sdk/pull/1361)
+ * Handle race between sending and await next event from other party
+   [\#1358](https://github.com/matrix-org/matrix-js-sdk/pull/1358)
+ * Add crypto.willUpdateDevices event and make
+   getStoredDevices/getStoredDevicesForUser synchronous
+   [\#1356](https://github.com/matrix-org/matrix-js-sdk/pull/1356)
+ * Remove redundant key backup setup path
+   [\#1355](https://github.com/matrix-org/matrix-js-sdk/pull/1355)
+
+Changes in [5.3.1-rc.4](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.3.1-rc.4) (2020-04-23)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.3.1-rc.3...v5.3.1-rc.4)
+
+ * Retry account data upload requests
+   [\#1347](https://github.com/matrix-org/matrix-js-sdk/pull/1347)
+ * Fix: handle filter not found
+   [\#1341](https://github.com/matrix-org/matrix-js-sdk/pull/1341)
+ * Make getAccountDataFromServer return null if not found
+   [\#1339](https://github.com/matrix-org/matrix-js-sdk/pull/1339)
+ * Fix setDefaultKeyId to fail if the request fails
+   [\#1337](https://github.com/matrix-org/matrix-js-sdk/pull/1337)
+ * Fix: don't do extra /filter request when enabling lazy loading of members
+   [\#1333](https://github.com/matrix-org/matrix-js-sdk/pull/1333)
+ * Reject attemptAuth promise if no auth flow found
+   [\#1331](https://github.com/matrix-org/matrix-js-sdk/pull/1331)
+ * Serialise Olm prekey decryptions
+   [\#1330](https://github.com/matrix-org/matrix-js-sdk/pull/1330)
+
+Changes in [5.3.1-rc.3](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.3.1-rc.3) (2020-04-17)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.3.1-rc.2...v5.3.1-rc.3)
+
+ * Fix cross-signing/SSSS reset
+   [\#1323](https://github.com/matrix-org/matrix-js-sdk/pull/1323)
+ * Fix: crash when backup key needs fixing from corruption issue
+   [\#1325](https://github.com/matrix-org/matrix-js-sdk/pull/1325)
+
+Changes in [5.3.1-rc.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.3.1-rc.2) (2020-04-16)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.3.1-rc.1...v5.3.1-rc.2)
+
+ * Implement QR code reciprocate for self-verification with untrusted MSK
+   [\#1321](https://github.com/matrix-org/matrix-js-sdk/pull/1321)
+
+Changes in [5.3.1-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.3.1-rc.1) (2020-04-15)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.3.0-rc.1...v5.3.1-rc.1)
+
+ * Adapt release script for riot-desktop
+   [\#1319](https://github.com/matrix-org/matrix-js-sdk/pull/1319)
+ * Fix: prevent spurious notifications from indexer
+   [\#1318](https://github.com/matrix-org/matrix-js-sdk/pull/1318)
+ * Always create our own user object
+   [\#1317](https://github.com/matrix-org/matrix-js-sdk/pull/1317)
+ * Fix incorrect backup key format in SSSS
+   [\#1311](https://github.com/matrix-org/matrix-js-sdk/pull/1311)
+ * Fix e2ee crash after refreshing after having received a cross-singing key
+   reset
+   [\#1315](https://github.com/matrix-org/matrix-js-sdk/pull/1315)
+ * Fix: catch send errors in SAS verifier
+   [\#1314](https://github.com/matrix-org/matrix-js-sdk/pull/1314)
+ * Clear cross-signing keys when detecting the keys have changed
+   [\#1312](https://github.com/matrix-org/matrix-js-sdk/pull/1312)
+ * Upgrade deps
+   [\#1310](https://github.com/matrix-org/matrix-js-sdk/pull/1310)
+
+Changes in [5.3.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.3.0-rc.1) (2020-04-08)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.2.0...v5.3.0-rc.1)
+
+ * Store key backup key in cache as Uint8Array
+   [\#1308](https://github.com/matrix-org/matrix-js-sdk/pull/1308)
+ * Use the correct request body for the /keys/query endpoint.
+   [\#1307](https://github.com/matrix-org/matrix-js-sdk/pull/1307)
+ * Avoid creating two devices on registration
+   [\#1305](https://github.com/matrix-org/matrix-js-sdk/pull/1305)
+ * Lower max-warnings to 81
+   [\#1306](https://github.com/matrix-org/matrix-js-sdk/pull/1306)
+ * Move key backup key creation before caching
+   [\#1303](https://github.com/matrix-org/matrix-js-sdk/pull/1303)
+ * Expose function to force-reset outgoing room key requests
+   [\#1298](https://github.com/matrix-org/matrix-js-sdk/pull/1298)
+ * Add isSelfVerification property to VerificationRequest
+   [\#1302](https://github.com/matrix-org/matrix-js-sdk/pull/1302)
+ * QR code reciprocation
+   [\#1297](https://github.com/matrix-org/matrix-js-sdk/pull/1297)
+ * Add ability to check symmetric SSSS key before we try to use it
+   [\#1294](https://github.com/matrix-org/matrix-js-sdk/pull/1294)
+ * Add some debug logging for events stuck to bottom of timeline
+   [\#1296](https://github.com/matrix-org/matrix-js-sdk/pull/1296)
+ * Fix: spontanous verification request cancellation under some circumstances
+   [\#1295](https://github.com/matrix-org/matrix-js-sdk/pull/1295)
+ * Receive private key for caching from the app layer
+   [\#1293](https://github.com/matrix-org/matrix-js-sdk/pull/1293)
+ * Track whether we have verified a user before
+   [\#1292](https://github.com/matrix-org/matrix-js-sdk/pull/1292)
+ * Fix: error during tests
+   [\#1222](https://github.com/matrix-org/matrix-js-sdk/pull/1222)
+ * Send .done event for to_device verification
+   [\#1288](https://github.com/matrix-org/matrix-js-sdk/pull/1288)
+ * Request the key backup key & restore backup
+   [\#1291](https://github.com/matrix-org/matrix-js-sdk/pull/1291)
+ * Make screen sharing works on Chrome using getDisplayMedia()
+   [\#1276](https://github.com/matrix-org/matrix-js-sdk/pull/1276)
+ * Fix isVerified returning false
+   [\#1289](https://github.com/matrix-org/matrix-js-sdk/pull/1289)
+ * Fix: verification gets cancelled when event gets duplicated
+   [\#1286](https://github.com/matrix-org/matrix-js-sdk/pull/1286)
+ * Use requestSecret on the client to request secrets
+   [\#1287](https://github.com/matrix-org/matrix-js-sdk/pull/1287)
+ * Allow guests to fetch TURN servers
+   [\#1277](https://github.com/matrix-org/matrix-js-sdk/pull/1277)
+
 Changes in [5.2.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.2.0) (2020-03-30)
 ================================================================================================
 [Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.2.0-rc.1...v5.2.0)

CONTRIBUTING.rst

@@ -28,10 +28,9 @@ use GitHub's pull request workflow to review the contribution, and either ask
 you to make any refinements needed or merge it and make them ourselves. The
 changes will then land on master when we next do a release.
 
-We use Travis for continuous integration, and all pull requests get
-automatically tested by Travis: if your change breaks the build, then the PR
-will show that there are failed checks, so please check back after a few
-minutes.
+We use continuous integration, and all pull requests get automatically tested:
+if your change breaks the build, then the PR will show that there are failed
+checks, so please check back after a few minutes.
 
 Code style
 ~~~~~~~~~~

README.md

@@ -30,7 +30,7 @@ This SDK targets Node 10 for compatibility, which translates to ES6. If you're u
 a bundler like webpack you'll likely have to transpile dependencies, including this
 SDK, to match your target browsers.
 
-Using `yarn` instead of `npm` is recommended. Please see the Yarn [install guide](https://yarnpkg.com/docs/install/) 
+Using `yarn` instead of `npm` is recommended. Please see the Yarn [install guide](https://classic.yarnpkg.com/en/docs/install) 
 if you do not have it already.
 
 ``yarn add matrix-js-sdk``
@@ -182,10 +182,8 @@ you can pass the result of the promise into it with something like:
   matrixClient.someMethod(arg1, arg2).nodeify(callback);
 ```
 
-The main thing to note is that it is an error to discard the result of a
-promise-returning function, as that will cause exceptions to go unobserved. If
-you have nothing better to do with the result, just call ``.done()`` on it. See
-http://documentup.com/kriskowal/q/#the-end for more information.
+The main thing to note is that it is problematic to discard the result of a
+promise-returning function, as that will cause exceptions to go unobserved.
 
 Methods which return a promise show this in their documentation.
 

examples/node/app.js

@@ -288,7 +288,7 @@ function printMemberList(room) {
 }
 
 function printRoomInfo(room) {
-    var eventDict = room.currentState.events;
+    var eventMap = room.currentState.events;
     var eTypeHeader = "    Event Type(state_key)    ";
     var sendHeader = "        Sender        ";
     // pad content to 100
@@ -300,14 +300,15 @@ function printRoomInfo(room) {
     var contentHeader = padSide + "Content" + padSide;
     print(eTypeHeader+sendHeader+contentHeader);
     print(new Array(100).join("-"));
-    Object.keys(eventDict).forEach(function(eventType) {
+    eventMap.keys().forEach(function(eventType) {
         if (eventType === "m.room.member") { return; } // use /members instead.
-        Object.keys(eventDict[eventType]).forEach(function(stateKey) {
+        var eventEventMap = eventMap.get(eventType);
+        eventEventMap.keys().forEach(function(stateKey) {
             var typeAndKey = eventType + (
                 stateKey.length > 0 ? "("+stateKey+")" : ""
             );
             var typeStr = fixWidth(typeAndKey, eTypeHeader.length);
-            var event = eventDict[eventType][stateKey];
+            var event = eventEventMap.get(stateKey);
             var sendStr = fixWidth(event.getSender(), sendHeader.length);
             var contentStr = fixWidth(
                 JSON.stringify(event.getContent()), contentHeader.length

jsdoc.json

@@ -18,6 +18,13 @@
     "readme": "README.md",
     "recurse": true,
     "verbose": true,
-    "template": "node_modules/better-docs"
+    "template": "node_modules/docdash"
+  },
+  "docdash": {
+    "static": true,
+    "private": false,
+    "search": true,
+    "collapse": true,
+    "typedefs": true
   }
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "matrix-js-sdk",
-  "version": "5.2.0",
+  "version": "9.2.0",
   "description": "Matrix Client-Server SDK for Javascript",
   "scripts": {
     "prepare": "yarn build",
@@ -13,10 +13,9 @@
     "build:compile-browser": "mkdirp dist && browserify -d src/browser-index.js -p [ tsify -p ./tsconfig.json ] -t [ babelify --sourceMaps=inline --presets [ @babel/preset-env @babel/preset-typescript ] ] | exorcist dist/browser-matrix.js.map > dist/browser-matrix.js",
     "build:minify-browser": "terser dist/browser-matrix.js --compress --mangle --source-map --output dist/browser-matrix.min.js",
     "gendoc": "jsdoc -c jsdoc.json -P package.json",
-    "lint": "yarn lint:types && yarn lint:ts && yarn lint:js",
-    "lint:js": "eslint --max-warnings 93 src spec",
+    "lint": "yarn lint:types && yarn lint:js",
+    "lint:js": "eslint --max-warnings 73 src spec",
     "lint:types": "tsc --noEmit",
-    "lint:ts": "tslint --project ./tsconfig.json -t stylish",
     "test": "jest spec/ --coverage --testEnvironment node",
     "test:watch": "jest spec/ --coverage --testEnvironment node --watch"
   },
@@ -35,6 +34,7 @@
   "author": "matrix.org",
   "license": "Apache-2.0",
   "files": [
+    "dist",
     "lib",
     "src",
     "git-revision.txt",
@@ -46,49 +46,50 @@
     "release.sh"
   ],
   "dependencies": {
-    "@babel/runtime": "^7.8.3",
+    "@babel/runtime": "^7.11.2",
     "another-json": "^0.2.0",
     "browser-request": "^0.3.3",
     "bs58": "^4.0.1",
-    "content-type": "^1.0.2",
-    "loglevel": "^1.6.4",
-    "qs": "^6.5.2",
-    "request": "^2.88.0",
-    "unhomoglyph": "^1.0.2"
+    "content-type": "^1.0.4",
+    "loglevel": "^1.7.0",
+    "qs": "^6.9.4",
+    "request": "^2.88.2",
+    "unhomoglyph": "^1.0.6"
   },
   "devDependencies": {
-    "@babel/cli": "^7.7.5",
-    "@babel/core": "^7.7.5",
-    "@babel/plugin-proposal-class-properties": "^7.7.4",
-    "@babel/plugin-proposal-numeric-separator": "^7.7.4",
-    "@babel/plugin-proposal-object-rest-spread": "^7.7.4",
-    "@babel/plugin-syntax-dynamic-import": "^7.7.4",
-    "@babel/plugin-transform-runtime": "^7.8.3",
-    "@babel/preset-env": "^7.7.6",
-    "@babel/preset-typescript": "^7.7.4",
-    "@babel/register": "^7.7.4",
+    "@babel/cli": "^7.11.6",
+    "@babel/core": "^7.11.6",
+    "@babel/plugin-proposal-class-properties": "^7.10.4",
+    "@babel/plugin-proposal-numeric-separator": "^7.10.4",
+    "@babel/plugin-proposal-object-rest-spread": "^7.11.0",
+    "@babel/plugin-syntax-dynamic-import": "^7.8.3",
+    "@babel/plugin-transform-runtime": "^7.11.5",
+    "@babel/preset-env": "^7.11.5",
+    "@babel/preset-typescript": "^7.10.4",
+    "@babel/register": "^7.11.5",
+    "@types/jest": "^26.0.14",
     "@types/node": "12",
-    "babel-eslint": "^10.0.3",
+    "@types/request": "^2.48.5",
+    "babel-eslint": "^10.1.0",
     "babel-jest": "^24.9.0",
     "babelify": "^10.0.0",
-    "better-docs": "^1.4.7",
-    "browserify": "^16.5.0",
-    "eslint": "^5.12.0",
-    "eslint-config-google": "^0.7.1",
-    "eslint-plugin-babel": "^5.3.0",
-    "eslint-plugin-jest": "^23.0.4",
+    "better-docs": "^2.3.2",
+    "browserify": "^16.5.2",
+    "docdash": "^1.2.0",
+    "eslint": "7.9.0",
+    "eslint-config-matrix-org": "^0.1.2",
+    "eslint-plugin-babel": "^5.3.1",
     "exorcist": "^1.0.1",
-    "fake-indexeddb": "^3.0.0",
+    "fake-indexeddb": "^3.1.2",
     "jest": "^24.9.0",
-    "jest-localstorage-mock": "^2.4.0",
-    "jsdoc": "^3.5.5",
+    "jest-localstorage-mock": "^2.4.3",
+    "jsdoc": "^3.6.6",
     "matrix-mock-request": "^1.2.3",
-    "olm": "https://packages.matrix.org/npm/olm/olm-3.1.4.tgz",
-    "rimraf": "^3.0.0",
-    "terser": "^4.4.3",
-    "tsify": "^4.0.1",
-    "tslint": "^5.20.1",
-    "typescript": "^3.7.3"
+    "olm": "https://packages.matrix.org/npm/olm/olm-3.2.1.tgz",
+    "rimraf": "^3.0.2",
+    "terser": "^4.8.0",
+    "tsify": "^4.0.2",
+    "typescript": "^3.9.7"
   },
   "jest": {
     "testEnvironment": "node"

release.sh

@@ -10,7 +10,7 @@
 #   npm; typically installed by Node.js
 #   yarn; install via brew (macOS) or similar (https://yarnpkg.com/docs/install/)
 #
-# Note: this script is also used to release matrix-react-sdk and riot-web.
+# Note: this script is also used to release matrix-react-sdk and element-web.
 
 set -e
 
@@ -38,6 +38,7 @@ $USAGE
     -c changelog_file:  specify name of file containing changelog
     -x:                 skip updating the changelog
     -z:                 skip generating the jsdoc
+    -n:                 skip publish to NPM
 EOF
 }
 
@@ -60,9 +61,10 @@ fi
 
 skip_changelog=
 skip_jsdoc=
+skip_npm=
 changelog_file="CHANGELOG.md"
 expected_npm_user="matrixdotorg"
-while getopts hc:u:xz f; do
+while getopts hc:u:xzn f; do
     case $f in
         h)
             help
@@ -77,6 +79,9 @@ while getopts hc:u:xz f; do
         z)
             skip_jsdoc=1
             ;;
+        n)
+            skip_npm=1
+            ;;
         u)
             expected_npm_user="$OPTARG"
             ;;
@@ -89,6 +94,14 @@ if [ $# -ne 1 ]; then
     exit 1
 fi
 
+# We use Git branch / commit dependencies for some packages, and Yarn seems
+# to have a hard time getting that right. See also
+# https://github.com/yarnpkg/yarn/issues/4734. As a workaround, we clean the
+# global cache here to ensure we get the right thing.
+yarn cache clean
+# Ensure all dependencies are updated
+yarn install --ignore-scripts
+
 if [ -z "$skip_changelog" ]; then
     # update_changelog doesn't have a --version flag
     update_changelog -h > /dev/null || (echo "github-changelog-generator is required: please install it"; exit)
@@ -96,10 +109,12 @@ fi
 
 # Login and publish continues to use `npm`, as it seems to have more clearly
 # defined options and semantics than `yarn` for writing to the registry.
-actual_npm_user=`npm whoami`;
-if [ $expected_npm_user != $actual_npm_user ]; then
-    echo "you need to be logged into npm as $expected_npm_user, but you are logged in as $actual_npm_user" >&2
-    exit 1
+if [ -z "$skip_npm" ]; then
+    actual_npm_user=`npm whoami`;
+    if [ $expected_npm_user != $actual_npm_user ]; then
+        echo "you need to be logged into npm as $expected_npm_user, but you are logged in as $actual_npm_user" >&2
+        exit 1
+    fi
 fi
 
 # ignore leading v on release
@@ -197,11 +212,6 @@ if [ $dodist -eq 0 ]; then
     pushd "$builddir"
     git clone "$projdir" .
     git checkout "$rel_branch"
-    # We use Git branch / commit dependencies for some packages, and Yarn seems
-    # to have a hard time getting that right. See also
-    # https://github.com/yarnpkg/yarn/issues/4734. As a workaround, we clean the
-    # global cache here to ensure we get the right thing.
-    yarn cache clean
     yarn install
     # We haven't tagged yet, so tell the dist script what version
     # it's building
@@ -298,11 +308,13 @@ rm "${latest_changes}"
 # defined options and semantics than `yarn` for writing to the registry.
 # Tag both releases and prereleases as `next` so the last stable release remains
 # the default.
-npm publish --tag next
-if [ $prerelease -eq 0 ]; then
-    # For a release, also add the default `latest` tag.
-    package=$(cat package.json | jq -er .name)
-    npm dist-tag add "$package@$release" latest
+if [ -z "$skip_npm" ]; then
+    npm publish --tag next
+    if [ $prerelease -eq 0 ]; then
+        # For a release, also add the default `latest` tag.
+        package=$(cat package.json | jq -er .name)
+        npm dist-tag add "$package@$release" latest
+    fi
 fi
 
 if [ -z "$skip_jsdoc" ]; then
@@ -318,6 +330,7 @@ if [ -z "$skip_jsdoc" ]; then
         $release index.html
     git add "$release"
     git commit --no-verify -m "Add jsdoc for $release" index.html "$release"
+    git push origin gh-pages
 fi
 
 # if it is a pre-release, leave it on the release branch for now.
@@ -330,16 +343,15 @@ fi
 echo "updating master branch"
 git checkout master
 git pull
-git merge "$rel_branch"
+git merge "$rel_branch" --no-edit
 
-# push master and docs (if generated) to github
+# push master to github
 git push origin master
-if [ -z "$skip_jsdoc" ]; then
-    git push origin gh-pages
-fi
 
-# finally, merge master back onto develop
-git checkout develop
-git pull
-git merge master
-git push origin develop
+# finally, merge master back onto develop (if it exists)
+if [ $(git branch -lr | grep origin/develop -c) -ge 1 ]; then
+    git checkout develop
+    git pull
+    git merge master --no-edit
+    git push origin develop
+fi

spec/TestClient.js

@@ -69,6 +69,9 @@ export function TestClient(
 
     this.deviceKeys = null;
     this.oneTimeKeys = {};
+    this._callEventHandler = {
+        calls: new Map(),
+    };
 }
 
 TestClient.prototype.toString = function() {
@@ -185,7 +188,7 @@ TestClient.prototype.expectKeyQuery = function(response) {
         200, (path, content) => {
             Object.keys(response.device_keys).forEach((userId) => {
                 expect(content.device_keys[userId]).toEqual(
-                    {},
+                    [],
                     "Expected key query for " + userId + ", got " +
                     Object.keys(content.device_keys),
                 );
@@ -230,3 +233,7 @@ TestClient.prototype.flushSync = function() {
         logger.log(`${this}: flushSync completed`);
     });
 };
+
+TestClient.prototype.isFallbackICEServerAllowed = function() {
+    return true;
+};

spec/browserify/setupTests.js

@@ -0,0 +1,23 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// stub for browser-matrix browserify tests
+global.XMLHttpRequest = jest.fn();
+
+afterAll(() => {
+    // clean up XMLHttpRequest mock
+    global.XMLHttpRequest = undefined;
+});

spec/browserify/sync-browserify.spec.js

@@ -0,0 +1,103 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// load XmlHttpRequest mock
+import "./setupTests";
+import "../../dist/browser-matrix"; // uses browser-matrix instead of the src
+import {MockStorageApi} from "../MockStorageApi";
+import {WebStorageSessionStore} from "../../src/store/session/webstorage";
+import MockHttpBackend from "matrix-mock-request";
+import {LocalStorageCryptoStore} from "../../src/crypto/store/localStorage-crypto-store";
+import * as utils from "../test-utils";
+
+const USER_ID = "@user:test.server";
+const DEVICE_ID = "device_id";
+const ACCESS_TOKEN = "access_token";
+const ROOM_ID = "!room_id:server.test";
+
+/* global matrixcs */
+
+describe("Browserify Test", function() {
+    let client;
+    let httpBackend;
+
+    async function createTestClient() {
+        const sessionStoreBackend = new MockStorageApi();
+        const sessionStore = new WebStorageSessionStore(sessionStoreBackend);
+        const httpBackend = new MockHttpBackend();
+
+        const options = {
+            baseUrl: "http://" + USER_ID + ".test.server",
+            userId: USER_ID,
+            accessToken: ACCESS_TOKEN,
+            deviceId: DEVICE_ID,
+            sessionStore: sessionStore,
+            request: httpBackend.requestFn,
+            cryptoStore: new LocalStorageCryptoStore(sessionStoreBackend),
+        };
+
+        const client = matrixcs.createClient(options);
+
+        httpBackend.when("GET", "/pushrules").respond(200, {});
+        httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
+
+        return { client, httpBackend };
+    }
+
+    beforeEach(async () => {
+        ({client, httpBackend} = await createTestClient());
+        await client.startClient();
+    });
+
+    afterEach(async () => {
+        client.stopClient();
+        await httpBackend.stop();
+    });
+
+    it("Sync", async function() {
+        const event = utils.mkMembership({
+            room: ROOM_ID,
+            mship: "join",
+            user: "@other_user:server.test",
+            name: "Displayname",
+        });
+
+        const syncData = {
+            next_batch: "batch1",
+            rooms: {
+                join: {},
+            },
+        };
+        syncData.rooms.join[ROOM_ID] = {
+            timeline: {
+                events: [
+                    event,
+                ],
+                limited: false,
+            },
+        };
+
+        httpBackend.when("GET", "/sync").respond(200, syncData);
+        await Promise.race([
+            Promise.all([
+                httpBackend.flushAllExpected(),
+            ]),
+            new Promise((_, reject) => {
+                client.once("sync.unexpectedError", reject);
+            }),
+        ]);
+    }, 20000); // additional timeout as this test can take quite a while
+});

spec/integ/devicelist-integ-spec.js

@@ -140,7 +140,7 @@ describe("DeviceList management:", function() {
 
     it("We should not get confused by out-of-order device query responses",
        () => {
-           // https://github.com/vector-im/riot-web/issues/3126
+           // https://github.com/vector-im/element-web/issues/3126
            aliceTestClient.expectKeyQuery({device_keys: {'@alice:localhost': {}}});
            return aliceTestClient.start().then(() => {
                aliceTestClient.httpBackend.when('GET', '/sync').respond(
@@ -271,7 +271,7 @@ describe("DeviceList management:", function() {
            });
        }).timeout(3000);
 
-    // https://github.com/vector-im/riot-web/issues/4983
+    // https://github.com/vector-im/element-web/issues/4983
     describe("Alice should know she has stale device lists", () => {
         beforeEach(async function() {
             await aliceTestClient.start();

spec/integ/matrix-client-crypto.spec.js

@@ -70,7 +70,7 @@ function expectAliQueryKeys() {
     aliTestClient.httpBackend.when("POST", "/keys/query")
             .respond(200, function(path, content) {
         expect(content.device_keys[bobUserId]).toEqual(
-            {},
+            [],
             "Expected Alice to key query for " + bobUserId + ", got " +
             Object.keys(content.device_keys),
         );
@@ -98,7 +98,7 @@ function expectBobQueryKeys() {
         "POST", "/keys/query",
     ).respond(200, function(path, content) {
         expect(content.device_keys[aliUserId]).toEqual(
-            {},
+            [],
             "Expected Bob to key query for " + aliUserId + ", got " +
             Object.keys(content.device_keys),
         );

spec/integ/matrix-client-methods.spec.js

@@ -347,8 +347,8 @@ describe("MatrixClient", function() {
 
             httpBackend.when("POST", "/keys/query").check(function(req) {
                 expect(req.data).toEqual({device_keys: {
-                    'boris': {},
-                    'chaz': {},
+                    'boris': [],
+                    'chaz': [],
                 }});
             }).respond(200, {
                 device_keys: {

spec/integ/matrix-client-opts.spec.js

@@ -3,6 +3,7 @@ import HttpBackend from "matrix-mock-request";
 import {MatrixClient} from "../../src/matrix";
 import {MatrixScheduler} from "../../src/scheduler";
 import {MemoryStore} from "../../src/store/memory";
+import {MatrixError} from "../../src/http-api";
 
 describe("MatrixClient opts", function() {
     const baseUrl = "http://localhost.or.something";
@@ -132,10 +133,10 @@ describe("MatrixClient opts", function() {
         });
 
         it("shouldn't retry sending events", function(done) {
-            httpBackend.when("PUT", "/txn1").fail(500, {
+            httpBackend.when("PUT", "/txn1").fail(500, new MatrixError({
                 errcode: "M_SOMETHING",
                 error: "Ruh roh",
-            });
+            }));
             client.sendTextMessage("!foo:bar", "a body", "txn1").then(function(res) {
                 expect(false).toBe(true, "sendTextMessage resolved but shouldn't");
             }, function(err) {

spec/integ/matrix-client-retrying.spec.js

@@ -90,7 +90,7 @@ describe("MatrixClient retrying", function() {
         // wait for the localecho of ev1 to be updated
         const p3 = new Promise((resolve, reject) => {
             room.on("Room.localEchoUpdated", (ev0) => {
-                if(ev0 === ev1) {
+                if (ev0 === ev1) {
                     resolve();
                 }
             });

spec/integ/megolm-integ.spec.js

@@ -346,7 +346,7 @@ describe("megolm", function() {
     });
 
     it("Alice receives a megolm message before the session keys", function() {
-        // https://github.com/vector-im/riot-web/issues/2273
+        // https://github.com/vector-im/element-web/issues/2273
         let roomKeyEncrypted;
 
         return aliceTestClient.start().then(() => {
@@ -726,7 +726,7 @@ describe("megolm", function() {
         });
     });
 
-    // https://github.com/vector-im/riot-web/issues/2676
+    // https://github.com/vector-im/element-web/issues/2676
     it("Alice should send to her other devices", function() {
         // for this test, we make the testOlmAccount be another of Alice's devices.
         // it ought to get included in messages Alice sends.

spec/olm-loader.js

@@ -16,6 +16,7 @@ limitations under the License.
 */
 
 import {logger} from '../src/logger';
+import * as utils from "../src/utils";
 
 // try to load the olm library.
 try {
@@ -24,3 +25,11 @@ try {
 } catch (e) {
     logger.warn("unable to run crypto tests: libolm not available");
 }
+
+// also try to set node crypto
+try {
+    const crypto = require('crypto');
+    utils.setCrypto(crypto);
+} catch (err) {
+    logger.log('nodejs was compiled without crypto support: some tests will fail');
+}

spec/unit/content-repo.spec.js

@@ -1,4 +1,4 @@
-import {getHttpUriForMxc, getIdenticonUri} from "../../src/content-repo";
+import {getHttpUriForMxc} from "../../src/content-repo";
 
 describe("ContentRepo", function() {
     const baseUrl = "https://my.home.server";
@@ -56,31 +56,4 @@ describe("ContentRepo", function() {
             );
         });
     });
-
-    describe("getIdenticonUri", function() {
-        it("should do nothing for null input", function() {
-            expect(getIdenticonUri(null)).toEqual(null);
-        });
-
-        it("should set w/h by default to 96", function() {
-            expect(getIdenticonUri(baseUrl, "foobar")).toEqual(
-                baseUrl + "/_matrix/media/unstable/identicon/foobar" +
-                "?width=96&height=96",
-            );
-        });
-
-        it("should be able to set custom w/h", function() {
-            expect(getIdenticonUri(baseUrl, "foobar", 32, 64)).toEqual(
-                baseUrl + "/_matrix/media/unstable/identicon/foobar" +
-                "?width=32&height=64",
-            );
-        });
-
-        it("should URL encode the identicon string", function() {
-            expect(getIdenticonUri(baseUrl, "foo#bar", 32, 64)).toEqual(
-                baseUrl + "/_matrix/media/unstable/identicon/foo%23bar" +
-                "?width=32&height=64",
-            );
-        });
-    });
 });

spec/unit/crypto.spec.js

@@ -10,6 +10,7 @@ import * as olmlib from "../../src/crypto/olmlib";
 import {sleep} from "../../src/utils";
 import {EventEmitter} from "events";
 import {CRYPTO_ENABLED} from "../../src/client";
+import {DeviceInfo} from "../../src/crypto/deviceinfo";
 
 const Olm = global.Olm;
 
@@ -26,6 +27,66 @@ describe("Crypto", function() {
         expect(Crypto.getOlmVersion()[0]).toEqual(3);
     });
 
+    describe("encrypted events", function() {
+        it("provides encryption information", async function() {
+            const client = (new TestClient(
+                "@alice:example.com", "deviceid",
+            )).client;
+            await client.initCrypto();
+
+            // unencrypted event
+            const event = {
+                getId: () => "$event_id",
+                getSenderKey: () => null,
+                getWireContent: () => {return {};},
+            };
+
+            let encryptionInfo = client.getEventEncryptionInfo(event);
+            expect(encryptionInfo.encrypted).toBeFalsy();
+
+            // unknown sender (e.g. deleted device), forwarded megolm key (untrusted)
+            event.getSenderKey = () => 'YmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmI';
+            event.getWireContent = () => {return {algorithm: olmlib.MEGOLM_ALGORITHM};};
+            event.getForwardingCurve25519KeyChain = () => ["not empty"];
+            event.isKeySourceUntrusted = () => false;
+            event.getClaimedEd25519Key =
+                () => 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA';
+
+            encryptionInfo = client.getEventEncryptionInfo(event);
+            expect(encryptionInfo.encrypted).toBeTruthy();
+            expect(encryptionInfo.authenticated).toBeFalsy();
+            expect(encryptionInfo.sender).toBeFalsy();
+
+            // known sender, megolm key from backup
+            event.getForwardingCurve25519KeyChain = () => [];
+            event.isKeySourceUntrusted = () => true;
+            const device = new DeviceInfo("FLIBBLE");
+            device.keys["curve25519:FLIBBLE"] =
+                'YmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmI';
+            device.keys["ed25519:FLIBBLE"] =
+                'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA';
+            client._crypto._deviceList.getDeviceByIdentityKey = () => device;
+
+            encryptionInfo = client.getEventEncryptionInfo(event);
+            expect(encryptionInfo.encrypted).toBeTruthy();
+            expect(encryptionInfo.authenticated).toBeFalsy();
+            expect(encryptionInfo.sender).toBeTruthy();
+            expect(encryptionInfo.mismatchedSender).toBeFalsy();
+
+            // known sender, trusted megolm key, but bad ed25519key
+            event.isKeySourceUntrusted = () => false;
+            device.keys["ed25519:FLIBBLE"] =
+                'BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB';
+
+            encryptionInfo = client.getEventEncryptionInfo(event);
+            expect(encryptionInfo.encrypted).toBeTruthy();
+            expect(encryptionInfo.authenticated).toBeTruthy();
+            expect(encryptionInfo.sender).toBeTruthy();
+            expect(encryptionInfo.mismatchedSender).toBeTruthy();
+
+            client.stopClient();
+        });
+    });
 
     describe('Session management', function() {
         const otkResponse = {
@@ -313,6 +374,10 @@ describe("Crypto", function() {
             // make a room key request, and record the transaction ID for the
             // sendToDevice call
             await aliceClient.cancelAndResendEventRoomKeyRequest(event);
+            // key requests get queued until the sync has finished, but we don't
+            // let the client set up enough for that to happen, so gut-wrench a bit
+            // to force it to send now.
+            aliceClient._crypto._outgoingRoomKeyRequestManager.sendQueuedRequests();
             jest.runAllTimers();
             await Promise.resolve();
             expect(aliceClient.sendToDevice).toBeCalledTimes(1);

spec/unit/crypto/CrossSigningInfo.spec.js

@@ -25,6 +25,8 @@ import {
 import {MemoryCryptoStore} from '../../../src/crypto/store/memory-crypto-store';
 import 'fake-indexeddb/auto';
 import 'jest-localstorage-mock';
+import {OlmDevice} from "../../../src/crypto/OlmDevice";
+import {logger} from '../../../src/logger';
 
 const userId = "@alice:example.com";
 
@@ -37,7 +39,7 @@ const testKey = new Uint8Array([
 ]);
 
 const types = [
-    { type: "master", shouldCache: false },
+    { type: "master", shouldCache: true },
     { type: "self_signing", shouldCache: true },
     { type: "user_signing", shouldCache: true },
     { type: "invalid", shouldCache: false },
@@ -50,7 +52,7 @@ const masterKeyPub = "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk";
 
 describe("CrossSigningInfo.getCrossSigningKey", function() {
     if (!global.Olm) {
-        console.warn('Not running megolm backup unit tests: libolm not present');
+        logger.warn('Not running megolm backup unit tests: libolm not present');
         return;
     }
 
@@ -82,9 +84,16 @@ describe("CrossSigningInfo.getCrossSigningKey", function() {
         const info = new CrossSigningInfo(userId, {
             getCrossSigningKey: () => testKey,
         });
-        const [pubKey, ab] = await info.getCrossSigningKey("master", masterKeyPub);
+        const [pubKey, pkSigning] = await info.getCrossSigningKey("master", masterKeyPub);
         expect(pubKey).toEqual(masterKeyPub);
-        expect(ab).toEqual({a: 106712, b: 106712});
+        // check that the pkSigning object corresponds to the pubKey
+        const signature = pkSigning.sign("message");
+        const util = new global.Olm.Utility();
+        try {
+            util.ed25519_verify(pubKey, "message", signature);
+        } finally {
+            util.free();
+        }
     });
 
     it.each(types)("should request a key from the cache callback (if set)" +
@@ -233,8 +242,9 @@ describe.each([
 
     it("should cache data to the store and retrieve it", async () => {
         await store.startup();
+        const olmDevice = new OlmDevice(store);
         const { getCrossSigningKeyCache, storeCrossSigningKeyCache } =
-          createCryptoStoreCacheCallbacks(store);
+              createCryptoStoreCacheCallbacks(store, olmDevice);
         await storeCrossSigningKeyCache("self_signing", testKey);
 
         // If we've not saved anything, don't expect anything
@@ -243,6 +253,6 @@ describe.each([
         expect(nokey).toBeNull();
 
         const key = await getCrossSigningKeyCache("self_signing", "");
-        expect(key).toEqual(testKey);
+        expect(new Uint8Array(key)).toEqual(testKey);
     });
 });

spec/unit/crypto/DeviceList.spec.js

@@ -72,6 +72,8 @@ describe('DeviceList', function() {
     function createTestDeviceList() {
         const baseApis = {
             downloadKeysForUsers: downloadSpy,
+            getUserId: () => '@test1:sw1v.org',
+            deviceId: 'HGKAWHRVJQ',
         };
         const mockOlm = {
             verifySignature: function(key, message, signature) {},

spec/unit/crypto/backup.spec.js

@@ -27,6 +27,7 @@ import {MockStorageApi} from "../../MockStorageApi";
 import * as testUtils from "../../test-utils";
 import {OlmDevice} from "../../../src/crypto/OlmDevice";
 import {Crypto} from "../../../src/crypto";
+import {resetCrossSigningKeys} from "./crypto-utils";
 
 const Olm = global.Olm;
 
@@ -332,7 +333,7 @@ describe("MegolmBackup", function() {
             client.on("crossSigning.getKey", function(e) {
                 e.done(privateKeys[e.type]);
             });
-            await client.resetCrossSigningKeys();
+            await resetCrossSigningKeys(client);
             let numCalls = 0;
             await new Promise((resolve, reject) => {
                 client._http.authedRequest = function(
@@ -517,6 +518,7 @@ describe("MegolmBackup", function() {
                 return megolmDecryption.decryptEvent(ENCRYPTED_EVENT);
             }).then((res) => {
                 expect(res.clearEvent.content).toEqual('testytest');
+                expect(res.untrusted).toBeTruthy(); // keys from backup are untrusted
             });
         });
 
@@ -546,7 +548,7 @@ describe("MegolmBackup", function() {
             const key = Uint8Array.from([1, 2, 3, 4, 5, 6, 7, 8]);
             await client._crypto.storeSessionBackupPrivateKey(key);
             const result = await client._crypto.getSessionBackupPrivateKey();
-            expect(result).toEqual(key);
+            expect(new Uint8Array(result)).toEqual(key);
         });
 
         it('caches session backup keys as it encounters them', async function() {

spec/unit/crypto/cross-signing.spec.js

@@ -20,6 +20,9 @@ import anotherjson from 'another-json';
 import * as olmlib from "../../../src/crypto/olmlib";
 import {TestClient} from '../../TestClient';
 import {HttpResponse, setHttpResponses} from '../../test-utils';
+import { resetCrossSigningKeys } from "./crypto-utils";
+import { MatrixError } from '../../../src/http-api';
+import {logger} from '../../../src/logger';
 
 async function makeTestClient(userInfo, options, keys) {
     if (!keys) keys = {};
@@ -47,7 +50,7 @@ async function makeTestClient(userInfo, options, keys) {
 
 describe("Cross Signing", function() {
     if (!global.Olm) {
-        console.warn('Not running megolm backup unit tests: libolm not present');
+        logger.warn('Not running megolm backup unit tests: libolm not present');
         return;
     }
 
@@ -66,11 +69,66 @@ describe("Cross Signing", function() {
             );
         });
         alice.uploadKeySignatures = async () => {};
+        alice.setAccountData = async () => {};
+        alice.getAccountDataFromServer = async () => {};
         // set Alice's cross-signing key
-        await alice.resetCrossSigningKeys();
+        await alice.bootstrapCrossSigning({
+            authUploadDeviceSigningKeys: async func => await func({}),
+        });
         expect(alice.uploadDeviceSigningKeys).toHaveBeenCalled();
     });
 
+    it("should abort bootstrap if device signing auth fails", async function() {
+        const alice = await makeTestClient(
+            {userId: "@alice:example.com", deviceId: "Osborne2"},
+        );
+        alice.uploadDeviceSigningKeys = async (auth, keys) => {
+            const errorResponse = {
+                session: "sessionId",
+                flows: [
+                    {
+                        stages: [
+                            "m.login.password",
+                        ],
+                    },
+                ],
+                params: {},
+            };
+
+            // If we're not just polling for flows, add on error rejecting the
+            // auth attempt.
+            if (auth) {
+                Object.assign(errorResponse, {
+                    completed: [],
+                    error: "Invalid password",
+                    errcode: "M_FORBIDDEN",
+                });
+            }
+
+            const error = new MatrixError(errorResponse);
+            error.httpStatus == 401;
+            throw error;
+        };
+        alice.uploadKeySignatures = async () => {};
+        alice.setAccountData = async () => {};
+        alice.getAccountDataFromServer = async () => { };
+        const authUploadDeviceSigningKeys = async func => await func({});
+
+        // Try bootstrap, expecting `authUploadDeviceSigningKeys` to pass
+        // through failure, stopping before actually applying changes.
+        let bootstrapDidThrow = false;
+        try {
+            await alice.bootstrapCrossSigning({
+                authUploadDeviceSigningKeys,
+            });
+        } catch (e) {
+            if (e.errcode === "M_FORBIDDEN") {
+                bootstrapDidThrow = true;
+            }
+        }
+        expect(bootstrapDidThrow).toBeTruthy();
+    });
+
     it("should upload a signature when a user is verified", async function() {
         const alice = await makeTestClient(
             {userId: "@alice:example.com", deviceId: "Osborne2"},
@@ -78,7 +136,7 @@ describe("Cross Signing", function() {
         alice.uploadDeviceSigningKeys = async () => {};
         alice.uploadKeySignatures = async () => {};
         // set Alice's cross-signing key
-        await alice.resetCrossSigningKeys();
+        await resetCrossSigningKeys(alice);
         // Alice downloads Bob's device key
         alice._crypto._deviceList.storeCrossSigningForUser("@bob:example.com", {
             keys: {
@@ -273,7 +331,7 @@ describe("Cross Signing", function() {
         alice.uploadDeviceSigningKeys = async () => {};
         alice.uploadKeySignatures = async () => {};
         // set Alice's cross-signing key
-        await alice.resetCrossSigningKeys();
+        await resetCrossSigningKeys(alice);
         // Alice downloads Bob's ssk and device key
         const bobMasterSigning = new global.Olm.PkSigning();
         const bobMasterPrivkey = bobMasterSigning.generate_seed();
@@ -363,7 +421,7 @@ describe("Cross Signing", function() {
         alice.uploadKeySignatures = async () => {};
 
         // set Alice's cross-signing key
-        await alice.resetCrossSigningKeys();
+        await resetCrossSigningKeys(alice);
 
         const selfSigningKey = new Uint8Array([
             0x1e, 0xf4, 0x01, 0x6d, 0x4f, 0xa1, 0x73, 0x66,
@@ -520,7 +578,7 @@ describe("Cross Signing", function() {
         alice.uploadDeviceSigningKeys = async () => {};
         alice.uploadKeySignatures = async () => {};
         // set Alice's cross-signing key
-        await alice.resetCrossSigningKeys();
+        await resetCrossSigningKeys(alice);
         // Alice downloads Bob's ssk and device key
         // (NOTE: device key is not signed by ssk)
         const bobMasterSigning = new global.Olm.PkSigning();
@@ -588,7 +646,7 @@ describe("Cross Signing", function() {
         );
         alice.uploadDeviceSigningKeys = async () => {};
         alice.uploadKeySignatures = async () => {};
-        await alice.resetCrossSigningKeys();
+        await resetCrossSigningKeys(alice);
         // Alice downloads Bob's keys
         const bobMasterSigning = new global.Olm.PkSigning();
         const bobMasterPrivkey = bobMasterSigning.generate_seed();
@@ -740,7 +798,7 @@ describe("Cross Signing", function() {
         bob.uploadDeviceSigningKeys = async () => {};
         bob.uploadKeySignatures = async () => {};
         // set Bob's cross-signing key
-        await bob.resetCrossSigningKeys();
+        await resetCrossSigningKeys(bob);
         alice._crypto._deviceList.storeDevicesForUser("@bob:example.com", {
             Dynabook: {
                 algorithms: ["m.olm.curve25519-aes-sha256", "m.megolm.v1.aes-sha"],
@@ -766,7 +824,7 @@ describe("Cross Signing", function() {
         let upgradePromise = new Promise((resolve) => {
             upgradeResolveFunc = resolve;
         });
-        await alice.resetCrossSigningKeys();
+        await resetCrossSigningKeys(alice);
         await upgradePromise;
 
         const bobTrust = alice.checkUserTrust("@bob:example.com");

spec/unit/crypto/crypto-utils.js

@@ -0,0 +1,44 @@
+import {IndexedDBCryptoStore} from '../../../src/crypto/store/indexeddb-crypto-store';
+
+
+// needs to be phased out and replaced with bootstrapSecretStorage,
+// but that is doing too much extra stuff for it to be an easy transition.
+export async function resetCrossSigningKeys(client, {
+    level,
+    authUploadDeviceSigningKeys = async func => await func(),
+} = {}) {
+    const crypto = client._crypto;
+
+    const oldKeys = Object.assign({}, crypto._crossSigningInfo.keys);
+    try {
+        await crypto._crossSigningInfo.resetKeys(level);
+        await crypto._signObject(crypto._crossSigningInfo.keys.master);
+        // write a copy locally so we know these are trusted keys
+        await crypto._cryptoStore.doTxn(
+            'readwrite', [IndexedDBCryptoStore.STORE_ACCOUNT],
+            (txn) => {
+                crypto._cryptoStore.storeCrossSigningKeys(
+                    txn, crypto._crossSigningInfo.keys);
+            },
+        );
+    } catch (e) {
+        // If anything failed here, revert the keys so we know to try again from the start
+        // next time.
+        crypto._crossSigningInfo.keys = oldKeys;
+        throw e;
+    }
+    crypto._baseApis.emit("crossSigning.keysChanged", {});
+    await crypto._afterCrossSigningLocalKeyChange();
+}
+
+export async function createSecretStorageKey() {
+    const decryption = new global.Olm.PkDecryption();
+    const storagePublicKey = decryption.generate_key();
+    const storagePrivateKey = decryption.get_private_key();
+    decryption.free();
+    return {
+        // `pubkey` not used anymore with symmetric 4S
+        keyInfo: { pubkey: storagePublicKey },
+        privateKey: storagePrivateKey,
+    };
+}

spec/unit/crypto/outgoing-room-key-requests.spec.js

@@ -0,0 +1,87 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {
+    IndexedDBCryptoStore,
+} from '../../../src/crypto/store/indexeddb-crypto-store';
+import {MemoryCryptoStore} from '../../../src/crypto/store/memory-crypto-store';
+import 'fake-indexeddb/auto';
+import 'jest-localstorage-mock';
+
+import {
+    ROOM_KEY_REQUEST_STATES,
+} from '../../../src/crypto/OutgoingRoomKeyRequestManager';
+
+const requests = [
+    {
+        requestId: "A",
+        requestBody: { session_id: "A", room_id: "A" },
+        state: ROOM_KEY_REQUEST_STATES.SENT,
+    },
+    {
+        requestId: "B",
+        requestBody: { session_id: "B", room_id: "B" },
+        state: ROOM_KEY_REQUEST_STATES.SENT,
+    },
+    {
+        requestId: "C",
+        requestBody: { session_id: "C", room_id: "C" },
+        state: ROOM_KEY_REQUEST_STATES.UNSENT,
+    },
+];
+
+describe.each([
+    ["IndexedDBCryptoStore",
+     () => new IndexedDBCryptoStore(global.indexedDB, "tests")],
+    ["LocalStorageCryptoStore",
+     () => new IndexedDBCryptoStore(undefined, "tests")],
+    ["MemoryCryptoStore", () => {
+        const store = new IndexedDBCryptoStore(undefined, "tests");
+        store._backend = new MemoryCryptoStore();
+        store._backendPromise = Promise.resolve(store._backend);
+        return store;
+    }],
+])("Outgoing room key requests [%s]", function(name, dbFactory) {
+    let store;
+
+    beforeAll(async () => {
+        store = dbFactory();
+        await store.startup();
+        await Promise.all(requests.map((request) =>
+            store.getOrAddOutgoingRoomKeyRequest(request),
+        ));
+    });
+
+    it("getAllOutgoingRoomKeyRequestsByState retrieves all entries in a given state",
+    async () => {
+        const r = await
+            store.getAllOutgoingRoomKeyRequestsByState(ROOM_KEY_REQUEST_STATES.SENT);
+        expect(r).toHaveLength(2);
+        requests.filter((e) => e.state == ROOM_KEY_REQUEST_STATES.SENT).forEach((e) => {
+            expect(r).toContainEqual(e);
+        });
+    });
+
+    test("getOutgoingRoomKeyRequestByState retrieves any entry in a given state",
+    async () => {
+        const r =
+            await store.getOutgoingRoomKeyRequestByState([ROOM_KEY_REQUEST_STATES.SENT]);
+        expect(r).not.toBeNull();
+        expect(r).not.toBeUndefined();
+        expect(r.state).toEqual(ROOM_KEY_REQUEST_STATES.SENT);
+        expect(requests).toContainEqual(r);
+    });
+});

spec/unit/crypto/secrets.spec.js

@@ -20,6 +20,9 @@ import {SECRET_STORAGE_ALGORITHM_V1_AES} from "../../../src/crypto/SecretStorage
 import {MatrixEvent} from "../../../src/models/event";
 import {TestClient} from '../../TestClient';
 import {makeTestClients} from './verification/util';
+import {encryptAES} from "../../../src/crypto/aes";
+import {resetCrossSigningKeys, createSecretStorageKey} from "./crypto-utils";
+import {logger} from '../../../src/logger';
 
 import * as utils from "../../../src/utils";
 
@@ -27,7 +30,7 @@ try {
     const crypto = require('crypto');
     utils.setCrypto(crypto);
 } catch (err) {
-    console.log('nodejs was compiled without crypto support');
+    logger.log('nodejs was compiled without crypto support');
 }
 
 async function makeTestClient(userInfo, options) {
@@ -49,9 +52,16 @@ async function makeTestClient(userInfo, options) {
     return client;
 }
 
+// Wrapper around pkSign to return a signed object. pkSign returns the
+// signature, rather than the signed object.
+function sign(obj, key, userId) {
+    olmlib.pkSign(obj, key, userId);
+    return obj;
+}
+
 describe("Secrets", function() {
     if (!global.Olm) {
-        console.warn('Not running megolm backup unit tests: libolm not present');
+        logger.warn('Not running megolm backup unit tests: libolm not present');
         return;
     }
 
@@ -182,9 +192,9 @@ describe("Secrets", function() {
                 }),
             ]);
         };
-        alice.resetCrossSigningKeys();
+        resetCrossSigningKeys(alice);
 
-        const newKeyId = await alice.addSecretStorageKey(
+        const { keyId: newKeyId } = await alice.addSecretStorageKey(
             SECRET_STORAGE_ALGORITHM_V1_AES,
         );
         // we don't await on this because it waits for the event to come down the sync
@@ -216,8 +226,8 @@ describe("Secrets", function() {
             ],
             {
                 cryptoCallbacks: {
-                    onSecretRequested: e => {
-                        expect(e.name).toBe("foo");
+                    onSecretRequested: (userId, deviceId, requestId, secretName, deviceTrust) => {
+                        expect(secretName).toBe("foo");
                         return "bar";
                     },
                 },
@@ -266,100 +276,399 @@ describe("Secrets", function() {
         expect(secret).toBe("bar");
     });
 
-    it("bootstraps when no storage or cross-signing keys locally", async function() {
-        const key = new Uint8Array(16);
-        for (let i = 0; i < 16; i++) key[i] = i;
-        const getKey = jest.fn(e => {
-            return [Object.keys(e.keys)[0], key];
-        });
-
-        const bob = await makeTestClient(
-            {
-                userId: "@bob:example.com",
-                deviceId: "bob1",
-            },
-            {
-                cryptoCallbacks: {
-                    getSecretStorageKey: getKey,
-                },
-            },
+    describe("bootstrap", function() {
+        // keys used in some of the tests
+        const XSK = new Uint8Array(
+            olmlib.decodeBase64("3lo2YdJugHjfE+Or7KJ47NuKbhE7AAGLgQ/dc19913Q="),
         );
-        bob.uploadDeviceSigningKeys = async () => {};
-        bob.uploadKeySignatures = async () => {};
-        bob.setAccountData = async function(eventType, contents, callback) {
-            const event = new MatrixEvent({
-                type: eventType,
-                content: contents,
+        const XSPubKey = "DRb8pFVJyEJ9OWvXeUoM0jq/C2Wt+NxzBZVuk2nRb+0";
+        const USK = new Uint8Array(
+            olmlib.decodeBase64("lKWi3hJGUie5xxHgySoz8PHFnZv6wvNaud/p2shN9VU="),
+        );
+        const USPubKey = "CUpoiTtHiyXpUmd+3ohb7JVxAlUaOG1NYs9Jlx8soQU";
+        const SSK = new Uint8Array(
+            olmlib.decodeBase64("1R6JVlXX99UcfUZzKuCDGQgJTw8ur1/ofgPD8pp+96M="),
+        );
+        const SSPubKey = "0DfNsRDzEvkCLA0gD3m7VAGJ5VClhjEsewI35xq873Q";
+        const SSSSKey = new Uint8Array(
+            olmlib.decodeBase64(
+                "XrmITOOdBhw6yY5Bh7trb/bgp1FRdIGyCUxxMP873R0=",
+            ),
+        );
+
+        it("bootstraps when no storage or cross-signing keys locally", async function() {
+            const key = new Uint8Array(16);
+            for (let i = 0; i < 16; i++) key[i] = i;
+            const getKey = jest.fn(e => {
+                return [Object.keys(e.keys)[0], key];
             });
-            this.store.storeAccountDataEvents([
-                event,
-            ]);
-            this.emit("accountData", event);
-        };
 
-        await bob.bootstrapSecretStorage();
-
-        const crossSigning = bob._crypto._crossSigningInfo;
-        const secretStorage = bob._crypto._secretStorage;
-
-        expect(crossSigning.getId()).toBeTruthy();
-        expect(await crossSigning.isStoredInSecretStorage(secretStorage)).toBeTruthy();
-        expect(await secretStorage.hasKey()).toBeTruthy();
-    });
-
-    it("bootstraps when cross-signing keys in secret storage", async function() {
-        const decryption = new global.Olm.PkDecryption();
-        const storagePublicKey = decryption.generate_key();
-        const storagePrivateKey = decryption.get_private_key();
-
-        const bob = await makeTestClient(
-            {
-                userId: "@bob:example.com",
-                deviceId: "bob1",
-            },
-            {
-                cryptoCallbacks: {
-                    getSecretStorageKey: async request => {
-                        const defaultKeyId = await bob.getDefaultSecretStorageKeyId();
-                        expect(Object.keys(request.keys)).toEqual([defaultKeyId]);
-                        return [defaultKeyId, storagePrivateKey];
+            const bob = await makeTestClient(
+                {
+                    userId: "@bob:example.com",
+                    deviceId: "bob1",
+                },
+                {
+                    cryptoCallbacks: {
+                        getSecretStorageKey: getKey,
                     },
                 },
-            },
-        );
+            );
+            bob.uploadDeviceSigningKeys = async () => {};
+            bob.uploadKeySignatures = async () => {};
+            bob.setAccountData = async function(eventType, contents, callback) {
+                const event = new MatrixEvent({
+                    type: eventType,
+                    content: contents,
+                });
+                this.store.storeAccountDataEvents([
+                    event,
+                ]);
+                this.emit("accountData", event);
+            };
 
-        bob.uploadDeviceSigningKeys = async () => {};
-        bob.uploadKeySignatures = async () => {};
-        bob.setAccountData = async function(eventType, contents, callback) {
-            const event = new MatrixEvent({
-                type: eventType,
-                content: contents,
+            await bob.bootstrapCrossSigning({
+                authUploadDeviceSigningKeys: async func => await func({}),
+            });
+            await bob.bootstrapSecretStorage({
+                createSecretStorageKey,
             });
-            this.store.storeAccountDataEvents([
-                event,
-            ]);
-            this.emit("accountData", event);
-        };
-        bob._crypto.checkKeyBackup = async () => {};
 
-        const crossSigning = bob._crypto._crossSigningInfo;
-        const secretStorage = bob._crypto._secretStorage;
+            const crossSigning = bob._crypto._crossSigningInfo;
+            const secretStorage = bob._crypto._secretStorage;
 
-        // Set up cross-signing keys from scratch with specific storage key
-        await bob.bootstrapSecretStorage({
-            createSecretStorageKey: async () => ({ pubkey: storagePublicKey }),
+            expect(crossSigning.getId()).toBeTruthy();
+            expect(await crossSigning.isStoredInSecretStorage(secretStorage))
+                .toBeTruthy();
+            expect(await secretStorage.hasKey()).toBeTruthy();
         });
 
-        // Clear local cross-signing keys and read from secret storage
-        bob._crypto._deviceList.storeCrossSigningForUser(
-            "@bob:example.com",
-            crossSigning.toStorage(),
-        );
-        crossSigning.keys = {};
-        await bob.bootstrapSecretStorage();
+        it("bootstraps when cross-signing keys in secret storage", async function() {
+            const decryption = new global.Olm.PkDecryption();
+            const storagePublicKey = decryption.generate_key();
+            const storagePrivateKey = decryption.get_private_key();
 
-        expect(crossSigning.getId()).toBeTruthy();
-        expect(await crossSigning.isStoredInSecretStorage(secretStorage)).toBeTruthy();
-        expect(await secretStorage.hasKey()).toBeTruthy();
+            const bob = await makeTestClient(
+                {
+                    userId: "@bob:example.com",
+                    deviceId: "bob1",
+                },
+                {
+                    cryptoCallbacks: {
+                        getSecretStorageKey: async request => {
+                            const defaultKeyId = await bob.getDefaultSecretStorageKeyId();
+                            expect(Object.keys(request.keys)).toEqual([defaultKeyId]);
+                            return [defaultKeyId, storagePrivateKey];
+                        },
+                    },
+                },
+            );
+
+            bob.uploadDeviceSigningKeys = async () => {};
+            bob.uploadKeySignatures = async () => {};
+            bob.setAccountData = async function(eventType, contents, callback) {
+                const event = new MatrixEvent({
+                    type: eventType,
+                    content: contents,
+                });
+                this.store.storeAccountDataEvents([
+                    event,
+                ]);
+                this.emit("accountData", event);
+            };
+            bob._crypto.checkKeyBackup = async () => {};
+
+            const crossSigning = bob._crypto._crossSigningInfo;
+            const secretStorage = bob._crypto._secretStorage;
+
+            // Set up cross-signing keys from scratch with specific storage key
+            await bob.bootstrapCrossSigning({
+                authUploadDeviceSigningKeys: async func => await func({}),
+            });
+            await bob.bootstrapSecretStorage({
+                createSecretStorageKey: async () => ({
+                    // `pubkey` not used anymore with symmetric 4S
+                    keyInfo: { pubkey: storagePublicKey },
+                    privateKey: storagePrivateKey,
+                }),
+            });
+
+            // Clear local cross-signing keys and read from secret storage
+            bob._crypto._deviceList.storeCrossSigningForUser(
+                "@bob:example.com",
+                crossSigning.toStorage(),
+            );
+            crossSigning.keys = {};
+            await bob.bootstrapCrossSigning({
+                authUploadDeviceSigningKeys: async func => await func({}),
+            });
+
+            expect(crossSigning.getId()).toBeTruthy();
+            expect(await crossSigning.isStoredInSecretStorage(secretStorage))
+                .toBeTruthy();
+            expect(await secretStorage.hasKey()).toBeTruthy();
+        });
+
+        it("adds passphrase checking if it's lacking", async function() {
+            let crossSigningKeys = {
+                master: XSK,
+                user_signing: USK,
+                self_signing: SSK,
+            };
+            const secretStorageKeys = {
+                key_id: SSSSKey,
+            };
+            const alice = await makeTestClient(
+                {userId: "@alice:example.com", deviceId: "Osborne2"},
+                {
+                    cryptoCallbacks: {
+                        getCrossSigningKey: t => crossSigningKeys[t],
+                        saveCrossSigningKeys: k => crossSigningKeys = k,
+                        getSecretStorageKey: ({keys}, name) => {
+                            for (const keyId of Object.keys(keys)) {
+                                if (secretStorageKeys[keyId]) {
+                                    return [keyId, secretStorageKeys[keyId]];
+                                }
+                            }
+                        },
+                    },
+                },
+            );
+            alice.store.storeAccountDataEvents([
+                new MatrixEvent({
+                    type: "m.secret_storage.default_key",
+                    content: {
+                        key: "key_id",
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.secret_storage.key.key_id",
+                    content: {
+                        algorithm: "m.secret_storage.v1.aes-hmac-sha2",
+                        passphrase: {
+                            algorithm: "m.pbkdf2",
+                            iterations: 500000,
+                            salt: "GbkvwKHVMveo1zGVSb2GMMdCinG2npJK",
+                        },
+                    },
+                }),
+                // we never use these values, other than checking that they
+                // exist, so just use dummy values
+                new MatrixEvent({
+                    type: "m.cross_signing.master",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.self_signing",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.user_signing",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+            ]);
+            alice._crypto._deviceList.storeCrossSigningForUser("@alice:example.com", {
+                keys: {
+                    master: {
+                        user_id: "@alice:example.com",
+                        usage: ["master"],
+                        keys: {
+                            [`ed25519:${XSPubKey}`]: XSPubKey,
+                        },
+                    },
+                    self_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["self_signing"],
+                        keys: {
+                            [`ed25519:${SSPubKey}`]: SSPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                    user_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["user_signing"],
+                        keys: {
+                            [`ed25519:${USPubKey}`]: USPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                },
+            });
+            alice.getKeyBackupVersion = async () => {
+                return {
+                    version: "1",
+                    algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
+                    auth_data: sign({
+                        public_key: "pxEXhg+4vdMf/kFwP4bVawFWdb0EmytL3eFJx++zQ0A",
+                    }, XSK, "@alice:example.com"),
+                };
+            };
+            alice.setAccountData = async function(name, data) {
+                const event = new MatrixEvent({
+                    type: name,
+                    content: data,
+                });
+                alice.store.storeAccountDataEvents([event]);
+                this.emit("accountData", event);
+            };
+
+            await alice.bootstrapSecretStorage();
+
+            expect(alice.getAccountData("m.secret_storage.default_key").getContent())
+                .toEqual({key: "key_id"});
+            const keyInfo = alice.getAccountData("m.secret_storage.key.key_id")
+                .getContent();
+            expect(keyInfo.algorithm)
+                .toEqual("m.secret_storage.v1.aes-hmac-sha2");
+            expect(keyInfo.passphrase).toEqual({
+                algorithm: "m.pbkdf2",
+                iterations: 500000,
+                salt: "GbkvwKHVMveo1zGVSb2GMMdCinG2npJK",
+            });
+            expect(keyInfo).toHaveProperty("iv");
+            expect(keyInfo).toHaveProperty("mac");
+            expect(alice.checkSecretStorageKey(secretStorageKeys.key_id, keyInfo))
+                .toBeTruthy();
+        });
+        it("fixes backup keys in the wrong format", async function() {
+            let crossSigningKeys = {
+                master: XSK,
+                user_signing: USK,
+                self_signing: SSK,
+            };
+            const secretStorageKeys = {
+                key_id: SSSSKey,
+            };
+            const alice = await makeTestClient(
+                {userId: "@alice:example.com", deviceId: "Osborne2"},
+                {
+                    cryptoCallbacks: {
+                        getCrossSigningKey: t => crossSigningKeys[t],
+                        saveCrossSigningKeys: k => crossSigningKeys = k,
+                        getSecretStorageKey: ({keys}, name) => {
+                            for (const keyId of Object.keys(keys)) {
+                                if (secretStorageKeys[keyId]) {
+                                    return [keyId, secretStorageKeys[keyId]];
+                                }
+                            }
+                        },
+                    },
+                },
+            );
+            alice.store.storeAccountDataEvents([
+                new MatrixEvent({
+                    type: "m.secret_storage.default_key",
+                    content: {
+                        key: "key_id",
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.secret_storage.key.key_id",
+                    content: {
+                        algorithm: "m.secret_storage.v1.aes-hmac-sha2",
+                        passphrase: {
+                            algorithm: "m.pbkdf2",
+                            iterations: 500000,
+                            salt: "GbkvwKHVMveo1zGVSb2GMMdCinG2npJK",
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.master",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.self_signing",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.user_signing",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.megolm_backup.v1",
+                    content: {
+                        encrypted: {
+                            key_id: await encryptAES(
+                                "123,45,6,7,89,1,234,56,78,90,12,34,5,67,8,90",
+                                secretStorageKeys.key_id, "m.megolm_backup.v1",
+                            ),
+                        },
+                    },
+                }),
+            ]);
+            alice._crypto._deviceList.storeCrossSigningForUser("@alice:example.com", {
+                keys: {
+                    master: {
+                        user_id: "@alice:example.com",
+                        usage: ["master"],
+                        keys: {
+                            [`ed25519:${XSPubKey}`]: XSPubKey,
+                        },
+                    },
+                    self_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["self_signing"],
+                        keys: {
+                            [`ed25519:${SSPubKey}`]: SSPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                    user_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["user_signing"],
+                        keys: {
+                            [`ed25519:${USPubKey}`]: USPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                },
+            });
+            alice.getKeyBackupVersion = async () => {
+                return {
+                    version: "1",
+                    algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
+                    auth_data: sign({
+                        public_key: "pxEXhg+4vdMf/kFwP4bVawFWdb0EmytL3eFJx++zQ0A",
+                    }, XSK, "@alice:example.com"),
+                };
+            };
+            alice.setAccountData = async function(name, data) {
+                const event = new MatrixEvent({
+                    type: name,
+                    content: data,
+                });
+                alice.store.storeAccountDataEvents([event]);
+                this.emit("accountData", event);
+            };
+
+            await alice.bootstrapSecretStorage();
+
+            const backupKey = alice.getAccountData("m.megolm_backup.v1")
+                .getContent();
+            expect(backupKey.encrypted).toHaveProperty("key_id");
+            expect(await alice.getSecret("m.megolm_backup.v1"))
+                .toEqual("ey0GB1kB6jhOWgwiBUMIWg==");
+        });
     });
 });

spec/unit/crypto/verification/sas.spec.js

@@ -22,6 +22,7 @@ import {DeviceInfo} from "../../../../src/crypto/deviceinfo";
 import {verificationMethods} from "../../../../src/crypto";
 import * as olmlib from "../../../../src/crypto/olmlib";
 import {logger} from "../../../../src/logger";
+import {resetCrossSigningKeys} from "../crypto-utils";
 
 const Olm = global.Olm;
 
@@ -44,7 +45,16 @@ describe("SAS verification", function() {
     });
 
     it("should error on an unexpected event", async function() {
-        const sas = new SAS({}, "@alice:example.com", "ABCDEFG");
+        //channel, baseApis, userId, deviceId, startEvent, request
+        const request = {
+            onVerifierCancelled: function() {},
+        };
+        const channel = {
+            send: function() {
+                return Promise.resolve();
+            },
+        };
+        const sas = new SAS(channel, {}, "@alice:example.com", "ABCDEFG", null, request);
         sas.handleEvent(new MatrixEvent({
             sender: "@alice:example.com",
             type: "es.inquisition",
@@ -172,11 +182,14 @@ describe("SAS verification", function() {
 
         it("should verify a key", async () => {
             let macMethod;
+            let keyAgreement;
             const origSendToDevice = bob.client.sendToDevice.bind(bob.client);
             bob.client.sendToDevice = function(type, map) {
                 if (type === "m.key.verification.accept") {
                     macMethod = map[alice.client.getUserId()][alice.client.deviceId]
                         .message_authentication_code;
+                    keyAgreement = map[alice.client.getUserId()][alice.client.deviceId]
+                        .key_agreement_protocol;
                 }
                 return origSendToDevice(type, map);
             };
@@ -203,6 +216,7 @@ describe("SAS verification", function() {
 
             // make sure that it uses the preferred method
             expect(macMethod).toBe("hkdf-hmac-sha256");
+            expect(keyAgreement).toBe("curve25519-hkdf-sha256");
 
             // make sure Alice and Bob verified each other
             const bobDevice
@@ -275,12 +289,12 @@ describe("SAS verification", function() {
             );
             alice.httpBackend.when('POST', '/keys/signatures/upload').respond(200, {});
             alice.httpBackend.flush(undefined, 2);
-            await alice.client.resetCrossSigningKeys();
+            await resetCrossSigningKeys(alice.client);
             bob.httpBackend.when('POST', '/keys/device_signing/upload').respond(200, {});
             bob.httpBackend.when('POST', '/keys/signatures/upload').respond(200, {});
             bob.httpBackend.flush(undefined, 2);
 
-            await bob.client.resetCrossSigningKeys();
+            await resetCrossSigningKeys(bob.client);
 
             bob.client._crypto._deviceList.storeCrossSigningForUser(
                 "@alice:example.com", {

spec/unit/crypto/verification/secret_request.spec.js

@@ -54,6 +54,7 @@ describe("self-verifications", () => {
             cacheCallbacks,
         );
         _crossSigningInfo.keys = {
+            master: { keys: { X: testKeyPub } },
             self_signing: { keys: { X: testKeyPub } },
             user_signing: { keys: { X: testKeyPub } },
         };
@@ -64,12 +65,20 @@ describe("self-verifications", () => {
             }),
         };
 
+        const storeSessionBackupPrivateKey = jest.fn();
+        const restoreKeyBackupWithCache = jest.fn(() => Promise.resolve());
+
         const client = {
             _crypto: {
                 _crossSigningInfo,
                 _secretStorage,
+                storeSessionBackupPrivateKey,
+                getSessionBackupPrivateKey: () => null,
             },
+            requestSecret: _secretStorage.request.bind(_secretStorage),
             getUserId: () => userId,
+            getKeyBackupVersion: () => Promise.resolve({}),
+            restoreKeyBackupWithCache,
         };
 
         const request = {
@@ -88,15 +97,20 @@ describe("self-verifications", () => {
 
         const result = await verification.done();
 
-        /* We should request, and store, two keys */
-        expect(cacheCallbacks.storeCrossSigningKeyCache.mock.calls.length).toBe(2);
-        expect(_secretStorage.request.mock.calls.length).toBe(2);
+        /* We should request, and store, 3 cross signing keys and the key backup key */
+        expect(cacheCallbacks.storeCrossSigningKeyCache.mock.calls.length).toBe(3);
+        expect(_secretStorage.request.mock.calls.length).toBe(4);
 
         expect(cacheCallbacks.storeCrossSigningKeyCache.mock.calls[0][1])
           .toEqual(testKey);
         expect(cacheCallbacks.storeCrossSigningKeyCache.mock.calls[1][1])
           .toEqual(testKey);
 
+        expect(storeSessionBackupPrivateKey.mock.calls[0][0])
+          .toEqual(testKey);
+
+        expect(restoreKeyBackupWithCache).toHaveBeenCalled();
+
         expect(result).toBeInstanceOf(Array);
         expect(result[0][0]).toBe(testKeyPub);
         expect(result[1][0]).toBe(testKeyPub);

spec/unit/crypto/verification/util.js

@@ -18,12 +18,13 @@ limitations under the License.
 import {TestClient} from '../../../TestClient';
 import {MatrixEvent} from "../../../../src/models/event";
 import nodeCrypto from "crypto";
+import {logger} from '../../../../src/logger';
 
 export async function makeTestClients(userInfos, options) {
     const clients = [];
     const clientMap = {};
     const sendToDevice = function(type, map) {
-        // console.log(this.getUserId(), "sends", type, map);
+        // logger.log(this.getUserId(), "sends", type, map);
         for (const [userId, devMap] of Object.entries(map)) {
             if (userId in clientMap) {
                 for (const [deviceId, msg] of Object.entries(devMap)) {
@@ -67,7 +68,7 @@ export async function makeTestClients(userInfos, options) {
         setImmediate(() => {
             for (const tc of clients) {
                 if (tc.client === this) { // eslint-disable-line babel/no-invalid-this
-                    console.log("sending remote echo!!");
+                    logger.log("sending remote echo!!");
                     tc.client.emit("Room.timeline", remoteEcho);
                 } else {
                     tc.client.emit("Room.timeline", event);

spec/unit/crypto/verification/verification_request.spec.js

@@ -119,6 +119,8 @@ async function distributeEvent(ownRequest, theirRequest, event) {
     await theirRequest.channel.handleEvent(event, theirRequest, true);
 }
 
+jest.useFakeTimers();
+
 describe("verification request unit tests", function() {
     beforeAll(function() {
         setupWebcrypto();
@@ -246,4 +248,38 @@ describe("verification request unit tests", function() {
         expect(bob1Request.done).toBe(true);
         expect(bob2Request.done).toBe(true);
     });
+
+    it("request times out after 10 minutes", async function() {
+        const alice = makeMockClient("@alice:matrix.tld", "device1");
+        const bob = makeMockClient("@bob:matrix.tld", "device1");
+        const aliceRequest = new VerificationRequest(
+            new InRoomChannel(alice, "!room", bob.getUserId()), new Map(), alice);
+        await aliceRequest.sendRequest();
+        const [requestEvent] = alice.popEvents();
+        await aliceRequest.channel.handleEvent(requestEvent, aliceRequest, true,
+            true, true);
+
+        expect(aliceRequest.cancelled).toBe(false);
+        expect(aliceRequest._cancellingUserId).toBe(undefined);
+        jest.advanceTimersByTime(10 * 60 * 1000);
+        expect(aliceRequest._cancellingUserId).toBe(alice.getUserId());
+    });
+
+    it("request times out 2 minutes after receipt", async function() {
+        const alice = makeMockClient("@alice:matrix.tld", "device1");
+        const bob = makeMockClient("@bob:matrix.tld", "device1");
+        const aliceRequest = new VerificationRequest(
+            new InRoomChannel(alice, "!room", bob.getUserId()), new Map(), alice);
+        await aliceRequest.sendRequest();
+        const [requestEvent] = alice.popEvents();
+        const bobRequest = new VerificationRequest(
+            new InRoomChannel(bob, "!room"), new Map(), bob);
+
+        await bobRequest.channel.handleEvent(requestEvent, bobRequest, true);
+
+        expect(bobRequest.cancelled).toBe(false);
+        expect(bobRequest._cancellingUserId).toBe(undefined);
+        jest.advanceTimersByTime(2 * 60 * 1000);
+        expect(bobRequest._cancellingUserId).toBe(bob.getUserId());
+    });
 });

spec/unit/filter-component.spec.js

@@ -0,0 +1,34 @@
+import {FilterComponent} from "../../src/filter-component";
+import {mkEvent} from '../test-utils';
+
+describe("Filter Component", function() {
+    describe("types", function() {
+        it("should filter out events with other types", function() {
+            const filter = new FilterComponent({ types: ['m.room.message'] });
+            const event = mkEvent({
+                type: 'm.room.member',
+                content: { },
+                room: 'roomId',
+                event: true,
+            });
+
+            const checkResult = filter.check(event);
+
+            expect(checkResult).toBe(false);
+        });
+
+        it("should validate events with the same type", function() {
+            const filter = new FilterComponent({ types: ['m.room.message'] });
+            const event = mkEvent({
+                type: 'm.room.message',
+                content: { },
+                room: 'roomId',
+                event: true,
+            });
+
+            const checkResult = filter.check(event);
+
+            expect(checkResult).toBe(true);
+        });
+    });
+});

spec/unit/interactive-auth.spec.js

@@ -97,7 +97,7 @@ describe("InteractiveAuth", function() {
         // first we expect a call to doRequest
         doRequest.mockImplementation(function(authData) {
             logger.log("request1", authData);
-            expect(authData).toEqual({});
+            expect(authData).toEqual(null); // first request should be null
             const err = new MatrixError({
                 session: "sessionId",
                 flows: [
@@ -143,4 +143,33 @@ describe("InteractiveAuth", function() {
             expect(stateUpdated).toBeCalledTimes(1);
         });
     });
+
+    it("should start an auth stage and reject if no auth flow", function() {
+        const doRequest = jest.fn();
+        const stateUpdated = jest.fn();
+
+        const ia = new InteractiveAuth({
+            matrixClient: new FakeClient(),
+            doRequest: doRequest,
+            stateUpdated: stateUpdated,
+        });
+
+        doRequest.mockImplementation(function(authData) {
+            logger.log("request1", authData);
+            expect(authData).toEqual(null); // first request should be null
+            const err = new MatrixError({
+                session: "sessionId",
+                flows: [],
+                params: {
+                    "logintype": { param: "aa" },
+                },
+            });
+            err.httpStatus = 401;
+            throw err;
+        });
+
+        return ia.attemptAuth().catch(function(error) {
+            expect(error.message).toBe('No appropriate authentication flow found');
+        });
+    });
 });

spec/unit/room-member.spec.js

@@ -33,12 +33,6 @@ describe("RoomMember", function() {
             expect(url.indexOf("flibble/wibble")).not.toEqual(-1);
         });
 
-        it("should return an identicon HTTP URL if allowDefault was set and there " +
-        "was no m.room.member event", function() {
-            const url = member.getAvatarUrl(hsUrl, 64, 64, "crop", true);
-            expect(url.indexOf("http")).toEqual(0); // don't care about form
-        });
-
         it("should return nothing if there is no m.room.member and allowDefault=false",
         function() {
             const url = member.getAvatarUrl(hsUrl, 64, 64, "crop", false);

spec/unit/room.spec.js

@@ -45,12 +45,6 @@ describe("Room", function() {
             expect(url.indexOf("flibble/wibble")).not.toEqual(-1);
         });
 
-        it("should return an identicon HTTP URL if allowDefault was set and there " +
-        "was no m.room.avatar event", function() {
-            const url = room.getAvatarUrl(hsUrl, 64, 64, "crop", true);
-            expect(url.indexOf("http")).toEqual(0); // don't care about form
-        });
-
         it("should return nothing if there is no m.room.avatar and allowDefault=false",
         function() {
             const url = room.getAvatarUrl(hsUrl, 64, 64, "crop", false);
@@ -1379,7 +1373,7 @@ describe("Room", function() {
             let hasThrown = false;
             try {
                 await room.loadMembersIfNeeded();
-            } catch(err) {
+            } catch (err) {
                 hasThrown = true;
             }
             expect(hasThrown).toEqual(true);

spec/unit/scheduler.spec.js

@@ -143,7 +143,7 @@ describe("MatrixScheduler", function() {
         deferA.reject({});
         try {
             await globalA;
-        } catch(err) {
+        } catch (err) {
             await Promise.resolve();
             expect(procCount).toEqual(2);
         }

spec/unit/sync-accumulator.spec.js

@@ -17,6 +17,39 @@ limitations under the License.
 
 import {SyncAccumulator} from "../../src/sync-accumulator";
 
+// The event body & unsigned object get frozen to assert that they don't get altered
+// by the impl
+const RES_WITH_AGE = {
+    next_batch: "abc",
+    rooms: {
+        invite: {},
+        leave: {},
+        join: {
+            "!foo:bar": {
+                account_data: { events: [] },
+                ephemeral: { events: [] },
+                unread_notifications: {},
+                timeline: {
+                    events: [
+                        Object.freeze({
+                            content: {
+                                body: "This thing is happening right now!",
+                            },
+                            origin_server_ts: 123456789,
+                            sender: "@alice:localhost",
+                            type: "m.room.message",
+                            unsigned: Object.freeze({
+                                age: 50,
+                            }),
+                        }),
+                    ],
+                    prev_batch: "something",
+                },
+            },
+        },
+    },
+};
+
 describe("SyncAccumulator", function() {
     let sa;
 
@@ -368,6 +401,39 @@ describe("SyncAccumulator", function() {
             expect(summary["m.joined_member_count"]).toEqual(5);
             expect(summary["m.heroes"]).toEqual(["@bob:bar"]);
         });
+
+        it("should return correctly adjusted age attributes", () => {
+            const delta = 1000;
+            const startingTs = 1000;
+
+            const oldDateNow = Date.now;
+            try {
+                Date.now = jest.fn();
+                Date.now.mockReturnValue(startingTs);
+
+                sa.accumulate(RES_WITH_AGE);
+
+                Date.now.mockReturnValue(startingTs + delta);
+
+                const output = sa.getJSON();
+                expect(output.roomsData.join["!foo:bar"].timeline.events[0].unsigned.age).toEqual(
+                    RES_WITH_AGE.rooms.join["!foo:bar"].timeline.events[0].unsigned.age + delta,
+                );
+                expect(Object.keys(output.roomsData.join["!foo:bar"].timeline.events[0])).toEqual(
+                    Object.keys(RES_WITH_AGE.rooms.join["!foo:bar"].timeline.events[0]),
+                );
+            } finally {
+                Date.now = oldDateNow;
+            }
+        });
+
+        it("should mangle age without adding extra keys", () => {
+            sa.accumulate(RES_WITH_AGE);
+            const output = sa.getJSON();
+            expect(Object.keys(output.roomsData.join["!foo:bar"].timeline.events[0])).toEqual(
+                Object.keys(RES_WITH_AGE.rooms.join["!foo:bar"].timeline.events[0]),
+            );
+        });
     });
 });
 

spec/unit/webrtc/call.spec.ts

@@ -0,0 +1,193 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {TestClient} from '../../TestClient';
+import {MatrixCall, CallErrorCode} from '../../../src/webrtc/call';
+
+const DUMMY_SDP = (
+    "v=0\r\n" +
+    "o=- 5022425983810148698 2 IN IP4 127.0.0.1\r\n" +
+    "s=-\r\nt=0 0\r\na=group:BUNDLE 0\r\n" +
+    "a=msid-semantic: WMS h3wAi7s8QpiQMH14WG3BnDbmlOqo9I5ezGZA\r\n" +
+    "m=audio 9 UDP/TLS/RTP/SAVPF 111 103 104 9 0 8 106 105 13 110 112 113 126\r\n" +
+    "c=IN IP4 0.0.0.0\r\na=rtcp:9 IN IP4 0.0.0.0\r\na=ice-ufrag:hLDR\r\n" +
+    "a=ice-pwd:bMGD9aOldHWiI+6nAq/IIlRw\r\n" +
+    "a=ice-options:trickle\r\n" +
+    "a=fingerprint:sha-256 E4:94:84:F9:4A:98:8A:56:F5:5F:FD:AF:72:B9:32:89:49:5C:4B:9A:" +
+        "4A:15:8E:41:8A:F3:69:E4:39:52:DC:D6\r\n" +
+    "a=setup:active\r\n" +
+    "a=mid:0\r\n" +
+    "a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level\r\n" +
+    "a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\r\n" +
+    "a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\r\n" +
+    "a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid\r\n" +
+    "a=extmap:5 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id\r\n" +
+    "a=extmap:6 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id\r\n" +
+    "a=sendrecv\r\n" +
+    "a=msid:h3wAi7s8QpiQMH14WG3BnDbmlOqo9I5ezGZA 4357098f-3795-4131-bff4-9ba9c0348c49\r\n" +
+    "a=rtcp-mux\r\n" +
+    "a=rtpmap:111 opus/48000/2\r\n" +
+    "a=rtcp-fb:111 transport-cc\r\n" +
+    "a=fmtp:111 minptime=10;useinbandfec=1\r\n" +
+    "a=rtpmap:103 ISAC/16000\r\n" +
+    "a=rtpmap:104 ISAC/32000\r\n" +
+    "a=rtpmap:9 G722/8000\r\n" +
+    "a=rtpmap:0 PCMU/8000\r\n" +
+    "a=rtpmap:8 PCMA/8000\r\n" +
+    "a=rtpmap:106 CN/32000\r\n" +
+    "a=rtpmap:105 CN/16000\r\n" +
+    "a=rtpmap:13 CN/8000\r\n" +
+    "a=rtpmap:110 telephone-event/48000\r\n" +
+    "a=rtpmap:112 telephone-event/32000\r\n" +
+    "a=rtpmap:113 telephone-event/16000\r\n" +
+    "a=rtpmap:126 telephone-event/8000\r\n" +
+    "a=ssrc:3619738545 cname:2RWtmqhXLdoF4sOi\r\n"
+);
+
+class MockRTCPeerConnection {
+    localDescription: RTCSessionDescription;
+
+    constructor() {
+        this.localDescription = {
+            sdp: DUMMY_SDP,
+            type: 'offer',
+            toJSON: function() {},
+        };
+    }
+
+    addEventListener() {}
+    createOffer() {
+        return Promise.resolve({});
+    }
+    setRemoteDescription() {
+        return Promise.resolve();
+    }
+    setLocalDescription() {
+        return Promise.resolve();
+    }
+    close() {}
+}
+
+describe('Call', function() {
+    let client;
+    let call;
+    let prevNavigator;
+    let prevDocument;
+    let prevWindow;
+
+    beforeEach(function() {
+        prevNavigator = global.navigator;
+        prevDocument = global.document;
+        prevWindow = global.window;
+
+        global.navigator = {
+            mediaDevices: {
+                // @ts-ignore Mock
+                getUserMedia: () => {
+                    return {
+                        getTracks: () => [],
+                        getAudioTracks: () => [],
+                        getVideoTracks: () => [],
+                    };
+                },
+            },
+        };
+
+        global.window = {
+            // @ts-ignore Mock
+            RTCPeerConnection: MockRTCPeerConnection,
+            // @ts-ignore Mock
+            RTCSessionDescription: {},
+            // @ts-ignore Mock
+            RTCIceCandidate: {},
+            getUserMedia: {},
+        };
+        // @ts-ignore Mock
+        global.document = {};
+
+        client = new TestClient("@alice:foo", "somedevice", "token", undefined, {});
+        // We just stub out sendEvent: we're not interested in testing the client's
+        // event sending code here
+        client.client.sendEvent = () => {};
+        call = new MatrixCall({
+            client: client.client,
+            roomId: '!foo:bar',
+        });
+        // call checks one of these is wired up
+        call.on('error', () => {});
+    });
+
+    afterEach(function() {
+        client.stop();
+        global.navigator = prevNavigator;
+        global.window = prevWindow;
+        global.document = prevDocument;
+    });
+
+    it('should ignore candidate events from non-matching party ID', async function() {
+        await call.placeVoiceCall();
+        await call.onAnswerReceived({
+            getContent: () => {
+                return {
+                    version: 0,
+                    call_id: call.callId,
+                    party_id: 'the_correct_party_id',
+                    answer: {
+                        sdp: DUMMY_SDP,
+                    },
+                };
+            },
+        });
+
+        call.peerConn.addIceCandidate = jest.fn();
+        call.onRemoteIceCandidatesReceived({
+            getContent: () => {
+                return {
+                    version: 0,
+                    call_id: call.callId,
+                    party_id: 'the_correct_party_id',
+                    candidates: [
+                        {
+                            candidate: '',
+                            sdpMid: '',
+                        },
+                    ],
+                };
+            },
+        });
+        expect(call.peerConn.addIceCandidate.mock.calls.length).toBe(1);
+
+        call.onRemoteIceCandidatesReceived({
+            getContent: () => {
+                return {
+                    version: 0,
+                    call_id: call.callId,
+                    party_id: 'some_other_party_id',
+                    candidates: [
+                        {
+                            candidate: '',
+                            sdpMid: '',
+                        },
+                    ],
+                };
+            },
+        });
+        expect(call.peerConn.addIceCandidate.mock.calls.length).toBe(1);
+
+        // Hangup to stop timers
+        call.hangup(CallErrorCode.UserHangup, true);
+    });
+});

src/@types/event.ts

@@ -0,0 +1,88 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export enum EventType {
+    // Room state events
+    RoomCanonicalAlias = "m.room.canonical_alias",
+    RoomCreate = "m.room.create",
+    RoomJoinRules = "m.room.join_rules",
+    RoomMember = "m.room.member",
+    RoomThirdPartyInvite = "m.room.third_party_invite",
+    RoomPowerLevels = "m.room.power_levels",
+    RoomName = "m.room.name",
+    RoomTopic = "m.room.topic",
+    RoomAvatar = "m.room.avatar",
+    RoomPinnedEvents = "m.room.pinned_events",
+    RoomEncryption = "m.room.encryption",
+    RoomHistoryVisibility = "m.room.history_visibility",
+    RoomGuestAccess = "m.room.guest_access",
+    RoomServerAcl = "m.room.server_acl",
+    RoomTombstone = "m.room.tombstone",
+    /**
+     * @deprecated Should not be used.
+     */
+    RoomAliases = "m.room.aliases", // deprecated https://matrix.org/docs/spec/client_server/r0.6.1#historical-events
+
+    // Room timeline events
+    RoomRedaction = "m.room.redaction",
+    RoomMessage = "m.room.message",
+    RoomMessageEncrypted = "m.room.encrypted",
+    Sticker = "m.sticker",
+    CallInvite = "m.call.invite",
+    CallCandidates = "m.call.candidates",
+    CallAnswer = "m.call.answer",
+    CallHangup = "m.call.hangup",
+    CallReject = "m.call.reject",
+    CallSelectAnswer = "m.call.select_answer",
+    CallNegotiate = "m.call.negotiate",
+    KeyVerificationRequest = "m.key.verification.request",
+    KeyVerificationStart = "m.key.verification.start",
+    KeyVerificationCancel = "m.key.verification.cancel",
+    KeyVerificationMac = "m.key.verification.mac",
+    // use of this is discouraged https://matrix.org/docs/spec/client_server/r0.6.1#m-room-message-feedback
+    RoomMessageFeedback = "m.room.message.feedback",
+
+    // Room ephemeral events
+    Typing = "m.typing",
+    Receipt = "m.receipt",
+    Presence = "m.presence",
+
+    // Room account_data events
+    FullyRead = "m.fully_read",
+    Tag = "m.tag",
+
+    // User account_data events
+    PushRules = "m.push_rules",
+    Direct = "m.direct",
+    IgnoredUserList = "m.ignored_user_list",
+
+    // to_device events
+    RoomKey = "m.room_key",
+    RoomKeyRequest = "m.room_key_request",
+    ForwardedRoomKey = "m.forwarded_room_key",
+    Dummy = "m.dummy",
+}
+
+export enum MsgType {
+    Text = "m.text",
+    Emote = "m.emote",
+    Notice = "m.notice",
+    Image = "m.image",
+    File = "m.file",
+    Audio = "m.audio",
+    Location = "m.location",
+    Video = "m.video",
+}

src/@types/global.d.ts

@@ -0,0 +1,48 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// this is needed to tell TS about global.Olm
+import * as Olm from "olm"; // eslint-disable-line @typescript-eslint/no-unused-vars
+
+export {};
+
+declare global {
+    namespace NodeJS {
+        interface Global {
+            localStorage: Storage;
+        }
+    }
+
+    interface MediaDevices {
+        // This is experimental and types don't know about it yet
+        // https://github.com/microsoft/TypeScript/issues/33232
+        getDisplayMedia(constraints: MediaStreamConstraints): Promise<MediaStream>;
+    }
+
+    interface HTMLAudioElement {
+        // sinkId & setSinkId are experimental and typescript doesn't know about them
+        sinkId: string;
+        setSinkId(outputId: string);
+    }
+
+    interface DummyInterfaceWeShouldntBeUsingThis {}
+
+    interface Navigator {
+        // We check for the webkit-prefixed getUserMedia to detect if we're
+        // on webkit: we should check if we still need to do this
+        webkitGetUserMedia: DummyInterfaceWeShouldntBeUsingThis;
+    }
+}

src/autodiscovery.js

@@ -502,10 +502,12 @@ export class AutoDiscovery {
             request(
                 { method: "GET", uri: url, timeout: 5000 },
                 (err, response, body) => {
-                    if (err || response.statusCode < 200 || response.statusCode >= 300) {
+                    if (err || response &&
+                        (response.statusCode < 200 || response.statusCode >= 300)
+                    ) {
                         let action = "FAIL_PROMPT";
                         let reason = (err ? err.message : null) || "General failure";
-                        if (response.statusCode === 404) {
+                        if (response && response.statusCode === 404) {
                             action = "IGNORE";
                             reason = AutoDiscovery.ERROR_MISSING_WELLKNOWN;
                         }

src/base-apis.js

@@ -66,7 +66,7 @@ function termsUrlForService(serviceType, baseUrl) {
  * callback that returns a Promise<String> of an identity access token to supply
  * with identity requests. If the object is unset, no access token will be
  * supplied.
- * See also https://github.com/vector-im/riot-web/issues/10615 which seeks to
+ * See also https://github.com/vector-im/element-web/issues/10615 which seeks to
  * replace the previous approach of manual access tokens params with this
  * callback throughout the SDK.
  *
@@ -206,9 +206,6 @@ MatrixBaseApis.prototype.register = function(
         inhibitLogin = undefined;
     }
 
-    if (auth === undefined || auth === null) {
-        auth = {};
-    }
     if (sessionId) {
         auth.session = sessionId;
     }
@@ -465,8 +462,26 @@ MatrixBaseApis.prototype.getFallbackAuthUrl = function(loginType, authSessionId)
  * room_alias: {string(opt)}}</code>
  * @return {module:http-api.MatrixError} Rejects: with an error response.
  */
-MatrixBaseApis.prototype.createRoom = function(options, callback) {
-    // valid options include: room_alias_name, visibility, invite
+MatrixBaseApis.prototype.createRoom = async function(options, callback) {
+    // some valid options include: room_alias_name, visibility, invite
+
+    // inject the id_access_token if inviting 3rd party addresses
+    const invitesNeedingToken = (options.invite_3pid || [])
+        .filter(i => !i.id_access_token);
+    if (
+        invitesNeedingToken.length > 0 &&
+        this.identityServer &&
+        this.identityServer.getAccessToken &&
+        await this.doesServerAcceptIdentityAccessToken()
+    ) {
+        const identityAccessToken = await this.identityServer.getAccessToken();
+        if (identityAccessToken) {
+            for (const invite of invitesNeedingToken) {
+                invite.id_access_token = identityAccessToken;
+            }
+        }
+    }
+
     return this._http.authedRequest(
         callback, "POST", "/createRoom", undefined, options,
     );
@@ -1757,7 +1772,7 @@ MatrixBaseApis.prototype.downloadKeysForUsers = function(userIds, opts) {
         content.token = opts.token;
     }
     userIds.forEach((u) => {
-        content.device_keys[u] = {};
+        content.device_keys[u] = [];
     });
 
     return this._http.authedRequest(undefined, "POST", "/keys/query", undefined, content);

src/browser-index.js

@@ -34,7 +34,7 @@ matrixcs.request(function(opts, fn) {
 let indexedDB;
 try {
     indexedDB = global.indexedDB;
-} catch(e) {}
+} catch (e) {}
 
 // if our browser (appears to) support indexeddb, use an indexeddb crypto store.
 if (indexedDB) {

src/content-repo.js

@@ -75,35 +75,3 @@ export function getHttpUriForMxc(baseUrl, mxc, width, height,
         (utils.keys(params).length === 0 ? "" :
         ("?" + utils.encodeParams(params))) + fragment;
 }
-
-/**
- * Get an identicon URL from an arbitrary string.
- * @param {string} baseUrl The base homeserver url which has a content repo.
- * @param {string} identiconString The string to create an identicon for.
- * @param {Number} width The desired width of the image in pixels. Default: 96.
- * @param {Number} height The desired height of the image in pixels. Default: 96.
- * @return {string} The complete URL to the identicon.
- * @deprecated This is no longer in the specification.
- */
-export function getIdenticonUri(baseUrl, identiconString, width, height) {
-    if (!identiconString) {
-        return null;
-    }
-    if (!width) {
-        width = 96;
-    }
-    if (!height) {
-        height = 96;
-    }
-    const params = {
-        width: width,
-        height: height,
-    };
-
-    const path = utils.encodeUri("/_matrix/media/unstable/identicon/$ident", {
-        $ident: identiconString,
-    });
-    return baseUrl + path +
-        (utils.keys(params).length === 0 ? "" :
-            ("?" + utils.encodeParams(params)));
-}

src/crypto/CrossSigning.js

@@ -24,6 +24,9 @@ import {decodeBase64, encodeBase64, pkSign, pkVerify} from './olmlib';
 import {EventEmitter} from 'events';
 import {logger} from '../logger';
 import {IndexedDBCryptoStore} from '../crypto/store/indexeddb-crypto-store';
+import {decryptAES, encryptAES} from './aes';
+
+const KEY_REQUEST_TIMEOUT_MS = 1000 * 60;
 
 function publicKeyFromKeyInfo(keyInfo) {
     // `keys` is an object with { [`ed25519:${pubKey}`]: pubKey }
@@ -55,17 +58,42 @@ export class CrossSigningInfo extends EventEmitter {
         this._cacheCallbacks = cacheCallbacks || {};
         this.keys = {};
         this.firstUse = true;
+        // This tracks whether we've ever verified this user with any identity.
+        // When you verify a user, any devices online at the time that receive
+        // the verifying signature via the homeserver will latch this to true
+        // and can use it in the future to detect cases where the user has
+        // become unverifed later for any reason.
+        this.crossSigningVerifiedBefore = false;
+    }
+
+    static fromStorage(obj, userId) {
+        const res = new CrossSigningInfo(userId);
+        for (const prop in obj) {
+            if (obj.hasOwnProperty(prop)) {
+                res[prop] = obj[prop];
+            }
+        }
+        return res;
+    }
+
+    toStorage() {
+        return {
+            keys: this.keys,
+            firstUse: this.firstUse,
+            crossSigningVerifiedBefore: this.crossSigningVerifiedBefore,
+        };
     }
 
     /**
      * Calls the app callback to ask for a private key
+     *
      * @param {string} type The key type ("master", "self_signing", or "user_signing")
      * @param {string} expectedPubkey The matching public key or undefined to use
      *     the stored public key for the given key type.
      * @returns {Array} An array with [ public key, Olm.PkSigning ]
      */
     async getCrossSigningKey(type, expectedPubkey) {
-        const shouldCache = ["self_signing", "user_signing"].indexOf(type) >= 0;
+        const shouldCache = ["master", "self_signing", "user_signing"].indexOf(type) >= 0;
 
         if (!this._callbacks.getCrossSigningKey) {
             throw new Error("No getCrossSigningKey callback supplied");
@@ -118,23 +146,6 @@ export class CrossSigningInfo extends EventEmitter {
         );
     }
 
-    static fromStorage(obj, userId) {
-        const res = new CrossSigningInfo(userId);
-        for (const prop in obj) {
-            if (obj.hasOwnProperty(prop)) {
-                res[prop] = obj[prop];
-            }
-        }
-        return res;
-    }
-
-    toStorage() {
-        return {
-            keys: this.keys,
-            firstUse: this.firstUse,
-        };
-    }
-
     /**
      * Check whether the private keys exist in secret storage.
      * XXX: This could be static, be we often seem to have an instance when we
@@ -170,12 +181,12 @@ export class CrossSigningInfo extends EventEmitter {
      * typically called in conjunction with the creation of new cross-signing
      * keys.
      *
-     * @param {object} keys The keys to store
+     * @param {Map} keys The keys to store
      * @param {SecretStorage} secretStorage The secret store using account data
      */
     static async storeInSecretStorage(keys, secretStorage) {
-        for (const type of Object.keys(keys)) {
-            const encodedKey = encodeBase64(keys[type]);
+        for (const [type, privateKey] of keys) {
+            const encodedKey = encodeBase64(privateKey);
             await secretStorage.store(`m.cross_signing.${type}`, encodedKey);
         }
     }
@@ -191,9 +202,50 @@ export class CrossSigningInfo extends EventEmitter {
      */
     static async getFromSecretStorage(type, secretStorage) {
         const encodedKey = await secretStorage.get(`m.cross_signing.${type}`);
+        if (!encodedKey) {
+            return null;
+        }
         return decodeBase64(encodedKey);
     }
 
+    /**
+     * Check whether the private keys exist in the local key cache.
+     *
+     * @param {string} [type] The type of key to get. One of "master",
+     * "self_signing", or "user_signing". Optional, will check all by default.
+     * @returns {boolean} True if all keys are stored in the local cache.
+     */
+    async isStoredInKeyCache(type) {
+        const cacheCallbacks = this._cacheCallbacks;
+        if (!cacheCallbacks) return false;
+        const types = type ? [type] : ["master", "self_signing", "user_signing"];
+        for (const t of types) {
+            if (!await cacheCallbacks.getCrossSigningKeyCache(t)) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    /**
+     * Get cross-signing private keys from the local cache.
+     *
+     * @returns {Map} A map from key type (string) to private key (Uint8Array)
+     */
+    async getCrossSigningKeysFromCache() {
+        const keys = new Map();
+        const cacheCallbacks = this._cacheCallbacks;
+        if (!cacheCallbacks) return keys;
+        for (const type of ["master", "self_signing", "user_signing"]) {
+            const privKey = await cacheCallbacks.getCrossSigningKeyCache(type);
+            if (!privKey) {
+                continue;
+            }
+            keys.set(type, privKey);
+        }
+        return keys;
+    }
+
     /**
      * Get the ID used to identify the user. This can also be used to test for
      * the existence of a given key type.
@@ -303,6 +355,13 @@ export class CrossSigningInfo extends EventEmitter {
         }
     }
 
+    /**
+     * unsets the keys, used when another session has reset the keys, to disable cross-signing
+     */
+    clearKeys() {
+        this.keys = {};
+    }
+
     setKeys(keys) {
         const signingKeys = {};
         if (keys.master) {
@@ -375,6 +434,14 @@ export class CrossSigningInfo extends EventEmitter {
         }
     }
 
+    updateCrossSigningVerifiedBefore(isCrossSigningVerified) {
+        // It is critical that this value latches forward from false to true but
+        // never back to false to avoid a downgrade attack.
+        if (!this.crossSigningVerifiedBefore && isCrossSigningVerified) {
+            this.crossSigningVerifiedBefore = true;
+        }
+    }
+
     async signObject(data, type) {
         if (!this.keys[type]) {
             throw new Error(
@@ -433,13 +500,13 @@ export class CrossSigningInfo extends EventEmitter {
             && this.getId("self_signing")
             && this.getId("self_signing") === userCrossSigning.getId("self_signing")
         ) {
-            return new UserTrustLevel(true, this.firstUse);
+            return new UserTrustLevel(true, true, this.firstUse);
         }
 
         if (!this.keys.user_signing) {
             // If there's no user signing key, they can't possibly be verified.
             // They may be TOFU trusted though.
-            return new UserTrustLevel(false, userCrossSigning.firstUse);
+            return new UserTrustLevel(false, false, userCrossSigning.firstUse);
         }
 
         let userTrusted;
@@ -451,7 +518,11 @@ export class CrossSigningInfo extends EventEmitter {
         } catch (e) {
             userTrusted = false;
         }
-        return new UserTrustLevel(userTrusted, userCrossSigning.firstUse);
+        return new UserTrustLevel(
+            userTrusted,
+            userCrossSigning.crossSigningVerifiedBefore,
+            userCrossSigning.firstUse,
+        );
     }
 
     /**
@@ -523,8 +594,9 @@ export const CrossSigningLevel = {
  * Represents the ways in which we trust a user
  */
 export class UserTrustLevel {
-    constructor(crossSigningVerified, tofu) {
+    constructor(crossSigningVerified, crossSigningVerifiedBefore, tofu) {
         this._crossSigningVerified = crossSigningVerified;
+        this._crossSigningVerifiedBefore = crossSigningVerifiedBefore;
         this._tofu = tofu;
     }
 
@@ -542,6 +614,14 @@ export class UserTrustLevel {
         return this._crossSigningVerified;
     }
 
+    /**
+     * @returns {bool} true if we ever verified this user before (at least for
+     * the history of verifications observed by this device).
+     */
+    wasCrossSigningVerified() {
+        return this._crossSigningVerifiedBefore;
+    }
+
     /**
      * @returns {bool} true if this user's key is trusted on first use
      */
@@ -602,10 +682,10 @@ export class DeviceTrustLevel {
     }
 }
 
-export function createCryptoStoreCacheCallbacks(store) {
+export function createCryptoStoreCacheCallbacks(store, olmdevice) {
     return {
-        getCrossSigningKeyCache: function(type, _expectedPublicKey) {
-            return new Promise((resolve) => {
+        getCrossSigningKeyCache: async function(type, _expectedPublicKey) {
+            const key = await new Promise((resolve) => {
                 return store.doTxn(
                     'readonly',
                     [IndexedDBCryptoStore.STORE_ACCOUNT],
@@ -614,8 +694,23 @@ export function createCryptoStoreCacheCallbacks(store) {
                     },
                 );
             });
+
+            if (key && key.ciphertext) {
+                const pickleKey = Buffer.from(olmdevice._pickleKey);
+                const decrypted = await decryptAES(key, pickleKey, type);
+                return decodeBase64(decrypted);
+            } else {
+                return key;
+            }
         },
-        storeCrossSigningKeyCache: function(type, key) {
+        storeCrossSigningKeyCache: async function(type, key) {
+            if (!(key instanceof Uint8Array)) {
+                throw new Error(
+                    `storeCrossSigningKeyCache expects Uint8Array, got ${key}`,
+                );
+            }
+            const pickleKey = Buffer.from(olmdevice._pickleKey);
+            key = await encryptAES(encodeBase64(key), pickleKey, type);
             return store.doTxn(
                 'readwrite',
                 [IndexedDBCryptoStore.STORE_ACCOUNT],
@@ -626,3 +721,96 @@ export function createCryptoStoreCacheCallbacks(store) {
         },
     };
 }
+
+/**
+ * Request cross-signing keys from another device during verification.
+ *
+ * @param {module:base-apis~MatrixBaseApis} baseApis base Matrix API interface
+ * @param {string} userId The user ID being verified
+ * @param {string} deviceId The device ID being verified
+ */
+export async function requestKeysDuringVerification(baseApis, userId, deviceId) {
+    // If this is a self-verification, ask the other party for keys
+    if (baseApis.getUserId() !== userId) {
+        return;
+    }
+    logger.log("Cross-signing: Self-verification done; requesting keys");
+    // This happens asynchronously, and we're not concerned about waiting for
+    // it.  We return here in order to test.
+    return new Promise((resolve, reject) => {
+        const client = baseApis;
+        const original = client._crypto._crossSigningInfo;
+
+        // We already have all of the infrastructure we need to validate and
+        // cache cross-signing keys, so instead of replicating that, here we set
+        // up callbacks that request them from the other device and call
+        // CrossSigningInfo.getCrossSigningKey() to validate/cache
+        const crossSigning = new CrossSigningInfo(
+            original.userId,
+            { getCrossSigningKey: async (type) => {
+                logger.debug("Cross-signing: requesting secret",
+                                type, deviceId);
+                const { promise } = client.requestSecret(
+                    `m.cross_signing.${type}`, [deviceId],
+                );
+                const result = await promise;
+                const decoded = decodeBase64(result);
+                return Uint8Array.from(decoded);
+            } },
+            original._cacheCallbacks,
+        );
+        crossSigning.keys = original.keys;
+
+        // XXX: get all keys out if we get one key out
+        // https://github.com/vector-im/element-web/issues/12604
+        // then change here to reject on the timeout
+        // Requests can be ignored, so don't wait around forever
+        const timeout = new Promise((resolve, reject) => {
+            setTimeout(
+                resolve,
+                KEY_REQUEST_TIMEOUT_MS,
+                new Error("Timeout"),
+            );
+        });
+
+        // also request and cache the key backup key
+        const backupKeyPromise = new Promise(async resolve => {
+            const cachedKey = await client._crypto.getSessionBackupPrivateKey();
+            if (!cachedKey) {
+                logger.info("No cached backup key found. Requesting...");
+                const secretReq = client.requestSecret(
+                    'm.megolm_backup.v1', [deviceId],
+                );
+                const base64Key = await secretReq.promise;
+                logger.info("Got key backup key, decoding...");
+                const decodedKey = decodeBase64(base64Key);
+                logger.info("Decoded backup key, storing...");
+                client._crypto.storeSessionBackupPrivateKey(
+                    Uint8Array.from(decodedKey),
+                );
+                logger.info("Backup key stored. Starting backup restore...");
+                const backupInfo = await client.getKeyBackupVersion();
+                // no need to await for this - just let it go in the bg
+                client.restoreKeyBackupWithCache(
+                    undefined, undefined, backupInfo,
+                ).then(() => {
+                    logger.info("Backup restored.");
+                });
+            }
+            resolve();
+        });
+
+        // We call getCrossSigningKey() for its side-effects
+        return Promise.race([
+            Promise.all([
+                crossSigning.getCrossSigningKey("master"),
+                crossSigning.getCrossSigningKey("self_signing"),
+                crossSigning.getCrossSigningKey("user_signing"),
+                backupKeyPromise,
+            ]),
+            timeout,
+        ]).then(resolve, reject);
+    }).catch((e) => {
+        logger.warn("Cross-signing: failure while requesting keys:", e);
+    });
+}

src/crypto/DeviceList.js

@@ -109,6 +109,9 @@ export class DeviceList extends EventEmitter {
         this._savePromiseTime = null;
         // The timer used to delay the save
         this._saveTimer = null;
+        // True if we have fetched data from the server or loaded a non-empty
+        // set of device data from the store
+        this._hasFetched = null;
     }
 
     /**
@@ -118,6 +121,7 @@ export class DeviceList extends EventEmitter {
         await this._cryptoStore.doTxn(
             'readonly', [IndexedDBCryptoStore.STORE_DEVICE_DATA], (txn) => {
                 this._cryptoStore.getEndToEndDeviceData(txn, (deviceData) => {
+                    this._hasFetched = Boolean(deviceData && deviceData.devices);
                     this._devices = deviceData ? deviceData.devices : {},
                     this._crossSigningInfo = deviceData ?
                         deviceData.crossSigningInfo || {} : {};
@@ -652,6 +656,7 @@ export class DeviceList extends EventEmitter {
         });
 
         const finished = (success) => {
+            this.emit("crypto.willUpdateDevices", users, !this._hasFetched);
             users.forEach((u) => {
                 this._dirty = true;
 
@@ -677,7 +682,8 @@ export class DeviceList extends EventEmitter {
                 }
             });
             this.saveIfDirty();
-            this.emit("crypto.devicesUpdated", users);
+            this.emit("crypto.devicesUpdated", users, !this._hasFetched);
+            this._hasFetched = true;
         };
 
         return prom;
@@ -784,7 +790,7 @@ class DeviceListUpdateSerialiser {
 
             // yield to other things that want to execute in between users, to
             // avoid wedging the CPU
-            // (https://github.com/vector-im/riot-web/issues/3158)
+            // (https://github.com/vector-im/element-web/issues/3158)
             //
             // of course we ought to do this in a web worker or similar, but
             // this serves as an easy solution for now.
@@ -824,7 +830,7 @@ class DeviceListUpdateSerialiser {
     }
 
     async _processQueryResponseForUser(
-        userId, dkResponse, crossSigningResponse, sskResponse,
+        userId, dkResponse, crossSigningResponse,
     ) {
         logger.log('got device keys for ' + userId + ':', dkResponse);
         logger.log('got cross-signing keys for ' + userId + ':', crossSigningResponse);
@@ -842,6 +848,7 @@ class DeviceListUpdateSerialiser {
 
             await _updateStoredDeviceKeysForUser(
                 this._olmDevice, userId, userStore, dkResponse || {},
+                this._baseApis.getUserId(), this._baseApis.deviceId,
             );
 
             // put the updates into the object that will be returned as our results
@@ -879,8 +886,9 @@ class DeviceListUpdateSerialiser {
 }
 
 
-async function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
-        userResult) {
+async function _updateStoredDeviceKeysForUser(
+    _olmDevice, userId, userStore, userResult, localUserId, localDeviceId,
+) {
     let updated = false;
 
     // remove any devices in the store which aren't in the response
@@ -890,6 +898,13 @@ async function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
         }
 
         if (!(deviceId in userResult)) {
+            if (userId === localUserId && deviceId === localDeviceId) {
+                logger.warn(
+                    `Local device ${deviceId} missing from sync, skipping removal`,
+                );
+                continue;
+            }
+
             logger.log("Device " + userId + ":" + deviceId +
                 " has been removed");
             delete userStore[deviceId];

src/crypto/EncryptionSetup.js

@@ -0,0 +1,357 @@
+import { logger } from "../logger";
+import {MatrixEvent} from "../models/event";
+import {EventEmitter} from "events";
+import {createCryptoStoreCacheCallbacks} from "./CrossSigning";
+import {IndexedDBCryptoStore} from './store/indexeddb-crypto-store';
+import {
+    PREFIX_UNSTABLE,
+} from "../http-api";
+
+/**
+ * Builds an EncryptionSetupOperation by calling any of the add.. methods.
+ * Once done, `buildOperation()` can be called which allows to apply to operation.
+ *
+ * This is used as a helper by Crypto to keep track of all the network requests
+ * and other side-effects of bootstrapping, so it can be applied in one go (and retried in the future)
+ * Also keeps track of all the private keys created during bootstrapping, so we don't need to prompt for them
+ * more than once.
+ */
+export class EncryptionSetupBuilder {
+    /**
+     * @param {Object.<String, MatrixEvent>} accountData pre-existing account data, will only be read, not written.
+     * @param {CryptoCallbacks} delegateCryptoCallbacks crypto callbacks to delegate to if the key isn't in cache yet
+     */
+    constructor(accountData, delegateCryptoCallbacks) {
+        this.accountDataClientAdapter = new AccountDataClientAdapter(accountData);
+        this.crossSigningCallbacks = new CrossSigningCallbacks();
+        this.ssssCryptoCallbacks = new SSSSCryptoCallbacks(delegateCryptoCallbacks);
+
+        this._crossSigningKeys = null;
+        this._keySignatures = null;
+        this._keyBackupInfo = null;
+    }
+
+    /**
+     * Adds new cross-signing public keys
+     *
+     * @param {function} authUpload Function called to await an interactive auth
+     * flow when uploading device signing keys.
+     * Args:
+     *     {function} A function that makes the request requiring auth. Receives
+     *     the auth data as an object. Can be called multiple times, first with
+     *     an empty authDict, to obtain the flows.
+     * @param {Object} keys the new keys
+     */
+    addCrossSigningKeys(authUpload, keys) {
+        this._crossSigningKeys = {authUpload, keys};
+    }
+
+    /**
+     * Adds the key backup info to be updated on the server
+     *
+     * Used either to create a new key backup, or add signatures
+     * from the new MSK.
+     *
+     * @param {Object} keyBackupInfo as received from/sent to the server
+     */
+    addSessionBackup(keyBackupInfo) {
+        this._keyBackupInfo = keyBackupInfo;
+    }
+
+    /**
+     * Adds the session backup private key to be updated in the local cache
+     *
+     * Used after fixing the format of the key
+     *
+     * @param {Uint8Array} privateKey
+     */
+    addSessionBackupPrivateKeyToCache(privateKey) {
+        this._sessionBackupPrivateKey = privateKey;
+    }
+
+    /**
+     * Add signatures from a given user and device/x-sign key
+     * Used to sign the new cross-signing key with the device key
+     *
+     * @param {String} userId
+     * @param {String} deviceId
+     * @param {String} signature
+     */
+    addKeySignature(userId, deviceId, signature) {
+        if (!this._keySignatures) {
+            this._keySignatures = {};
+        }
+        const userSignatures = this._keySignatures[userId] || {};
+        this._keySignatures[userId] = userSignatures;
+        userSignatures[deviceId] = signature;
+    }
+
+
+    /**
+     * @param {String} type
+     * @param {Object} content
+     * @return {Promise}
+     */
+    setAccountData(type, content) {
+        return this.accountDataClientAdapter.setAccountData(type, content);
+    }
+
+    /**
+     * builds the operation containing all the parts that have been added to the builder
+     * @return {EncryptionSetupOperation}
+     */
+    buildOperation() {
+        const accountData = this.accountDataClientAdapter._values;
+        return new EncryptionSetupOperation(
+            accountData,
+            this._crossSigningKeys,
+            this._keyBackupInfo,
+            this._keySignatures,
+        );
+    }
+
+    /**
+     * Stores the created keys locally.
+     *
+     * This does not yet store the operation in a way that it can be restored,
+     * but that is the idea in the future.
+     *
+     * @param  {Crypto} crypto
+     * @return {Promise}
+     */
+    async persist(crypto) {
+        // store private keys in cache
+        if (this._crossSigningKeys) {
+            const cacheCallbacks = createCryptoStoreCacheCallbacks(
+                crypto._cryptoStore, crypto._olmDevice);
+            for (const type of ["master", "self_signing", "user_signing"]) {
+                logger.log(`Cache ${type} cross-signing private key locally`);
+                const privateKey = this.crossSigningCallbacks.privateKeys.get(type);
+                await cacheCallbacks.storeCrossSigningKeyCache(type, privateKey);
+            }
+            // store own cross-sign pubkeys as trusted
+            await crypto._cryptoStore.doTxn(
+                'readwrite', [IndexedDBCryptoStore.STORE_ACCOUNT],
+                (txn) => {
+                    crypto._cryptoStore.storeCrossSigningKeys(
+                        txn, this._crossSigningKeys.keys);
+                },
+            );
+        }
+        // store session backup key in cache
+        if (this._sessionBackupPrivateKey) {
+            await crypto.storeSessionBackupPrivateKey(this._sessionBackupPrivateKey);
+        }
+    }
+}
+
+/**
+ * Can be created from EncryptionSetupBuilder, or
+ * (in a follow-up PR, not implemented yet) restored from storage, to retry.
+ *
+ * It does not have knowledge of any private keys, unlike the builder.
+ */
+export class EncryptionSetupOperation {
+    /**
+     * @param  {Map<String, Object>} accountData
+     * @param  {Object} crossSigningKeys
+     * @param  {Object} keyBackupInfo
+     * @param  {Object} keySignatures
+     */
+    constructor(accountData, crossSigningKeys, keyBackupInfo, keySignatures) {
+        this._accountData = accountData;
+        this._crossSigningKeys = crossSigningKeys;
+        this._keyBackupInfo = keyBackupInfo;
+        this._keySignatures = keySignatures;
+    }
+
+    /**
+     * Runs the (remaining part of, in the future) operation by sending requests to the server.
+     * @param  {Crypto} crypto
+     */
+    async apply(crypto) {
+        const baseApis = crypto._baseApis;
+        // upload cross-signing keys
+        if (this._crossSigningKeys) {
+            const keys = {};
+            for (const [name, key] of Object.entries(this._crossSigningKeys.keys)) {
+                keys[name + "_key"] = key;
+            }
+
+            // We must only call `uploadDeviceSigningKeys` from inside this auth
+            // helper to ensure we properly handle auth errors.
+            await this._crossSigningKeys.authUpload(authDict => {
+                return baseApis.uploadDeviceSigningKeys(authDict, keys);
+            });
+
+            // pass the new keys to the main instance of our own CrossSigningInfo.
+            crypto._crossSigningInfo.setKeys(this._crossSigningKeys.keys);
+        }
+        // set account data
+        if (this._accountData) {
+            for (const [type, content] of this._accountData) {
+                await baseApis.setAccountData(type, content);
+            }
+        }
+        // upload first cross-signing signatures with the new key
+        // (e.g. signing our own device)
+        if (this._keySignatures) {
+            await baseApis.uploadKeySignatures(this._keySignatures);
+        }
+        // need to create/update key backup info
+        if (this._keyBackupInfo) {
+            if (this._keyBackupInfo.version) {
+                // session backup signature
+                // The backup is trusted because the user provided the private key.
+                // Sign the backup with the cross signing key so the key backup can
+                // be trusted via cross-signing.
+                await baseApis._http.authedRequest(
+                    undefined, "PUT", "/room_keys/version/" + this._keyBackupInfo.version,
+                    undefined, {
+                        algorithm: this._keyBackupInfo.algorithm,
+                        auth_data: this._keyBackupInfo.auth_data,
+                    },
+                    {prefix: PREFIX_UNSTABLE},
+                );
+            } else {
+                // add new key backup
+                await baseApis._http.authedRequest(
+                    undefined, "POST", "/room_keys/version",
+                    undefined, this._keyBackupInfo,
+                    {prefix: PREFIX_UNSTABLE},
+                );
+            }
+        }
+    }
+}
+
+
+/**
+ * Catches account data set by SecretStorage during bootstrapping by
+ * implementing the methods related to account data in MatrixClient
+ */
+class AccountDataClientAdapter extends EventEmitter {
+    /**
+     * @param  {Object.<String, MatrixEvent>} accountData existing account data
+     */
+    constructor(accountData) {
+        super();
+        this._existingValues = accountData;
+        this._values = new Map();
+    }
+
+    /**
+     * @param  {String} type
+     * @return {Promise<Object>} the content of the account data
+     */
+    getAccountDataFromServer(type) {
+        return Promise.resolve(this.getAccountData(type));
+    }
+
+    /**
+     * @param  {String} type
+     * @return {Object} the content of the account data
+     */
+    getAccountData(type) {
+        const modifiedValue = this._values.get(type);
+        if (modifiedValue) {
+            return modifiedValue;
+        }
+        const existingValue = this._existingValues[type];
+        if (existingValue) {
+            return existingValue.getContent();
+        }
+        return null;
+    }
+
+    /**
+     * @param {String} type
+     * @param {Object} content
+     * @return {Promise}
+     */
+    setAccountData(type, content) {
+        const lastEvent = this._values.get(type);
+        this._values.set(type, content);
+        // ensure accountData is emitted on the next tick,
+        // as SecretStorage listens for it while calling this method
+        // and it seems to rely on this.
+        return Promise.resolve().then(() => {
+            const event = new MatrixEvent({type, content});
+            this.emit("accountData", event, lastEvent);
+        });
+    }
+}
+
+/**
+ * Catches the private cross-signing keys set during bootstrapping
+ * by both cache callbacks (see createCryptoStoreCacheCallbacks) as non-cache callbacks.
+ * See CrossSigningInfo constructor
+ */
+class CrossSigningCallbacks {
+    constructor() {
+        this.privateKeys = new Map();
+    }
+
+    // cache callbacks
+    getCrossSigningKeyCache(type, expectedPublicKey) {
+        return this.getCrossSigningKey(type, expectedPublicKey);
+    }
+
+    storeCrossSigningKeyCache(type, key) {
+        this.privateKeys.set(type, key);
+        return Promise.resolve();
+    }
+
+    // non-cache callbacks
+    getCrossSigningKey(type, _expectedPubkey) {
+        return Promise.resolve(this.privateKeys.get(type));
+    }
+
+    saveCrossSigningKeys(privateKeys) {
+        for (const [type, privateKey] of Object.entries(privateKeys)) {
+            this.privateKeys.set(type, privateKey);
+        }
+    }
+}
+
+/**
+ * Catches the 4S private key set during bootstrapping by implementing
+ * the SecretStorage crypto callbacks
+ */
+class SSSSCryptoCallbacks {
+    constructor(delegateCryptoCallbacks) {
+        this._privateKeys = new Map();
+        this._delegateCryptoCallbacks = delegateCryptoCallbacks;
+    }
+
+    async getSecretStorageKey({ keys }, name) {
+        for (const keyId of Object.keys(keys)) {
+            const privateKey = this._privateKeys.get(keyId);
+            if (privateKey) {
+                return [keyId, privateKey];
+            }
+        }
+        // if we don't have the key cached yet, ask
+        // for it to the general crypto callbacks and cache it
+        if (this._delegateCryptoCallbacks) {
+            const result = await this._delegateCryptoCallbacks.
+                getSecretStorageKey({keys}, name);
+            if (result) {
+                const [keyId, privateKey] = result;
+                this._privateKeys.set(keyId, privateKey);
+            }
+            return result;
+        }
+    }
+
+    addPrivateKey(keyId, keyInfo, privKey) {
+        this._privateKeys.set(keyId, privKey);
+        // Also pass along to application to cache if it wishes
+        if (
+            this._delegateCryptoCallbacks &&
+            this._delegateCryptoCallbacks.cacheSecretStorageKey
+        ) {
+            this._delegateCryptoCallbacks.cacheSecretStorageKey(keyId, keyInfo, privKey);
+        }
+    }
+}

src/crypto/OlmDevice.js

@@ -36,9 +36,15 @@ function checkPayloadLength(payloadString) {
         // Note that even if we manage to do the encryption, the message send may fail,
         // because by the time we've wrapped the ciphertext in the event object, it may
         // exceed 65K. But at least we won't just fail with "abort()" in that case.
-        throw new Error("Message too long (" + payloadString.length + " bytes). " +
+        const err = new Error("Message too long (" + payloadString.length + " bytes). " +
                         "The maximum for an encrypted message is " +
                         MAX_PLAINTEXT_LENGTH + " bytes.");
+        // TODO: [TypeScript] We should have our own error types
+        err.data = {
+            errcode: "M_TOO_LARGE",
+            error: "Payload too large for encrypted message",
+        };
+        throw err;
     }
 }
 
@@ -105,6 +111,9 @@ export function OlmDevice(cryptoStore) {
     // Keep track of sessions that we're starting, so that we don't start
     // multiple sessions for the same device at the same time.
     this._sessionsInProgress = {};
+
+    // Used by olm to serialise prekey message decryptions
+    this._olmPrekeyPromise = Promise.resolve();
 }
 
 /**
@@ -135,7 +144,7 @@ OlmDevice.prototype.init = async function(opts = {}) {
     try {
         if (fromExportedDevice) {
             if (pickleKey) {
-                console.warn(
+                logger.warn(
                     'ignoring opts.pickleKey'
                     + ' because opts.fromExportedDevice is present.',
                 );
@@ -468,6 +477,36 @@ OlmDevice.prototype.generateOneTimeKeys = function(numKeys) {
     );
 };
 
+/**
+ * Generate a new fallback keys
+ *
+ * @return {Promise} Resolved once the account is saved back having generated the key
+ */
+OlmDevice.prototype.generateFallbackKey = async function() {
+    await this._cryptoStore.doTxn(
+        'readwrite', [IndexedDBCryptoStore.STORE_ACCOUNT],
+        (txn) => {
+            this._getAccount(txn, (account) => {
+                account.generate_fallback_key();
+                this._storeAccount(txn, account);
+            });
+        },
+    );
+};
+
+OlmDevice.prototype.getFallbackKey = async function() {
+    let result;
+    await this._cryptoStore.doTxn(
+        'readonly', [IndexedDBCryptoStore.STORE_ACCOUNT],
+        (txn) => {
+            this._getAccount(txn, (account) => {
+                result = JSON.parse(account.fallback_key());
+            });
+        },
+    );
+    return result;
+};
+
 /**
  * Generate a new outbound session
  *
@@ -983,11 +1022,12 @@ OlmDevice.prototype._getInboundGroupSession = function(
  * @param {Object<string, string>} keysClaimed Other keys the sender claims.
  * @param {boolean} exportFormat true if the megolm keys are in export format
  *    (ie, they lack an ed25519 signature)
+ * @param {Object} [extraSessionData={}] any other data to be include with the session
  */
 OlmDevice.prototype.addInboundGroupSession = async function(
     roomId, senderKey, forwardingCurve25519KeyChain,
     sessionId, sessionKey, keysClaimed,
-    exportFormat,
+    exportFormat, extraSessionData = {},
 ) {
     await this._cryptoStore.doTxn(
         'readwrite', [
@@ -1029,12 +1069,17 @@ OlmDevice.prototype.addInboundGroupSession = async function(
                             }
                         }
 
-                        const sessionData = {
+                        logger.info(
+                            "Storing megolm session " + senderKey + "/" + sessionId +
+                            " with first index " + session.first_known_index(),
+                        );
+
+                        const sessionData = Object.assign({}, extraSessionData, {
                             room_id: roomId,
                             session: session.pickle(this._pickleKey),
                             keysClaimed: keysClaimed,
                             forwardingCurve25519KeyChain: forwardingCurve25519KeyChain,
-                        };
+                        });
 
                         this._cryptoStore.storeEndToEndInboundGroupSession(
                             senderKey, sessionId, sessionData, txn,
@@ -1210,6 +1255,7 @@ OlmDevice.prototype.decryptGroupMessage = async function(
                         forwardingCurve25519KeyChain: (
                             sessionData.forwardingCurve25519KeyChain || []
                         ),
+                        untrusted: sessionData.untrusted,
                     };
                 },
             );

src/crypto/OutgoingRoomKeyRequestManager.js

@@ -58,7 +58,7 @@ const SEND_KEY_REQUESTS_DELAY_MS = 500;
  *
  * @enum {number}
  */
-const ROOM_KEY_REQUEST_STATES = {
+export const ROOM_KEY_REQUEST_STATES = {
     /** request not yet sent */
     UNSENT: 0,
 
@@ -97,10 +97,6 @@ export class OutgoingRoomKeyRequestManager {
      */
     start() {
         this._clientRunning = true;
-
-        // set the timer going, to handle any requests which didn't get sent
-        // on the previous run of the client.
-        this._startTimer();
     }
 
     /**
@@ -113,7 +109,14 @@ export class OutgoingRoomKeyRequestManager {
     }
 
     /**
-     * Send off a room key request, if we haven't already done so.
+     * Send any requests that have been queued
+     */
+    sendQueuedRequests() {
+        this._startTimer();
+    }
+
+    /**
+     * Queue up a room key request, if we haven't already queued or sent one.
      *
      * The `requestBody` is compared (with a deep-equality check) against
      * previous queued or sent requests and if it matches, no change is made.
@@ -129,7 +132,7 @@ export class OutgoingRoomKeyRequestManager {
      *    pending list (or we have established that a similar request already
      *    exists)
      */
-    async sendRoomKeyRequest(requestBody, recipients, resend=false) {
+    async queueRoomKeyRequest(requestBody, recipients, resend=false) {
         const req = await this._cryptoStore.getOutgoingRoomKeyRequest(
             requestBody,
         );
@@ -184,7 +187,7 @@ export class OutgoingRoomKeyRequestManager {
                         // in state ROOM_KEY_REQUEST_STATES.SENT, so we must have
                         // raced with another tab to mark the request cancelled.
                         // Try again, to make sure the request is resent.
-                        return await this.sendRoomKeyRequest(
+                        return await this.queueRoomKeyRequest(
                             requestBody, recipients, resend,
                         );
                     }
@@ -220,9 +223,6 @@ export class OutgoingRoomKeyRequestManager {
                 throw new Error('unhandled state: ' + req.state);
             }
         }
-        // some of the branches require the timer to be started.  Just start it
-        // all the time, because it doesn't hurt to start it.
-        this._startTimer();
     }
 
     /**
@@ -327,6 +327,21 @@ export class OutgoingRoomKeyRequestManager {
         );
     }
 
+    /**
+     * Find anything in `sent` state, and kick it around the loop again.
+     * This is intended for situations where something substantial has changed, and we
+     * don't really expect the other end to even care about the cancellation.
+     * For example, after initialization or self-verification.
+     * @return {Promise} An array of `queueRoomKeyRequest` outputs.
+     */
+    async cancelAndResendAllOutgoingRequests() {
+        const outgoings = await this._cryptoStore.getAllOutgoingRoomKeyRequestsByState(
+            ROOM_KEY_REQUEST_STATES.SENT,
+        );
+        return Promise.all(outgoings.map(({ requestBody, recipients }) =>
+            this.queueRoomKeyRequest(requestBody, recipients, true)));
+    }
+
     // start the background timer to send queued requests, if the timer isn't
     // already running
     _startTimer() {
@@ -366,15 +381,12 @@ export class OutgoingRoomKeyRequestManager {
             return Promise.resolve();
         }
 
-        logger.log("Looking for queued outgoing room key requests");
-
         return this._cryptoStore.getOutgoingRoomKeyRequestByState([
             ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING,
             ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING_AND_WILL_RESEND,
             ROOM_KEY_REQUEST_STATES.UNSENT,
         ]).then((req) => {
             if (!req) {
-                logger.log("No more outgoing room key requests");
                 this._sendOutgoingRoomKeyRequestsTimer = null;
                 return;
             }
@@ -398,7 +410,6 @@ export class OutgoingRoomKeyRequestManager {
             }).catch((e) => {
                 logger.error("Error sending room key request; will retry later.", e);
                 this._sendOutgoingRoomKeyRequestsTimer = null;
-                this._startTimer();
             });
         });
     }

src/crypto/SecretStorage.js

@@ -17,26 +17,24 @@ limitations under the License.
 import {EventEmitter} from 'events';
 import {logger} from '../logger';
 import * as olmlib from './olmlib';
-import {pkVerify} from './olmlib';
 import {randomString} from '../randomstring';
 import {encryptAES, decryptAES} from './aes';
+import {encodeBase64} from "./olmlib";
 
 export const SECRET_STORAGE_ALGORITHM_V1_AES
     = "m.secret_storage.v1.aes-hmac-sha2";
-// don't use curve25519 for writing data.
-export const SECRET_STORAGE_ALGORITHM_V1_CURVE25519
-    = "m.secret_storage.v1.curve25519-aes-sha2";
+
+const ZERO_STR = "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0";
 
 /**
  * Implements Secure Secret Storage and Sharing (MSC1946)
  * @module crypto/SecretStorage
  */
 export class SecretStorage extends EventEmitter {
-    constructor(baseApis, cryptoCallbacks, crossSigningInfo) {
+    constructor(baseApis, cryptoCallbacks) {
         super();
         this._baseApis = baseApis;
         this._cryptoCallbacks = cryptoCallbacks;
-        this._crossSigningInfo = crossSigningInfo;
         this._requests = {};
         this._incomingRequests = {};
     }
@@ -50,7 +48,7 @@ export class SecretStorage extends EventEmitter {
     }
 
     setDefaultKeyId(keyId) {
-        return new Promise((resolve) => {
+        return new Promise(async (resolve, reject) => {
             const listener = (ev) => {
                 if (
                     ev.getType() === 'm.secret_storage.default_key' &&
@@ -62,10 +60,15 @@ export class SecretStorage extends EventEmitter {
             };
             this._baseApis.on('accountData', listener);
 
-            this._baseApis.setAccountData(
-                'm.secret_storage.default_key',
-                { key: keyId },
-            );
+            try {
+                await this._baseApis.setAccountData(
+                    'm.secret_storage.default_key',
+                    { key: keyId },
+                );
+            } catch (e) {
+                this._baseApis.removeListener('accountData', listener);
+                reject(e);
+            }
         });
     }
 
@@ -78,31 +81,29 @@ export class SecretStorage extends EventEmitter {
      * @param {string} [keyId] the ID of the key.  If not given, a random
      *     ID will be generated.
      *
-     * @return {string} the ID of the key
+     * @return {object} An object with:
+     *     keyId: {string} the ID of the key
+     *     keyInfo: {object} details about the key (iv, mac, passphrase)
      */
     async addKey(algorithm, opts, keyId) {
-        const keyData = {algorithm};
+        const keyInfo = {algorithm};
 
         if (!opts) opts = {};
 
         if (opts.name) {
-            keyData.name = opts.name;
+            keyInfo.name = opts.name;
         }
 
-        switch (algorithm) {
-        case SECRET_STORAGE_ALGORITHM_V1_AES:
-        {
-            const decryption = new global.Olm.PkDecryption();
-            try {
-                if (opts.passphrase) {
-                    keyData.passphrase = opts.passphrase;
-                }
-            } finally {
-                decryption.free();
+        if (algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+            if (opts.passphrase) {
+                keyInfo.passphrase = opts.passphrase;
             }
-            break;
-        }
-        default:
+            if (opts.key) {
+                const {iv, mac} = await SecretStorage._calculateKeyCheck(opts.key);
+                keyInfo.iv = iv;
+                keyInfo.mac = mac;
+            }
+        } else {
             throw new Error(`Unknown key algorithm ${opts.algorithm}`);
         }
 
@@ -116,40 +117,14 @@ export class SecretStorage extends EventEmitter {
             );
         }
 
-        await this._crossSigningInfo.signObject(keyData, 'master');
-
-        await this._baseApis.setAccountData(
-            `m.secret_storage.key.${keyId}`, keyData,
-        );
-
-        return keyId;
-    }
-
-    /**
-     * Signs a given secret storage key with the cross-signing master key.
-     *
-     * @param {string} [keyId = default key's ID] The ID of the key to sign.
-     *     Defaults to the default key ID if not provided.
-     */
-    async signKey(keyId) {
-        if (!keyId) {
-            keyId = await this.getDefaultKeyId();
-        }
-        if (!keyId) {
-            throw new Error("signKey requires a key ID");
-        }
-
-        const keyInfo = await this._baseApis.getAccountDataFromServer(
-            `m.secret_storage.key.${keyId}`,
-        );
-        if (!keyInfo) {
-            throw new Error(`Key ${keyId} does not exist in account data`);
-        }
-
-        await this._crossSigningInfo.signObject(keyInfo, 'master');
         await this._baseApis.setAccountData(
             `m.secret_storage.key.${keyId}`, keyInfo,
         );
+
+        return {
+            keyId,
+            keyInfo,
+        };
     }
 
     /**
@@ -185,15 +160,32 @@ export class SecretStorage extends EventEmitter {
         return !!(await this.getKey(keyId));
     }
 
-    async keyNeedsUpgrade(keyId) {
-        const keyInfo = await this.getKey(keyId);
-        if (keyInfo && keyInfo[1].algorithm === SECRET_STORAGE_ALGORITHM_V1_CURVE25519) {
-            return true;
+    /**
+     * Check whether a key matches what we expect based on the key info
+     *
+     * @param {Uint8Array} key the key to check
+     * @param {object} info the key info
+     *
+     * @return {boolean} whether or not the key matches
+     */
+    async checkKey(key, info) {
+        if (info.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+            if (info.mac) {
+                const {mac} = await SecretStorage._calculateKeyCheck(key, info.iv);
+                return info.mac.replace(/=+$/g, '') === mac.replace(/=+$/g, '');
+            } else {
+                // if we have no information, we have to assume the key is right
+                return true;
+            }
         } else {
-            return false;
+            throw new Error("Unknown algorithm");
         }
     }
 
+    static async _calculateKeyCheck(key, iv) {
+        return await encryptAES(ZERO_STR, key, "", iv);
+    }
+
     /**
      * Store an encrypted secret on the server
      *
@@ -227,15 +219,11 @@ export class SecretStorage extends EventEmitter {
             }
 
             // encrypt secret, based on the algorithm
-            switch (keyInfo.algorithm) {
-            case SECRET_STORAGE_ALGORITHM_V1_AES:
-            {
+            if (keyInfo.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
                 const keys = {[keyId]: keyInfo};
                 const [, encryption] = await this._getSecretStorageKey(keys, name);
                 encrypted[keyId] = await encryption.encrypt(secret);
-                break;
-            }
-            default:
+            } else {
                 logger.warn("unknown algorithm for secret storage key " + keyId
                             + ": " + keyInfo.algorithm);
                 // do nothing if we don't understand the encryption algorithm
@@ -264,7 +252,7 @@ export class SecretStorage extends EventEmitter {
         ) {
             const hasKey = await this.hasKey(keys[0]);
             if (hasKey) {
-                console.log("Fixing up passthrough secret: " + name);
+                logger.log("Fixing up passthrough secret: " + name);
                 await this.storePassthrough(name, keys[0]);
                 const newData = await this._baseApis.getAccountDataFromServer(name);
                 return newData;
@@ -301,27 +289,19 @@ export class SecretStorage extends EventEmitter {
                 "m.secret_storage.key." + keyId,
             );
             const encInfo = secretInfo.encrypted[keyId];
-            switch (keyInfo.algorithm) {
-            case SECRET_STORAGE_ALGORITHM_V1_AES:
+            // only use keys we understand the encryption algorithm of
+            if (keyInfo.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
                 if (encInfo.iv && encInfo.ciphertext && encInfo.mac) {
                     keys[keyId] = keyInfo;
                 }
-                break;
-            case SECRET_STORAGE_ALGORITHM_V1_CURVE25519:
-                if (
-                    keyInfo.pubkey && (
-                        (encInfo.ciphertext && encInfo.mac && encInfo.ephemeral) ||
-                        encInfo.passthrough
-                    )
-                ) {
-                    keys[keyId] = keyInfo;
-                }
-                break;
-            default:
-                // do nothing if we don't understand the encryption algorithm
             }
         }
 
+        if (Object.keys(keys).length === 0) {
+            throw new Error(`Could not decrypt ${name} because none of ` +
+                `the keys it is encrypted with are for a supported algorithm`);
+        }
+
         let keyId;
         let decryption;
         try {
@@ -331,8 +311,9 @@ export class SecretStorage extends EventEmitter {
             const encInfo = secretInfo.encrypted[keyId];
 
             // We don't actually need the decryption object if it's a passthrough
-            // since we just want to return the key itself.
-            if (encInfo.passthrough) return decryption.get_private_key();
+            // since we just want to return the key itself. It must be base64
+            // encoded, since this is how a key would normally be stored.
+            if (encInfo.passthrough) return encodeBase64(decryption.get_private_key());
 
             return await decryption.decrypt(encInfo);
         } finally {
@@ -366,8 +347,7 @@ export class SecretStorage extends EventEmitter {
 
         const ret = {};
 
-        // check if secret is encrypted by a known/trusted secret and
-        // encryption looks sane
+        // filter secret encryption keys with supported algorithm
         for (const keyId of Object.keys(secretInfo.encrypted)) {
             // get key information from key storage
             const keyInfo = await this._baseApis.getAccountDataFromServer(
@@ -376,49 +356,11 @@ export class SecretStorage extends EventEmitter {
             if (!keyInfo) continue;
             const encInfo = secretInfo.encrypted[keyId];
 
-            // We don't actually need the decryption object if it's a passthrough
-            // since we just want to return the key itself.
-            if (encInfo.passthrough) {
-                try {
-                    pkVerify(
-                        keyInfo,
-                        this._crossSigningInfo.getId('master'),
-                        this._crossSigningInfo.userId,
-                    );
-                } catch (e) {
-                    // not trusted, so move on to the next key
-                    continue;
-                }
-                ret[keyId] = keyInfo;
-                continue;
-            }
-
-            switch (keyInfo.algorithm) {
-            case SECRET_STORAGE_ALGORITHM_V1_AES:
+            // only use keys we understand the encryption algorithm of
+            if (keyInfo.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
                 if (encInfo.iv && encInfo.ciphertext && encInfo.mac) {
                     ret[keyId] = keyInfo;
                 }
-                break;
-            case SECRET_STORAGE_ALGORITHM_V1_CURVE25519:
-                if (keyInfo.pubkey && encInfo.ciphertext && encInfo.mac
-                    && encInfo.ephemeral) {
-                    if (checkKey) {
-                        try {
-                            pkVerify(
-                                keyInfo,
-                                this._crossSigningInfo.getId('master'),
-                                this._crossSigningInfo.userId,
-                            );
-                        } catch (e) {
-                            // not trusted, so move on to the next key
-                            continue;
-                        }
-                    }
-                    ret[keyId] = keyInfo;
-                }
-                break;
-            default:
-                // do nothing if we don't understand the encryption algorithm
             }
         }
         return Object.keys(ret).length ? ret : null;
@@ -436,6 +378,7 @@ export class SecretStorage extends EventEmitter {
         const requestId = this._baseApis.makeTxnId();
 
         const requestControl = this._requests[requestId] = {
+            name,
             devices,
         };
         const promise = new Promise((resolve, reject) => {
@@ -519,13 +462,13 @@ export class SecretStorage extends EventEmitter {
             if (!this._cryptoCallbacks.onSecretRequested) {
                 return;
             }
-            const secret = await this._cryptoCallbacks.onSecretRequested({
-                user_id: sender,
-                device_id: deviceId,
-                request_id: content.request_id,
-                name: content.name,
-                device_trust: this._baseApis.checkDeviceTrust(sender, deviceId),
-            });
+            const secret = await this._cryptoCallbacks.onSecretRequested(
+                sender,
+                deviceId,
+                content.request_id,
+                content.name,
+                this._baseApis.checkDeviceTrust(sender, deviceId),
+            );
             if (secret) {
                 logger.info(`Preparing ${content.name} secret for ${deviceId}`);
                 const payload = {
@@ -545,7 +488,7 @@ export class SecretStorage extends EventEmitter {
                     this._baseApis,
                     {
                         [sender]: [
-                            await this._baseApis.getStoredDevice(sender, deviceId),
+                            this._baseApis.getStoredDevice(sender, deviceId),
                         ],
                     },
                 );
@@ -555,7 +498,7 @@ export class SecretStorage extends EventEmitter {
                     this._baseApis.deviceId,
                     this._baseApis._crypto._olmDevice,
                     sender,
-                    this._baseApis._crypto.getStoredDevice(sender, deviceId),
+                    this._baseApis.getStoredDevice(sender, deviceId),
                     payload,
                 );
                 const contentMap = {
@@ -599,6 +542,10 @@ export class SecretStorage extends EventEmitter {
                 return;
             }
 
+            logger.log(
+                `Successfully received secret ${requestControl.name} ` +
+                `from ${deviceInfo.deviceId}`,
+            );
             requestControl.resolve(content.secret);
         }
     }
@@ -622,9 +569,7 @@ export class SecretStorage extends EventEmitter {
             throw new Error("App returned unknown key from getSecretStorageKey!");
         }
 
-        switch (keys[keyId].algorithm) {
-        case SECRET_STORAGE_ALGORITHM_V1_AES:
-        {
+        if (keys[keyId].algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
             const decryption = {
                 encrypt: async function(secret) {
                     return await encryptAES(secret, privateKey, name);
@@ -634,36 +579,7 @@ export class SecretStorage extends EventEmitter {
                 },
             };
             return [keyId, decryption];
-        }
-        case SECRET_STORAGE_ALGORITHM_V1_CURVE25519:
-        {
-            const pkDecryption = new global.Olm.PkDecryption();
-            let pubkey;
-            try {
-                pubkey = pkDecryption.init_with_private_key(privateKey);
-            } catch (e) {
-                pkDecryption.free();
-                throw new Error("getSecretStorageKey callback returned invalid key");
-            }
-            if (pubkey !== keys[keyId].pubkey) {
-                pkDecryption.free();
-                throw new Error(
-                    "getSecretStorageKey callback returned incorrect key",
-                );
-            }
-            const decryption = {
-                free: pkDecryption.free.bind(pkDecryption),
-                decrypt: async function(encInfo) {
-                    return pkDecryption.decrypt(
-                        encInfo.ephemeral, encInfo.mac, encInfo.ciphertext,
-                    );
-                },
-                // needed for passthrough
-                get_private_key: pkDecryption.get_private_key.bind(pkDecryption),
-            };
-            return [keyId, decryption];
-        }
-        default:
+        } else {
             throw new Error("Unknown key type: " + keys[keyId].algorithm);
         }
     }

src/crypto/aes.js

@@ -29,14 +29,20 @@ const zerosalt = new Uint8Array(8);
  * @param {string} data the plaintext to encrypt
  * @param {Uint8Array} key the encryption key to use
  * @param {string} name the name of the secret
+ * @param {string} ivStr the initialization vector to use
  */
-async function encryptNode(data, key, name) {
+async function encryptNode(data, key, name, ivStr) {
     const crypto = getCrypto();
     if (!crypto) {
         throw new Error("No usable crypto implementation");
     }
 
-    const iv = crypto.randomBytes(16);
+    let iv;
+    if (ivStr) {
+        iv = decodeBase64(ivStr);
+    } else {
+        iv = crypto.randomBytes(16);
+    }
 
     // clear bit 63 of the IV to stop us hitting the 64-bit counter boundary
     // (which would mean we wouldn't be able to decrypt on Android). The loss
@@ -78,9 +84,9 @@ async function decryptNode(data, key, name) {
     const [aesKey, hmacKey] = deriveKeysNode(key, name);
 
     const hmac = crypto.createHmac("sha256", hmacKey)
-        .update(data.ciphertext, "base64").digest("base64");
+        .update(data.ciphertext, "base64").digest("base64").replace(/=+$/g, '');
 
-    if (hmac !== data.mac) {
+    if (hmac !== data.mac.replace(/=+$/g, '')) {
         throw new Error(`Error decrypting secret ${name}: bad MAC`);
     }
 
@@ -112,10 +118,16 @@ function deriveKeysNode(key, name) {
  * @param {string} data the plaintext to encrypt
  * @param {Uint8Array} key the encryption key to use
  * @param {string} name the name of the secret
+ * @param {string} ivStr the initialization vector to use
  */
-async function encryptBrowser(data, key, name) {
-    const iv = new Uint8Array(16);
-    window.crypto.getRandomValues(iv);
+async function encryptBrowser(data, key, name, ivStr) {
+    let iv;
+    if (ivStr) {
+        iv = decodeBase64(ivStr);
+    } else {
+        iv = new Uint8Array(16);
+        window.crypto.getRandomValues(iv);
+    }
 
     // clear bit 63 of the IV to stop us hitting the 64-bit counter boundary
     // (which would mean we wouldn't be able to decrypt on Android). The loss

src/crypto/algorithms/megolm.js

@@ -311,7 +311,7 @@ MegolmEncryption.prototype._ensureOutboundSession = async function(
                         }
 
                         await this._shareKeyWithDevices(
-                            session, key, payload, retryDevices, failedDevices,
+                            session, key, payload, retryDevices, failedDevices, 30000,
                         );
 
                         await this._notifyFailedOlmDevices(session, key, failedDevices);
@@ -521,6 +521,33 @@ MegolmEncryption.prototype._encryptAndSendKeysToDevices = function(
     }
 
     return Promise.all(promises).then(() => {
+        // prune out any devices that encryptMessageForDevice could not encrypt for,
+        // in which case it will have just not added anything to the ciphertext object.
+        // There's no point sending messages to devices if we couldn't encrypt to them,
+        // since that's effectively a blank message.
+        for (const userId of Object.keys(contentMap)) {
+            for (const deviceId of Object.keys(contentMap[userId])) {
+                if (Object.keys(contentMap[userId][deviceId].ciphertext).length === 0) {
+                    logger.log(
+                        "No ciphertext for device " +
+                        userId + ":" + deviceId + ": pruning",
+                    );
+                    delete contentMap[userId][deviceId];
+                }
+            }
+            // No devices left for that user? Strip that too.
+            if (Object.keys(contentMap[userId]).length === 0) {
+                logger.log("Pruned all devices for user " + userId);
+                delete contentMap[userId];
+            }
+        }
+
+        // Is there anything left?
+        if (Object.keys(contentMap).length === 0) {
+            logger.log("No users left to send to: aborting");
+            return;
+        }
+
         return this._baseApis.sendToDevice("m.room.encrypted", contentMap).then(() => {
             // store that we successfully uploaded the keys of the current slice
             for (const userId of Object.keys(contentMap)) {
@@ -988,7 +1015,7 @@ MegolmEncryption.prototype._getDevicesInRoom = async function(room) {
     // with them, which means that they will have announced any new devices via
     // device_lists in their /sync response.  This cache should then be maintained
     // using all the device_lists changes and left fields.
-    // See https://github.com/vector-im/riot-web/issues/2305 for details.
+    // See https://github.com/vector-im/element-web/issues/2305 for details.
     const devices = await this._crypto.downloadKeys(roomMembers, false);
     const blocked = {};
     // remove any blocked devices
@@ -1082,7 +1109,7 @@ MegolmDecryption.prototype.decryptEvent = async function(event) {
     //
     // then, if the key turns up while decryption is in progress (and
     // decryption fails), we will schedule a retry.
-    // (fixes https://github.com/vector-im/riot-web/issues/5001)
+    // (fixes https://github.com/vector-im/element-web/issues/5001)
     this._addEventToPendingList(event);
 
     let res;
@@ -1174,6 +1201,7 @@ MegolmDecryption.prototype.decryptEvent = async function(event) {
         senderCurve25519Key: res.senderKey,
         claimedEd25519Key: res.keysClaimed.ed25519,
         forwardingCurve25519KeyChain: res.forwardingCurve25519KeyChain,
+        untrusted: res.untrusted,
     };
 };
 
@@ -1296,7 +1324,6 @@ MegolmDecryption.prototype.onRoomKeyEvent = function(event) {
         keysClaimed = event.getKeysClaimed();
     }
 
-    logger.log(`Received and adding key for megolm session ${senderKey}|${sessionId}`);
     return this._olmDevice.addInboundGroupSession(
         content.room_id, senderKey, forwardingKeyChain, sessionId,
         content.session_key, keysClaimed,
@@ -1522,8 +1549,11 @@ MegolmDecryption.prototype._buildKeyForwardingMessage = async function(
  * @inheritdoc
  *
  * @param {module:crypto/OlmDevice.MegolmSessionData} session
+ * @param {object} [opts={}] options for the import
+ * @param {boolean} [opts.untrusted] whether the key should be considered as untrusted
+ * @param {string} [opts.source] where the key came from
  */
-MegolmDecryption.prototype.importRoomKey = function(session) {
+MegolmDecryption.prototype.importRoomKey = function(session, opts = {}) {
     return this._olmDevice.addInboundGroupSession(
         session.room_id,
         session.sender_key,
@@ -1532,8 +1562,9 @@ MegolmDecryption.prototype.importRoomKey = function(session) {
         session.session_key,
         session.sender_claimed_keys,
         true,
+        opts.untrusted ? { untrusted: opts.untrusted } : {},
     ).then(() => {
-        if (this._crypto.backupInfo) {
+        if (this._crypto.backupInfo && opts.source !== "backup") {
             // don't wait for it to complete
             this._crypto.backupGroupSession(
                 session.room_id,
@@ -1555,7 +1586,8 @@ MegolmDecryption.prototype.importRoomKey = function(session) {
 };
 
 /**
- * Have another go at decrypting events after we receive a key
+ * Have another go at decrypting events after we receive a key. Resolves once
+ * decryption has been re-attempted on all events.
  *
  * @private
  * @param {String} senderKey
@@ -1574,21 +1606,17 @@ MegolmDecryption.prototype._retryDecryption = async function(senderKey, sessionI
         return true;
     }
 
-    pending.delete(sessionId);
-    if (pending.size === 0) {
-        this._pendingEvents[senderKey];
-    }
+    logger.debug("Retrying decryption on events", [...pending]);
 
     await Promise.all([...pending].map(async (ev) => {
         try {
-            await ev.attemptDecryption(this._crypto);
+            await ev.attemptDecryption(this._crypto, true);
         } catch (e) {
             // don't die if something goes wrong
         }
     }));
 
-    // ev.attemptDecryption will re-add to this._pendingEvents if an event
-    // couldn't be decrypted
+    // If decrypted successfully, they'll have been removed from _pendingEvents
     return !((this._pendingEvents[senderKey] || {})[sessionId]);
 };
 

src/crypto/algorithms/olm.js

@@ -264,6 +264,25 @@ OlmDecryption.prototype.decryptEvent = async function(event) {
  */
 OlmDecryption.prototype._decryptMessage = async function(
     theirDeviceIdentityKey, message,
+) {
+    // This is a wrapper that serialises decryptions of prekey messages, because
+    // otherwise we race between deciding we have no active sessions for the message
+    // and creating a new one, which we can only do once because it removes the OTK.
+    if (message.type !== 0) {
+        // not a prekey message: we can safely just try & decrypt it
+        return this._reallyDecryptMessage(theirDeviceIdentityKey, message);
+    } else {
+        const myPromise = this._olmDevice._olmPrekeyPromise.then(() => {
+            return this._reallyDecryptMessage(theirDeviceIdentityKey, message);
+        });
+        // we want the error, but don't propagate it to the next decryption
+        this._olmDevice._olmPrekeyPromise = myPromise.catch(() => {});
+        return await myPromise;
+    }
+};
+
+OlmDecryption.prototype._reallyDecryptMessage = async function(
+    theirDeviceIdentityKey, message,
 ) {
     const sessionIds = await this._olmDevice.getSessionIdsForDevice(
         theirDeviceIdentityKey,

src/crypto/dehydration.ts

@@ -0,0 +1,288 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {decodeBase64, encodeBase64} from './olmlib';
+import {IndexedDBCryptoStore} from '../crypto/store/indexeddb-crypto-store';
+import {decryptAES, encryptAES} from './aes';
+import anotherjson from "another-json";
+import {logger} from '../logger';
+
+// FIXME: these types should eventually go in a different file
+type Signatures = Record<string, Record<string, string>>;
+
+interface DeviceKeys {
+    algorithms: Array<string>;
+    device_id: string; // eslint-disable-line camelcase
+    user_id: string; // eslint-disable-line camelcase
+    keys: Record<string, string>;
+    signatures?: Signatures;
+}
+
+interface OneTimeKey {
+    key: string;
+    fallback?: boolean;
+    signatures?: Signatures;
+}
+
+export const DEHYDRATION_ALGORITHM = "org.matrix.msc2697.v1.olm.libolm_pickle";
+
+const oneweek = 7 * 24 * 60 * 60 * 1000;
+
+export class DehydrationManager {
+    private inProgress = false;
+    private timeoutId: any;
+    private key: Uint8Array;
+    private keyInfo: {[props: string]: any};
+    private deviceDisplayName: string;
+    constructor(private crypto) {
+        this.getDehydrationKeyFromCache();
+    }
+    async getDehydrationKeyFromCache(): Promise<void> {
+        return await this.crypto._cryptoStore.doTxn(
+            'readonly',
+            [IndexedDBCryptoStore.STORE_ACCOUNT],
+            (txn) => {
+                this.crypto._cryptoStore.getSecretStorePrivateKey(
+                    txn,
+                    async (result) => {
+                        if (result) {
+                            const {key, keyInfo, deviceDisplayName, time} = result;
+                            const pickleKey = Buffer.from(this.crypto._olmDevice._pickleKey);
+                            const decrypted = await decryptAES(key, pickleKey, DEHYDRATION_ALGORITHM);
+                            this.key = decodeBase64(decrypted);
+                            this.keyInfo = keyInfo;
+                            this.deviceDisplayName = deviceDisplayName;
+                            const now = Date.now();
+                            const delay = Math.max(1, time + oneweek - now);
+                            this.timeoutId = global.setTimeout(
+                                this.dehydrateDevice.bind(this), delay,
+                            );
+                        }
+                    },
+                    "dehydration",
+                );
+            },
+        );
+    }
+
+    /** set the key, and queue periodic dehydration to the server in the background */
+    async setKeyAndQueueDehydration(
+        key: Uint8Array, keyInfo: {[props: string]: any} = {},
+        deviceDisplayName: string = undefined,
+    ): Promise<void> {
+        const matches = await this.setKey(key, keyInfo, deviceDisplayName);
+        if (!matches) {
+            // start dehydration in the background
+            this.dehydrateDevice();
+        }
+    }
+
+    async setKey(
+        key: Uint8Array, keyInfo: {[props: string]: any} = {},
+        deviceDisplayName: string = undefined,
+    ): Promise<boolean> {
+        if (!key) {
+            // unsetting the key -- cancel any pending dehydration task
+            if (this.timeoutId) {
+                global.clearTimeout(this.timeoutId);
+                this.timeoutId = undefined;
+            }
+            // clear storage
+            await this.crypto._cryptoStore.doTxn(
+                'readwrite',
+                [IndexedDBCryptoStore.STORE_ACCOUNT],
+                (txn) => {
+                    this.crypto._cryptoStore.storeSecretStorePrivateKey(
+                        txn, "dehydration", null,
+                    );
+                },
+            );
+            this.key = undefined;
+            this.keyInfo = undefined;
+            return;
+        }
+
+        // Check to see if it's the same key as before.  If it's different,
+        // dehydrate a new device.  If it's the same, we can keep the same
+        // device.  (Assume that keyInfo and deviceDisplayName will be the
+        // same if the key is the same.)
+        let matches: boolean = this.key && key.length == this.key.length;
+        for (let i = 0; matches && i < key.length; i++) {
+            if (key[i] != this.key[i]) {
+                matches = false;
+            }
+        }
+        if (!matches) {
+            this.key = key;
+            this.keyInfo = keyInfo;
+            this.deviceDisplayName = deviceDisplayName;
+        }
+        return matches;
+    }
+
+    /** returns the device id of the newly created dehydrated device */
+    async dehydrateDevice(): Promise<string> {
+        if (this.inProgress) {
+            logger.log("Dehydration already in progress -- not starting new dehydration");
+            return;
+        }
+        this.inProgress = true;
+        if (this.timeoutId) {
+            global.clearTimeout(this.timeoutId);
+            this.timeoutId = undefined;
+        }
+        try {
+            const pickleKey = Buffer.from(this.crypto._olmDevice._pickleKey);
+
+            // update the crypto store with the timestamp
+            const key = await encryptAES(encodeBase64(this.key), pickleKey, DEHYDRATION_ALGORITHM);
+            await this.crypto._cryptoStore.doTxn(
+                'readwrite',
+                [IndexedDBCryptoStore.STORE_ACCOUNT],
+                (txn) => {
+                    this.crypto._cryptoStore.storeSecretStorePrivateKey(
+                        txn, "dehydration",
+                        {
+                            keyInfo: this.keyInfo,
+                            key,
+                            deviceDisplayName: this.deviceDisplayName,
+                            time: Date.now(),
+                        },
+                    );
+                },
+            );
+            logger.log("Attempting to dehydrate device");
+
+            logger.log("Creating account");
+            // create the account and all the necessary keys
+            const account = new global.Olm.Account();
+            account.create();
+            const e2eKeys = JSON.parse(account.identity_keys());
+
+            const maxKeys = account.max_number_of_one_time_keys();
+            // FIXME: generate in small batches?
+            account.generate_one_time_keys(maxKeys / 2);
+            account.generate_fallback_key();
+            const otks: Record<string, string> = JSON.parse(account.one_time_keys());
+            const fallbacks: Record<string, string> = JSON.parse(account.fallback_key());
+            account.mark_keys_as_published();
+
+            // dehydrate the account and store it on the server
+            const pickledAccount = account.pickle(new Uint8Array(this.key));
+
+            const deviceData: {[props: string]: any} = {
+                algorithm: DEHYDRATION_ALGORITHM,
+                account: pickledAccount,
+            };
+            if (this.keyInfo.passphrase) {
+                deviceData.passphrase = this.keyInfo.passphrase;
+            }
+
+            logger.log("Uploading account to server");
+            const dehydrateResult = await this.crypto._baseApis._http.authedRequest(
+                undefined,
+                "PUT",
+                "/dehydrated_device",
+                undefined,
+                {
+                    device_data: deviceData,
+                    initial_device_display_name: this.deviceDisplayName,
+                },
+                {
+                    prefix: "/_matrix/client/unstable/org.matrix.msc2697.v2",
+                },
+            );
+
+            // send the keys to the server
+            const deviceId = dehydrateResult.device_id;
+            logger.log("Preparing device keys", deviceId);
+            const deviceKeys: DeviceKeys = {
+                algorithms: this.crypto._supportedAlgorithms,
+                device_id: deviceId,
+                user_id: this.crypto._userId,
+                keys: {
+                    [`ed25519:${deviceId}`]: e2eKeys.ed25519,
+                    [`curve25519:${deviceId}`]: e2eKeys.curve25519,
+                },
+            };
+            const deviceSignature = account.sign(anotherjson.stringify(deviceKeys));
+            deviceKeys.signatures = {
+                [this.crypto._userId]: {
+                    [`ed25519:${deviceId}`]: deviceSignature,
+                },
+            };
+            if (this.crypto._crossSigningInfo.getId("self_signing")) {
+                await this.crypto._crossSigningInfo.signObject(deviceKeys, "self_signing");
+            }
+
+            logger.log("Preparing one-time keys");
+            const oneTimeKeys = {};
+            for (const [keyId, key] of Object.entries(otks.curve25519)) {
+                const k: OneTimeKey = {key};
+                const signature = account.sign(anotherjson.stringify(k));
+                k.signatures = {
+                    [this.crypto._userId]: {
+                        [`ed25519:${deviceId}`]: signature,
+                    },
+                };
+                oneTimeKeys[`signed_curve25519:${keyId}`] = k;
+            }
+
+            logger.log("Preparing fallback keys");
+            const fallbackKeys = {};
+            for (const [keyId, key] of Object.entries(fallbacks.curve25519)) {
+                const k: OneTimeKey = {key, fallback: true};
+                const signature = account.sign(anotherjson.stringify(k));
+                k.signatures = {
+                    [this.crypto._userId]: {
+                        [`ed25519:${deviceId}`]: signature,
+                    },
+                };
+                fallbackKeys[`signed_curve25519:${keyId}`] = k;
+            }
+
+            logger.log("Uploading keys to server");
+            await this.crypto._baseApis._http.authedRequest(
+                undefined,
+                "POST",
+                "/keys/upload/" + encodeURI(deviceId),
+                undefined,
+                {
+                    "device_keys": deviceKeys,
+                    "one_time_keys": oneTimeKeys,
+                    "org.matrix.msc2732.fallback_keys": fallbackKeys,
+                },
+            );
+            logger.log("Done dehydrating");
+
+            // dehydrate again in a week
+            this.timeoutId = global.setTimeout(
+                this.dehydrateDevice.bind(this), oneweek,
+            );
+
+            return deviceId;
+        } finally {
+            this.inProgress = false;
+        }
+    }
+
+    private stop() {
+        if (this.timeoutId) {
+            global.clearTimeout(this.timeoutId);
+            this.timeoutId = undefined;
+        }
+    }
+}

src/crypto/olmlib.js

@@ -207,6 +207,25 @@ export async function ensureOlmSessionsForDevices(
         for (const deviceInfo of devices) {
             const deviceId = deviceInfo.deviceId;
             const key = deviceInfo.getIdentityKey();
+
+            if (key === olmDevice.deviceCurve25519Key) {
+                // We should never be trying to start a session with ourself.
+                // Apart from talking to yourself being the first sign of madness,
+                // olm sessions can't do this because they get confused when
+                // they get a message and see that the 'other side' has started a
+                // new chain when this side has an active sender chain.
+                // If you see this message being logged in the wild, we should find
+                // the thing that is trying to send Olm messages to itself and fix it.
+                logger.info("Attempted to start session with ourself! Ignoring");
+                // We must fill in the section in the return value though, as callers
+                // expect it to be there.
+                result[userId][deviceId] = {
+                    device: deviceInfo,
+                    sessionId: null,
+                };
+                continue;
+            }
+
             if (!olmDevice._sessionsInProgress[key]) {
                 // pre-emptively mark the session as in-progress to avoid race
                 // conditions.  If we find that we already have a session, then
@@ -238,6 +257,11 @@ export async function ensureOlmSessionsForDevices(
                 delete resolveSession[key];
             }
             if (sessionId === null || force) {
+                if (force) {
+                    logger.info("Forcing new Olm session for " + userId + ":" + deviceId);
+                } else {
+                    logger.info("Making new Olm session for " + userId + ":" + deviceId);
+                }
                 devicesWithoutSession.push([userId, deviceId]);
             }
             result[userId][deviceId] = {
@@ -277,6 +301,14 @@ export async function ensureOlmSessionsForDevices(
             const deviceInfo = devices[j];
             const deviceId = deviceInfo.deviceId;
             const key = deviceInfo.getIdentityKey();
+
+            if (key === olmDevice.deviceCurve25519Key) {
+                // We've already logged about this above. Skip here too
+                // otherwise we'll log saying there are no one-time keys
+                // which will be confusing.
+                continue;
+            }
+
             if (result[userId][deviceId].sessionId && !force) {
                 // we already have a result for this device
                 continue;

src/crypto/recoverykey.js

@@ -59,8 +59,8 @@ export function decodeRecoveryKey(recoverykey) {
         throw new Error("Incorrect length");
     }
 
-    return result.slice(
+    return Uint8Array.from(result.slice(
         OLM_RECOVERY_KEY_PREFIX.length,
         OLM_RECOVERY_KEY_PREFIX.length + global.Olm.PRIVATE_KEY_LENGTH,
-    );
+    ));
 }

src/crypto/store/indexeddb-crypto-store-backend.js

@@ -131,7 +131,7 @@ export class Backend {
 
         cursorReq.onsuccess = (ev) => {
             const cursor = ev.target.result;
-            if(!cursor) {
+            if (!cursor) {
                 // no match found
                 callback(null);
                 return;
@@ -203,6 +203,23 @@ export class Backend {
         return promiseifyTxn(txn).then(() => result);
     }
 
+    /**
+     *
+     * @param {Number} wantedState
+     * @return {Promise<Array<*>>} All elements in a given state
+     */
+    getAllOutgoingRoomKeyRequestsByState(wantedState) {
+        return new Promise((resolve, reject) => {
+            const txn = this._db.transaction("outgoingRoomKeyRequests", "readonly");
+            const store = txn.objectStore("outgoingRoomKeyRequests");
+            const index = store.index("state");
+            const request = index.getAll(wantedState);
+
+            request.onsuccess = (ev) => resolve(ev.target.result);
+            request.onerror = (ev) => reject(ev.target.error);
+        });
+    }
+
     getOutgoingRoomKeyRequestsByTarget(userId, deviceId, wantedStates) {
         let stateIndex = 0;
         const results = [];

src/crypto/store/indexeddb-crypto-store.js

@@ -225,6 +225,17 @@ export class IndexedDBCryptoStore {
         return this._backend.getOutgoingRoomKeyRequestByState(wantedStates);
     }
 
+    /**
+     * Look for room key requests by state –
+     * unlike above, return a list of all entries in one state.
+     *
+     * @param {Number} wantedState
+     * @return {Promise<Array<*>>} Returns an array of requests in the given state
+     */
+    getAllOutgoingRoomKeyRequestsByState(wantedState) {
+        return this._backend.getAllOutgoingRoomKeyRequestsByState(wantedState);
+    }
+
     /**
      * Look for room key requests by target device and state
      *

src/crypto/store/localStorage-crypto-store.js

@@ -369,7 +369,7 @@ export class LocalStorageCryptoStore extends MemoryCryptoStore {
 
     getSecretStorePrivateKey(txn, func, type) {
         const key = getJsonItem(this.store, E2E_PREFIX + `ssss_cache.${type}`);
-        func(key ? Uint8Array.from(key) : key);
+        func(key);
     }
 
     storeCrossSigningKeys(txn, keys) {
@@ -380,7 +380,7 @@ export class LocalStorageCryptoStore extends MemoryCryptoStore {
 
     storeSecretStorePrivateKey(txn, type, key) {
         setJsonItem(
-            this.store, E2E_PREFIX + `ssss_cache.${type}`, Array.from(key),
+            this.store, E2E_PREFIX + `ssss_cache.${type}`, key,
         );
     }
 

src/crypto/store/memory-crypto-store.js

@@ -166,6 +166,19 @@ export class MemoryCryptoStore {
         return Promise.resolve(null);
     }
 
+    /**
+     *
+     * @param {Number} wantedState
+     * @return {Promise<Array<*>>} All OutgoingRoomKeyRequests in state
+     */
+    getAllOutgoingRoomKeyRequestsByState(wantedState) {
+        return Promise.resolve(
+            this._outgoingRoomKeyRequests.filter(
+                (r) => r.state == wantedState,
+            ),
+        );
+    }
+
     getOutgoingRoomKeyRequestsByTarget(userId, deviceId, wantedStates) {
         const results = [];
 

src/crypto/verification/Base.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2018 New Vector Ltd
+Copyright 2020 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -24,8 +25,7 @@ import {EventEmitter} from 'events';
 import {logger} from '../../logger';
 import {DeviceInfo} from '../deviceinfo';
 import {newTimeoutError} from "./Error";
-import {CrossSigningInfo} from "../CrossSigning";
-import {decodeBase64} from "../olmlib";
+import {requestKeysDuringVerification} from "../CrossSigning";
 
 const timeoutException = new Error("Verification timed out");
 
@@ -78,8 +78,6 @@ export class VerificationBase extends EventEmitter {
         this._transactionTimeoutTimer = null;
     }
 
-    static keyRequestTimeoutMs = 1000 * 60;
-
     get initiatedByMe() {
         // if there is no start event yet,
         // we probably want to send it,
@@ -121,6 +119,11 @@ export class VerificationBase extends EventEmitter {
         if (this._done) {
             return Promise.reject(new Error("Verification is already done"));
         }
+        const existingEvent = this.request.getEventFromOtherParty(type);
+        if (existingEvent) {
+            return Promise.resolve(existingEvent);
+        }
+
         this._expectedEvent = type;
         return new Promise((resolve, reject) => {
             this._resolveEvent = resolve;
@@ -192,63 +195,7 @@ export class VerificationBase extends EventEmitter {
         if (!this._done) {
             this.request.onVerifierFinished();
             this._resolve();
-
-            //#region Cross-signing keys request
-            // If this is a self-verification, ask the other party for keys
-            if (this._baseApis.getUserId() !== this.userId) {
-                return;
-            }
-            console.log("VerificationBase.done: Self-verification done; requesting keys");
-            /* This happens asynchronously, and we're not concerned about
-             * waiting for it.  We return here in order to test. */
-            return new Promise((resolve, reject) => {
-                const client = this._baseApis;
-                const original = client._crypto._crossSigningInfo;
-                const storage = client._crypto._secretStorage;
-
-                /* We already have all of the infrastructure we need to validate and
-                 * cache cross-signing keys, so instead of replicating that, here we
-                 * set up callbacks that request them from the other device and call
-                 * CrossSigningInfo.getCrossSigningKey() to validate/cache */
-                const crossSigning = new CrossSigningInfo(
-                    original.userId,
-                    { getCrossSigningKey: async (type) => {
-                        console.debug("VerificationBase.done: requesting secret",
-                                      type, this.deviceId);
-                        const { promise } =
-                            storage.request(`m.cross_signing.${type}`, [this.deviceId]);
-                        const result = await promise;
-                        const decoded = decodeBase64(result);
-                        return Uint8Array.from(decoded);
-                    } },
-                    original._cacheCallbacks,
-                );
-                crossSigning.keys = original.keys;
-
-                // XXX: get all keys out if we get one key out
-                // https://github.com/vector-im/riot-web/issues/12604
-                // then change here to reject on the timeout
-                /* Requests can be ignored, so don't wait around forever */
-                const timeout = new Promise((resolve, reject) => {
-                    setTimeout(
-                        resolve,
-                        VerificationBase.keyRequestTimeoutMs,
-                        new Error("Timeout"),
-                    );
-                });
-
-                /* We call getCrossSigningKey() for its side-effects */
-                return Promise.race([
-                    Promise.all([
-                        crossSigning.getCrossSigningKey("self_signing"),
-                        crossSigning.getCrossSigningKey("user_signing"),
-                    ]),
-                    timeout,
-                ]).then(resolve, reject);
-            }).catch((e) => {
-                console.warn("VerificationBase: failure while requesting keys:", e);
-            });
-            //#endregion
+            return requestKeysDuringVerification(this._baseApis, this.userId, this.deviceId);
         }
     }
 
@@ -256,6 +203,7 @@ export class VerificationBase extends EventEmitter {
         this._endTimer(); // always kill the activity timer
         if (!this._done) {
             this.cancelled = true;
+            this.request.onVerifierCancelled();
             if (this.userId && this.deviceId) {
                 // send a cancellation to the other user (if it wasn't
                 // cancelled by the other user)
@@ -338,7 +286,7 @@ export class VerificationBase extends EventEmitter {
 
         for (const [keyId, keyInfo] of Object.entries(keys)) {
             const deviceId = keyId.split(':', 2)[1];
-            const device = await this._baseApis.getStoredDevice(userId, deviceId);
+            const device = this._baseApis.getStoredDevice(userId, deviceId);
             if (device) {
                 await verifier(keyId, device, keyInfo);
                 verifiedDevices.push(deviceId);

src/crypto/verification/QRCode.js

@@ -23,8 +23,10 @@ limitations under the License.
 import {VerificationBase as Base} from "./Base";
 import {
     newKeyMismatchError,
-    newUserMismatchError,
+    newUserCancelledError,
 } from './Error';
+import {encodeUnpaddedBase64, decodeBase64} from "../olmlib";
+import {logger} from '../../logger';
 
 export const SHOW_QR_CODE_METHOD = "m.qr_code.show.v1";
 export const SCAN_QR_CODE_METHOD = "m.qr_code.scan.v1";
@@ -49,52 +51,51 @@ export class ReciprocateQRCode extends Base {
                 "with this method yet.");
         }
 
-        const targetUserId = this.startEvent.getSender();
-        if (!this.userId) {
-            console.log("Asking to confirm user ID");
-            this.userId = await new Promise((resolve, reject) => {
-                this.emit("confirm_user_id", {
-                    userId: targetUserId,
-                    confirm: resolve, // takes a userId
-                    cancel: () => reject(newUserMismatchError()),
-                });
-            });
-        } else if (targetUserId !== this.userId) {
-            throw newUserMismatchError({
-                expected: this.userId,
-                actual: targetUserId,
-            });
-        }
-
-        if (this.startEvent.getContent()['secret'] !== this.request.encodedSharedSecret) {
+        const {qrCodeData} = this.request;
+        // 1. check the secret
+        if (this.startEvent.getContent()['secret'] !== qrCodeData.encodedSharedSecret) {
             throw newKeyMismatchError();
         }
 
-        // If we've gotten this far, verify the user's master cross signing key
-        const xsignInfo = this._baseApis.getStoredCrossSigningForUser(this.userId);
-        if (!xsignInfo) throw new Error("Missing cross signing info");
-
-        const masterKey = xsignInfo.getId("master");
-        const masterKeyId = `ed25519:${masterKey}`;
-        const keys = {[masterKeyId]: masterKey};
-
-        const devices = (await this._baseApis.getStoredDevicesForUser(this.userId)) || [];
-        const targetDevice = devices.find(d => {
-            return d.deviceId === this.request.targetDevice.deviceId;
+        // 2. ask if other user shows shield as well
+        await new Promise((resolve, reject) => {
+            this.reciprocateQREvent = {
+                confirm: resolve,
+                cancel: () => reject(newUserCancelledError()),
+            };
+            this.emit("show_reciprocate_qr", this.reciprocateQREvent);
         });
-        if (!targetDevice) throw new Error("Device not found, somehow");
-        keys[`ed25519:${targetDevice.deviceId}`] = targetDevice.getFingerprint();
 
-        if (this.request.requestingUserId === this.request.receivingUserId) {
-            delete keys[masterKeyId];
+        // 3. determine key to sign / mark as trusted
+        const keys = {};
+
+        switch (qrCodeData.mode) {
+            case MODE_VERIFY_OTHER_USER: {
+                // add master key to keys to be signed, only if we're not doing self-verification
+                const masterKey = qrCodeData.otherUserMasterKey;
+                keys[`ed25519:${masterKey}`] = masterKey;
+                break;
+            }
+            case MODE_VERIFY_SELF_TRUSTED: {
+                const deviceId = this.request.targetDevice.deviceId;
+                keys[`ed25519:${deviceId}`] = qrCodeData.otherDeviceKey;
+                break;
+            }
+            case MODE_VERIFY_SELF_UNTRUSTED: {
+                const masterKey = qrCodeData.myMasterKey;
+                keys[`ed25519:${masterKey}`] = masterKey;
+                break;
+            }
         }
 
+        // 4. sign the key (or mark own MSK as verified in case of MODE_VERIFY_SELF_TRUSTED)
         await this._verifyKeys(this.userId, keys, (keyId, device, keyInfo) => {
+            // make sure the device has the expected keys
             const targetKey = keys[keyId];
             if (!targetKey) throw newKeyMismatchError();
 
             if (keyInfo !== targetKey) {
-                console.error("key ID from key info does not match");
+                logger.error("key ID from key info does not match");
                 throw newKeyMismatchError();
             }
             for (const deviceKeyId in device.keys) {
@@ -102,12 +103,204 @@ export class ReciprocateQRCode extends Base {
                 const deviceTargetKey = keys[deviceKeyId];
                 if (!deviceTargetKey) throw newKeyMismatchError();
                 if (device.keys[deviceKeyId] !== deviceTargetKey) {
-                    console.error("master key does not match");
+                    logger.error("master key does not match");
                     throw newKeyMismatchError();
                 }
             }
-
-            // Otherwise it is probably fine
         });
     }
 }
+
+const CODE_VERSION = 0x02; // the version of binary QR codes we support
+const BINARY_PREFIX = "MATRIX"; // ASCII, used to prefix the binary format
+const MODE_VERIFY_OTHER_USER = 0x00; // Verifying someone who isn't us
+const MODE_VERIFY_SELF_TRUSTED = 0x01; // We trust the master key
+const MODE_VERIFY_SELF_UNTRUSTED = 0x02; // We do not trust the master key
+
+export class QRCodeData {
+    constructor(
+        mode, sharedSecret, otherUserMasterKey,
+        otherDeviceKey, myMasterKey, buffer,
+    ) {
+        this._sharedSecret = sharedSecret;
+        this._mode = mode;
+        this._otherUserMasterKey = otherUserMasterKey;
+        this._otherDeviceKey = otherDeviceKey;
+        this._myMasterKey = myMasterKey;
+        this._buffer = buffer;
+    }
+
+    static async create(request, client) {
+        const sharedSecret = QRCodeData._generateSharedSecret();
+        const mode = QRCodeData._determineMode(request, client);
+        let otherUserMasterKey = null;
+        let otherDeviceKey = null;
+        let myMasterKey = null;
+        if (mode === MODE_VERIFY_OTHER_USER) {
+            const otherUserCrossSigningInfo =
+                client.getStoredCrossSigningForUser(request.otherUserId);
+            otherUserMasterKey = otherUserCrossSigningInfo.getId("master");
+        } else if (mode === MODE_VERIFY_SELF_TRUSTED) {
+            otherDeviceKey = await QRCodeData._getOtherDeviceKey(request, client);
+        } else if (mode === MODE_VERIFY_SELF_UNTRUSTED) {
+            const myUserId = client.getUserId();
+            const myCrossSigningInfo = client.getStoredCrossSigningForUser(myUserId);
+            myMasterKey = myCrossSigningInfo.getId("master");
+        }
+        const qrData = QRCodeData._generateQrData(
+            request, client, mode,
+            sharedSecret,
+            otherUserMasterKey,
+            otherDeviceKey,
+            myMasterKey,
+        );
+        const buffer = QRCodeData._generateBuffer(qrData);
+        return new QRCodeData(mode, sharedSecret,
+            otherUserMasterKey, otherDeviceKey, myMasterKey, buffer);
+    }
+
+    get buffer() {
+        return this._buffer;
+    }
+
+    get mode() {
+        return this._mode;
+    }
+
+    /**
+     * only set when mode is MODE_VERIFY_SELF_TRUSTED
+     * @return {string} device key of other party at time of generating QR code
+     */
+    get otherDeviceKey() {
+        return this._otherDeviceKey;
+    }
+
+    /**
+     * only set when mode is MODE_VERIFY_OTHER_USER
+     * @return {string} master key of other party at time of generating QR code
+     */
+    get otherUserMasterKey() {
+        return this._otherUserMasterKey;
+    }
+
+    /**
+     * only set when mode is MODE_VERIFY_SELF_UNTRUSTED
+     * @return {string} own master key at time of generating QR code
+     */
+    get myMasterKey() {
+        return this._myMasterKey;
+    }
+
+    /**
+     * The unpadded base64 encoded shared secret.
+     */
+    get encodedSharedSecret() {
+        return this._sharedSecret;
+    }
+
+    static _generateSharedSecret() {
+        const secretBytes = new Uint8Array(11);
+        global.crypto.getRandomValues(secretBytes);
+        return encodeUnpaddedBase64(secretBytes);
+    }
+
+    static async _getOtherDeviceKey(request, client) {
+        const myUserId = client.getUserId();
+        const otherDevice = request.targetDevice;
+        const otherDeviceId = otherDevice ? otherDevice.deviceId : null;
+        const device = client.getStoredDevice(myUserId, otherDeviceId);
+        if (!device) {
+            throw new Error("could not find device " + otherDeviceId);
+        }
+        const key = device.getFingerprint();
+        return key;
+    }
+
+    static _determineMode(request, client) {
+        const myUserId = client.getUserId();
+        const otherUserId = request.otherUserId;
+
+        let mode = MODE_VERIFY_OTHER_USER;
+        if (myUserId === otherUserId) {
+            // Mode changes depending on whether or not we trust the master cross signing key
+            const myTrust = client.checkUserTrust(myUserId);
+            if (myTrust.isCrossSigningVerified()) {
+                mode = MODE_VERIFY_SELF_TRUSTED;
+            } else {
+                mode = MODE_VERIFY_SELF_UNTRUSTED;
+            }
+        }
+        return mode;
+    }
+
+    static _generateQrData(request, client, mode,
+        encodedSharedSecret, otherUserMasterKey,
+        otherDeviceKey, myMasterKey,
+    ) {
+        const myUserId = client.getUserId();
+        const transactionId = request.channel.transactionId;
+        const qrData = {
+            prefix: BINARY_PREFIX,
+            version: CODE_VERSION,
+            mode,
+            transactionId,
+            firstKeyB64: '', // worked out shortly
+            secondKeyB64: '', // worked out shortly
+            secretB64: encodedSharedSecret,
+        };
+
+        const myCrossSigningInfo = client.getStoredCrossSigningForUser(myUserId);
+
+        if (mode === MODE_VERIFY_OTHER_USER) {
+            // First key is our master cross signing key
+            qrData.firstKeyB64 = myCrossSigningInfo.getId("master");
+            // Second key is the other user's master cross signing key
+            qrData.secondKeyB64 = otherUserMasterKey;
+        } else if (mode === MODE_VERIFY_SELF_TRUSTED) {
+            // First key is our master cross signing key
+            qrData.firstKeyB64 = myCrossSigningInfo.getId("master");
+            qrData.secondKeyB64 = otherDeviceKey;
+        } else if (mode === MODE_VERIFY_SELF_UNTRUSTED) {
+            // First key is our device's key
+            qrData.firstKeyB64 = client.getDeviceEd25519Key();
+            // Second key is what we think our master cross signing key is
+            qrData.secondKeyB64 = myMasterKey;
+        }
+        return qrData;
+    }
+
+    static _generateBuffer(qrData) {
+        let buf = Buffer.alloc(0); // we'll concat our way through life
+
+        const appendByte = (b) => {
+            const tmpBuf = Buffer.from([b]);
+            buf = Buffer.concat([buf, tmpBuf]);
+        };
+        const appendInt = (i) => {
+            const tmpBuf = Buffer.alloc(2);
+            tmpBuf.writeInt16BE(i, 0);
+            buf = Buffer.concat([buf, tmpBuf]);
+        };
+        const appendStr = (s, enc, withLengthPrefix = true) => {
+            const tmpBuf = Buffer.from(s, enc);
+            if (withLengthPrefix) appendInt(tmpBuf.byteLength);
+            buf = Buffer.concat([buf, tmpBuf]);
+        };
+        const appendEncBase64 = (b64) => {
+            const b = decodeBase64(b64);
+            const tmpBuf = Buffer.from(b);
+            buf = Buffer.concat([buf, tmpBuf]);
+        };
+
+        // Actually build the buffer for the QR code
+        appendStr(qrData.prefix, "ascii", false);
+        appendByte(qrData.version);
+        appendByte(qrData.mode);
+        appendStr(qrData.transactionId, "utf-8");
+        appendEncBase64(qrData.firstKeyB64);
+        appendEncBase64(qrData.secondKeyB64);
+        appendEncBase64(qrData.secretB64);
+
+        return buf;
+    }
+}

src/crypto/verification/SAS.js

@@ -175,11 +175,33 @@ function calculateMAC(olmSAS, method) {
     };
 }
 
+const calculateKeyAgreement = {
+    "curve25519-hkdf-sha256": function(sas, olmSAS, bytes) {
+        const ourInfo = `${sas._baseApis.getUserId()}|${sas._baseApis.deviceId}|`
+              + `${sas.ourSASPubKey}|`;
+        const theirInfo = `${sas.userId}|${sas.deviceId}|${sas.theirSASPubKey}|`;
+        const sasInfo =
+            "MATRIX_KEY_VERIFICATION_SAS|"
+              + (sas.initiatedByMe ? ourInfo + theirInfo : theirInfo + ourInfo)
+              + sas._channel.transactionId;
+        return olmSAS.generate_bytes(sasInfo, bytes);
+    },
+    "curve25519": function(sas, olmSAS, bytes) {
+        const ourInfo = `${sas._baseApis.getUserId()}${sas._baseApis.deviceId}`;
+        const theirInfo = `${sas.userId}${sas.deviceId}`;
+        const sasInfo =
+            "MATRIX_KEY_VERIFICATION_SAS"
+              + (sas.initiatedByMe ? ourInfo + theirInfo : theirInfo + ourInfo)
+              + sas._channel.transactionId;
+        return olmSAS.generate_bytes(sasInfo, bytes);
+    },
+};
+
 /* lists of algorithms/methods that are supported.  The key agreement, hashes,
  * and MAC lists should be sorted in order of preference (most preferred
  * first).
  */
-const KEY_AGREEMENT_LIST = ["curve25519"];
+const KEY_AGREEMENT_LIST = ["curve25519-hkdf-sha256", "curve25519"];
 const HASHES_LIST = ["sha256"];
 const MAC_LIST = ["hkdf-hmac-sha256", "hmac-sha256"];
 const SAS_LIST = Object.keys(sasGenerators);
@@ -291,12 +313,14 @@ export class SAS extends Base {
         if (typeof content.commitment !== "string") {
             throw newInvalidMessageError();
         }
+        const keyAgreement = content.key_agreement_protocol;
         const macMethod = content.message_authentication_code;
         const hashCommitment = content.commitment;
         const olmSAS = new global.Olm.SAS();
         try {
-            this._send("m.key.verification.key", {
-                key: olmSAS.get_pubkey(),
+            this.ourSASPubKey = olmSAS.get_pubkey();
+            await this._send("m.key.verification.key", {
+                key: this.ourSASPubKey,
             });
 
 
@@ -308,19 +332,20 @@ export class SAS extends Base {
             if (olmutil.sha256(commitmentStr) !== hashCommitment) {
                 throw newMismatchedCommitmentError();
             }
+            this.theirSASPubKey = content.key;
             olmSAS.set_their_key(content.key);
 
-            const sasInfo = "MATRIX_KEY_VERIFICATION_SAS"
-                  + this._baseApis.getUserId() + this._baseApis.deviceId
-                  + this.userId + this.deviceId
-                  + this._channel.transactionId;
-            const sasBytes = olmSAS.generate_bytes(sasInfo, 6);
+            const sasBytes = calculateKeyAgreement[keyAgreement](this, olmSAS, 6);
             const verifySAS = new Promise((resolve, reject) => {
                 this.sasEvent = {
                     sas: generateSas(sasBytes, sasMethods),
-                    confirm: () => {
-                        this._sendMAC(olmSAS, macMethod);
-                        resolve();
+                    confirm: async () => {
+                        try {
+                            await this._sendMAC(olmSAS, macMethod);
+                            resolve();
+                        } catch (err) {
+                            reject(err);
+                        }
                     },
                     cancel: () => reject(newUserCancelledError()),
                     mismatch: () => reject(newMismatchedSASError()),
@@ -377,7 +402,7 @@ export class SAS extends Base {
         const olmSAS = new global.Olm.SAS();
         try {
             const commitmentStr = olmSAS.get_pubkey() + anotherjson.stringify(content);
-            this._send("m.key.verification.accept", {
+            await this._send("m.key.verification.accept", {
                 key_agreement_protocol: keyAgreement,
                 hash: hashMethod,
                 message_authentication_code: macMethod,
@@ -390,22 +415,24 @@ export class SAS extends Base {
             let e = await this._waitForEvent("m.key.verification.key");
             // FIXME: make sure event is properly formed
             content = e.getContent();
+            this.theirSASPubKey = content.key;
             olmSAS.set_their_key(content.key);
-            this._send("m.key.verification.key", {
-                key: olmSAS.get_pubkey(),
+            this.ourSASPubKey = olmSAS.get_pubkey();
+            await this._send("m.key.verification.key", {
+                key: this.ourSASPubKey,
             });
 
-            const sasInfo = "MATRIX_KEY_VERIFICATION_SAS"
-                  + this.userId + this.deviceId
-                  + this._baseApis.getUserId() + this._baseApis.deviceId
-                  + this._channel.transactionId;
-            const sasBytes = olmSAS.generate_bytes(sasInfo, 6);
+            const sasBytes = calculateKeyAgreement[keyAgreement](this, olmSAS, 6);
             const verifySAS = new Promise((resolve, reject) => {
                 this.sasEvent = {
                     sas: generateSas(sasBytes, sasMethods),
-                    confirm: () => {
-                        this._sendMAC(olmSAS, macMethod);
-                        resolve();
+                    confirm: async () => {
+                        try {
+                            await this._sendMAC(olmSAS, macMethod);
+                            resolve();
+                        } catch (err) {
+                            reject(err);
+                        }
                     },
                     cancel: () => reject(newUserCancelledError()),
                     mismatch: () => reject(newMismatchedSASError()),
@@ -461,7 +488,7 @@ export class SAS extends Base {
             keyList.sort().join(","),
             baseInfo + "KEY_IDS",
         );
-        this._send("m.key.verification.mac", { mac, keys });
+        return this._send("m.key.verification.mac", { mac, keys });
     }
 
     async _checkMAC(olmSAS, content, method) {

src/crypto/verification/request/InRoomChannel.js

@@ -44,11 +44,6 @@ export class InRoomChannel {
         this._requestEventId = null;
     }
 
-    /** Whether this channel needs m.key.verification.done messages to be sent after a successful verification */
-    get needsDoneMessage() {
-        return true;
-    }
-
     get receiveStartFromOtherDevices() {
         return true;
     }
@@ -183,6 +178,11 @@ export class InRoomChannel {
      * @returns {Promise} a promise that resolves when any requests as an anwser to the passed-in event are sent.
      */
     async handleEvent(event, request, isLiveEvent) {
+        // prevent processing the same event multiple times, as under
+        // some circumstances Room.timeline can get emitted twice for the same event
+        if (request.hasEventId(event.getId())) {
+            return;
+        }
         const type = InRoomChannel.getEventType(event);
         // do validations that need state (roomId, userId),
         // ignore if invalid

src/crypto/verification/request/ToDeviceChannel.js

@@ -61,10 +61,6 @@ export class ToDeviceChannel {
         return this._deviceId;
     }
 
-    get needsDoneMessage() {
-        return false;
-    }
-
     static getEventType(event) {
         return event.getType();
     }
@@ -360,4 +356,12 @@ export class ToDeviceRequests {
             }
         }
     }
+
+    getRequestsInProgress(userId) {
+        const requestsByTxnId = this._requestsByUserId.get(userId);
+        if (requestsByTxnId) {
+            return Array.from(requestsByTxnId.values()).filter(r => r.pending);
+        }
+        return [];
+    }
 }

src/crypto/verification/request/VerificationRequest.js

@@ -23,17 +23,19 @@ import {
     newUnexpectedMessageError,
     newUnknownMethodError,
 } from "../Error";
-import * as olmlib from "../../olmlib";
+import {QRCodeData, SCAN_QR_CODE_METHOD} from "../QRCode";
+
+// How long after the event's timestamp that the request times out
+const TIMEOUT_FROM_EVENT_TS = 10 * 60 * 1000; // 10 minutes
+
+// How long after we receive the event that the request times out
+const TIMEOUT_FROM_EVENT_RECEIPT = 2 * 60 * 1000; // 2 minutes
 
-// the recommended amount of time before a verification request
-// should be (automatically) cancelled without user interaction
-// and ignored.
-const VERIFICATION_REQUEST_TIMEOUT = 10 * 60 * 1000; //10m
 // to avoid almost expired verification notifications
 // from showing a notification and almost immediately
 // disappearing, also ignore verification requests that
 // are this amount of time away from expiring.
-const VERIFICATION_REQUEST_MARGIN = 3 * 1000; //3s
+const VERIFICATION_REQUEST_MARGIN = 3 * 1000; // 3 seconds
 
 
 export const EVENT_PREFIX = "m.key.verification.";
@@ -70,10 +72,19 @@ export class VerificationRequest extends EventEmitter {
         this._eventsByThem = new Map();
         this._observeOnly = false;
         this._timeoutTimer = null;
-        this._sharedSecret = null; // used for QR codes
         this._accepting = false;
         this._declining = false;
         this._verifierHasFinished = false;
+        this._cancelled = false;
+        this._chosenMethod = null;
+        // we keep a copy of the QR Code data (including other user master key) around
+        // for QR reciprocate verification, to protect against
+        // cross-signing identity reset between the .ready and .start event
+        // and signing the wrong key after .start
+        this._qrCodeData = null;
+
+        // The timestamp when we received the request event from the other side
+        this._requestReceivedAt = null;
     }
 
     /**
@@ -154,12 +165,31 @@ export class VerificationRequest extends EventEmitter {
         return this._commonMethods;
     }
 
+    /** the method picked in the .start event */
+    get chosenMethod() {
+        return this._chosenMethod;
+    }
+
+    calculateEventTimeout(event) {
+        let effectiveExpiresAt = this.channel.getTimestamp(event)
+            + TIMEOUT_FROM_EVENT_TS;
+
+        if (this._requestReceivedAt && !this.initiatedByMe &&
+            this.phase <= PHASE_REQUESTED
+        ) {
+            const expiresAtByReceipt = this._requestReceivedAt
+                + TIMEOUT_FROM_EVENT_RECEIPT;
+            effectiveExpiresAt = Math.min(effectiveExpiresAt, expiresAtByReceipt);
+        }
+
+        return Math.max(0, effectiveExpiresAt - Date.now());
+    }
+
     /** The current remaining amount of ms before the request should be automatically cancelled */
     get timeout() {
         const requestEvent = this._getEventByEither(REQUEST_TYPE);
         if (requestEvent) {
-            const elapsed = Date.now() - this.channel.getTimestamp(requestEvent);
-            return Math.max(0, VERIFICATION_REQUEST_TIMEOUT - elapsed);
+            return this.calculateEventTimeout(requestEvent);
         }
         return 0;
     }
@@ -201,14 +231,20 @@ export class VerificationRequest extends EventEmitter {
             this._phase !== PHASE_CANCELLED;
     }
 
+    /** Only set after a .ready if the other party can scan a QR code */
+    get qrCodeData() {
+        return this._qrCodeData;
+    }
+
     /** Checks whether the other party supports a given verification method.
      *  This is useful when setting up the QR code UI, as it is somewhat asymmetrical:
      *  if the other party supports SCAN_QR, we should show a QR code in the UI, and vice versa.
      *  For methods that need to be supported by both ends, use the `methods` property.
      *  @param {string} method the method to check
+     *  @param {boolean} force to check even if the phase is not ready or started yet, internal usage
      *  @return {bool} whether or not the other party said the supported the method */
-    otherPartySupportsMethod(method) {
-        if (!this.ready && !this.started) {
+    otherPartySupportsMethod(method, force = false) {
+        if (!force && !this.ready && !this.started) {
             return false;
         }
         const theirMethodEvent = this._eventsByThem.get(REQUEST_TYPE) ||
@@ -286,6 +322,10 @@ export class VerificationRequest extends EventEmitter {
         return this.channel.userId;
     }
 
+    get isSelfVerification() {
+        return this._client.getUserId() === this.otherUserId;
+    }
+
     /**
      * The id of the user that cancelled the request,
      * only defined when phase is PHASE_CANCELLED
@@ -315,14 +355,6 @@ export class VerificationRequest extends EventEmitter {
         return this._observeOnly;
     }
 
-    /**
-     * The unpadded base64 encoded shared secret. Primarily used for QR code
-     * verification.
-     */
-    get encodedSharedSecret() {
-        if (!this._sharedSecret) this._generateSharedSecret();
-        return this._sharedSecret;
-    }
 
     /**
      * Gets which device the verification should be started with
@@ -369,6 +401,7 @@ export class VerificationRequest extends EventEmitter {
                 if (!this._verifier) {
                     throw newUnknownMethodError();
                 }
+                this._chosenMethod = method;
             }
         }
         return this._verifier;
@@ -382,7 +415,6 @@ export class VerificationRequest extends EventEmitter {
         if (!this.observeOnly && this._phase === PHASE_UNSENT) {
             const methods = [...this._verificationMethods.keys()];
             await this.channel.send(REQUEST_TYPE, {methods});
-            this._generateSharedSecret();
         }
     }
 
@@ -415,16 +447,9 @@ export class VerificationRequest extends EventEmitter {
             this._accepting = true;
             this.emit("change");
             await this.channel.send(READY_TYPE, {methods});
-            this._generateSharedSecret();
         }
     }
 
-    _generateSharedSecret() {
-        const secretBytes = new Uint8Array(8);
-        global.crypto.getRandomValues(secretBytes);
-        this._sharedSecret = olmlib.encodeUnpaddedBase64(secretBytes);
-    }
-
     /**
      * Can be used to listen for state changes until the callback returns true.
      * @param {Function} fn callback to evaluate whether the request is in the desired state.
@@ -520,7 +545,7 @@ export class VerificationRequest extends EventEmitter {
         }
 
         const cancelEvent = this._getEventByEither(CANCEL_TYPE);
-        if (cancelEvent && phase() !== PHASE_DONE) {
+        if ((this._cancelled || cancelEvent) && phase() !== PHASE_DONE) {
             transitions.push({phase: PHASE_CANCELLED, event: cancelEvent});
             return transitions;
         }
@@ -559,6 +584,14 @@ export class VerificationRequest extends EventEmitter {
             const {method} = event.getContent();
             if (!this._verifier && !this.observeOnly) {
                 this._verifier = this._createVerifier(method, event);
+                if (!this._verifier) {
+                    this.cancel({
+                        code: "m.unknown_method",
+                        reason: `Unknown method: ${method}`,
+                    });
+                } else {
+                    this._chosenMethod = method;
+                }
             }
         }
     }
@@ -580,10 +613,9 @@ export class VerificationRequest extends EventEmitter {
             return false;
         }
         const oldEvent = this._verifier.startEvent;
-        const isSelfVerification = this.channel.userId === this._client.getUserId();
 
         let oldRaceIdentifier;
-        if (isSelfVerification) {
+        if (this.isSelfVerification) {
             // if the verifier does not have a startEvent,
             // it is because it's still sending and we are on the initator side
             // we know we are sending a .start event because we already
@@ -603,7 +635,7 @@ export class VerificationRequest extends EventEmitter {
         }
 
         let newRaceIdentifier;
-        if (isSelfVerification) {
+        if (this.isSelfVerification) {
             const newContent = newEvent.getContent();
             newRaceIdentifier = newContent && newContent.from_device;
         } else {
@@ -612,6 +644,20 @@ export class VerificationRequest extends EventEmitter {
         return newRaceIdentifier < oldRaceIdentifier;
     }
 
+    hasEventId(eventId) {
+        for (const event of this._eventsByUs.values()) {
+            if (event.getId() === eventId) {
+                return true;
+            }
+        }
+        for (const event of this._eventsByThem.values()) {
+            if (event.getId() === eventId) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     /**
      * Changes the state of the request and verifier in response to a key verification event.
      * @param {string} type the "symbolic" event type, as returned by the `getEventType` function on the channel.
@@ -637,6 +683,18 @@ export class VerificationRequest extends EventEmitter {
             }
         }
 
+        // This assumes verification won't need to send an event with
+        // the same type for the same party twice.
+        // This is true for QR and SAS verification, and was
+        // added here to prevent verification getting cancelled
+        // when the server duplicates an event (https://github.com/matrix-org/synapse/issues/3365)
+        const isDuplicateEvent = isSentByUs ?
+            this._eventsByUs.has(type) :
+            this._eventsByThem.has(type);
+        if (isDuplicateEvent) {
+            return;
+        }
+
         const oldPhase = this.phase;
         this._addEvent(type, event, isSentByUs);
 
@@ -650,7 +708,7 @@ export class VerificationRequest extends EventEmitter {
                 if (this._verifier.canSwitchStartEvent(event) && newEventWinsRace) {
                     this._verifier.switchStartEvent(event);
                 } else if (!isRemoteEcho) {
-                     if (type === CANCEL_TYPE || (this._verifier.events
+                    if (type === CANCEL_TYPE || (this._verifier.events
                         && this._verifier.events.includes(type))) {
                         this._verifier.handleEvent(event);
                     }
@@ -658,6 +716,20 @@ export class VerificationRequest extends EventEmitter {
             }
 
             if (newTransitions.length) {
+                // create QRCodeData if the other side can scan
+                // important this happens before emitting a phase change,
+                // so listeners can rely on it being there already
+                // We only do this for live events because it is important that
+                // we sign the keys that were in the QR code, and not the keys
+                // we happen to have at some later point in time.
+                if (isLiveEvent && newTransitions.some(t => t.phase === PHASE_READY)) {
+                    const shouldGenerateQrCode =
+                        this.otherPartySupportsMethod(SCAN_QR_CODE_METHOD, true);
+                    if (shouldGenerateQrCode) {
+                        this._qrCodeData = await QRCodeData.create(this, this._client);
+                    }
+                }
+
                 const lastTransition = newTransitions[newTransitions.length - 1];
                 const {phase} = lastTransition;
 
@@ -682,7 +754,7 @@ export class VerificationRequest extends EventEmitter {
 
     _setupTimeout(phase) {
         const shouldTimeout = !this._timeoutTimer && !this.observeOnly &&
-            phase === PHASE_REQUESTED && this.initiatedByMe;
+            phase === PHASE_REQUESTED;
 
         if (shouldTimeout) {
             this._timeoutTimer = setTimeout(this._cancelOnTimeout, this.timeout);
@@ -701,7 +773,17 @@ export class VerificationRequest extends EventEmitter {
 
     _cancelOnTimeout = () => {
         try {
-            this.cancel({reason: "Other party didn't accept in time", code: "m.timeout"});
+            if (this.initiatedByMe) {
+                this.cancel({
+                    reason: "Other party didn't accept in time",
+                    code: "m.timeout",
+                });
+            } else {
+                this.cancel({
+                    reason: "User didn't accept in time",
+                    code: "m.timeout",
+                });
+            }
         } catch (err) {
             logger.error("Error while cancelling verification request", err);
         }
@@ -738,16 +820,8 @@ export class VerificationRequest extends EventEmitter {
         if (!isLiveEvent) {
             this._observeOnly = true;
         }
-        // a timestamp is not provided on all to_device events
-        const timestamp = this.channel.getTimestamp(event);
-        if (Number.isFinite(timestamp)) {
-            const elapsed = Date.now() - timestamp;
-            // don't allow interaction on old requests
-            if (elapsed > (VERIFICATION_REQUEST_TIMEOUT - VERIFICATION_REQUEST_MARGIN) ||
-                elapsed < -(VERIFICATION_REQUEST_TIMEOUT / 2)
-            ) {
-                this._observeOnly = true;
-            }
+        if (this.calculateEventTimeout(event) < VERIFICATION_REQUEST_MARGIN) {
+            this._observeOnly = true;
         }
     }
 
@@ -766,6 +840,8 @@ export class VerificationRequest extends EventEmitter {
                     this._eventsByThem.delete(type);
                 }
             }
+            // also remember when we received the request event
+            this._requestReceivedAt = Date.now();
         }
     }
 
@@ -806,15 +882,26 @@ export class VerificationRequest extends EventEmitter {
         return true;
     }
 
-    onVerifierFinished() {
-        if (this.channel.needsDoneMessage) {
-            // verification in DM requires a done message
-            this.channel.send("m.key.verification.done", {});
-        }
-        this._verifierHasFinished = true;
+    onVerifierCancelled() {
+        this._cancelled = true;
+        // move to cancelled phase
         const newTransitions = this._applyPhaseTransitions();
         if (newTransitions.length) {
             this._setPhase(newTransitions[newTransitions.length - 1].phase);
         }
     }
+
+    onVerifierFinished() {
+        this.channel.send("m.key.verification.done", {});
+        this._verifierHasFinished = true;
+        // move to .done phase
+        const newTransitions = this._applyPhaseTransitions();
+        if (newTransitions.length) {
+            this._setPhase(newTransitions[newTransitions.length - 1].phase);
+        }
+    }
+
+    getEventFromOtherParty(type) {
+        return this._eventsByThem.get(type);
+    }
 }

src/filter-component.js

@@ -108,8 +108,9 @@ FilterComponent.prototype._checkFields =
         }
 
         const allowed_values = self[name];
-        if (allowed_values) {
-            if (!allowed_values.map(match_func)) {
+        if (allowed_values && allowed_values.length > 0) {
+            const anyMatch = allowed_values.some(match_func);
+            if (!anyMatch) {
                 return false;
             }
         }

src/filter.js

@@ -56,13 +56,6 @@ Filter.LAZY_LOADING_MESSAGES_FILTER = {
     lazy_load_members: true,
 };
 
-Filter.LAZY_LOADING_SYNC_FILTER = {
-    room: {
-        state: Filter.LAZY_LOADING_MESSAGES_FILTER,
-    },
-};
-
-
 /**
  * Get the ID of this filter on your homeserver (if known)
  * @return {?Number} The filter ID
@@ -96,6 +89,7 @@ Filter.prototype.setDefinition = function(definition) {
     //     "state": {
     //       "types": ["m.room.*"],
     //       "not_rooms": ["!726s6s6q:example.com"],
+    //       "lazy_load_members": true,
     //     },
     //     "timeline": {
     //       "limit": 10,
@@ -177,6 +171,10 @@ Filter.prototype.setTimelineLimit = function(limit) {
     setProp(this.definition, "room.timeline.limit", limit);
 };
 
+Filter.prototype.setLazyLoadMembers = function(enabled) {
+    setProp(this.definition, "room.state.lazy_load_members", !!enabled);
+};
+
 /**
  * Control whether left rooms should be included in responses.
  * @param {boolean} includeLeave True to make rooms the user has left appear

src/http-api.js

@@ -276,6 +276,9 @@ MatrixHttpApi.prototype = {
                         callbacks.clearTimeout(xhr.timeout_timer);
                         var resp;
                         try {
+                            if (xhr.status === 0) {
+                                throw new AbortError();
+                            }
                             if (!xhr.responseText) {
                                 throw new Error('No response body.');
                             }
@@ -648,12 +651,10 @@ MatrixHttpApi.prototype = {
 
         const self = this;
         if (this.opts.extraParams) {
-            for (const key in this.opts.extraParams) {
-                if (!this.opts.extraParams.hasOwnProperty(key)) {
-                    continue;
-                }
-                queryParams[key] = this.opts.extraParams[key];
-            }
+            queryParams = {
+              ...queryParams,
+              ...this.opts.extraParams,
+            };
         }
 
         const headers = utils.extend({}, opts.headers || {});
@@ -789,6 +790,17 @@ const requestCallback = function(
     userDefinedCallback = userDefinedCallback || function() {};
 
     return function(err, response, body) {
+        if (err) {
+            // the unit tests use matrix-mock-request, which throw the string "aborted" when aborting a request.
+            // See https://github.com/matrix-org/matrix-mock-request/blob/3276d0263a561b5b8326b47bae720578a2c7473a/src/index.js#L48
+            const aborted = err.name === "AbortError" || err === "aborted";
+            if (!aborted && !(err instanceof MatrixError)) {
+                // browser-request just throws normal Error objects,
+                // not `TypeError`s like fetch does. So just assume any
+                // error is due to the connection.
+                err = new ConnectionError("request failed", err);
+            }
+        }
         if (!err) {
             try {
                 if (response.statusCode >= 400) {
@@ -876,7 +888,7 @@ function getResponseContentType(response) {
 
     try {
         return parseContentType(contentType);
-    } catch(e) {
+    } catch (e) {
         throw new Error(`Error parsing Content-Type '${contentType}': ${e}`);
     }
 }
@@ -892,12 +904,76 @@ function getResponseContentType(response) {
  * @prop {Object} data The raw Matrix error JSON used to construct this object.
  * @prop {integer} httpStatus The numeric HTTP status code given
  */
-export function MatrixError(errorJson) {
-    errorJson = errorJson || {};
-    this.errcode = errorJson.errcode;
-    this.name = errorJson.errcode || "Unknown error code";
-    this.message = errorJson.error || "Unknown message";
-    this.data = errorJson;
+export class MatrixError extends Error {
+    constructor(errorJson) {
+        errorJson = errorJson || {};
+        super(`MatrixError: ${errorJson.errcode}`);
+        this.errcode = errorJson.errcode;
+        this.name = errorJson.errcode || "Unknown error code";
+        this.message = errorJson.error || "Unknown message";
+        this.data = errorJson;
+    }
+}
+
+/**
+ * Construct a ConnectionError. This is a JavaScript Error indicating
+ * that a request failed because of some error with the connection, either
+ * CORS was not correctly configured on the server, the server didn't response,
+ * the request timed out, or the internet connection on the client side went down.
+ * @constructor
+ */
+export class ConnectionError extends Error {
+    constructor(message, cause = undefined) {
+        super(message + (cause ? `: ${cause.message}` : ""));
+        this._cause = cause;
+    }
+
+    get name() {
+        return "ConnectionError";
+    }
+
+    get cause() {
+        return this._cause;
+    }
+}
+
+export class AbortError extends Error {
+    constructor() {
+        super("Operation aborted");
+    }
+
+    get name() {
+        return "AbortError";
+    }
+}
+
+/**
+ * Retries a network operation run in a callback.
+ * @param  {number}   maxAttempts maximum attempts to try
+ * @param  {Function} callback    callback that returns a promise of the network operation. If rejected with ConnectionError, it will be retried by calling the callback again.
+ * @return {any} the result of the network operation
+ * @throws {ConnectionError} If after maxAttempts the callback still throws ConnectionError
+ */
+export async function retryNetworkOperation(maxAttempts, callback) {
+    let attempts = 0;
+    let lastConnectionError = null;
+    while (attempts < maxAttempts) {
+        try {
+            if (attempts > 0) {
+                const timeout = 1000 * Math.pow(2, attempts);
+                logger.log(`network operation failed ${attempts} times,` +
+                    ` retrying in ${timeout}ms...`);
+                await new Promise(r => setTimeout(r, timeout));
+            }
+            return await callback();
+        } catch (err) {
+            if (err instanceof ConnectionError) {
+                attempts += 1;
+                lastConnectionError = err;
+            } else {
+                throw err;
+            }
+        }
+    }
+    throw lastConnectionError;
 }
-MatrixError.prototype = Object.create(Error.prototype);
-MatrixError.prototype.constructor = MatrixError;

src/index.ts

@@ -22,6 +22,7 @@ matrixcs.request(request);
 utils.runPolyfills();
 
 try {
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
     const crypto = require('crypto');
     utils.setCrypto(crypto);
 } catch (err) {

src/interactive-auth.js

@@ -143,11 +143,19 @@ InteractiveAuth.prototype = {
             this._resolveFunc = resolve;
             this._rejectFunc = reject;
 
-            // if we have no flows, try a request (we'll have
-            // just a session ID in _data if resuming)
-            if (!this._data.flows) {
+            const hasFlows = this._data && this._data.flows;
+
+            // if we have no flows, try a request to acquire the flows
+            if (!hasFlows) {
                 if (this._busyChangedCallback) this._busyChangedCallback(true);
-                this._doRequest(this._data).finally(() => {
+                // use the existing sessionid, if one is present.
+                let auth = null;
+                if (this._data.session) {
+                    auth = {
+                        session: this._data.session,
+                    };
+                }
+                this._doRequest(auth).finally(() => {
                     if (this._busyChangedCallback) this._busyChangedCallback(false);
                 });
             } else {
@@ -163,6 +171,8 @@ InteractiveAuth.prototype = {
      */
     poll: async function() {
         if (!this._data.session) return;
+        // likewise don't poll if there is no auth session in progress
+        if (!this._resolveFunc) return;
         // if we currently have a request in flight, there's no point making
         // another just to check what the status is
         if (this._submitPromise) return;
@@ -184,7 +194,11 @@ InteractiveAuth.prototype = {
                 }
                 authDict = {
                     type: EMAIL_STAGE_TYPE,
+                    // TODO: Remove `threepid_creds` once servers support proper UIA
+                    // See https://github.com/matrix-org/synapse/issues/5665
+                    // See https://github.com/matrix-org/matrix-doc/issues/2220
                     threepid_creds: creds,
+                    threepidCreds: creds,
                 };
             }
         }
@@ -262,11 +276,16 @@ InteractiveAuth.prototype = {
             }
         }
 
-        // use the sessionid from the last request.
-        const auth = {
-            session: this._data.session,
-        };
-        utils.extend(auth, authData);
+        // use the sessionid from the last request, if one is present.
+        let auth;
+        if (this._data.session) {
+            auth = {
+                session: this._data.session,
+            };
+            utils.extend(auth, authData);
+        } else {
+            auth = authData;
+        }
 
         try {
             // NB. the 'background' flag is deprecated by the busyChanged
@@ -318,10 +337,12 @@ InteractiveAuth.prototype = {
         try {
             const result = await this._requestCallback(auth, background);
             this._resolveFunc(result);
+            this._resolveFunc = null;
+            this._rejectFunc = null;
         } catch (error) {
             // sometimes UI auth errors don't come with flows
             const errorFlows = error.data ? error.data.flows : null;
-            const haveFlows = Boolean(this._data.flows) || Boolean(errorFlows);
+            const haveFlows = this._data.flows || Boolean(errorFlows);
             if (error.httpStatus !== 401 || !error.data || !haveFlows) {
                 // doesn't look like an interactive-auth failure.
                 if (!background) {
@@ -347,7 +368,13 @@ InteractiveAuth.prototype = {
                 error.data.session = this._data.session;
             }
             this._data = error.data;
-            this._startNextAuthStage();
+            try {
+                this._startNextAuthStage();
+            } catch (e) {
+                this._rejectFunc(e);
+                this._resolveFunc = null;
+                this._rejectFunc = null;
+            }
 
             if (
                 !this._emailSid &&
@@ -379,8 +406,10 @@ InteractiveAuth.prototype = {
                     // (or not being registered, depending on what we're trying
                     // to do) or it could be a network failure. Either way, pass
                     // the failure up as the user can't complete auth if we can't
-                    // send the email, foe whatever reason.
+                    // send the email, for whatever reason.
                     this._rejectFunc(e);
+                    this._resolveFunc = null;
+                    this._rejectFunc = null;
                 } finally {
                     this._requestingEmailToken = false;
                 }
@@ -408,7 +437,7 @@ InteractiveAuth.prototype = {
             return;
         }
 
-        if (this._data.errcode || this._data.error) {
+        if (this._data && this._data.errcode || this._data.error) {
             this._stateUpdatedCallback(nextStage, {
                 errcode: this._data.errcode || "",
                 error: this._data.error || "",

src/logger.ts

@@ -40,11 +40,13 @@ log.methodFactory = function(methodName, logLevel, loggerName) {
             methodName === "warn" ||
             methodName === "trace" ||
             methodName === "info";
+        /* eslint-disable no-console */
         if (supportedByConsole) {
             return console[methodName](...args);
         } else {
             return console.log(...args);
         }
+        /* eslint-enable no-console */
     };
 };
 

src/matrix.ts

@@ -1,7 +1,7 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
 Copyright 2017 Vector Creations Ltd
-Copyright 2019 The Matrix.org Foundation C.I.C.
+Copyright 2019, 2020 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -16,8 +16,15 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+import type Request from "request";
+
 import {MemoryCryptoStore} from "./crypto/store/memory-crypto-store";
+import {LocalStorageCryptoStore} from "./crypto/store/localStorage-crypto-store";
+import {IndexedDBCryptoStore} from "./crypto/store/indexeddb-crypto-store";
 import {MemoryStore} from "./store/memory";
+import {StubStore} from "./store/stub";
+import {LocalIndexedDBStoreBackend} from "./store/indexeddb-local-backend";
+import {RemoteIndexedDBStoreBackend} from "./store/indexeddb-remote-backend";
 import {MatrixScheduler} from "./scheduler";
 import {MatrixClient} from "./client";
 
@@ -89,6 +96,10 @@ export function wrapRequest(wrapper) {
     };
 }
 
+type Store =
+    StubStore | MemoryStore | LocalIndexedDBStoreBackend | RemoteIndexedDBStoreBackend;
+
+type CryptoStore = MemoryCryptoStore | LocalStorageCryptoStore | IndexedDBCryptoStore;
 
 let cryptoStoreFactory = () => new MemoryCryptoStore;
 
@@ -102,6 +113,80 @@ export function setCryptoStoreFactory(fac) {
     cryptoStoreFactory = fac;
 }
 
+export interface ICreateClientOpts {
+    baseUrl: string;
+    idBaseUrl?: string;
+    store?: Store;
+    cryptoStore?: CryptoStore;
+    scheduler?: MatrixScheduler;
+    request?: Request;
+    userId?: string;
+    deviceId?: string;
+    accessToken?: string;
+    identityServer?: any;
+    localTimeoutMs?: number;
+    useAuthorizationHeader?: boolean;
+    timelineSupport?: boolean;
+    queryParams?: Record<string, unknown>;
+    deviceToImport?: {
+        olmDevice: {
+            pickledAccount: string;
+            sessions: Array<Record<string, any>>;
+            pickleKey: string;
+        };
+        userId: string;
+        deviceId: string;
+    };
+    pickleKey?: string;
+    sessionStore?: any;
+    unstableClientRelationAggregation?: boolean;
+    verificationMethods?: Array<any>;
+    forceTURN?: boolean;
+    fallbackICEServerAllowed?: boolean;
+    cryptoCallbacks?: ICryptoCallbacks;
+}
+
+export interface ICryptoCallbacks {
+    getCrossSigningKey?: (keyType: string, pubKey: Uint8Array) => Promise<Uint8Array>;
+    saveCrossSigningKeys?: (keys: Record<string, Uint8Array>) => void;
+    shouldUpgradeDeviceVerifications?: (
+        users: Record<string, any>
+    ) => Promise<string[]>;
+    getSecretStorageKey?: (
+        keys: {keys: Record<string, ISecretStorageKeyInfo>}, name: string
+    ) => Promise<[string, Uint8Array] | null>;
+    cacheSecretStorageKey?: (
+        keyId: string, keyInfo: ISecretStorageKeyInfo, key: Uint8Array
+    ) => void;
+    onSecretRequested?: (
+        userId: string, deviceId: string,
+        requestId: string, secretName: string, deviceTrust: IDeviceTrustLevel
+    ) => Promise<string>;
+    getDehydrationKey?: (
+        keyInfo: ISecretStorageKeyInfo,
+        checkFunc: (Uint8Array) => void,
+    ) => Promise<Uint8Array>;
+}
+
+// TODO: Move this to `SecretStorage` once converted
+export interface ISecretStorageKeyInfo {
+    passphrase?: {
+        algorithm: "m.pbkdf2";
+        iterations: number;
+        salt: string;
+    };
+    iv?: string;
+    mac?: string;
+}
+
+// TODO: Move this to `CrossSigning` once converted
+export interface IDeviceTrustLevel {
+    isVerified(): boolean;
+    isCrossSigningVerified(): boolean;
+    isLocallyVerified(): boolean;
+    isTofu(): boolean;
+}
+
 /**
  * Construct a Matrix Client. Similar to {@link module:client.MatrixClient}
  * except that the 'request', 'store' and 'scheduler' dependencies are satisfied.
@@ -125,10 +210,10 @@ export function setCryptoStoreFactory(fac) {
  * @see {@link module:client.MatrixClient} for the full list of options for
  * <code>opts</code>.
  */
-export function createClient(opts) {
+export function createClient(opts: ICreateClientOpts | string) {
     if (typeof opts === "string") {
         opts = {
-            "baseUrl": opts,
+            "baseUrl": opts as string,
         };
     }
     opts.request = opts.request || requestInstance;
@@ -167,7 +252,7 @@ export function createClient(opts) {
  * @param {requestCallback} callback The request callback.
  */
 
- /**
+/**
   * The request callback interface for performing HTTP requests. This matches the
   * API for the {@link https://github.com/request/request#requestoptions-callback|
   * request NPM module}. The SDK will implement a callback which meets this

src/models/event-timeline-set.js

@@ -647,7 +647,8 @@ EventTimelineSet.prototype.compareEventOrdering = function(eventId1, eventId2) {
     if (timeline1 === timeline2) {
         // both events are in the same timeline - figure out their
         // relative indices
-        let idx1, idx2;
+        let idx1;
+        let idx2;
         const events = timeline1.getEvents();
         for (let idx = 0; idx < events.length &&
              (idx1 === undefined || idx2 === undefined); idx++) {
@@ -805,7 +806,7 @@ EventTimelineSet.prototype.aggregateRelations = function(event) {
             this.room,
         );
         isNewRelations = true;
-        relatesToEvent = this.findEventById(relatesToEventId);
+        relatesToEvent = this.findEventById(relatesToEventId) || this.room.getPendingEvent(relatesToEventId);
         if (relatesToEvent) {
             relationsWithEventType.setTargetEvent(relatesToEvent);
         }

src/models/event.js

@@ -87,7 +87,7 @@ export const MatrixEvent = function(
     // amount of needless string duplication. This can save moderate amounts of
     // memory (~10% on a 350MB heap).
     // 'membership' at the event level (rather than the content level) is a legacy
-    // field that Riot never otherwise looks at, but it will still take up a lot
+    // field that Element never otherwise looks at, but it will still take up a lot
     // of space if we don't intern it.
     ["state_key", "type", "sender", "room_id", "membership"].forEach((prop) => {
         if (!event[prop]) {
@@ -144,6 +144,10 @@ export const MatrixEvent = function(
      */
     this._forwardingCurve25519KeyChain = [];
 
+    /* where the decryption key is untrusted
+     */
+    this._untrusted = null;
+
     /* if we have a process decrypting this event, a Promise which resolves
      * when it is finished. Normally null.
      */
@@ -160,12 +164,24 @@ export const MatrixEvent = function(
      * so it can be easily accessed from the timeline.
      */
     this.verificationRequest = null;
+
+    /* The txnId with which this event was sent if it was during this session,
+       allows for a unique ID which does not change when the event comes back down sync.
+     */
+    this._txnId = null;
+
+    /* Set an approximate timestamp for the event relative the local clock.
+     * This will inherently be approximate because it doesn't take into account
+     * the time between the server putting the 'age' field on the event as it sent
+     * it to us and the time we're now constructing this event, but that's better
+     * than assuming the local clock is in sync with the origin HS's clock.
+     */
+    this._localTimestamp = Date.now() - this.getAge();
 };
 utils.inherits(MatrixEvent, EventEmitter);
 
 
 utils.extend(MatrixEvent.prototype, {
-
     /**
      * Get the event_id for this event.
      * @return {string} The event ID, e.g. <code>$143350589368169JsLZx:localhost
@@ -303,11 +319,12 @@ utils.extend(MatrixEvent.prototype, {
 
     /**
      * Get the age of the event when this function was called.
-     * Relies on the local clock being in sync with the clock of the original homeserver.
+     * This is the 'age' field adjusted according to how long this client has
+     * had the event.
      * @return {Number} The age of this event in milliseconds.
      */
     getLocalAge: function() {
-        return Date.now() - this.getTs();
+        return Date.now() - this._localTimestamp;
     },
 
     /**
@@ -390,11 +407,12 @@ utils.extend(MatrixEvent.prototype, {
      * @internal
      *
      * @param {module:crypto} crypto crypto module
+     * @param {bool} isRetry True if this is a retry (enables more logging)
      *
      * @returns {Promise} promise which resolves (to undefined) when the decryption
      * attempt is completed.
      */
-    attemptDecryption: async function(crypto) {
+    attemptDecryption: async function(crypto, isRetry) {
         // start with a couple of sanity checks.
         if (!this.isEncrypted()) {
             throw new Error("Attempt to decrypt event which isn't encrypted");
@@ -406,7 +424,7 @@ utils.extend(MatrixEvent.prototype, {
         ) {
             // we may want to just ignore this? let's start with rejecting it.
             throw new Error(
-                "Attempt to decrypt event which has already been encrypted",
+                "Attempt to decrypt event which has already been decrypted",
             );
         }
 
@@ -424,7 +442,7 @@ utils.extend(MatrixEvent.prototype, {
             return this._decryptionPromise;
         }
 
-        this._decryptionPromise = this._decryptionLoop(crypto);
+        this._decryptionPromise = this._decryptionLoop(crypto, isRetry);
         return this._decryptionPromise;
     },
 
@@ -469,7 +487,7 @@ utils.extend(MatrixEvent.prototype, {
         return recipients;
     },
 
-    _decryptionLoop: async function(crypto) {
+    _decryptionLoop: async function(crypto, isRetry) {
         // make sure that this method never runs completely synchronously.
         // (doing so would mean that we would clear _decryptionPromise *before*
         // it is set in attemptDecryption - and hence end up with a stuck
@@ -486,13 +504,18 @@ utils.extend(MatrixEvent.prototype, {
                     res = this._badEncryptedMessage("Encryption not enabled");
                 } else {
                     res = await crypto.decryptEvent(this);
+                    if (isRetry) {
+                        logger.info(`Decrypted event on retry (id=${this.getId()})`);
+                    }
                 }
             } catch (e) {
                 if (e.name !== "DecryptionError") {
                     // not a decryption error: log the whole exception as an error
                     // (and don't bother with a retry)
+                    const re = isRetry ? 're' : '';
                     logger.error(
-                        `Error decrypting event (id=${this.getId()}): ${e.stack || e}`,
+                        `Error ${re}decrypting event ` +
+                        `(id=${this.getId()}): ${e.stack || e}`,
                     );
                     this._decryptionPromise = null;
                     this._retryDecryption = false;
@@ -593,6 +616,7 @@ utils.extend(MatrixEvent.prototype, {
             decryptionResult.claimedEd25519Key || null;
         this._forwardingCurve25519KeyChain =
             decryptionResult.forwardingCurve25519KeyChain || [];
+        this._untrusted = decryptionResult.untrusted || false;
     },
 
     /**
@@ -611,7 +635,7 @@ utils.extend(MatrixEvent.prototype, {
      * @return {boolean} True if this event is encrypted.
      */
     isEncrypted: function() {
-        return this.event.type === "m.room.encrypted";
+        return !this.isState() && this.event.type === "m.room.encrypted";
     },
 
     /**
@@ -683,6 +707,16 @@ utils.extend(MatrixEvent.prototype, {
         return this._forwardingCurve25519KeyChain;
     },
 
+    /**
+     * Whether the decryption key was obtained from an untrusted source. If so,
+     * we cannot verify the authenticity of the message.
+     *
+     * @return {boolean}
+     */
+    isKeySourceUntrusted: function() {
+        return this._untrusted;
+    },
+
     getUnsigned: function() {
         return this.event.unsigned || {};
     },
@@ -879,6 +913,8 @@ utils.extend(MatrixEvent.prototype, {
     /**
      * Set an event that replaces the content of this event, through an m.replace relation.
      *
+     * @fires module:models/event.MatrixEvent#"Event.replaced"
+     *
      * @param {MatrixEvent?} newEvent the event with the replacing content, if any.
      */
     makeReplaced(newEvent) {
@@ -1064,6 +1100,14 @@ utils.extend(MatrixEvent.prototype, {
     setVerificationRequest: function(request) {
         this.verificationRequest = request;
     },
+
+    setTxnId(txnId) {
+        this._txnId = txnId;
+    },
+
+    getTxnId() {
+        return this._txnId;
+    },
 });
 
 

src/models/relations.js

@@ -16,6 +16,7 @@ limitations under the License.
 
 import {EventEmitter} from 'events';
 import {EventStatus} from '../models/event';
+import {logger} from '../logger';
 
 /**
  * A container for relation events that supports easy access to common ways of
@@ -60,7 +61,7 @@ export class Relations extends EventEmitter {
 
         const relation = event.getRelation();
         if (!relation) {
-            console.error("Event must have relation info");
+            logger.error("Event must have relation info");
             return;
         }
 
@@ -68,7 +69,7 @@ export class Relations extends EventEmitter {
         const eventType = event.getType();
 
         if (this.relationType !== relationType || this.eventType !== eventType) {
-            console.error("Event relation info doesn't match this container");
+            logger.error("Event relation info doesn't match this container");
             return;
         }
 
@@ -104,7 +105,7 @@ export class Relations extends EventEmitter {
 
         const relation = event.getRelation();
         if (!relation) {
-            console.error("Event must have relation info");
+            logger.error("Event must have relation info");
             return;
         }
 
@@ -112,7 +113,7 @@ export class Relations extends EventEmitter {
         const eventType = event.getType();
 
         if (this.relationType !== relationType || this.eventType !== eventType) {
-            console.error("Event relation info doesn't match this container");
+            logger.error("Event relation info doesn't match this container");
             return;
         }
 

src/models/room-member.js

@@ -20,7 +20,7 @@ limitations under the License.
  */
 
 import {EventEmitter} from "events";
-import {getHttpUriForMxc, getIdenticonUri} from "../content-repo";
+import {getHttpUriForMxc} from "../content-repo";
 import * as utils from "../utils";
 
 /**
@@ -274,10 +274,6 @@ RoomMember.prototype.getAvatarUrl =
     );
     if (httpUrl) {
         return httpUrl;
-    } else if (allowDefault) {
-        return getIdenticonUri(
-            baseUrl, this.userId, width, height,
-        );
     }
     return null;
 };
@@ -286,9 +282,9 @@ RoomMember.prototype.getAvatarUrl =
  * @return {string} the mxc avatar url
  */
 RoomMember.prototype.getMxcAvatarUrl = function() {
-    if(this.events.member) {
+    if (this.events.member) {
         return this.events.member.getDirectionalContent().avatar_url;
-    } else if(this.user) {
+    } else if (this.user) {
         return this.user.avatarUrl;
     }
     return null;

src/models/room-state.js

@@ -68,9 +68,7 @@ export function RoomState(roomId, oobMemberFlags = undefined) {
     this.members = {
         // userId: RoomMember
     };
-    this.events = {
-        // eventType: { stateKey: MatrixEvent }
-    };
+    this.events = new Map(); // Map<eventType, Map<stateKey, MatrixEvent>>
     this.paginationToken = null;
 
     this._sentinels = {
@@ -211,14 +209,14 @@ RoomState.prototype.getSentinelMember = function(userId) {
  * <code>undefined</code>, else a single event (or null if no match found).
  */
 RoomState.prototype.getStateEvents = function(eventType, stateKey) {
-    if (!this.events[eventType]) {
+    if (!this.events.has(eventType)) {
         // no match
         return stateKey === undefined ? [] : null;
     }
     if (stateKey === undefined) { // return all values
-        return utils.values(this.events[eventType]);
+        return Array.from(this.events.get(eventType).values());
     }
-    const event = this.events[eventType][stateKey];
+    const event = this.events.get(eventType).get(stateKey);
     return event ? event : null;
 };
 
@@ -238,9 +236,8 @@ RoomState.prototype.clone = function() {
     const status = this._oobMemberFlags.status;
     this._oobMemberFlags.status = OOB_STATUS_NOTSTARTED;
 
-    Object.values(this.events).forEach((eventsByStateKey) => {
-        const eventsForType = Object.values(eventsByStateKey);
-        copy.setStateEvents(eventsForType);
+    Array.from(this.events.values()).forEach((eventsByStateKey) => {
+        copy.setStateEvents(Array.from(eventsByStateKey.values()));
     });
 
     // Ugly hack: see above
@@ -276,8 +273,8 @@ RoomState.prototype.clone = function() {
  */
 RoomState.prototype.setUnknownStateEvents = function(events) {
     const unknownStateEvents = events.filter((event) => {
-        return this.events[event.getType()] === undefined ||
-            this.events[event.getType()][event.getStateKey()] === undefined;
+        return !this.events.has(event.getType()) ||
+            !this.events.get(event.getType()).has(event.getStateKey());
     });
 
     this.setStateEvents(unknownStateEvents);
@@ -306,6 +303,7 @@ RoomState.prototype.setStateEvents = function(stateEvents) {
             return;
         }
 
+        const lastStateEvent = self._getStateEventMatching(event);
         self._setStateEvent(event);
         if (event.getType() === "m.room.member") {
             _updateDisplayNameCache(
@@ -313,7 +311,7 @@ RoomState.prototype.setStateEvents = function(stateEvents) {
             );
             _updateThirdPartyTokenCache(self, event);
         }
-        self.emit("RoomState.events", event, self);
+        self.emit("RoomState.events", event, self, lastStateEvent);
     });
 
     // update higher level data structures. This needs to be done AFTER the
@@ -385,10 +383,15 @@ RoomState.prototype._getOrCreateMember = function(userId, event) {
 };
 
 RoomState.prototype._setStateEvent = function(event) {
-    if (this.events[event.getType()] === undefined) {
-        this.events[event.getType()] = {};
+    if (!this.events.has(event.getType())) {
+        this.events.set(event.getType(), new Map());
     }
-    this.events[event.getType()][event.getStateKey()] = event;
+    this.events.get(event.getType()).set(event.getStateKey(), event);
+};
+
+RoomState.prototype._getStateEventMatching = function(event) {
+    if (!this.events.has(event.getType())) return null;
+    return this.events.get(event.getType()).get(event.getStateKey());
 };
 
 RoomState.prototype._updateMember = function(member) {
@@ -670,7 +673,7 @@ RoomState.prototype._maySendEventOfType = function(eventType, userId, state) {
         const userPowerLevel = power_levels.users && power_levels.users[userId];
         if (Number.isFinite(userPowerLevel)) {
             powerLevel = userPowerLevel;
-        } else if(Number.isFinite(power_levels.users_default)) {
+        } else if (Number.isFinite(power_levels.users_default)) {
             powerLevel = power_levels.users_default;
         }
 
@@ -769,8 +772,12 @@ function _updateDisplayNameCache(roomState, userId, displayName) {
  * @param {MatrixEvent} event The matrix event which caused this event to fire.
  * @param {RoomState} state The room state whose RoomState.events dictionary
  * was updated.
+ * @param {MatrixEvent} prevEvent The event being replaced by the new state, if
+ * known. Note that this can differ from `getPrevContent()` on the new state event
+ * as this is the store's view of the last state, not the previous state provided
+ * by the server.
  * @example
- * matrixClient.on("RoomState.events", function(event, state){
+ * matrixClient.on("RoomState.events", function(event, state, prevEvent){
  *   var newStateEvent = event;
  * });
  */

src/models/room.js

@@ -23,7 +23,7 @@ limitations under the License.
 import {EventEmitter} from "events";
 import {EventTimelineSet} from "./event-timeline-set";
 import {EventTimeline} from "./event-timeline";
-import {getHttpUriForMxc, getIdenticonUri} from "../content-repo";
+import {getHttpUriForMxc} from "../content-repo";
 import * as utils from "../utils";
 import {EventStatus, MatrixEvent} from "./event";
 import {RoomMember} from "./room-member";
@@ -122,6 +122,10 @@ export function Room(roomId, client, myUserId, opts) {
     opts = opts || {};
     opts.pendingEventOrdering = opts.pendingEventOrdering || "chronological";
 
+    // In some cases, we add listeners for every displayed Matrix event, so it's
+    // common to have quite a few more than the default limit.
+    this.setMaxListeners(100);
+
     this.reEmitter = new ReEmitter(this);
 
     if (["chronological", "detached"].indexOf(opts.pendingEventOrdering) === -1) {
@@ -209,7 +213,7 @@ utils.inherits(Room, EventEmitter);
 Room.prototype.getVersion = function() {
     const createEvent = this.currentState.getStateEvents("m.room.create", "");
     if (!createEvent) {
-        logger.warn("Room " + this.room_id + " does not have an m.room.create event");
+        logger.warn("Room " + this.roomId + " does not have an m.room.create event");
         return '1';
     }
     const ver = createEvent.getContent()['room_version'];
@@ -354,21 +358,33 @@ Room.prototype.getPendingEvents = function() {
 
 /**
  * Check whether the pending event list contains a given event by ID.
+ * If pending event ordering is not "detached" then this returns false.
  *
  * @param {string} eventId The event ID to check for.
  * @return {boolean}
- * @throws If <code>opts.pendingEventOrdering</code> was not 'detached'
  */
 Room.prototype.hasPendingEvent = function(eventId) {
     if (this._opts.pendingEventOrdering !== "detached") {
-        throw new Error(
-            "Cannot call hasPendingEvent with pendingEventOrdering == " +
-                this._opts.pendingEventOrdering);
+        return false;
     }
 
     return this._pendingEventList.some(event => event.getId() === eventId);
 };
 
+/**
+ * Get a specific event from the pending event list, if configured, null otherwise.
+ *
+ * @param {string} eventId The event ID to check for.
+ * @return {MatrixEvent}
+ */
+Room.prototype.getPendingEvent = function(eventId) {
+    if (this._opts.pendingEventOrdering !== "detached") {
+        return null;
+    }
+
+    return this._pendingEventList.find(event => event.getId() === eventId);
+};
+
 /**
  * Get the live unfiltered timeline for this room.
  *
@@ -675,7 +691,7 @@ Room.prototype.hasUnverifiedDevices = async function() {
     }
     const e2eMembers = await this.getEncryptionTargetMembers();
     for (const member of e2eMembers) {
-        const devices = await this._client.getStoredDevicesForUser(member.userId);
+        const devices = this._client.getStoredDevicesForUser(member.userId);
         if (devices.some((device) => device.isUnverified())) {
             return true;
         }
@@ -814,10 +830,6 @@ Room.prototype.getAvatarUrl = function(baseUrl, width, height, resizeMethod,
         return getHttpUriForMxc(
             baseUrl, mainUrl, width, height, resizeMethod,
         );
-    } else if (allowDefault) {
-        return getIdenticonUri(
-            baseUrl, this.roomId, width, height,
-        );
     }
 
     return null;
@@ -1278,6 +1290,11 @@ Room.prototype._handleRemoteEcho = function(remoteEvent, localEvent) {
     const newEventId = remoteEvent.getId();
     const oldStatus = localEvent.status;
 
+    logger.debug(
+        `Got remote echo for event ${oldEventId} -> ${newEventId} ` +
+        `old status ${oldStatus}`,
+    );
+
     // no longer pending
     delete this._txnToEvent[remoteEvent.getUnsigned().transaction_id];
 
@@ -1347,7 +1364,10 @@ ALLOWED_TRANSITIONS[EventStatus.CANCELLED] =
  * @fires module:client~MatrixClient#event:"Room.localEchoUpdated"
  */
 Room.prototype.updatePendingEvent = function(event, newStatus, newEventId) {
-    logger.log(`setting pendingEvent status to ${newStatus} in ${event.getRoomId()}`);
+    logger.log(
+        `setting pendingEvent status to ${newStatus} in ${event.getRoomId()} ` +
+        `event ID ${event.getId()} -> ${newEventId}`,
+    );
 
     // if the message was sent, we expect an event id
     if (newStatus == EventStatus.SENT && !newEventId) {
@@ -1785,8 +1805,9 @@ Room.prototype.addAccountData = function(events) {
         if (event.getType() === "m.tag") {
             this.addTags(event);
         }
+        const lastEvent = this.accountData[event.getType()];
         this.accountData[event.getType()] = event;
-        this.emit("Room.accountData", event, this);
+        this.emit("Room.accountData", event, this, lastEvent);
     }
 };
 
@@ -1891,14 +1912,14 @@ function calculateRoomName(room, userId, ignoreRoomNameEvent) {
     // let's try to figure out who was here before
     let leftNames = otherNames;
     // if we didn't have heroes, try finding them in the room state
-    if(!leftNames.length) {
+    if (!leftNames.length) {
         leftNames = room.currentState.getMembers().filter((m) => {
             return m.userId !== userId &&
                 m.membership !== "invite" &&
                 m.membership !== "join";
         }).map((m) => m.name);
     }
-    if(leftNames.length) {
+    if (leftNames.length) {
         return `Empty room (was ${memberNamesToRoomName(leftNames)})`;
     } else {
         return "Empty room";
@@ -1983,8 +2004,10 @@ function memberNamesToRoomName(names, count = (names.length + 1)) {
  * @event module:client~MatrixClient#"Room.accountData"
  * @param {event} event The account_data event
  * @param {Room} room The room whose account_data was updated.
+ * @param {MatrixEvent} prevEvent The event being replaced by
+ * the new account data, if known.
  * @example
- * matrixClient.on("Room.accountData", function(event, room){
+ * matrixClient.on("Room.accountData", function(event, room, oldEvent){
  *   if (event.getType() === "m.room.colorscheme") {
  *       applyColorScheme(event.getContents());
  *   }

src/models/user.js

@@ -129,7 +129,11 @@ User.prototype.setPresenceEvent = function(event) {
  */
 User.prototype.setDisplayName = function(name) {
     const oldName = this.displayName;
-    this.displayName = name;
+    if (typeof name === "string") {
+        this.displayName = name;
+    } else {
+        this.displayName = undefined;
+    }
     if (name !== oldName) {
         this._updateModifiedTime();
     }
@@ -142,7 +146,11 @@ User.prototype.setDisplayName = function(name) {
  * @param {string} name The new display name.
  */
 User.prototype.setRawDisplayName = function(name) {
-    this.rawDisplayName = name;
+    if (typeof name === "string") {
+        this.rawDisplayName = name;
+    } else {
+        this.rawDisplayName = undefined;
+    }
 };
 
 

src/pushprocessor.js

@@ -16,6 +16,7 @@ limitations under the License.
 */
 
 import {escapeRegExp, globToRegexp, isNullOrUndefined} from "./utils";
+import {logger} from './logger';
 
 /**
  * @module pushprocessor
@@ -23,9 +24,8 @@ import {escapeRegExp, globToRegexp, isNullOrUndefined} from "./utils";
 
 const RULEKINDS_IN_ORDER = ['override', 'content', 'room', 'sender', 'underride'];
 
-// The default override rules to apply when calculating actions for an event. These
-// defaults apply under no other circumstances to avoid confusing the client with server
-// state. We do this for two reasons:
+// The default override rules to apply to the push rules that arrive from the server.
+// We do this for two reasons:
 //   1. Synapse is unlikely to send us the push rule in an incremental sync - see
 //      https://github.com/matrix-org/synapse/pull/4867#issuecomment-481446072 for
 //      more details.
@@ -85,12 +85,15 @@ export function PushProcessor(client) {
         // $glob: RegExp,
     };
 
-    const matchingRuleFromKindSet = (ev, kindset, device) => {
+    const matchingRuleFromKindSet = (ev, kindset) => {
         for (let ruleKindIndex = 0;
                 ruleKindIndex < RULEKINDS_IN_ORDER.length;
                 ++ruleKindIndex) {
             const kind = RULEKINDS_IN_ORDER[ruleKindIndex];
             const ruleset = kindset[kind];
+            if (!ruleset) {
+                continue;
+            }
 
             for (let ruleIndex = 0; ruleIndex < ruleset.length; ++ruleIndex) {
                 const rule = ruleset[ruleIndex];
@@ -98,7 +101,7 @@ export function PushProcessor(client) {
                     continue;
                 }
 
-                const rawrule = templateRuleToRaw(kind, rule, device);
+                const rawrule = templateRuleToRaw(kind, rule);
                 if (!rawrule) {
                     continue;
                 }
@@ -112,7 +115,7 @@ export function PushProcessor(client) {
         return null;
     };
 
-    const templateRuleToRaw = function(kind, tprule, device) {
+    const templateRuleToRaw = function(kind, tprule) {
         const rawrule = {
             'rule_id': tprule.rule_id,
             'actions': tprule.actions,
@@ -154,19 +157,12 @@ export function PushProcessor(client) {
                 });
                 break;
         }
-        if (device) {
-            rawrule.conditions.push({
-                'kind': 'device',
-                'profile_tag': device,
-            });
-        }
         return rawrule;
     };
 
     const eventFulfillsCondition = function(cond, ev) {
         const condition_functions = {
             "event_match": eventFulfillsEventMatchCondition,
-            "device": eventFulfillsDeviceCondition,
             "contains_display_name": eventFulfillsDisplayNameCondition,
             "room_member_count": eventFulfillsRoomMemberCountCondition,
             "sender_notification_permission": eventFulfillsSenderNotifPermCondition,
@@ -258,10 +254,6 @@ export function PushProcessor(client) {
         return content.body.search(pat) > -1;
     };
 
-    const eventFulfillsDeviceCondition = function(cond, ev) {
-        return false; // XXX: Allow a profile tag to be set for the web client instance
-    };
-
     const eventFulfillsEventMatchCondition = function(cond, ev) {
         if (!cond.key) {
             return false;
@@ -326,23 +318,13 @@ export function PushProcessor(client) {
     };
 
     const matchingRuleForEventWithRulesets = function(ev, rulesets) {
-        if (!rulesets || !rulesets.device) {
+        if (!rulesets) {
             return null;
         }
-        if (ev.getSender() == client.credentials.userId) {
+        if (ev.getSender() === client.credentials.userId) {
             return null;
         }
 
-        const allDevNames = Object.keys(rulesets.device);
-        for (let i = 0; i < allDevNames.length; ++i) {
-            const devname = allDevNames[i];
-            const devrules = rulesets.device[devname];
-
-            const matchingRule = matchingRuleFromKindSet(devrules, devname);
-            if (matchingRule) {
-                return matchingRule;
-            }
-        }
         return matchingRuleFromKindSet(ev, rulesets.global);
     };
 
@@ -364,33 +346,6 @@ export function PushProcessor(client) {
         return actionObj;
     };
 
-    const applyRuleDefaults = function(clientRuleset) {
-        // Deep clone the object before we mutate it
-        const ruleset = JSON.parse(JSON.stringify(clientRuleset));
-
-        if (!clientRuleset['global']) {
-            clientRuleset['global'] = {};
-        }
-        if (!clientRuleset['global']['override']) {
-            clientRuleset['global']['override'] = [];
-        }
-
-        // Apply default overrides
-        const globalOverrides = clientRuleset['global']['override'];
-        for (const override of DEFAULT_OVERRIDE_RULES) {
-            const existingRule = globalOverrides
-                .find((r) => r.rule_id === override.rule_id);
-
-            if (!existingRule) {
-                const ruleId = override.rule_id;
-                console.warn(`Adding default global override for ${ruleId}`);
-                globalOverrides.push(override);
-            }
-        }
-
-        return ruleset;
-    };
-
     this.ruleMatchesEvent = function(rule, ev) {
         let ret = true;
         for (let i = 0; i < rule.conditions.length; ++i) {
@@ -410,8 +365,7 @@ export function PushProcessor(client) {
      * @return {PushAction}
      */
     this.actionsForEvent = function(ev) {
-        const rules = applyRuleDefaults(client.pushRules);
-        return pushActionsForEventAndRulesets(ev, rules);
+        return pushActionsForEventAndRulesets(ev, client.pushRules);
     };
 
     /**
@@ -421,7 +375,7 @@ export function PushProcessor(client) {
      * @return {object} The push rule, or null if no such rule was found
      */
     this.getPushRuleById = function(ruleId) {
-        for (const scope of ['device', 'global']) {
+        for (const scope of ['global']) {
             if (client.pushRules[scope] === undefined) continue;
 
             for (const kind of RULEKINDS_IN_ORDER) {
@@ -476,18 +430,25 @@ PushProcessor.rewriteDefaultRules = function(incomingRules) {
     if (!newRules.global) newRules.global = {};
     if (!newRules.global.override) newRules.global.override = [];
 
-    // Fix default override rules
-    newRules.global.override = newRules.global.override.map(r => {
-        const defaultRule = DEFAULT_OVERRIDE_RULES.find(d => d.rule_id === r.rule_id);
-        if (!defaultRule) return r;
+    // Merge the client-level defaults with the ones from the server
+    const globalOverrides = newRules.global.override;
+    for (const override of DEFAULT_OVERRIDE_RULES) {
+        const existingRule = globalOverrides
+            .find((r) => r.rule_id === override.rule_id);
 
-        // Copy over the actions, default, and conditions. Don't touch the user's
-        // preference.
-        r.default = defaultRule.default;
-        r.conditions = defaultRule.conditions;
-        r.actions = defaultRule.actions;
-        return r;
-    });
+        if (existingRule) {
+            // Copy over the actions, default, and conditions. Don't touch the user's
+            // preference.
+            existingRule.default = override.default;
+            existingRule.conditions = override.conditions;
+            existingRule.actions = override.actions;
+        } else {
+            // Add the rule
+            const ruleId = override.rule_id;
+            logger.warn(`Adding default global override for ${ruleId}`);
+            globalOverrides.push(override);
+        }
+    }
 
     return newRules;
 };

src/realtime-callbacks.js

@@ -185,8 +185,8 @@ function _runCallbacks() {
  */
 function binarySearch(array, func) {
     // min is inclusive, max exclusive.
-    let min = 0,
-        max = array.length;
+    let min = 0;
+    let max = array.length;
 
     while (min < max) {
         const mid = (min + max) >> 1;

src/store/indexeddb-local-backend.js

@@ -226,7 +226,7 @@ LocalIndexedDBStoreBackend.prototype = {
                 account_data: {
                     events: accountData,
                 },
-            });
+            }, true);
         });
     },
 
@@ -416,7 +416,7 @@ LocalIndexedDBStoreBackend.prototype = {
     },
 
     syncToDatabase: function(userTuples) {
-        const syncData = this._syncAccumulator.getJSON();
+        const syncData = this._syncAccumulator.getJSON(true);
 
         return Promise.all([
             this._persistUserPresenceEvents(userTuples),

src/store/memory.js

@@ -25,6 +25,15 @@ limitations under the License.
 import {User} from "../models/user";
 import * as utils from "../utils";
 
+function isValidFilterId(filterId) {
+    const isValidStr = typeof filterId === "string" &&
+        !!filterId &&
+        filterId !== "undefined" && // exclude these as we've serialized undefined in localStorage before
+        filterId !== "null";
+
+    return isValidStr || typeof filterId === "number";
+}
+
 /**
  * Construct a new in-memory data store for the Matrix Client.
  * @constructor
@@ -273,8 +282,17 @@ MemoryStore.prototype = {
         if (!this.localStorage) {
             return null;
         }
+        const key = "mxjssdk_memory_filter_" + filterName;
+        // XXX Storage.getItem doesn't throw ...
+        // or are we using something different
+        // than window.localStorage in some cases
+        // that does throw?
+        // that would be very naughty
         try {
-            return this.localStorage.getItem("mxjssdk_memory_filter_" + filterName);
+            const value = this.localStorage.getItem(key);
+            if (isValidFilterId(value)) {
+                return value;
+            }
         } catch (e) {}
         return null;
     },
@@ -288,8 +306,13 @@ MemoryStore.prototype = {
         if (!this.localStorage) {
             return;
         }
+        const key = "mxjssdk_memory_filter_" + filterName;
         try {
-            this.localStorage.setItem("mxjssdk_memory_filter_" + filterName, filterId);
+            if (isValidFilterId(filterId)) {
+                this.localStorage.setItem(key, filterId);
+            } else {
+                this.localStorage.removeItem(key);
+            }
         } catch (e) {}
     },
 

src/sync-accumulator.js

@@ -87,8 +87,8 @@ export class SyncAccumulator {
         };
     }
 
-    accumulate(syncResponse) {
-        this._accumulateRooms(syncResponse);
+    accumulate(syncResponse, fromDatabase) {
+        this._accumulateRooms(syncResponse, fromDatabase);
         this._accumulateGroups(syncResponse);
         this._accumulateAccountData(syncResponse);
         this.nextBatch = syncResponse.next_batch;
@@ -107,35 +107,36 @@ export class SyncAccumulator {
     /**
      * Accumulate incremental /sync room data.
      * @param {Object} syncResponse the complete /sync JSON
+     * @param {boolean} fromDatabase True if the sync response is one saved to the database
      */
-    _accumulateRooms(syncResponse) {
+    _accumulateRooms(syncResponse, fromDatabase) {
         if (!syncResponse.rooms) {
             return;
         }
         if (syncResponse.rooms.invite) {
             Object.keys(syncResponse.rooms.invite).forEach((roomId) => {
                 this._accumulateRoom(
-                    roomId, "invite", syncResponse.rooms.invite[roomId],
+                    roomId, "invite", syncResponse.rooms.invite[roomId], fromDatabase,
                 );
             });
         }
         if (syncResponse.rooms.join) {
             Object.keys(syncResponse.rooms.join).forEach((roomId) => {
                 this._accumulateRoom(
-                    roomId, "join", syncResponse.rooms.join[roomId],
+                    roomId, "join", syncResponse.rooms.join[roomId], fromDatabase,
                 );
             });
         }
         if (syncResponse.rooms.leave) {
             Object.keys(syncResponse.rooms.leave).forEach((roomId) => {
                 this._accumulateRoom(
-                    roomId, "leave", syncResponse.rooms.leave[roomId],
+                    roomId, "leave", syncResponse.rooms.leave[roomId], fromDatabase,
                 );
             });
         }
     }
 
-    _accumulateRoom(roomId, category, data) {
+    _accumulateRoom(roomId, category, data, fromDatabase) {
         // Valid /sync state transitions
         //       +--------+ <======+            1: Accept an invite
         //   +== | INVITE |        | (5)        2: Leave a room
@@ -159,7 +160,7 @@ export class SyncAccumulator {
                     delete this.inviteRooms[roomId];
                 }
                 // (3)
-                this._accumulateJoinState(roomId, data);
+                this._accumulateJoinState(roomId, data, fromDatabase);
                 break;
             case "leave":
                 if (this.inviteRooms[roomId]) { // (4)
@@ -203,7 +204,7 @@ export class SyncAccumulator {
     }
 
     // Accumulate timeline and state events in a room.
-    _accumulateJoinState(roomId, data) {
+    _accumulateJoinState(roomId, data, fromDatabase) {
         // We expect this function to be called a lot (every /sync) so we want
         // this to be fast. /sync stores events in an array but we often want
         // to clobber based on type/state_key. Rather than convert arrays to
@@ -337,8 +338,20 @@ export class SyncAccumulator {
                 setState(currentData._currentState, e);
                 // append the event to the timeline. The back-pagination token
                 // corresponds to the first event in the timeline
+                let transformedEvent;
+                if (!fromDatabase) {
+                    transformedEvent = Object.assign({}, e);
+                    if (transformedEvent.unsigned !== undefined) {
+                        transformedEvent.unsigned = Object.assign({}, transformedEvent.unsigned);
+                    }
+                    const age = e.unsigned ? e.unsigned.age : e.age;
+                    if (age !== undefined) transformedEvent._localTs = Date.now() - age;
+                } else {
+                    transformedEvent = e;
+                }
+
                 currentData._timeline.push({
-                    event: e,
+                    event: transformedEvent,
                     token: index === 0 ? data.timeline.prev_batch : null,
                 });
             });
@@ -405,6 +418,7 @@ export class SyncAccumulator {
      * represents all room data that should be stored. This should be paired
      * with the sync token which represents the most recent /sync response
      * provided to accumulate().
+     * @param {boolean} forDatabase True to generate a sync to be saved to storage
      * @return {Object} An object with a "nextBatch", "roomsData" and "accountData"
      * keys.
      * The "nextBatch" key is a string which represents at what point in the
@@ -414,7 +428,7 @@ export class SyncAccumulator {
      * /sync response from the 'rooms' key onwards. The "accountData" key is
      * a list of raw events which represent global account data.
      */
-    getJSON() {
+    getJSON(forDatabase) {
         const data = {
             join: {},
             invite: {},
@@ -486,7 +500,28 @@ export class SyncAccumulator {
                     }
                     roomJson.timeline.prev_batch = msgData.token;
                 }
-                roomJson.timeline.events.push(msgData.event);
+
+                let transformedEvent;
+                if (!forDatabase && msgData.event._localTs) {
+                    // This means we have to copy each event so we can fix it up to
+                    // set a correct 'age' parameter whilst keeping the local timestamp
+                    // on our stored event. If this turns out to be a bottleneck, it could
+                    // be optimised either by doing this in the main process after the data
+                    // has been structured-cloned to go between the worker & main process,
+                    // or special-casing data from saved syncs to read the local timstamp
+                    // directly rather than turning it into age to then immediately be
+                    // transformed back again into a local timestamp.
+                    transformedEvent = Object.assign({}, msgData.event);
+                    if (transformedEvent.unsigned !== undefined) {
+                        transformedEvent.unsigned = Object.assign({}, transformedEvent.unsigned);
+                    }
+                    delete transformedEvent._localTs;
+                    transformedEvent.unsigned = transformedEvent.unsigned || {};
+                    transformedEvent.unsigned.age = Date.now() - msgData.event._localTs;
+                } else {
+                    transformedEvent = msgData.event;
+                }
+                roomJson.timeline.events.push(transformedEvent);
             });
 
             // Add state data: roll back current state to the start of timeline,

src/sync.js

@@ -475,9 +475,9 @@ SyncApi.prototype.sync = function() {
 
     this._running = true;
 
-    if (global.document) {
+    if (global.window) {
         this._onOnlineBound = this._onOnline.bind(this);
-        global.document.addEventListener("online", this._onOnlineBound, false);
+        global.window.addEventListener("online", this._onOnlineBound, false);
     }
 
     let savedSyncPromise = Promise.resolve();
@@ -511,6 +511,12 @@ SyncApi.prototype.sync = function() {
         checkLazyLoadStatus(); // advance to the next stage
     }
 
+    function buildDefaultFilter() {
+        const filter = new Filter(client.credentials.userId);
+        filter.setTimelineLimit(self.opts.initialSyncLimit);
+        return filter;
+    }
+
     const checkLazyLoadStatus = async () => {
         debuglog("Checking lazy load status...");
         if (this.opts.lazyLoadMembers && client.isGuest()) {
@@ -520,19 +526,11 @@ SyncApi.prototype.sync = function() {
             debuglog("Checking server lazy load support...");
             const supported = await client.doesServerSupportLazyLoading();
             if (supported) {
-                try {
-                    debuglog("Creating and storing lazy load sync filter...");
-                    this.opts.filter = await client.createFilter(
-                        Filter.LAZY_LOADING_SYNC_FILTER,
-                    );
-                    debuglog("Created and stored lazy load sync filter");
-                } catch (err) {
-                    logger.error(
-                        "Creating and storing lazy load sync filter failed",
-                        err,
-                    );
-                    throw err;
+                debuglog("Enabling lazy load on sync filter...");
+                if (!this.opts.filter) {
+                    this.opts.filter = buildDefaultFilter();
                 }
+                this.opts.filter.setLazyLoadMembers(true);
             } else {
                 debuglog("LL: lazy loading requested but not supported " +
                     "by server, so disabling");
@@ -575,8 +573,7 @@ SyncApi.prototype.sync = function() {
         if (self.opts.filter) {
             filter = self.opts.filter;
         } else {
-            filter = new Filter(client.credentials.userId);
-            filter.setTimelineLimit(self.opts.initialSyncLimit);
+            filter = buildDefaultFilter();
         }
 
         let filterId;
@@ -646,8 +643,8 @@ SyncApi.prototype.sync = function() {
  */
 SyncApi.prototype.stop = function() {
     debuglog("SyncApi.stop");
-    if (global.document) {
-        global.document.removeEventListener("online", this._onOnlineBound, false);
+    if (global.window) {
+        global.window.removeEventListener("online", this._onOnlineBound, false);
         this._onOnlineBound = undefined;
     }
     this._running = false;
@@ -704,7 +701,7 @@ SyncApi.prototype._syncFromCache = async function(savedSync) {
 
     try {
         await this._processSyncResponse(syncEventData, data);
-    } catch(e) {
+    } catch (e) {
         logger.error("Error processing cached sync", e.stack || e);
     }
 
@@ -777,7 +774,7 @@ SyncApi.prototype._sync = async function(syncOptions) {
 
     try {
         await this._processSyncResponse(syncEventData, data);
-    } catch(e) {
+    } catch (e) {
         // log the exception with stack if we have it, else fall back
         // to the plain description
         logger.error("Caught /sync error", e.stack || e);
@@ -897,7 +894,7 @@ SyncApi.prototype._onSyncError = function(err, syncOptions) {
     logger.error("/sync error %s", err);
     logger.error(err);
 
-    if(this._shouldAbortSync(err)) {
+    if (this._shouldAbortSync(err)) {
         return;
     }
 
@@ -1026,18 +1023,23 @@ SyncApi.prototype._processSyncResponse = async function(
     // handle non-room account_data
     if (data.account_data && utils.isArray(data.account_data.events)) {
         const events = data.account_data.events.map(client.getEventMapper());
+        const prevEventsMap = events.reduce((m, c) => {
+            m[c.getId()] = client.store.getAccountData(c.getType());
+            return m;
+        }, {});
         client.store.storeAccountDataEvents(events);
         events.forEach(
             function(accountDataEvent) {
                 // Honour push rules that come down the sync stream but also
                 // honour push rules that were previously cached. Base rules
-                // will be updated when we recieve push rules via getPushRules
+                // will be updated when we receive push rules via getPushRules
                 // (see SyncApi.prototype.sync) before syncing over the network.
                 if (accountDataEvent.getType() === 'm.push_rules') {
                     const rules = accountDataEvent.getContent();
                     client.pushRules = PushProcessor.rewriteDefaultRules(rules);
                 }
-                client.emit("accountData", accountDataEvent);
+                const prevEvent = prevEventsMap[accountDataEvent.getId()];
+                client.emit("accountData", accountDataEvent, prevEvent);
                 return accountDataEvent;
             },
         );
@@ -1359,6 +1361,16 @@ SyncApi.prototype._processSyncResponse = async function(
         const currentCount = data.device_one_time_keys_count.signed_curve25519 || 0;
         this.opts.crypto.updateOneTimeKeyCount(currentCount);
     }
+    if (this.opts.crypto && data["org.matrix.msc2732.device_unused_fallback_key_types"]) {
+        // The presence of device_unused_fallback_key_types indicates that the
+        // server supports fallback keys. If there's no unused
+        // signed_curve25519 fallback key we need a new one.
+        const unusedFallbackKeys = data["org.matrix.msc2732.device_unused_fallback_key_types"];
+        this.opts.crypto.setNeedsNewFallback(
+            unusedFallbackKeys instanceof Array &&
+            !unusedFallbackKeys.includes("signed_curve25519"),
+        );
+    }
 };
 
 /**

src/timeline-window.js

@@ -396,7 +396,8 @@ TimelineWindow.prototype.getEvents = function() {
         // (Note that both this._start.index and this._end.index are relative
         // to their respective timelines' BaseIndex).
         //
-        let startIndex = 0, endIndex = events.length;
+        let startIndex = 0;
+        let endIndex = events.length;
         if (timeline === this._start.timeline) {
             startIndex = this._start.index + timeline.getBaseIndex();
         }

src/utils.ts

@@ -49,7 +49,7 @@ export function encodeParams(params: Record<string, string>): string {
  * @return {string} The result of replacing all template variables e.g. '/foo/baz'.
  */
 export function encodeUri(pathTemplate: string,
-                          variables: Record<string, string>): string {
+    variables: Record<string, string>): string {
     for (const key in variables) {
         if (!variables.hasOwnProperty(key)) {
             continue;
@@ -85,7 +85,7 @@ export function map<T, S>(array: T[], fn: (t: T) => S): S[] {
  * @return {Array} A new array with the results of the function.
  */
 export function filter<T>(array: T[],
-                          fn: (t: T, i?: number, a?: T[]) => boolean): T[] {
+    fn: (t: T, i?: number, a?: T[]) => boolean): T[] {
     const results: T[] = [];
     for (let i = 0; i < array.length; i++) {
         if (fn(array[i], i, array)) {
@@ -151,10 +151,10 @@ export function forEach<T>(array: T[], fn: (t: T, i: number) => void) {
  * the given function.
  */
 export function findElement<T>(
-        array: T[],
-        fn: (t: T, i?: number, a?: T[]) => boolean,
-        reverse?: boolean
-    ) {
+    array: T[],
+    fn: (t: T, i?: number, a?: T[]) => boolean,
+    reverse?: boolean,
+) {
     let i;
     if (reverse) {
         for (i = array.length - 1; i >= 0; i--) {
@@ -182,10 +182,10 @@ export function findElement<T>(
  * @return {boolean} True if an element was removed.
  */
 export function removeElement<T>(
-        array: T[],
-        fn: (t: T, i?: number, a?: T[]) => boolean,
-        reverse?: boolean
-    ) {
+    array: T[],
+    fn: (t: T, i?: number, a?: T[]) => boolean,
+    reverse?: boolean,
+) {
     let i;
     let removed;
     if (reverse) {
@@ -265,7 +265,7 @@ export function checkObjectHasNoAdditionalKeys(obj: object, allowedKeys: string[
  * @param {Object} obj The object to deep copy.
  * @return {Object} A copy of the object without any references to the original.
  */
-export function deepCopy(obj: object): object {
+export function deepCopy<T>(obj: T): T {
     return JSON.parse(JSON.stringify(obj));
 }
 
@@ -293,7 +293,7 @@ export function deepCompare(x: any, y: any): boolean {
 
     // special-case NaN (since NaN !== NaN)
     if (typeof x === 'number' && isNaN(x) && isNaN(y)) {
-         return true;
+        return true;
     }
 
     // special-case null (since typeof null == 'object', but null.constructor
@@ -368,10 +368,11 @@ export function deepCompare(x: any, y: any): boolean {
  *
  * @return {Object} target
  */
-export function extend() {
-    const target = arguments[0] || {};
-    for (let i = 1; i < arguments.length; i++) {
-        const source = arguments[i];
+export function extend(...restParams) {
+    const target = restParams[0] || {};
+    for (let i = 1; i < restParams.length; i++) {
+        const source = restParams[i];
+        if (!source) continue;
         for (const propName in source) { // eslint-disable-line guard-for-in
             target[propName] = source[propName];
         }
@@ -388,36 +389,36 @@ export function runPolyfills() {
     // SOURCE:
     // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/filter
     if (!Array.prototype.filter) {
-      Array.prototype.filter = function(fun: Function/*, thisArg*/) {
-        if (this === void 0 || this === null) {
-          throw new TypeError();
-        }
-
-        const t = Object(this);
-        const len = t.length >>> 0;
-        if (typeof fun !== 'function') {
-          throw new TypeError();
-        }
-
-        const res = [];
-        const thisArg = arguments.length >= 2 ? arguments[1] : void 0;
-        for (let i = 0; i < len; i++) {
-          if (i in t) {
-            const val = t[i];
-
-            // NOTE: Technically this should Object.defineProperty at
-            //       the next index, as push can be affected by
-            //       properties on Object.prototype and Array.prototype.
-            //       But that method's new, and collisions should be
-            //       rare, so use the more-compatible alternative.
-            if (fun.call(thisArg, val, i, t)) {
-              res.push(val);
+        // eslint-disable-next-line no-extend-native
+        Array.prototype.filter = function(fun: Function/*, thisArg*/, ...restProps) {
+            if (this === void 0 || this === null) {
+                throw new TypeError();
             }
-          }
-        }
 
-        return res;
-      };
+            const t = Object(this);
+            const len = t.length >>> 0;
+            if (typeof fun !== 'function') {
+                throw new TypeError();
+            }
+
+            const res = [];
+            const thisArg = restProps ? restProps[0] : void 0;
+            for (let i = 0; i < len; i++) {
+                if (i in t) {
+                    const val = t[i];
+
+                    // NOTE: Technically this should Object.defineProperty at
+                    //       the next index, as push can be affected by
+                    //       properties on Object.prototype and Array.prototype.
+                    //       But that method's new, and collisions should be
+                    //       rare, so use the more-compatible alternative.
+                    if (fun.call(thisArg, val, i, t)) {
+                        res.push(val);
+                    }
+                }
+            }
+            return res;
+        };
     }
 
     //                Array.prototype.map
@@ -427,151 +428,156 @@ export function runPolyfills() {
     // Production steps of ECMA-262, Edition 5, 15.4.4.19
     // Reference: http://es5.github.io/#x15.4.4.19
     if (!Array.prototype.map) {
-      Array.prototype.map = function(callback, thisArg) {
-        let T, k;
+        // eslint-disable-next-line no-extend-native
+        Array.prototype.map = function(callback, thisArg) {
+            let T;
+            let k;
 
-        if (this === null || this === undefined) {
-          throw new TypeError(' this is null or not defined');
-        }
+            if (this === null || this === undefined) {
+                throw new TypeError(' this is null or not defined');
+            }
 
-        // 1. Let O be the result of calling ToObject passing the |this|
-        //    value as the argument.
-        const O = Object(this);
+            // 1. Let O be the result of calling ToObject passing the |this|
+            //    value as the argument.
+            const O = Object(this);
 
-        // 2. Let lenValue be the result of calling the Get internal
-        //    method of O with the argument "length".
-        // 3. Let len be ToUint32(lenValue).
-        const len = O.length >>> 0;
+            // 2. Let lenValue be the result of calling the Get internal
+            //    method of O with the argument "length".
+            // 3. Let len be ToUint32(lenValue).
+            const len = O.length >>> 0;
 
-        // 4. If IsCallable(callback) is false, throw a TypeError exception.
-        // See: http://es5.github.com/#x9.11
-        if (typeof callback !== 'function') {
-          throw new TypeError(callback + ' is not a function');
-        }
+            // 4. If IsCallable(callback) is false, throw a TypeError exception.
+            // See: http://es5.github.com/#x9.11
+            if (typeof callback !== 'function') {
+                throw new TypeError(callback + ' is not a function');
+            }
 
-        // 5. If thisArg was supplied, let T be thisArg; else let T be undefined.
-        if (arguments.length > 1) {
-          T = thisArg;
-        }
+            // 5. If thisArg was supplied, let T be thisArg; else let T be undefined.
+            if (arguments.length > 1) {
+                T = thisArg;
+            }
 
-        // 6. Let A be a new array created as if by the expression new Array(len)
-        //    where Array is the standard built-in constructor with that name and
-        //    len is the value of len.
-        const A = new Array(len);
+            // 6. Let A be a new array created as if by the expression new Array(len)
+            //    where Array is the standard built-in constructor with that name and
+            //    len is the value of len.
+            const A = new Array(len);
 
-        // 7. Let k be 0
-        k = 0;
+            // 7. Let k be 0
+            k = 0;
 
-        // 8. Repeat, while k < len
-        while (k < len) {
-          let kValue, mappedValue;
+            // 8. Repeat, while k < len
+            while (k < len) {
+                let kValue;
+                let mappedValue;
 
-          // a. Let Pk be ToString(k).
-          //   This is implicit for LHS operands of the in operator
-          // b. Let kPresent be the result of calling the HasProperty internal
-          //    method of O with argument Pk.
-          //   This step can be combined with c
-          // c. If kPresent is true, then
-          if (k in O) {
-            // i. Let kValue be the result of calling the Get internal
-            //    method of O with argument Pk.
-            kValue = O[k];
+                // a. Let Pk be ToString(k).
+                //   This is implicit for LHS operands of the in operator
+                // b. Let kPresent be the result of calling the HasProperty internal
+                //    method of O with argument Pk.
+                //   This step can be combined with c
+                // c. If kPresent is true, then
+                if (k in O) {
+                    // i. Let kValue be the result of calling the Get internal
+                    //    method of O with argument Pk.
+                    kValue = O[k];
 
-            // ii. Let mappedValue be the result of calling the Call internal
-            //     method of callback with T as the this value and argument
-            //     list containing kValue, k, and O.
-            mappedValue = callback.call(T, kValue, k, O);
+                    // ii. Let mappedValue be the result of calling the Call internal
+                    //     method of callback with T as the this value and argument
+                    //     list containing kValue, k, and O.
+                    mappedValue = callback.call(T, kValue, k, O);
 
-            // iii. Call the DefineOwnProperty internal method of A with arguments
-            // Pk, Property Descriptor
-            // { Value: mappedValue,
-            //   Writable: true,
-            //   Enumerable: true,
-            //   Configurable: true },
-            // and false.
+                    // iii. Call the DefineOwnProperty internal method of A with arguments
+                    // Pk, Property Descriptor
+                    // { Value: mappedValue,
+                    //   Writable: true,
+                    //   Enumerable: true,
+                    //   Configurable: true },
+                    // and false.
 
-            // In browsers that support Object.defineProperty, use the following:
-            // Object.defineProperty(A, k, {
-            //   value: mappedValue,
-            //   writable: true,
-            //   enumerable: true,
-            //   configurable: true
-            // });
+                    // In browsers that support Object.defineProperty, use the following:
+                    // Object.defineProperty(A, k, {
+                    //   value: mappedValue,
+                    //   writable: true,
+                    //   enumerable: true,
+                    //   configurable: true
+                    // });
 
-            // For best browser support, use the following:
-            A[k] = mappedValue;
-          }
-          // d. Increase k by 1.
-          k++;
-        }
+                    // For best browser support, use the following:
+                    A[k] = mappedValue;
+                }
+                // d. Increase k by 1.
+                k++;
+            }
 
-        // 9. return A
-        return A;
-      };
+            // 9. return A
+            return A;
+        };
     }
 
     //                Array.prototype.forEach
     // ========================================================
     // SOURCE:
-  // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/forEach
+    // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/forEach
     // Production steps of ECMA-262, Edition 5, 15.4.4.18
     // Reference: http://es5.github.io/#x15.4.4.18
     if (!Array.prototype.forEach) {
-      Array.prototype.forEach = function(callback, thisArg) {
-        let T, k;
+        // eslint-disable-next-line no-extend-native
+        Array.prototype.forEach = function(callback, thisArg) {
+            let T;
+            let k;
 
-        if (this === null || this === undefined) {
-          throw new TypeError(' this is null or not defined');
-        }
+            if (this === null || this === undefined) {
+                throw new TypeError(' this is null or not defined');
+            }
 
-        // 1. Let O be the result of calling ToObject passing the |this| value as the
-        // argument.
-        const O = Object(this);
+            // 1. Let O be the result of calling ToObject passing the |this| value as the
+            // argument.
+            const O = Object(this);
 
-        // 2. Let lenValue be the result of calling the Get internal method of O with the
-        // argument "length".
-        // 3. Let len be ToUint32(lenValue).
-        const len = O.length >>> 0;
+            // 2. Let lenValue be the result of calling the Get internal method of O with the
+            // argument "length".
+            // 3. Let len be ToUint32(lenValue).
+            const len = O.length >>> 0;
 
-        // 4. If IsCallable(callback) is false, throw a TypeError exception.
-        // See: http://es5.github.com/#x9.11
-        if (typeof callback !== "function") {
-          throw new TypeError(callback + ' is not a function');
-        }
+            // 4. If IsCallable(callback) is false, throw a TypeError exception.
+            // See: http://es5.github.com/#x9.11
+            if (typeof callback !== "function") {
+                throw new TypeError(callback + ' is not a function');
+            }
 
-        // 5. If thisArg was supplied, let T be thisArg; else let T be undefined.
-        if (arguments.length > 1) {
-          T = thisArg;
-        }
+            // 5. If thisArg was supplied, let T be thisArg; else let T be undefined.
+            if (arguments.length > 1) {
+                T = thisArg;
+            }
 
-        // 6. Let k be 0
-        k = 0;
+            // 6. Let k be 0
+            k = 0;
 
-        // 7. Repeat, while k < len
-        while (k < len) {
-          let kValue;
+            // 7. Repeat, while k < len
+            while (k < len) {
+                let kValue;
 
-          // a. Let Pk be ToString(k).
-          //   This is implicit for LHS operands of the in operator
-          // b. Let kPresent be the result of calling the HasProperty internal
-          //    method of O with
-          //    argument Pk.
-          //   This step can be combined with c
-          // c. If kPresent is true, then
-          if (k in O) {
-            // i. Let kValue be the result of calling the Get internal method of O with
-            // argument Pk
-            kValue = O[k];
+                // a. Let Pk be ToString(k).
+                //   This is implicit for LHS operands of the in operator
+                // b. Let kPresent be the result of calling the HasProperty internal
+                //    method of O with
+                //    argument Pk.
+                //   This step can be combined with c
+                // c. If kPresent is true, then
+                if (k in O) {
+                    // i. Let kValue be the result of calling the Get internal method of O with
+                    // argument Pk
+                    kValue = O[k];
 
-            // ii. Call the Call internal method of callback with T as the this value and
-            // argument list containing kValue, k, and O.
-            callback.call(T, kValue, k, O);
-          }
-          // d. Increase k by 1.
-          k++;
-        }
+                    // ii. Call the Call internal method of callback with T as the this value and
+                    // argument list containing kValue, k, and O.
+                    callback.call(T, kValue, k, O);
+                }
+                // d. Increase k by 1.
+                k++;
+            }
         // 8. return undefined
-      };
+        };
     }
 }
 
@@ -655,7 +661,10 @@ export function isNumber(value: any): boolean {
  * @return {string} a string with the hidden characters removed
  */
 export function removeHiddenChars(str: string): string {
-    return unhomoglyph(str.normalize('NFD').replace(removeHiddenCharsRegex, ''));
+    if (typeof str === "string") {
+        return unhomoglyph(str.normalize('NFD').replace(removeHiddenCharsRegex, ''));
+    }
+    return "";
 }
 
 // Regex matching bunch of unicode control characters and otherwise misleading/invisible characters.
@@ -665,6 +674,7 @@ export function removeHiddenChars(str: string): string {
 // LTR/RTL and other directional formatting marks U+202A - U+202F
 // Combining characters U+0300 - U+036F
 // Zero width no-break space (BOM) U+FEFF
+// eslint-disable-next-line no-misleading-character-class
 const removeHiddenCharsRegex = /[\u2000-\u200F\u202A-\u202F\u0300-\u036f\uFEFF\s]/g;
 
 export function escapeRegExp(string: string): string {
@@ -723,9 +733,9 @@ export function defer() {
 }
 
 export async function promiseMapSeries<T>(
-        promises: Promise<T>[],
-        fn: (t: T) => void
-    ): Promise<void> {
+    promises: Promise<T>[],
+    fn: (t: T) => void,
+): Promise<void> {
     for (const o of await promises) {
         await fn(await o);
     }

src/webrtc/callEventHandler.ts

@@ -0,0 +1,274 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import MatrixEvent from '../models/event';
+import {logger} from '../logger';
+import { createNewMatrixCall, MatrixCall, CallErrorCode, CallState, CallDirection } from './call';
+import { EventType } from '../@types/event';
+import { MatrixClient } from '../client';
+
+// Don't ring unless we'd be ringing for at least 3 seconds: the user needs some
+// time to press the 'accept' button
+const RING_GRACE_PERIOD = 3000;
+
+export class CallEventHandler {
+    client: MatrixClient;
+    calls: Map<string, MatrixCall>;
+    callEventBuffer: MatrixEvent[];
+    candidateEventsByCall: Map<string, Array<MatrixEvent>>;
+
+    constructor(client: MatrixClient) {
+        this.client = client;
+        this.calls = new Map<string, MatrixCall>();
+        // The sync code always emits one event at a time, so it will patiently
+        // wait for us to finish processing a call invite before delivering the
+        // next event, even if that next event is a hangup. We therefore accumulate
+        // all our call events and then process them on the 'sync' event, ie.
+        // each time a sync has completed. This way, we can avoid emitting incoming
+        // call events if we get both the invite and answer/hangup in the same sync.
+        // This happens quite often, eg. replaying sync from storage, catchup sync
+        // after loading and after we've been offline for a bit.
+        this.callEventBuffer = [];
+        this.candidateEventsByCall = new Map<string, Array<MatrixEvent>>();
+        this.client.on("sync", this.evaluateEventBuffer);
+        this.client.on("event", this.onEvent);
+    }
+
+    public stop() {
+        this.client.removeListener("sync", this.evaluateEventBuffer);
+        this.client.removeListener("event", this.onEvent);
+    }
+
+    private evaluateEventBuffer = () => {
+        if (this.client.getSyncState() === "SYNCING") {
+            // don't process any events until they are all decrypted
+            if (this.callEventBuffer.some((e) => e.isBeingDecrypted())) return;
+
+            const ignoreCallIds = new Set<String>();
+            // inspect the buffer and mark all calls which have been answered
+            // or hung up before passing them to the call event handler.
+            for (const ev of this.callEventBuffer) {
+                if (ev.getType() === EventType.CallAnswer ||
+                        ev.getType() === EventType.CallHangup) {
+                    ignoreCallIds.add(ev.getContent().call_id);
+                }
+            }
+            // now loop through the buffer chronologically and inject them
+            for (const e of this.callEventBuffer) {
+                if (
+                    e.getType() === EventType.CallInvite &&
+                    ignoreCallIds.has(e.getContent().call_id)
+                ) {
+                    // This call has previously been answered or hung up: ignore it
+                    continue;
+                }
+                try {
+                    this.handleCallEvent(e);
+                } catch (e) {
+                    logger.error("Caught exception handling call event", e);
+                }
+            }
+            this.callEventBuffer = [];
+        }
+    }
+
+    private onEvent = (event: MatrixEvent) => {
+        // any call events or ones that might be once they're decrypted
+        if (event.getType().indexOf("m.call.") === 0 || event.isBeingDecrypted()) {
+            // queue up for processing once all events from this sync have been
+            // processed (see above).
+            this.callEventBuffer.push(event);
+        }
+
+        if (event.isBeingDecrypted() || event.isDecryptionFailure()) {
+            // add an event listener for once the event is decrypted.
+            event.once("Event.decrypted", () => {
+                if (event.getType().indexOf("m.call.") === -1) return;
+
+                if (this.callEventBuffer.includes(event)) {
+                    // we were waiting for that event to decrypt, so recheck the buffer
+                    this.evaluateEventBuffer();
+                } else {
+                    // This one wasn't buffered so just run the event handler for it
+                    // straight away
+                    try {
+                        this.handleCallEvent(event);
+                    } catch (e) {
+                        logger.error("Caught exception handling call event", e);
+                    }
+                }
+            });
+        }
+    }
+
+    private handleCallEvent(event: MatrixEvent) {
+        const content = event.getContent();
+        let call = content.call_id ? this.calls.get(content.call_id) : undefined;
+        //console.info("RECV %s content=%s", event.getType(), JSON.stringify(content));
+
+        if (event.getType() === EventType.CallInvite) {
+            if (event.getSender() === this.client.credentials.userId) {
+                return; // ignore invites you send
+            }
+
+            if (event.getLocalAge() > content.lifetime - RING_GRACE_PERIOD) {
+                return; // expired call
+            }
+
+            if (call && call.state === CallState.Ended) {
+                return; // stale/old invite event
+            }
+            if (call) {
+                logger.log(
+                    `WARN: Already have a MatrixCall with id ${content.call_id} but got an ` +
+                    `invite. Clobbering.`,
+                );
+            }
+
+            call = createNewMatrixCall(this.client, event.getRoomId(), {
+                forceTURN: this.client._forceTURN,
+            });
+            if (!call) {
+                logger.log(
+                    "Incoming call ID " + content.call_id + " but this client " +
+                    "doesn't support WebRTC",
+                );
+                // don't hang up the call: there could be other clients
+                // connected that do support WebRTC and declining the
+                // the call on their behalf would be really annoying.
+                return;
+            }
+
+            call.callId = content.call_id;
+            call.initWithInvite(event);
+            this.calls.set(call.callId, call);
+
+            // if we stashed candidate events for that call ID, play them back now
+            if (this.candidateEventsByCall.get(call.callId)) {
+                for (const ev of this.candidateEventsByCall.get(call.callId)) {
+                    call.onRemoteIceCandidatesReceived(ev);
+                }
+            }
+
+            // Were we trying to call that user (room)?
+            let existingCall;
+            for (const thisCall of this.calls.values()) {
+                const isCalling = [CallState.WaitLocalMedia, CallState.CreateOffer, CallState.InviteSent].includes(
+                    thisCall.state,
+                );
+
+                if (
+                    call.roomId === thisCall.roomId &&
+                    thisCall.direction === CallDirection.Outbound &&
+                    isCalling
+                ) {
+                    existingCall = thisCall;
+                    break;
+                }
+            }
+
+            if (existingCall) {
+                // If we've only got to wait_local_media or create_offer and
+                // we've got an invite, pick the incoming call because we know
+                // we haven't sent our invite yet otherwise, pick whichever
+                // call has the lowest call ID (by string comparison)
+                if (existingCall.state === CallState.WaitLocalMedia ||
+                        existingCall.state === CallState.CreateOffer ||
+                        existingCall.callId > call.callId) {
+                    logger.log(
+                        "Glare detected: answering incoming call " + call.callId +
+                        " and canceling outgoing call " + existingCall.callId,
+                    );
+                    existingCall.replacedBy(call);
+                    call.answer();
+                } else {
+                    logger.log(
+                        "Glare detected: rejecting incoming call " + call.callId +
+                        " and keeping outgoing call " + existingCall.callId,
+                    );
+                    call.hangup(CallErrorCode.Replaced, true);
+                }
+            } else {
+                this.client.emit("Call.incoming", call);
+            }
+        } else if (event.getType() === EventType.CallAnswer) {
+            if (!call) {
+                return;
+            }
+            if (event.getSender() === this.client.credentials.userId) {
+                if (call.state === CallState.Ringing) {
+                    call.onAnsweredElsewhere(content);
+                }
+            } else {
+                call.onAnswerReceived(event);
+            }
+        } else if (event.getType() === EventType.CallCandidates) {
+            if (event.getSender() === this.client.credentials.userId) {
+                return;
+            }
+            if (!call) {
+                // store the candidates; we may get a call eventually.
+                if (!this.candidateEventsByCall.has(content.call_id)) {
+                    this.candidateEventsByCall.set(content.call_id, []);
+                }
+                this.candidateEventsByCall.get(content.call_id).push(event);
+            } else {
+                call.onRemoteIceCandidatesReceived(event);
+            }
+        } else if ([EventType.CallHangup, EventType.CallReject].includes(event.getType())) {
+            // Note that we also observe our own hangups here so we can see
+            // if we've already rejected a call that would otherwise be valid
+            if (!call) {
+                // if not live, store the fact that the call has ended because
+                // we're probably getting events backwards so
+                // the hangup will come before the invite
+                call = createNewMatrixCall(this.client, event.getRoomId());
+                if (call) {
+                    call.callId = content.call_id;
+                    call.initWithHangup(event);
+                    this.calls.set(content.call_id, call);
+                }
+            } else {
+                if (call.state !== CallState.Ended) {
+                    if (event.getType() === EventType.CallHangup) {
+                        call.onHangupReceived(content);
+                    } else {
+                        call.onRejectReceived(content);
+                    }
+                    this.calls.delete(content.call_id);
+                }
+            }
+        } else if (event.getType() === EventType.CallSelectAnswer) {
+            if (!call) return;
+
+            if (event.getContent().party_id === call.ourPartyId) {
+                // Ignore remote echo
+                return;
+            }
+
+            call.onSelectAnswerReceived(event);
+        } else if (event.getType() === EventType.CallNegotiate) {
+            if (!call) return;
+
+            if (event.getContent().party_id === call.ourPartyId) {
+                // Ignore remote echo
+                return;
+            }
+
+            call.onNegotiateReceived(event);
+        }
+    }
+}

tsconfig.json

@@ -2,6 +2,7 @@
   "compilerOptions": {
     "experimentalDecorators": true,
     "emitDecoratorMetadata": true,
+    "esModuleInterop": true,
     "module": "commonjs",
     "moduleResolution": "node",
     "target": "es2016",

tslint.json

@@ -1,72 +0,0 @@
-{
-  "rules": {
-    "class-name": false,
-    "comment-format": [
-      true
-    ],
-    "curly": false,
-    "eofline": false,
-    "forin": false,
-    "indent": [
-      true,
-      "spaces"
-    ],
-    "label-position": true,
-    "max-line-length": false,
-    "member-access": false,
-    "member-ordering": [
-      true,
-      "static-after-instance",
-      "variables-before-functions"
-    ],
-    "no-arg": true,
-    "no-bitwise": false,
-    "no-console": false,
-    "no-construct": true,
-    "no-debugger": true,
-    "no-duplicate-variable": true,
-    "no-empty": false,
-    "no-eval": true,
-    "no-inferrable-types": true,
-    "no-shadowed-variable": true,
-    "no-string-literal": false,
-    "no-switch-case-fall-through": true,
-    "no-trailing-whitespace": true,
-    "no-unused-expression": true,
-    "no-use-before-declare": false,
-    "no-var-keyword": true,
-    "object-literal-sort-keys": false,
-    "one-line": [
-      true,
-      "check-open-brace",
-      "check-catch",
-      "check-else",
-      "check-whitespace"
-    ],
-    "quotemark": false,
-    "radix": true,
-    "semicolon": [
-      "always"
-    ],
-    "triple-equals": [],
-    "typedef-whitespace": [
-      true,
-      {
-        "call-signature": "nospace",
-        "index-signature": "nospace",
-        "parameter": "nospace",
-        "property-declaration": "nospace",
-        "variable-declaration": "nospace"
-      }
-    ],
-    "variable-name": false,
-    "whitespace": [
-      true,
-      "check-branch",
-      "check-decl",
-      "check-operator",
-      "check-separator",
-      "check-type"
-    ]
-  }
-}