v32.2.0

This can get quite spammy in rooms with legacy calls and isn't really
neccesary: remove it to cut down on the log spam.

.eslintignore

@@ -0,0 +1 @@
+_docs

.eslintrc.js

@@ -103,11 +103,8 @@ module.exports = {
             },
         },
         {
-            // We don't need amazing docs in our spec files
             files: ["src/**/*.ts"],
             rules: {
-                "tsdoc/syntax": "error",
-                // We use some select jsdoc rules as the tsdoc linter has only one rule
                 "jsdoc/no-types": "error",
                 "jsdoc/empty-tags": "error",
                 "jsdoc/check-property-names": "error",

.github/CODEOWNERS

@@ -1,6 +1,15 @@
-*                         @matrix-org/element-web
-/.github/workflows/**     @matrix-org/element-web-app-team
-/package.json             @matrix-org/element-web-app-team
-/yarn.lock                @matrix-org/element-web-app-team
+*                         @matrix-org/element-web-reviewers
+/.github/workflows/**     @matrix-org/element-web-team
+/package.json             @matrix-org/element-web-team
+/yarn.lock                @matrix-org/element-web-team
 /src/webrtc               @matrix-org/element-call-reviewers
+/src/matrixrtc               @matrix-org/element-call-reviewers
 /spec/*/webrtc            @matrix-org/element-call-reviewers
+/spec/*/matrixrtc            @matrix-org/element-call-reviewers
+
+/src/crypto               @matrix-org/element-crypto-web-reviewers
+/src/rust-crypto          @matrix-org/element-crypto-web-reviewers
+/spec/integ/crypto        @matrix-org/element-crypto-web-reviewers
+/spec/unit/crypto.spec.ts @matrix-org/element-crypto-web-reviewers
+/spec/unit/crypto         @matrix-org/element-crypto-web-reviewers
+/spec/unit/rust-crypto    @matrix-org/element-crypto-web-reviewers

.github/PULL_REQUEST_TEMPLATE.md

@@ -2,12 +2,7 @@
 
 ## Checklist
 
--   [ ] Tests written for new code (and old code if feasible)
--   [ ] Linter and other CI checks pass
--   [ ] Sign-off given on the changes (see [CONTRIBUTING.md](https://github.com/matrix-org/matrix-js-sdk/blob/develop/CONTRIBUTING.md))
-
-<!--
-If you would like to specify text for the changelog entry other than your PR title, add the following:
-
-Notes: Add super cool feature
--->
+-   [ ] Tests written for new code (and old code if feasible).
+-   [ ] New or updated `public`/`exported` symbols have accurate [TSDoc](https://tsdoc.org/) documentation.
+-   [ ] Linter and other CI checks pass.
+-   [ ] Sign-off given on the changes (see [CONTRIBUTING.md](https://github.com/matrix-org/matrix-js-sdk/blob/develop/CONTRIBUTING.md)).

.github/actions/sign-release-tarball/action.yml

@@ -0,0 +1,28 @@
+name: Sign Release Tarball
+description: Generates signature for release tarball and uploads it as a release asset
+inputs:
+    gpg-fingerprint:
+        description: Fingerprint of the GPG key to use for signing the tarball.
+        required: true
+    upload-url:
+        description: GitHub release upload URL to upload the signature file to.
+        required: true
+runs:
+    using: composite
+    steps:
+        - name: Generate tarball signature
+          shell: bash
+          run: |
+              git -c tar.tar.gz.command='gzip -cn' archive --format=tar.gz --prefix="${REPO#*/}-${VERSION#v}/" -o "/tmp/${VERSION}.tar.gz" "${VERSION}"
+              gpg -u "$GPG_FINGERPRINT" --armor --output "${VERSION}.tar.gz.asc" --detach-sig "/tmp/${VERSION}.tar.gz"
+              rm "/tmp/${VERSION}.tar.gz"
+          env:
+              GPG_FINGERPRINT: ${{ inputs.gpg-fingerprint }}
+              REPO: ${{ github.repository }}
+
+        - name: Upload tarball signature
+          if: ${{ inputs.upload-url }}
+          uses: shogo82148/actions-upload-release-asset@8f032eff0255912cc9c8455797fd6d72f25c7ab7 # v1
+          with:
+              upload_url: ${{ inputs.upload-url }}
+              asset_path: ${{ env.VERSION }}.tar.gz.asc

.github/actions/upload-release-assets/action.yml

@@ -0,0 +1,41 @@
+name: Upload release assets
+description: Uploads assets to an existing release and optionally signs them
+inputs:
+    gpg-fingerprint:
+        description: Fingerprint of the GPG key to use for signing the assets, if any.
+        required: false
+    upload-url:
+        description: GitHub release upload URL to upload the assets to.
+        required: true
+    asset-path:
+        description: |
+            The path to the asset you want to upload, if any. You can use glob patterns here.
+            Will be GPG signed and an `.asc` file included in the release artifacts if `gpg-fingerprint` is set.
+        required: true
+runs:
+    using: composite
+    steps:
+        - name: Sign assets
+          if: inputs.gpg-fingerprint
+          shell: bash
+          run: |
+              for FILE in $ASSET_PATH
+              do
+                  gpg -u "$GPG_FINGERPRINT" --armor --output "$FILE".asc --detach-sig "$FILE"
+              done
+          env:
+              GPG_FINGERPRINT: ${{ inputs.gpg-fingerprint }}
+              ASSET_PATH: ${{ inputs.asset-path }}
+
+        - name: Upload asset signatures
+          if: inputs.gpg-fingerprint
+          uses: shogo82148/actions-upload-release-asset@8f032eff0255912cc9c8455797fd6d72f25c7ab7 # v1
+          with:
+              upload_url: ${{ inputs.upload-url }}
+              asset_path: ${{ inputs.asset-path }}.asc
+
+        - name: Upload assets
+          uses: shogo82148/actions-upload-release-asset@8f032eff0255912cc9c8455797fd6d72f25c7ab7 # v1
+          with:
+              upload_url: ${{ inputs.upload-url }}
+              asset_path: ${{ inputs.asset-path }}

.github/release-drafter.yml

@@ -0,0 +1,35 @@
+name-template: "v$RESOLVED_VERSION"
+tag-template: "v$RESOLVED_VERSION"
+change-template: "* $TITLE ([#$NUMBER]($URL)). Contributed by @$AUTHOR."
+categories:
+    - title: "🚨 BREAKING CHANGES"
+      label: "X-Breaking-Change"
+    - title: "🦖 Deprecations"
+      label: "T-Deprecation"
+    - title: "✨ Features"
+      label: "T-Enhancement"
+    - title: "🐛 Bug Fixes"
+      label: "T-Defect"
+    - title: "🧰 Maintenance"
+      label: "Dependencies"
+      collapse-after: 5
+change-title-escapes: '\<*_&' # You can add # and @ to disable mentions, and add ` to disable code blocks.
+version-resolver:
+    major:
+        labels:
+            - "X-Breaking-Change"
+    default: minor
+exclude-labels:
+    - "T-Task"
+    - "X-Reverted"
+    - "backport staging"
+exclude-contributors:
+    - "RiotRobot"
+template: |
+    $CHANGES
+#no-changes-template: ""
+prerelease: true
+prerelease-identifier: rc
+include-pre-releases: false
+stable-ref: master
+staging-ref: staging

.github/workflows/cypress.yml

@@ -1,31 +0,0 @@
-# Triggers after the "Downstream artifacts" build has finished, to run the
-# cypress tests (with access to repo secrets)
-
-name: matrix-react-sdk Cypress End to End Tests
-on:
-    workflow_run:
-        workflows: ["Build downstream artifacts"]
-        types:
-            - completed
-
-concurrency:
-    group: ${{ github.workflow }}-${{ github.event.workflow_run.head_branch || github.run_id }}
-    cancel-in-progress: ${{ github.event.workflow_run.event == 'pull_request' }}
-
-jobs:
-    cypress:
-        name: Cypress
-        uses: matrix-org/matrix-react-sdk/.github/workflows/cypress.yaml@v3.77.1
-        permissions:
-            actions: read
-            issues: read
-            statuses: write
-            pull-requests: read
-        secrets:
-            # secrets are not automatically shared with called workflows, so share the cypress dashboard key, and the Kiwi login details
-            CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
-            TCMS_USERNAME: ${{ secrets.TCMS_USERNAME }}
-            TCMS_PASSWORD: ${{ secrets.TCMS_PASSWORD }}
-        with:
-            react-sdk-repository: matrix-org/matrix-react-sdk
-            rust-crypto: true

.github/workflows/docs-pr-netlify.yaml

@@ -11,18 +11,16 @@ jobs:
         if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'pull_request'
         runs-on: ubuntu-latest
         steps:
-            # There's a 'download artifact' action, but it hasn't been updated for the workflow_run action
-            # (https://github.com/actions/download-artifact/issues/60) so instead we get this mess:
             - name: 📥 Download artifact
-              uses: dawidd6/action-download-artifact@246dbf436b23d7c49e21a7ab8204ca9ecd1fe615 # v2
+              uses: actions/download-artifact@v4
               with:
-                  workflow: static_analysis.yml
-                  run_id: ${{ github.event.workflow_run.id }}
+                  github-token: ${{ secrets.ELEMENT_BOT_TOKEN }}
+                  run-id: ${{ github.event.workflow_run.id }}
                   name: docs
                   path: docs
 
             - name: 📤 Deploy to Netlify
-              uses: matrix-org/netlify-pr-preview@v2
+              uses: matrix-org/netlify-pr-preview@v3
               with:
                   path: docs
                   owner: ${{ github.event.workflow_run.head_repository.owner.login }}
@@ -32,3 +30,4 @@ jobs:
                   site_id: ${{ secrets.NETLIFY_SITE_ID }}
                   desc: Documentation preview
                   deployment_env: PR Documentation Preview
+        environment: PR Documentation Preview

.github/workflows/downstream-artifacts.yml

@@ -1,27 +0,0 @@
-name: Build downstream artifacts
-on:
-    # We only want the Rust Crypto Cypress tests on merge queue to prevent regressions
-    # from creeping in. They take a long time to run and consume 4 concurrent runners.
-    # Anyone working on Rust Crypto is able to run the tests locally if required.
-    merge_group:
-        types: [checks_requested]
-
-    # For now at least, we don't run this or the cypress-tests against pushes
-    # to develop or master.
-    #
-    # Note that if we later choose to do so, we'll need to find a way to stop
-    # the results in Cypress Cloud from clobbering those from the 'develop'
-    # branch of matrix-react-sdk.
-    #
-    #push:
-    #    branches: [develop, master]
-concurrency:
-    group: ${{ github.workflow }}-${{ github.ref }}
-    cancel-in-progress: true
-jobs:
-    build-element-web:
-        name: Build element-web
-        uses: matrix-org/matrix-react-sdk/.github/workflows/element-web.yaml@v3.77.1
-        with:
-            matrix-js-sdk-sha: ${{ github.sha }}
-            react-sdk-repository: matrix-org/matrix-react-sdk

.github/workflows/downstream-end-to-end-tests.yml

@@ -0,0 +1,33 @@
+# Triggers after the "Downstream artifacts" build has finished, to run the
+# matrix-react-sdk playwright tests (with access to repo secrets)
+
+name: matrix-react-sdk End to End Tests
+on:
+    merge_group:
+        types: [checks_requested]
+
+    pull_request: {}
+
+    # For now at least, we don't run this or the downstream-end-to-end-tests against pushes
+    # to develop or master.
+    #
+    #push:
+    #    branches: [develop, master]
+
+concurrency:
+    group: ${{ github.workflow }}-${{ github.event.workflow_run.head_branch || github.run_id }}
+    cancel-in-progress: ${{ github.event.workflow_run.event == 'pull_request' }}
+
+jobs:
+    playwright:
+        name: Playwright
+        uses: matrix-org/matrix-react-sdk/.github/workflows/end-to-end-tests.yaml@develop
+        permissions:
+            actions: read
+            issues: read
+            pull-requests: read
+        with:
+            react-sdk-repository: matrix-org/matrix-react-sdk
+            # We only want to run the playwright tests on merge queue to prevent regressions
+            # from creeping in. They take a long time to run and consume multiple concurrent runners.
+            skip: ${{ github.event_name != 'merge_group' }}

.github/workflows/notify-downstream.yaml

@@ -20,7 +20,7 @@ jobs:
         runs-on: ubuntu-latest
         steps:
             - name: Notify matrix-react-sdk repo that a new SDK build is on develop so it can CI against it
-              uses: peter-evans/repository-dispatch@26b39ed245ab8f31526069329e112ab2fb224588 # v2
+              uses: peter-evans/repository-dispatch@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # v3
               with:
                   token: ${{ secrets.ELEMENT_BOT_TOKEN }}
                   repository: ${{ matrix.repo }}

.github/workflows/pull_request.yaml

@@ -14,11 +14,18 @@ jobs:
         name: Preview Changelog
         runs-on: ubuntu-latest
         steps:
-            - uses: matrix-org/allchange@main
+            - uses: mheap/github-action-required-labels@132879b972cb7f2ac593006455875098e73cc7f2 # v5
               if: github.event_name != 'merge_group'
               with:
-                  ghToken: ${{ secrets.GITHUB_TOKEN }}
-                  requireLabel: true
+                  labels: |
+                      X-Breaking-Change
+                      T-Deprecation
+                      T-Enhancement
+                      T-Defect
+                      T-Task
+                      Dependencies
+                  mode: minimum
+                  count: 1
 
     prevent-blocked:
         name: Prevent Blocked
@@ -27,7 +34,7 @@ jobs:
             pull-requests: read
         steps:
             - name: Add notice
-              uses: actions/github-script@v6
+              uses: actions/github-script@v7
               if: contains(github.event.pull_request.labels.*.name, 'X-Blocked')
               with:
                   script: |
@@ -39,7 +46,8 @@ jobs:
         if: github.event.action == 'opened'
         steps:
             - name: Check membership
-              uses: tspascoal/get-user-teams-membership@37c08f7b52a72ca95d12af2e7ab2553ca9adf13b # v2
+              if: github.event.pull_request.user.login != 'renovate[bot]'
+              uses: tspascoal/get-user-teams-membership@57e9f42acd78f4d0f496b3be4368fc5f62696662 # v3
               id: teams
               with:
                   username: ${{ github.event.pull_request.user.login }}
@@ -48,8 +56,8 @@ jobs:
                   GITHUB_TOKEN: ${{ secrets.ELEMENT_BOT_TOKEN }}
 
             - name: Add label
-              if: ${{ steps.teams.outputs.isTeamMember == 'false' }}
-              uses: actions/github-script@v6
+              if: steps.teams.outputs.isTeamMember == 'false'
+              uses: actions/github-script@v7
               with:
                   script: |
                       github.rest.issues.addLabels({
@@ -68,7 +76,7 @@ jobs:
             github.event.pull_request.head.repo.full_name != github.repository
         steps:
             - name: Close pull request
-              uses: actions/github-script@v6
+              uses: actions/github-script@v7
               with:
                   script: |
                       github.rest.issues.createComment({

.github/workflows/release-drafter-workflow.yml

@@ -0,0 +1,89 @@
+# Workflow used by other workflows to generate draft releases.
+name: Release Drafter Reusable
+on:
+    workflow_call:
+        inputs:
+            include-changes:
+                description: Project to include changelog entries from in this release.
+                type: string
+                required: false
+concurrency: release-drafter-action
+jobs:
+    draft:
+        runs-on: ubuntu-latest
+        steps:
+            - name: 🧮 Checkout code
+              uses: actions/checkout@v4
+              with:
+                  ref: staging
+                  fetch-depth: 0
+
+            - uses: actions/setup-node@v4
+              with:
+                  node-version-file: package.json
+                  cache: "yarn"
+
+            - name: Install Deps
+              run: "yarn install --frozen-lockfile"
+
+            - uses: t3chguy/release-drafter@105e541c2c3d857f032bd522c0764694758fabad
+              id: draft-release
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+              with:
+                  disable-autolabeler: true
+
+            - name: Get actions scripts
+              uses: actions/checkout@v4
+              with:
+                  repository: matrix-org/matrix-js-sdk
+                  persist-credentials: false
+                  path: .action-repo
+                  sparse-checkout: |
+                      .github/actions
+                      scripts/release
+
+            - name: Ingest upstream changes
+              if: inputs.include-changes
+              uses: actions/github-script@v7
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+                  RELEASE_ID: ${{ steps.draft-release.outputs.id }}
+                  DEPENDENCY: ${{ inputs.include-changes }}
+                  VERSION: ${{ steps.draft-release.outputs.tag_name }}
+              with:
+                  retries: 3
+                  script: |
+                      const { RELEASE_ID: releaseId, DEPENDENCY, VERSION } = process.env;
+                      const { owner, repo } = context.repo;
+                      const script = require("./.action-repo/scripts/release/merge-release-notes.js");
+
+                      let deps = [];
+                      if (DEPENDENCY.includes("/")) {
+                          deps.push(DEPENDENCY.replace("$VERSION", VERSION))
+                      } else {
+                          const fromVersion = JSON.parse((await github.request(`https://raw.githubusercontent.com/${owner}/${repo}/master/package.json`)).data).dependencies[DEPENDENCY];
+                          const toVersion = require("./package.json").dependencies[DEPENDENCY];
+
+                          if (toVersion.endsWith("#develop")) {
+                              core.warning(`${DEPENDENCY} will be kept at ${fromVersion}`, { title: "Develop dependency found" });
+                          } else {
+                              deps.push([DEPENDENCY, fromVersion, toVersion]);
+                          }
+                      }
+
+                      if (deps.length) {
+                          const notes = await script({
+                              github,
+                              releaseId,
+                              dependencies: deps,
+                          });
+
+                          await github.rest.repos.updateRelease({
+                              owner,
+                              repo,
+                              release_id: releaseId,
+                              body: notes,
+                              tag_name: VERSION,
+                          });
+                      }

.github/workflows/release-drafter.yml

@@ -0,0 +1,13 @@
+# Generates the draft release for the js-sdk
+# Normally triggered whenever anything is merged to the staging branch, but
+# also has a workflow dispatch trigger in case it needs running manually due
+# to failures / workflow updates etc.
+name: Release Drafter
+on:
+    push:
+        branches: [staging]
+    workflow_dispatch: {}
+concurrency: ${{ github.workflow }}
+jobs:
+    draft:
+        uses: matrix-org/matrix-js-sdk/.github/workflows/release-drafter-workflow.yml@develop

.github/workflows/release-gitflow.yml

@@ -0,0 +1,85 @@
+# Gitflow merge-back master->develop
+name: Merge master -> develop
+on:
+    push:
+        branches: [master]
+    workflow_call:
+        secrets:
+            ELEMENT_BOT_TOKEN:
+                required: true
+        inputs:
+            dependencies:
+                description: List of dependencies to reset.
+                type: string
+                required: false
+concurrency: ${{ github.workflow }}
+jobs:
+    merge:
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v4
+              with:
+                  token: ${{ secrets.ELEMENT_BOT_TOKEN }}
+                  fetch-depth: 0
+
+            - name: Get actions scripts
+              uses: actions/checkout@v4
+              with:
+                  repository: matrix-org/matrix-js-sdk
+                  persist-credentials: false
+                  path: .action-repo
+                  sparse-checkout: |
+                      scripts/release
+
+            - uses: actions/setup-node@v4
+              with:
+                  cache: "yarn"
+
+            - name: Install Deps
+              run: "yarn install --frozen-lockfile"
+
+            - name: Set up git
+              run: |
+                  git config --global user.email "releases@riot.im"
+                  git config --global user.name "RiotRobot"
+
+            - name: Merge to develop
+              run: |
+                  git checkout develop
+                  git merge -X ours master
+
+            - name: Run post-merge-master script to revert package.json fields
+              run: ./.action-repo/scripts/release/post-merge-master.sh
+
+            - name: Reset dependencies
+              if: inputs.dependencies
+              run: |
+                  while IFS= read -r PACKAGE; do
+                      [ -z "$PACKAGE" ] && continue
+
+                      CURRENT_VERSION=$(cat package.json | jq -r .dependencies[\"$PACKAGE\"])
+                      echo "Current $PACKAGE version is $CURRENT_VERSION"
+
+                      if [ "$CURRENT_VERSION" == "null" ]
+                      then
+                          echo "Unable to find $PACKAGE in package.json"
+                          exit 1
+                      fi
+
+                      if [ "$CURRENT_VERSION" == "develop" ]
+                      then
+                          echo "Not updating dependency $PACKAGE"
+                          continue
+                      fi
+
+                      echo "Resetting $1 to develop branch..."
+                      yarn add "github:matrix-org/$PACKAGE#develop"
+                      git add -u
+                      git commit -m "Reset $PACKAGE back to develop branch"
+                  done <<< "$DEPENDENCIES"
+              env:
+                  DEPENDENCIES: ${{ inputs.dependencies }}
+                  FINAL: ${{ inputs.final }}
+
+            - name: Push changes
+              run: git push origin develop

.github/workflows/release-make.yml

@@ -0,0 +1,331 @@
+name: Release Make
+on:
+    workflow_call:
+        secrets:
+            ELEMENT_BOT_TOKEN:
+                required: true
+            NPM_TOKEN:
+                required: false
+            GPG_PASSPHRASE:
+                required: false
+            GPG_PRIVATE_KEY:
+                required: false
+        inputs:
+            final:
+                description: Make final release
+                required: true
+                default: false
+                type: boolean
+            npm:
+                description: Publish to npm
+                type: boolean
+                default: false
+            downstreams:
+                description: List of github projects (owner/repo) which should have their dependency bumped to the newly released version (in JSON string array string syntax)
+                type: string
+                required: false
+            include-changes:
+                description: Project to include changelog entries from in this release.
+                type: string
+                required: false
+            gpg-fingerprint:
+                description: Fingerprint of the GPG key to use for signing the git tag and assets, if any.
+                type: string
+                required: false
+            asset-path:
+                description: |
+                    The path to the asset you want to upload, if any. You can use glob patterns here.
+                    Will be GPG signed and an `.asc` file included in the release artifacts if `gpg-fingerprint` is set.
+                type: string
+                required: false
+            expected-asset-count:
+                description: The number of expected assets, including signatures, excluding generated zip & tarball.
+                type: number
+                required: false
+jobs:
+    release:
+        name: Release
+        runs-on: ubuntu-latest
+        environment: Release
+        steps:
+            - name: Load GPG key
+              id: gpg
+              if: inputs.gpg-fingerprint
+              uses: crazy-max/ghaction-import-gpg@01dd5d3ca463c7f10f7f4f7b4f177225ac661ee4 # v6
+              with:
+                  gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
+                  passphrase: ${{ secrets.GPG_PASSPHRASE }}
+                  fingerprint: ${{ inputs.gpg-fingerprint }}
+
+            - name: Get draft release
+              id: draft-release
+              uses: cardinalby/git-get-release-action@5172c3a026600b1d459b117738c605fabc9e4e44 # v1
+              env:
+                  GITHUB_TOKEN: ${{ github.token }}
+              with:
+                  draft: true
+                  latest: true
+
+            - uses: actions/checkout@v4
+              with:
+                  ref: staging
+                  token: ${{ secrets.ELEMENT_BOT_TOKEN }}
+                  fetch-depth: 0
+
+            - name: Get actions scripts
+              uses: actions/checkout@v4
+              with:
+                  repository: matrix-org/matrix-js-sdk
+                  persist-credentials: false
+                  path: .action-repo
+                  sparse-checkout: |
+                      .github/actions
+                      scripts/release
+
+            - name: Prepare variables
+              id: prepare
+              run: |
+                  echo "VERSION=$VERSION" >> $GITHUB_ENV
+
+                  HAS_DIST=0
+                  jq -e .scripts.dist package.json >/dev/null 2>&1 && HAS_DIST=1
+                  echo "has-dist-script=$HAS_DIST" >> $GITHUB_OUTPUT
+              env:
+                  VERSION: ${{ steps.draft-release.outputs.tag_name }}
+
+            - name: Finalise version
+              if: inputs.final
+              run: echo "VERSION=$(echo $VERSION | cut -d- -f1)" >> $GITHUB_ENV
+
+            - name: Check version number not in use
+              uses: actions/github-script@v7
+              with:
+                  script: |
+                      const { VERSION } = process.env;
+                      github.rest.repos.getReleaseByTag({
+                          owner: context.repo.owner,
+                          repo: context.repo.repo,
+                          tag: VERSION,
+                      }).then(() => {
+                          core.setFailed(`Version ${VERSION} already exists`);
+                      }).catch(() => {
+                        // This is fine, we expect there to not be any release with this version yet
+                      });
+
+            - name: Set up git
+              run: |
+                  git config --global user.email "releases@riot.im"
+                  git config --global user.name "RiotRobot"
+
+            - uses: actions/setup-node@v4
+              with:
+                  cache: "yarn"
+
+            - name: Install dependencies
+              run: "yarn install --frozen-lockfile"
+
+            - name: Handle develop dependencies
+              run: |
+                  ret=0
+                  cat package.json | jq -r '.dependencies | to_entries | .[] | "\(.key) \(.value)"' | grep '#develop$' | while read -r dep ; do
+                      IFS=" "
+                      PACKAGE=${dep[0]}
+                      VERSION=${dep[1]}
+
+                      echo "::warning title=Develop dependency found::$DEPENDENCY will be kept at $VERSION"
+                      yarn upgrade "$PACKAGE@$VERSION" --exact
+                      git add -u
+                      git commit -m "Keep $PACKAGE at $VERSION"
+                  done
+
+            - name: Bump package.json version
+              run: yarn version --no-git-tag-version --new-version "${VERSION#v}"
+
+            - name: Add to CHANGELOG.md
+              if: inputs.final
+              run: |
+                  mv CHANGELOG.md CHANGELOG.md.old
+                  HEADER="Changes in [${VERSION#v}](https://github.com/${{ github.repository }}/releases/tag/$VERSION) ($(date '+%Y-%m-%d'))"
+
+                  {
+                      echo "$HEADER"
+                      printf '=%.0s' $(seq ${#HEADER})
+                      echo ""
+                      echo "$RELEASE_NOTES"
+                      echo ""
+                  } > CHANGELOG.md
+
+                  cat CHANGELOG.md.old >> CHANGELOG.md
+                  rm CHANGELOG.md.old
+                  git add CHANGELOG.md
+              env:
+                  RELEASE_NOTES: ${{ steps.draft-release.outputs.body }}
+
+            - name: Run pre-release script to update package.json fields
+              run: |
+                  ./.action-repo/scripts/release/pre-release.sh
+                  git add package.json
+
+            - name: Commit changes
+              run: git commit -m "$VERSION"
+
+            - name: Build assets
+              if: steps.prepare.outputs.has-dist-script == '1'
+              run: DIST_VERSION="$VERSION" yarn dist
+
+            - name: Upload release assets & signatures
+              if: inputs.asset-path
+              uses: ./.action-repo/.github/actions/upload-release-assets
+              with:
+                  gpg-fingerprint: ${{ inputs.gpg-fingerprint }}
+                  upload-url: ${{ steps.draft-release.outputs.upload_url }}
+                  asset-path: ${{ inputs.asset-path }}
+
+            - name: Create signed tag
+              if: inputs.gpg-fingerprint
+              run: |
+                  GIT_COMMITTER_EMAIL="$SIGNING_ID" GPG_TTY=$(tty) git tag -u "$SIGNING_ID" -m "Release $VERSION" "$VERSION"
+              env:
+                  SIGNING_ID: ${{ steps.gpg.outputs.email }}
+
+            - name: Generate & upload tarball signature
+              if: inputs.gpg-fingerprint
+              uses: ./.action-repo/.github/actions/sign-release-tarball
+              with:
+                  gpg-fingerprint: ${{ inputs.gpg-fingerprint }}
+                  upload-url: ${{ steps.draft-release.outputs.upload_url }}
+
+            # We defer pushing changes until after the release assets are built,
+            # signed & uploaded to improve the atomicity of this action.
+            - name: Push changes to staging
+              run: |
+                  git push origin staging $TAG
+                  git reset --hard
+              env:
+                  TAG: ${{ inputs.gpg-fingerprint && env.VERSION || '' }}
+
+            - name: Validate tarball signature
+              if: inputs.gpg-fingerprint
+              run: |
+                  wget https://github.com/$GITHUB_REPOSITORY/archive/refs/tags/$VERSION.tar.gz
+                  gpg --verify "$VERSION.tar.gz.asc" "$VERSION.tar.gz"
+
+            - name: Validate release has expected assets
+              if: inputs.expected-asset-count
+              uses: actions/github-script@v7
+              env:
+                  RELEASE_ID: ${{ steps.draft-release.outputs.id }}
+                  EXPECTED_ASSET_COUNT: ${{ inputs.expected-asset-count }}
+              with:
+                  retries: 3
+                  script: |
+                      const { RELEASE_ID: release_id, EXPECTED_ASSET_COUNT } = process.env;
+                      const { owner, repo } = context.repo;
+
+                      const { data: release } = await github.rest.repos.getRelease({
+                          owner,
+                          repo,
+                          release_id,
+                      });
+
+                      if (release.assets.length !== parseInt(EXPECTED_ASSET_COUNT, 10)) {
+                          core.setFailed(`Found ${release.assets.length} assets but expected ${EXPECTED_ASSET_COUNT}`);
+                      }
+
+            - name: Merge to master
+              if: inputs.final
+              run: |
+                  git checkout master
+                  git merge -X theirs staging
+                  git push origin master
+
+            - name: Publish release
+              uses: actions/github-script@v7
+              env:
+                  RELEASE_ID: ${{ steps.draft-release.outputs.id }}
+                  FINAL: ${{ inputs.final }}
+              with:
+                  retries: 3
+                  github-token: ${{ secrets.ELEMENT_BOT_TOKEN }}
+                  script: |
+                      const { RELEASE_ID: release_id, RELEASE_NOTES, VERSION, FINAL } = process.env;
+                      const { owner, repo } = context.repo;
+
+                      const opts = {
+                          owner,
+                          repo,
+                          release_id,
+                          tag_name: VERSION,
+                          name: VERSION,
+                          draft: false,
+                          body: RELEASE_NOTES,
+                      };
+
+                      if (FINAL == "true") {
+                          opts.prerelease = false;
+                          opts.make_latest = true;
+                      }
+
+                      github.rest.repos.updateRelease(opts);
+
+    npm:
+        name: Publish to npm
+        needs: release
+        if: inputs.npm
+        uses: matrix-org/matrix-js-sdk/.github/workflows/release-npm.yml@develop
+        secrets:
+            NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+    post-release:
+        name: Post release steps
+        needs: release
+        runs-on: ubuntu-latest
+        steps:
+            - id: repository
+              run: echo "REPO=${GITHUB_REPOSITORY#*/}" >> $GITHUB_OUTPUT
+
+            - name: Advance release blocker labels
+              uses: garganshu/github-label-updater@3770d15ebfed2fe2cb06a241047bc340f774a7d1 # v1.0.0
+              with:
+                  owner: ${{ github.repository_owner }}
+                  repo: ${{ steps.repository.outputs.REPO }}
+                  token: ${{ secrets.GITHUB_TOKEN }}
+                  filter-labels: X-Upcoming-Release-Blocker
+                  remove-labels: X-Upcoming-Release-Blocker
+                  add-labels: X-Release-Blocker
+
+    #            - name: Wait for master->develop gitflow merge
+    #              if: inputs.final
+    #              uses: t3chguy/wait-on-check-action@18541021811b56544d90e0f073401c2b99e249d6 # fork
+    #              with:
+    #                  ref: master
+    #                  repo-token: ${{ secrets.GITHUB_TOKEN }}
+    #                  wait-interval: 10
+    #                  check-name: merge
+    #                  allowed-conclusions: success
+
+    bump-downstreams:
+        name: Update npm dependency in downstream projects
+        needs: npm
+        runs-on: ubuntu-latest
+        if: inputs.downstreams
+        strategy:
+            matrix:
+                repo: ${{ fromJSON(inputs.downstreams) }}
+        steps:
+            - uses: actions/checkout@v4
+              with:
+                  repository: ${{ matrix.repo }}
+                  ref: staging
+                  token: ${{ secrets.ELEMENT_BOT_TOKEN }}
+
+            - name: Bump dependency
+              env:
+                  DEPENDENCY: ${{ needs.npm.outputs.id }}
+              run: |
+                  git config --global user.email "releases@riot.im"
+                  git config --global user.name "RiotRobot"
+                  yarn upgrade "$DEPENDENCY" --exact
+                  git add package.json yarn.lock
+                  git commit -am"Upgrade dependency to $DEPENDENCY"
+                  git push origin staging

.github/workflows/release-npm.yml

@@ -1,20 +1,27 @@
-# Must only be called from `release#published` triggers
 name: Publish to npm
 on:
     workflow_call:
         secrets:
             NPM_TOKEN:
                 required: true
+        outputs:
+            id:
+                description: "The npm package@version string we published"
+                value: ${{ jobs.npm.outputs.id }}
 jobs:
     npm:
         name: Publish to npm
         runs-on: ubuntu-latest
+        outputs:
+            id: ${{ steps.npm-publish.outputs.id }}
         steps:
             - name: 🧮 Checkout code
-              uses: actions/checkout@v3
+              uses: actions/checkout@v4
+              with:
+                  ref: staging
 
             - name: 🔧 Yarn cache
-              uses: actions/setup-node@v3
+              uses: actions/setup-node@v4
               with:
                   cache: "yarn"
                   registry-url: "https://registry.npmjs.org"
@@ -24,7 +31,7 @@ jobs:
 
             - name: 🚀 Publish to npm
               id: npm-publish
-              uses: JS-DevTools/npm-publish@5a85faf05d2ade2d5b6682bfe5359915d5159c6c # v2.2.1
+              uses: JS-DevTools/npm-publish@19c28f1ef146469e409470805ea4279d47c3d35c # v3.1.1
               with:
                   token: ${{ secrets.NPM_TOKEN }}
                   access: public
@@ -32,7 +39,7 @@ jobs:
                   ignore-scripts: false
 
             - name: 🎖️ Add `latest` dist-tag to final releases
-              if: github.event.release.prerelease == false && steps.npm-publish.outputs.id
+              if: steps.npm-publish.outputs.id && !contains(steps.npm-publish.outputs.id, '-rc.')
               run: npm dist-tag add "$release" latest
               env:
                   NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

.github/workflows/release.yml

@@ -1,52 +1,72 @@
 name: Release Process
 on:
-    release:
-        types: [published]
-concurrency: ${{ github.workflow }}-${{ github.ref }}
+    workflow_dispatch:
+        inputs:
+            mode:
+                description: What type of release
+                required: true
+                default: rc
+                type: choice
+                options:
+                    - rc
+                    - final
+            docs:
+                description: Publish docs
+                required: true
+                type: boolean
+                default: true
+            npm:
+                description: Publish to npm
+                required: true
+                type: boolean
+                default: true
+concurrency: ${{ github.workflow }}
 jobs:
-    jsdoc:
+    release:
+        uses: matrix-org/matrix-js-sdk/.github/workflows/release-make.yml@develop
+        secrets: inherit
+        with:
+            final: ${{ inputs.mode == 'final' }}
+            npm: ${{ inputs.npm }}
+            downstreams: '["matrix-org/matrix-react-sdk", "element-hq/element-web"]'
+
+    docs:
         name: Publish Documentation
+        needs: release
+        if: inputs.docs
         runs-on: ubuntu-latest
         steps:
             - name: 🧮 Checkout code
-              uses: actions/checkout@v3
-
-            - name: 🧮 Checkout gh-pages
-              uses: actions/checkout@v3
-              with:
-                  ref: gh-pages
-                  path: _docs
+              uses: actions/checkout@v4
 
             - name: 🔧 Yarn cache
-              uses: actions/setup-node@v3
+              uses: actions/setup-node@v4
               with:
                   cache: "yarn"
 
             - name: 🔨 Install dependencies
               run: "yarn install --frozen-lockfile"
 
-            - name: 🔨 Install symlinks
-              run: |
-                  sudo apt-get update
-                  sudo apt-get install -y symlinks
-
             - name: 📖 Generate docs
-              run: |
-                  yarn tpv purge --yes --out _docs --stale --major 10
-                  yarn gendoc
-                  symlinks -rc _docs
+              run: yarn gendoc
 
-            - name: 🚀 Deploy
-              run: |
-                  git config --global user.email "releases@riot.im"
-                  git config --global user.name "RiotRobot"
-                  git add . --all
-                  git commit -m "Update docs"
-                  git push
-              working-directory: _docs
+            - name: Upload artifact
+              uses: actions/upload-pages-artifact@v3
+              with:
+                  path: _docs
 
-    npm:
-        name: Publish
-        uses: matrix-org/matrix-js-sdk/.github/workflows/release-npm.yml@develop
-        secrets:
-            NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+    docs-deploy:
+        environment:
+            name: github-pages
+            url: ${{ steps.deployment.outputs.page_url }}
+        runs-on: ubuntu-latest
+        needs: docs
+        # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+        permissions:
+            contents: read
+            pages: write
+            id-token: write
+        steps:
+            - name: Deploy to GitHub Pages
+              id: deployment
+              uses: actions/deploy-pages@v4

.github/workflows/sonarcloud.yml

@@ -5,19 +5,23 @@ on:
         secrets:
             SONAR_TOKEN:
                 required: true
+            ELEMENT_BOT_TOKEN:
+                required: true
         inputs:
-            extra_args:
-                type: string
+            sharded:
+                type: boolean
                 required: false
-                description: "Extra args to pass to SonarCloud"
+                description: "Whether to combine multiple LCOV and jest-sonar-report files in coverage artifact"
 jobs:
     sonarqube:
         runs-on: ubuntu-latest
-        if: github.event.workflow_run.conclusion == 'success'
+        if: |
+            github.event.workflow_run.conclusion == 'success' &&
+            github.event.workflow_run.event != 'merge_group'
         steps:
             # We create the status here and then update it to success/failure in the `report` stage
-            # This provides an easy link to this workflow_run from the PR before Cypress is done.
-            - uses: Sibz/github-status-action@faaa4d96fecf273bd762985e0e7f9f933c774918 # v1
+            # This provides an easy link to this workflow_run from the PR before Sonarcloud is done.
+            - uses: Sibz/github-status-action@071b5370da85afbb16637d6eed8524a06bc2053e # v1
               with:
                   authToken: ${{ secrets.GITHUB_TOKEN }}
                   state: pending
@@ -25,24 +29,53 @@ jobs:
                   sha: ${{ github.event.workflow_run.head_sha }}
                   target_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
 
+            - name: "🧮 Checkout code"
+              uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+              with:
+                  repository: ${{ github.event.workflow_run.head_repository.full_name }}
+                  ref: ${{ github.event.workflow_run.head_branch }} # checkout commit that triggered this workflow
+                  fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis
+
+            - name: 📥 Download artifact
+              uses: actions/download-artifact@v4
+              if: ${{ !inputs.sharded }}
+              with:
+                  github-token: ${{ secrets.ELEMENT_BOT_TOKEN }}
+                  run-id: ${{ github.event.workflow_run.id }}
+                  name: coverage
+                  path: coverage
+            - name: 📥 Download sharded artifacts
+              uses: actions/download-artifact@v4
+              if: inputs.sharded
+              with:
+                  github-token: ${{ secrets.ELEMENT_BOT_TOKEN }}
+                  run-id: ${{ github.event.workflow_run.id }}
+                  pattern: coverage-*
+                  path: coverage
+                  merge-multiple: true
+
+            - id: extra_args
+              run: |
+                  coverage=$(find coverage -type f -name '*lcov.info' -printf '%h/%f,' | tr -d '\r\n' | sed 's/,$//g')
+                  echo "sonar.javascript.lcov.reportPaths=$coverage" >> sonar-project.properties
+                  reports=$(find coverage -type f -name 'jest-sonar-report*.xml' -printf '%h/%f,' | tr -d '\r\n' | sed 's/,$//g')
+                  echo "sonar.testExecutionReportPaths=$reports" >> sonar-project.properties
+
             - name: "🩻 SonarCloud Scan"
               id: sonarcloud
-              uses: matrix-org/sonarcloud-workflow-action@v2.5
+              uses: matrix-org/sonarcloud-workflow-action@v3.2
               # workflow_run fails report against the develop commit always, we don't want that for PRs
               continue-on-error: ${{ github.event.workflow_run.head_branch != 'develop' }}
               with:
+                  skip_checkout: true
                   repository: ${{ github.event.workflow_run.head_repository.full_name }}
                   is_pr: ${{ github.event.workflow_run.event == 'pull_request' }}
                   version_cmd: "cat package.json | jq -r .version"
                   branch: ${{ github.event.workflow_run.head_branch }}
                   revision: ${{ github.event.workflow_run.head_sha }}
                   token: ${{ secrets.SONAR_TOKEN }}
-                  coverage_run_id: ${{ github.event.workflow_run.id }}
-                  coverage_workflow_name: tests.yml
-                  coverage_extract_path: coverage
-                  extra_args: ${{ inputs.extra_args }}
 
-            - uses: Sibz/github-status-action@faaa4d96fecf273bd762985e0e7f9f933c774918 # v1
+            - uses: Sibz/github-status-action@071b5370da85afbb16637d6eed8524a06bc2053e # v1
               if: always()
               with:
                   authToken: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/sonarqube.yml

@@ -8,38 +8,12 @@ concurrency:
     group: ${{ github.workflow }}-${{ github.event.workflow_run.head_branch }}
     cancel-in-progress: true
 jobs:
-    # This is a workaround for https://github.com/SonarSource/SonarJS/issues/578
-    prepare:
-        name: Prepare
-        if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event != 'merge_group'
-        runs-on: ubuntu-latest
-        outputs:
-            reportPaths: ${{ steps.extra_args.outputs.reportPaths }}
-            testExecutionReportPaths: ${{ steps.extra_args.outputs.testExecutionReportPaths }}
-        steps:
-            # There's a 'download artifact' action, but it hasn't been updated for the workflow_run action
-            # (https://github.com/actions/download-artifact/issues/60) so instead we get this mess:
-            - name: 📥 Download artifact
-              uses: dawidd6/action-download-artifact@246dbf436b23d7c49e21a7ab8204ca9ecd1fe615 # v2
-              with:
-                  workflow: tests.yaml
-                  run_id: ${{ github.event.workflow_run.id }}
-                  name: coverage
-                  path: coverage
-
-            - id: extra_args
-              run: |
-                  coverage=$(find coverage -type f -name '*lcov.info' | tr '\n' ',' | sed 's/,$//g')
-                  echo "reportPaths=$coverage" >> $GITHUB_OUTPUT
-                  reports=$(find coverage -type f -name 'jest-sonar-report*.xml' | tr '\n' ',' | sed 's/,$//g')
-                  echo "testExecutionReportPaths=$reports" >> $GITHUB_OUTPUT
-
     sonarqube:
         name: 🩻 SonarQube
         if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event != 'merge_group'
-        needs: prepare
         uses: matrix-org/matrix-js-sdk/.github/workflows/sonarcloud.yml@develop
         secrets:
             SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+            ELEMENT_BOT_TOKEN: ${{ secrets.ELEMENT_BOT_TOKEN }}
         with:
-            extra_args: -Dsonar.javascript.lcov.reportPaths=${{ needs.prepare.outputs.reportPaths }} -Dsonar.testExecutionReportPaths=${{ needs.prepare.outputs.testExecutionReportPaths }}
+            sharded: true

.github/workflows/static_analysis.yml

@@ -13,9 +13,9 @@ jobs:
         name: "Typescript Syntax Check"
         runs-on: ubuntu-latest
         steps:
-            - uses: actions/checkout@v3
+            - uses: actions/checkout@v4
 
-            - uses: actions/setup-node@v3
+            - uses: actions/setup-node@v4
               with:
                   cache: "yarn"
 
@@ -39,9 +39,9 @@ jobs:
         name: "ESLint"
         runs-on: ubuntu-latest
         steps:
-            - uses: actions/checkout@v3
+            - uses: actions/checkout@v4
 
-            - uses: actions/setup-node@v3
+            - uses: actions/setup-node@v4
               with:
                   cache: "yarn"
 
@@ -51,13 +51,29 @@ jobs:
             - name: Run Linter
               run: "yarn run lint:js"
 
+    workflow_lint:
+        name: "Workflow Lint"
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v4
+
+            - uses: actions/setup-node@v4
+              with:
+                  cache: "yarn"
+
+            - name: Install Deps
+              run: "yarn install --frozen-lockfile"
+
+            - name: Run Linter
+              run: "yarn lint:workflows"
+
     docs:
         name: "JSDoc Checker"
         runs-on: ubuntu-latest
         steps:
-            - uses: actions/checkout@v3
+            - uses: actions/checkout@v4
 
-            - uses: actions/setup-node@v3
+            - uses: actions/setup-node@v4
               with:
                   cache: "yarn"
 
@@ -67,17 +83,26 @@ jobs:
             - name: Generate Docs
               run: "yarn run gendoc --treatWarningsAsErrors"
 
-            # Upload artifact duplicates symlink contents so we do this to save 75% space
-            - name: Flatten symlink and write _redirects
-              run: |
-                  find _docs -mindepth 1 -maxdepth 1 ! -type f ! -name stable -printf '/%f/* /stable/:splat\n' > _docs/_redirects
-                  find _docs -mindepth 1 -maxdepth 1 -type l -delete
-                  find _docs -mindepth 1 -maxdepth 1 -type d -execdir mv {} stable \; -quit
-
             - name: Upload Artifact
-              uses: actions/upload-artifact@v3
+              uses: actions/upload-artifact@v4
               with:
                   name: docs
                   path: _docs
                   # We'll only use this in a workflow_run, then we're done with it
                   retention-days: 1
+
+    analyse_dead_code:
+        name: "Analyse Dead Code"
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v4
+
+            - uses: actions/setup-node@v4
+              with:
+                  cache: "yarn"
+
+            - name: Install Deps
+              run: "yarn install --frozen-lockfile"
+
+            - name: Run linter
+              run: "yarn run lint:knip"

.github/workflows/tests.yml

@@ -12,19 +12,20 @@ env:
     ENABLE_COVERAGE: ${{ github.event_name != 'merge_group' }}
 jobs:
     jest:
-        name: "Jest [${{ matrix.specs }}] (Node ${{ matrix.node }})"
+        name: "Jest [${{ matrix.specs }}] (Node ${{ matrix.node == '*' && 'latest' || matrix.node }})"
         runs-on: ubuntu-latest
         timeout-minutes: 10
         strategy:
             matrix:
-                specs: [browserify, integ, unit]
-                node: [18, latest]
+                specs: [integ, unit]
+                node: [18, "lts/*", 21]
         steps:
             - name: Checkout code
-              uses: actions/checkout@v3
+              uses: actions/checkout@v4
 
             - name: Setup Node
-              uses: actions/setup-node@v3
+              id: setupNode
+              uses: actions/setup-node@v4
               with:
                   cache: "yarn"
                   node-version: ${{ matrix.node }}
@@ -32,13 +33,9 @@ jobs:
             - name: Install dependencies
               run: "yarn install"
 
-            - name: Build
-              if: matrix.specs == 'browserify'
-              run: "yarn build"
-
             - name: Get number of CPU cores
               id: cpu-cores
-              uses: SimenB/github-actions-cpu-cores@410541432439795d30db6501fb1d8178eb41e502 # v1
+              uses: SimenB/github-actions-cpu-cores@97ba232459a8e02ff6121db9362b09661c875ab8 # v2
 
             - name: Run tests
               run: |
@@ -55,13 +52,13 @@ jobs:
 
             - name: Move coverage files into place
               if: env.ENABLE_COVERAGE == 'true'
-              run: mv coverage/lcov.info coverage/${{ matrix.node }}-${{ matrix.specs }}.lcov.info
+              run: mv coverage/lcov.info coverage/${{ steps.setupNode.outputs.node-version }}-${{ matrix.specs }}.lcov.info
 
             - name: Upload Artifact
               if: env.ENABLE_COVERAGE == 'true'
-              uses: actions/upload-artifact@v3
+              uses: actions/upload-artifact@v4
               with:
-                  name: coverage
+                  name: coverage-${{ matrix.specs }}-${{ matrix.node == 'lts/*' && 'lts' || matrix.node }}
                   path: |
                       coverage
                       !coverage/lcov-report
@@ -85,7 +82,7 @@ jobs:
         steps:
             - name: Skip SonarCloud on merge queues
               if: env.ENABLE_COVERAGE == 'false'
-              uses: Sibz/github-status-action@faaa4d96fecf273bd762985e0e7f9f933c774918 # v1
+              uses: Sibz/github-status-action@071b5370da85afbb16637d6eed8524a06bc2053e # v1
               with:
                   authToken: ${{ secrets.GITHUB_TOKEN }}
                   state: success

.github/workflows/triage-incoming.yml

@@ -0,0 +1,14 @@
+name: Move new issues into Issue triage board
+
+on:
+    issues:
+        types: [opened]
+
+jobs:
+    automate-project-columns-next:
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/add-to-project@main
+              with:
+                  project-url: https://github.com/orgs/element-hq/projects/120
+                  github-token: ${{ secrets.ELEMENT_BOT_TOKEN }}

.github/workflows/triage-labelled.yml

@@ -0,0 +1,11 @@
+name: Move labelled issues to correct projects
+
+on:
+    issues:
+        types: [labeled]
+
+jobs:
+    call-triage-labelled:
+        uses: vector-im/element-web/.github/workflows/triage-labelled.yml@develop
+        secrets:
+            ELEMENT_BOT_TOKEN: ${{ secrets.ELEMENT_BOT_TOKEN }}

.github/workflows/upgrade_dependencies.yml

@@ -1,38 +0,0 @@
-name: Upgrade Dependencies
-on:
-    workflow_dispatch: {}
-    workflow_call:
-        secrets:
-            ELEMENT_BOT_TOKEN:
-                required: true
-jobs:
-    upgrade:
-        runs-on: ubuntu-latest
-        steps:
-            - uses: actions/checkout@v3
-
-            - uses: actions/setup-node@v3
-              with:
-                  cache: "yarn"
-
-            - name: Upgrade
-              run: yarn upgrade && yarn install
-
-            - name: Create Pull Request
-              id: cpr
-              uses: peter-evans/create-pull-request@153407881ec5c347639a548ade7d8ad1d6740e38 # v5
-              with:
-                  token: ${{ secrets.ELEMENT_BOT_TOKEN }}
-                  branch: actions/upgrade-deps
-                  delete-branch: true
-                  title: Upgrade dependencies
-                  labels: |
-                      Dependencies
-                      T-Task
-
-            - name: Enable automerge
-              run: gh pr merge --merge --auto "$PR_NUMBER"
-              if: steps.cpr.outputs.pull-request-operation == 'created'
-              env:
-                  GH_TOKEN: ${{ secrets.ELEMENT_BOT_TOKEN }}
-                  PR_NUMBER: ${{ steps.cpr.outputs.pull-request-number }}

.husky/pre-commit

@@ -0,0 +1,3 @@
+#!/usr/bin/env sh
+
+npx lint-staged

.lintstagedrc

@@ -0,0 +1,4 @@
+{
+    "*.(ts|tsx)": ["eslint --fix", "prettier --write"],
+    "*.(py|md|yaml)": ["prettier --write"]
+}

.prettierignore

@@ -25,5 +25,6 @@ out
 # This file is owned, parsed, and generated by allchange, which doesn't comply with prettier
 /CHANGELOG.md
 
-# This file is also autogenerated
+# These files are also autogenerated
 /spec/test-utils/test-data/index.ts
+/spec/test-utils/test_indexeddb_cryptostore_dump/dump.json

CHANGELOG.md

@@ -1,3 +1,313 @@
+Changes in [32.2.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v32.2.0) (2024-05-07)
+==================================================================================================
+## ✨ Features
+
+* Use a different error code for UTDs when user was not in the room ([#4172](https://github.com/matrix-org/matrix-js-sdk/pull/4172)). Contributed by @uhoreg.
+* Modernize window.crypto access constants ([#4169](https://github.com/matrix-org/matrix-js-sdk/pull/4169)). Contributed by @turt2live.
+* Improve compliance with MSC3266 ([#4155](https://github.com/matrix-org/matrix-js-sdk/pull/4155)). Contributed by @AndrewFerr.
+* Add comment to make clear that RoomStateEvent.Events does not update related objects in the js-sdk ([#4152](https://github.com/matrix-org/matrix-js-sdk/pull/4152)). Contributed by @toger5.
+* Crypto: use a new error code for UTDs from device-relative historical events ([#4139](https://github.com/matrix-org/matrix-js-sdk/pull/4139)). Contributed by @richvdh.
+
+## 🐛 Bug Fixes
+
+* Element-R: Fix rust migration when ssss secret are stored not encryted in cache (old legacy behavior) ([#4168](https://github.com/matrix-org/matrix-js-sdk/pull/4168)). Contributed by @BillCarsonFr.
+
+
+Changes in [32.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v32.1.0) (2024-04-23)
+==================================================================================================
+## ✨ Features
+
+* Add support for device dehydration v2 (Element R) ([#4062](https://github.com/matrix-org/matrix-js-sdk/pull/4062)). Contributed by @uhoreg.
+* OIDC improvements in prep of OIDC-QR reciprocation ([#4149](https://github.com/matrix-org/matrix-js-sdk/pull/4149)). Contributed by @t3chguy.
+
+## 🐛 Bug Fixes
+
+* Validate backup private key before migrating it ([#4114](https://github.com/matrix-org/matrix-js-sdk/pull/4114)). Contributed by @BillCarsonFr.
+* ElementR| Retry query backup until it works during migration to avoid spurious correption error popup ([#4113](https://github.com/matrix-org/matrix-js-sdk/pull/4113)). Contributed by @BillCarsonFr.
+
+
+Changes in [32.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v32.0.0) (2024-04-09)
+==================================================================================================
+## 🚨 BREAKING CHANGES
+
+* Remove various deprecated methods \& re-exports ([#4125](https://github.com/matrix-org/matrix-js-sdk/pull/4125)). Contributed by @t3chguy.
+* Remove the logic that throws when the lazy loading options has changed. ([#4124](https://github.com/matrix-org/matrix-js-sdk/pull/4124)). Contributed by @langleyd.
+* Fix highlights from threads disappearing on new messages ([#4106](https://github.com/matrix-org/matrix-js-sdk/pull/4106)). Contributed by @dbkr.
+
+## ✨ Features
+
+* Add new `decryptExistingEvent` test helper ([#4133](https://github.com/matrix-org/matrix-js-sdk/pull/4133)). Contributed by @richvdh.
+* Improve types for `sendEvent` ([#4108](https://github.com/matrix-org/matrix-js-sdk/pull/4108)). Contributed by @t3chguy.
+* Remove various deprecated methods \& re-exports ([#4125](https://github.com/matrix-org/matrix-js-sdk/pull/4125)). Contributed by @t3chguy.
+* Add new enum for verification methods. ([#4129](https://github.com/matrix-org/matrix-js-sdk/pull/4129)). Contributed by @richvdh.
+* Add some test utils in a new entrypoint ([#4127](https://github.com/matrix-org/matrix-js-sdk/pull/4127)). Contributed by @richvdh.
+* Improve types for `sendStateEvent` ([#4105](https://github.com/matrix-org/matrix-js-sdk/pull/4105)). Contributed by @t3chguy.
+
+## 🐛 Bug Fixes
+
+* Improve types for `IPowerLevelsContent` and `hasSufficientPowerLevelFor` ([#4128](https://github.com/matrix-org/matrix-js-sdk/pull/4128)). Contributed by @galash13.
+* Remove the logic that throws when the lazy loading options has changed. ([#4124](https://github.com/matrix-org/matrix-js-sdk/pull/4124)). Contributed by @langleyd.
+* Fix highlights from threads disappearing on new messages ([#4106](https://github.com/matrix-org/matrix-js-sdk/pull/4106)). Contributed by @dbkr.
+* Extend logic for local notification processing to threads ([#4111](https://github.com/matrix-org/matrix-js-sdk/pull/4111)). Contributed by @dbkr.
+* Fix public rooms post request search params and body ([#4110](https://github.com/matrix-org/matrix-js-sdk/pull/4110)). Contributed by @ajbura.
+* Fix bugs with the first reply to a thread ([#4104](https://github.com/matrix-org/matrix-js-sdk/pull/4104)). Contributed by @dbkr.
+
+
+Changes in [31.6.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v31.6.1) (2024-03-28)
+==================================================================================================
+## 🐛 Bug Fixes
+
+* Fix merging of default push rules ([#4136](https://github.com/matrix-org/matrix-js-sdk/pull/4136)).
+
+
+Changes in [31.6.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v31.6.0) (2024-03-26)
+==================================================================================================
+## ✨ Features
+
+* Introduce Membership TS type (take 2) ([#4107](https://github.com/matrix-org/matrix-js-sdk/pull/4107)). Contributed by @andybalaam.
+* fix automatic DM avatar with functional members ([#4017](https://github.com/matrix-org/matrix-js-sdk/pull/4017)). Contributed by @HarHarLinks.
+* Export types describing all specced media event formats ([#4092](https://github.com/matrix-org/matrix-js-sdk/pull/4092)). Contributed by @t3chguy.
+* Add `.m.rule.is_room_mention` push rule to DEFAULT\_OVERRIDE\_RULES ([#4100](https://github.com/matrix-org/matrix-js-sdk/pull/4100)). Contributed by @t3chguy.
+* Make sending ContentLoaded optional for a widgetClient ([#4086](https://github.com/matrix-org/matrix-js-sdk/pull/4086)). Contributed by @toger5.
+
+## 🐛 Bug Fixes
+
+* Migrate own identity local trust to rust crypto ([#4090](https://github.com/matrix-org/matrix-js-sdk/pull/4090)). Contributed by @BillCarsonFr.
+* Fix race condition with sliding sync extensions ([#4089](https://github.com/matrix-org/matrix-js-sdk/pull/4089)). Contributed by @zzorba.
+
+
+Changes in [31.5.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v31.5.0) (2024-03-12)
+==================================================================================================
+## ✨ Features
+
+* Update MSC2965 OIDC Discovery implementation ([#4064](https://github.com/matrix-org/matrix-js-sdk/pull/4064)). Contributed by @t3chguy.
+
+## 🐛 Bug Fixes
+
+* Add basic retry for rust crypto outgoing requests ([#4061](https://github.com/matrix-org/matrix-js-sdk/pull/4061)). Contributed by @BillCarsonFr.
+
+
+Changes in [31.4.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v31.4.0) (2024-02-27)
+==================================================================================================
+## ✨ Features
+
+* Validate `account_management_uri` and `account_management_actions_supported` from OIDC Issuer well-known ([#4074](https://github.com/matrix-org/matrix-js-sdk/pull/4074)). Contributed by @t3chguy.
+* Allow specifying OIDC url state parameter for passing data to callback ([#4068](https://github.com/matrix-org/matrix-js-sdk/pull/4068)). Contributed by @t3chguy.
+* Add getAuthIssuer method for MSC2965 ([#4071](https://github.com/matrix-org/matrix-js-sdk/pull/4071)). Contributed by @t3chguy.
+* Allow specifying more OIDC client metadata for dynamic registration ([#4070](https://github.com/matrix-org/matrix-js-sdk/pull/4070)). Contributed by @t3chguy.
+* Add unread marker event type ([#4069](https://github.com/matrix-org/matrix-js-sdk/pull/4069)). Contributed by @dbkr.
+* Add "AsJson" forms of the key import/export methods ([#4057](https://github.com/matrix-org/matrix-js-sdk/pull/4057)). Contributed by @andybalaam.
+
+## 🐛 Bug Fixes
+
+* Ignore memberships of users that are not in the call ([#4065](https://github.com/matrix-org/matrix-js-sdk/pull/4065)). Contributed by @toger5.
+* Await encrypted messages ([#4063](https://github.com/matrix-org/matrix-js-sdk/pull/4063)). Contributed by @toger5.
+* ElementR | Ensure own user and device trust are updated after migration before giving back control to the app. ([#4059](https://github.com/matrix-org/matrix-js-sdk/pull/4059)). Contributed by @BillCarsonFr.
+* Bump matrix-sdk-crypto-wasm to 4.5.0 ([#4060](https://github.com/matrix-org/matrix-js-sdk/pull/4060)). Contributed by @andybalaam.
+
+
+Changes in [31.3.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v31.3.0) (2024-02-13)
+==================================================================================================
+## ✨ Features
+
+* Add expire\_ts compatibility to matrixRTC ([#4032](https://github.com/matrix-org/matrix-js-sdk/pull/4032)). Contributed by @toger5.
+* Element-R: support for migration of the room list from legacy crypto ([#4036](https://github.com/matrix-org/matrix-js-sdk/pull/4036)). Contributed by @richvdh.
+* Element-R: check persistent room list for encryption config ([#4035](https://github.com/matrix-org/matrix-js-sdk/pull/4035)). Contributed by @richvdh.
+* Support optional MSC3860 redirects ([#4007](https://github.com/matrix-org/matrix-js-sdk/pull/4007)). Contributed by @turt2live.
+
+## 🐛 Bug Fixes
+
+* WebR: migrate the megolm session imported flag ([#4037](https://github.com/matrix-org/matrix-js-sdk/pull/4037)). Contributed by @BillCarsonFr.
+* ElementR: fix emoji verification stalling when both ends hit start at the same time ([#4004](https://github.com/matrix-org/matrix-js-sdk/pull/4004)). Contributed by @uhoreg.
+* Dependencies: Bump wasm bindings version to 4.3.0 ([#4042](https://github.com/matrix-org/matrix-js-sdk/pull/4042)). Contributed by @BillCarsonFr.
+* Element R: emit events when devices have changed ([#4019](https://github.com/matrix-org/matrix-js-sdk/pull/4019)). Contributed by @uhoreg.
+* ElementR: report invalid keys rather than failing to restore from backup ([#4006](https://github.com/matrix-org/matrix-js-sdk/pull/4006)). Contributed by @uhoreg.
+* Make `timeline` a getter ([#4022](https://github.com/matrix-org/matrix-js-sdk/pull/4022)). Contributed by @florianduros.
+* Implement getting verification cancellation info in Rust crypto ([#3947](https://github.com/matrix-org/matrix-js-sdk/pull/3947)). Contributed by @uhoreg.
+* Fix crypto migration for megolm sessions with no sender key ([#4024](https://github.com/matrix-org/matrix-js-sdk/pull/4024)). Contributed by @richvdh.
+
+
+Changes in [31.2.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v31.2.0) (2024-01-31)
+==================================================================================================
+## ✨ Features
+
+* Emit events during migration from libolm ([#3982](https://github.com/matrix-org/matrix-js-sdk/pull/3982)). Contributed by @richvdh.
+* Support for migration from from libolm ([#3978](https://github.com/matrix-org/matrix-js-sdk/pull/3978)). Contributed by @richvdh.
+
+## 🐛 Bug Fixes
+
+* ElementR | backup: call expensive `roomKeyCounts` less often ([#4015](https://github.com/matrix-org/matrix-js-sdk/pull/4015)). Contributed by @BillCarsonFr.
+* Decrypt and Import full backups in chunk with progress ([#4005](https://github.com/matrix-org/matrix-js-sdk/pull/4005)). Contributed by @BillCarsonFr.
+* Fix new threads not appearing. ([#4009](https://github.com/matrix-org/matrix-js-sdk/pull/4009)). Contributed by @dbkr.
+
+
+Changes in [31.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v31.1.0) (2024-01-19)
+==================================================================================================
+## ✨ Features
+
+* Broaden spec version support ([#4016](https://github.com/matrix-org/matrix-js-sdk/pull/4016)). Contributed by @RiotRobot.
+
+
+Changes in [31.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v31.0.0) (2024-01-16)
+==================================================================================================
+## 🚨 BREAKING CHANGES
+
+* Bump minimum spec version to v1.5 ([#3970](https://github.com/matrix-org/matrix-js-sdk/pull/3970)). Contributed by @richvdh.
+
+## ✨ Features
+
+* Bump minimum spec version to v1.5 ([#3970](https://github.com/matrix-org/matrix-js-sdk/pull/3970)). Contributed by @richvdh.
+* Send authenticated /versions request ([#3968](https://github.com/matrix-org/matrix-js-sdk/pull/3968)). Contributed by @dbkr.
+
+## 🐛 Bug Fixes
+
+* Revert "Bump matrix-sdk-crypto-wasm to 3.6.0" ([#3991](https://github.com/matrix-org/matrix-js-sdk/pull/3991)). Contributed by @andybalaam.
+* #22606 Fix "Remove" button  to users without "m.room.redaction" ([#3981](https://github.com/matrix-org/matrix-js-sdk/pull/3981)). Contributed by @rashmitpankhania.
+* ElementR: Ensure Encryption order per room ([#3973](https://github.com/matrix-org/matrix-js-sdk/pull/3973)). Contributed by @BillCarsonFr.
+* Element-R: fix `bootstrapSecretStorage` not resetting key backup when requested ([#3976](https://github.com/matrix-org/matrix-js-sdk/pull/3976)). Contributed by @uhoreg.
+
+
+Changes in [30.3.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v30.3.0) (2023-12-19)
+==================================================================================================
+## ✨ Features
+
+* Element-R: disable sending room key requests ([#3939](https://github.com/matrix-org/matrix-js-sdk/pull/3939)). Contributed by @richvdh.
+
+## 🐛 Bug Fixes
+
+* Fix notifications appearing for old events ([#3946](https://github.com/matrix-org/matrix-js-sdk/pull/3946)). Contributed by @dbkr.
+* Don't back up keys that we got from backup ([#3934](https://github.com/matrix-org/matrix-js-sdk/pull/3934)). Contributed by @uhoreg.
+* Fix upload with empty Content-Type ([#3918](https://github.com/matrix-org/matrix-js-sdk/pull/3918)). Contributed by @JakubOnderka.
+* Prevent phantom notifications from events not in a room's timeline ([#3942](https://github.com/matrix-org/matrix-js-sdk/pull/3942)). Contributed by @dbkr.
+
+
+Changes in [30.2.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v30.2.0) (2023-12-05)
+==================================================================================================
+## ✨ Features
+
+* Only await key query after lazy members resolved ([#3902](https://github.com/matrix-org/matrix-js-sdk/pull/3902)). Contributed by @BillCarsonFr.
+
+## 🐛 Bug Fixes
+
+* Rewrite receipt-handling code ([#3901](https://github.com/matrix-org/matrix-js-sdk/pull/3901)). Contributed by @andybalaam.
+* Explicitly free some Rust-side objects ([#3911](https://github.com/matrix-org/matrix-js-sdk/pull/3911)). Contributed by @richvdh.
+* Fix type for TimestampToEventResponse.origin\_server\_ts ([#3906](https://github.com/matrix-org/matrix-js-sdk/pull/3906)). Contributed by @Half-Shot.
+
+
+Changes in [30.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v30.1.0) (2023-11-21)
+==================================================================================================
+## ✨ Features
+
+* Rotate per-participant keys when a member leaves ([#3833](https://github.com/matrix-org/matrix-js-sdk/pull/3833)). Contributed by @dbkr.
+* Add E2EE for embedded mode of Element Call ([#3667](https://github.com/matrix-org/matrix-js-sdk/pull/3667)). Contributed by @SimonBrandner.
+
+## 🐛 Bug Fixes
+
+* Shorten TimelineWindow when an event is removed ([#3862](https://github.com/matrix-org/matrix-js-sdk/pull/3862)). Contributed by @andybalaam.
+* Ignore receipts pointing at missing or invalid events ([#3817](https://github.com/matrix-org/matrix-js-sdk/pull/3817)). Contributed by @andybalaam.
+* Fix members being loaded from server on initial sync (defeating lazy loading) ([#3830](https://github.com/matrix-org/matrix-js-sdk/pull/3830)). Contributed by @BillCarsonFr.
+
+
+Changes in [30.0.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v30.0.1) (2023-11-13)
+==================================================================================================
+
+## 🐛 Bug Fixes
+ * Ensure `setUserCreator` is called when a store is assigned ([\#3867](https://github.com/matrix-org/matrix-js-sdk/pull/3867)). Fixes vector-im/element-web#26520. Contributed by @MidhunSureshR.
+
+Changes in [30.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v30.0.0) (2023-11-07)
+==================================================================================================
+
+## 🚨 BREAKING CHANGES
+ * Refactor & make base64 functions browser-safe ([\#3818](https://github.com/matrix-org/matrix-js-sdk/pull/3818)).
+ * `IndexedDBStore.startup()` must be called after using it on `sdk.createClient` now.
+
+## 🦖 Deprecations
+ * Deprecate `MatrixEvent.toJSON` ([\#3801](https://github.com/matrix-org/matrix-js-sdk/pull/3801)).
+
+## ✨ Features
+ * Element-R: Add the git sha of the binding crate to `CryptoApi#getVersion` ([\#3838](https://github.com/matrix-org/matrix-js-sdk/pull/3838)). Contributed by @florianduros.
+ * Element-R: Wire up `globalBlacklistUnverifiedDevices` field to rust crypto encryption settings ([\#3790](https://github.com/matrix-org/matrix-js-sdk/pull/3790)). Fixes vector-im/element-web#26315. Contributed by @florianduros.
+ * Element-R: Wire up room rotation ([\#3807](https://github.com/matrix-org/matrix-js-sdk/pull/3807)). Fixes vector-im/element-web#26318. Contributed by @florianduros.
+ * Element-R: Add current version of the rust-sdk and vodozemac ([\#3825](https://github.com/matrix-org/matrix-js-sdk/pull/3825)). Contributed by @florianduros.
+ * Element-R: Wire up room history visibility ([\#3805](https://github.com/matrix-org/matrix-js-sdk/pull/3805)). Fixes vector-im/element-web#26319. Contributed by @florianduros.
+ * Element-R: log when we send to-device messages ([\#3810](https://github.com/matrix-org/matrix-js-sdk/pull/3810)).
+
+## 🐛 Bug Fixes
+ * Fix reemitter not being correctly wired on user objects created in storage classes ([\#3796](https://github.com/matrix-org/matrix-js-sdk/pull/3796)). Contributed by @MidhunSureshR.
+ * Element-R: silence log errors when viewing a pending event ([\#3824](https://github.com/matrix-org/matrix-js-sdk/pull/3824)).
+ * Don't emit a closed event if the indexeddb is closed by Element ([\#3832](https://github.com/matrix-org/matrix-js-sdk/pull/3832)). Fixes vector-im/element-web#25941. Contributed by @dhenneke.
+ * Element-R: silence log errors when viewing a decryption failure ([\#3821](https://github.com/matrix-org/matrix-js-sdk/pull/3821)).
+
+Changes in [29.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v29.1.0) (2023-10-24)
+==================================================================================================
+
+## ✨ Features
+ * OIDC: refresh tokens ([\#3764](https://github.com/matrix-org/matrix-js-sdk/pull/3764)). Contributed by @kerryarchibald.
+ * OIDC: add `prompt` param to auth url creation ([\#3794](https://github.com/matrix-org/matrix-js-sdk/pull/3794)). Contributed by @kerryarchibald.
+ * Allow applications to specify their own logger instance ([\#3792](https://github.com/matrix-org/matrix-js-sdk/pull/3792)). Fixes #1899.
+ * Export AutoDiscoveryError and fix type of ALL_ERRORS ([\#3768](https://github.com/matrix-org/matrix-js-sdk/pull/3768)).
+
+## 🐛 Bug Fixes
+ * Fix sending call member events on leave ([\#3799](https://github.com/matrix-org/matrix-js-sdk/pull/3799)). Fixes vector-im/element-call#1763.
+ * Don't use event.sender in CallMembership ([\#3793](https://github.com/matrix-org/matrix-js-sdk/pull/3793)).
+ * Element-R: Don't mark QR code verification as done until it's done ([\#3791](https://github.com/matrix-org/matrix-js-sdk/pull/3791)). Fixes vector-im/element-web#26293.
+ * Element-R: Connect device to key backup when crypto is created ([\#3784](https://github.com/matrix-org/matrix-js-sdk/pull/3784)). Fixes vector-im/element-web#26316. Contributed by @florianduros.
+ * Element-R: Avoid errors in `VerificationRequest.generateQRCode` when QR code is unavailable ([\#3779](https://github.com/matrix-org/matrix-js-sdk/pull/3779)). Fixes vector-im/element-web#26300. Contributed by @florianduros.
+ * ElementR: Check key backup when user identity changes ([\#3760](https://github.com/matrix-org/matrix-js-sdk/pull/3760)). Fixes vector-im/element-web#26244. Contributed by @florianduros.
+ * Element-R: emit `VerificationRequestReceived` on incoming request ([\#3762](https://github.com/matrix-org/matrix-js-sdk/pull/3762)). Fixes vector-im/element-web#26245.
+
+Changes in [29.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v29.0.0) (2023-10-10)
+==================================================================================================
+
+## 🚨 BREAKING CHANGES
+ * Remove browserify builds ([\#3759](https://github.com/matrix-org/matrix-js-sdk/pull/3759)).
+
+## ✨ Features
+ * Export AutoDiscoveryError and fix type of ALL_ERRORS ([\#3768](https://github.com/matrix-org/matrix-js-sdk/pull/3768)).
+ * Support for stable MSC3882 get_login_token ([\#3416](https://github.com/matrix-org/matrix-js-sdk/pull/3416)). Contributed by @hughns.
+ * Remove IsUserMention and IsRoomMention from DEFAULT_OVERRIDE_RULES ([\#3752](https://github.com/matrix-org/matrix-js-sdk/pull/3752)). Contributed by @kerryarchibald.
+
+## 🐛 Bug Fixes
+ * Fix a case where joinRoom creates a duplicate Room object ([\#3747](https://github.com/matrix-org/matrix-js-sdk/pull/3747)).
+ * Add membershipID to call memberships ([\#3745](https://github.com/matrix-org/matrix-js-sdk/pull/3745)).
+ * Fix the warning for messages from unsigned devices ([\#3743](https://github.com/matrix-org/matrix-js-sdk/pull/3743)).
+ * Stop keep alive, when sync was stoped ([\#3720](https://github.com/matrix-org/matrix-js-sdk/pull/3720)). Contributed by @finsterwalder.
+
+Changes in [28.2.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v28.2.0) (2023-09-26)
+==================================================================================================
+
+## 🦖 Deprecations
+ * Implement `getEncryptionInfoForEvent` and deprecate `getEventEncryptionInfo` ([\#3693](https://github.com/matrix-org/matrix-js-sdk/pull/3693)).
+ * **The Browserify artifact is being deprecated, scheduled for removal in the October 10th release cycle. (#3189)**
+
+## ✨ Features
+ * Delete knocked room when knock membership changes ([\#3729](https://github.com/matrix-org/matrix-js-sdk/pull/3729)). Contributed by @maheichyk.
+ * Introduce MatrixRTCSession lower level group call primitive ([\#3663](https://github.com/matrix-org/matrix-js-sdk/pull/3663)).
+ * Sync knock rooms ([\#3703](https://github.com/matrix-org/matrix-js-sdk/pull/3703)). Contributed by @maheichyk.
+
+## 🐛 Bug Fixes
+ * Dont access indexed db when undefined ([\#3707](https://github.com/matrix-org/matrix-js-sdk/pull/3707)). Contributed by @finsterwalder.
+ * Don't reset unread count when adding a synthetic receipt ([\#3706](https://github.com/matrix-org/matrix-js-sdk/pull/3706)). Fixes #3684. Contributed by @andybalaam.
+
+Changes in [28.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v28.1.0) (2023-09-12)
+============================================================================================================
+
+## 🦖 Deprecations
+ * Deprecate `MatrixClient.checkUserTrust` ([\#3691](https://github.com/matrix-org/matrix-js-sdk/pull/3691)).
+ * Deprecate `MatrixClient.{prepare,create}KeyBackupVersion` in favour of new `CryptoApi.resetKeyBackup` API ([\#3689](https://github.com/matrix-org/matrix-js-sdk/pull/3689)).
+ * **The Browserify artifact is being deprecated, scheduled for removal in the October 10th release cycle. (#3189)**
+
+## ✨ Features
+ * Allow calls without ICE/TURN/STUN servers ([\#3695](https://github.com/matrix-org/matrix-js-sdk/pull/3695)).
+ * Emit summary update event ([\#3687](https://github.com/matrix-org/matrix-js-sdk/pull/3687)). Fixes vector-im/element-web#26033.
+ * ElementR: Update `CryptoApi.userHasCrossSigningKeys` ([\#3646](https://github.com/matrix-org/matrix-js-sdk/pull/3646)). Contributed by @florianduros.
+ * Add `join_rule` field to /publicRooms response ([\#3673](https://github.com/matrix-org/matrix-js-sdk/pull/3673)). Contributed by @charlynguyen.
+ * Use sender instead of content.creator field on m.room.create events ([\#3675](https://github.com/matrix-org/matrix-js-sdk/pull/3675)).
+
+## 🐛 Bug Fixes
+ * Provide better error for ICE Server SyntaxError ([\#3694](https://github.com/matrix-org/matrix-js-sdk/pull/3694)). Fixes vector-im/element-web#21804.
+ * Legacy crypto: re-check key backup after `bootstrapSecretStorage` ([\#3692](https://github.com/matrix-org/matrix-js-sdk/pull/3692)). Fixes vector-im/element-web#26115.
+
 Changes in [28.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v28.0.0) (2023-08-29)
 ==================================================================================================
 

README.md

@@ -21,28 +21,6 @@ endpoints from before Matrix 1.1, for example.
 
 # Quickstart
 
-## In a browser
-
-### Note, the browserify build has been deprecated. Please use a bundler like webpack or vite instead.
-
-Download the browser version from
-https://github.com/matrix-org/matrix-js-sdk/releases/latest and add that as a
-`<script>` to your page. There will be a global variable `matrixcs`
-attached to `window` through which you can access the SDK. See below for how to
-include libolm to enable end-to-end-encryption.
-
-The browser bundle supports recent versions of browsers. Typically this is ES2015
-or `> 0.5%, last 2 versions, Firefox ESR, not dead` if using
-[browserlists](https://github.com/browserslist/browserslist).
-
-Please check [the working browser example](examples/browser) for more information.
-
-## In Node.js
-
-Ensure you have the latest LTS version of Node.js installed.
-This library relies on `fetch` which is available in Node from v18.0.0 - it should work fine also with polyfills.
-If you wish to use a ponyfill or adapter of some sort then pass it as `fetchFn` to the MatrixClient constructor options.
-
 Using `yarn` instead of `npm` is recommended. Please see the Yarn [install guide](https://classic.yarnpkg.com/en/docs/install)
 if you do not have it already.
 
@@ -59,8 +37,6 @@ client.publicRooms(function (err, data) {
 See below for how to include libolm to enable end-to-end-encryption. Please check
 [the Node.js terminal app](examples/node) for a more complex example.
 
-You can also use the sdk with [Deno](https://deno.land/) (`import npm:matrix-js-sdk`) but its not officialy supported.
-
 To start the client:
 
 ```javascript
@@ -70,7 +46,7 @@ await client.startClient({ initialSyncLimit: 10 });
 You can perform a call to `/sync` to get the current state of the client:
 
 ```javascript
-client.once("sync", function (state, prevState, res) {
+client.once(ClientEvent.sync, function (state, prevState, res) {
     if (state === "PREPARED") {
         console.log("prepared");
     } else {
@@ -95,7 +71,7 @@ client.sendEvent("roomId", "m.room.message", content, "", (err, res) => {
 To listen for message events:
 
 ```javascript
-client.on("Room.timeline", function (event, room, toStartOfTimeline) {
+client.on(RoomEvent.Timeline, function (event, room, toStartOfTimeline) {
     if (event.getType() !== "m.room.message") {
         return; // only use messages
     }
@@ -118,7 +94,7 @@ Object.keys(client.store.rooms).forEach((roomId) => {
 This SDK provides a full object model around the Matrix Client-Server API and emits
 events for incoming data and state changes. Aside from wrapping the HTTP API, it:
 
--   Handles syncing (via `/initialSync` and `/events`)
+-   Handles syncing (via `/sync`)
 -   Handles the generation of "friendly" room and member names.
 -   Handles historical `RoomMember` information (e.g. display names).
 -   Manages room member state across multiple events (e.g. it handles typing, power
@@ -139,29 +115,29 @@ events for incoming data and state changes. Aside from wrapping the HTTP API, it
 -   Handles room initial sync on accepting invites.
 -   Handles WebRTC calling.
 
-Later versions of the SDK will:
-
--   Expose a `RoomSummary` which would be suitable for a recents page.
--   Provide different pluggable storage layers (e.g. local storage, database-backed)
-
 # Usage
 
-## Conventions
+## Supported platforms
 
-### Emitted events
+`matrix-js-sdk` can be used in either Node.js applications (ensure you have the latest LTS version of Node.js installed),
+or in browser applications, via a bundler such as Webpack or Vite.
 
-The SDK will emit events using an `EventEmitter`. It also
-emits object models (e.g. `Rooms`, `RoomMembers`) when they
-are updated.
+You can also use the sdk with [Deno](https://deno.land/) (`import npm:matrix-js-sdk`) but its not officialy supported.
+
+## Emitted events
+
+The SDK raises notifications to the application using
+[`EventEmitter`s](https://nodejs.org/api/events.html#class-eventemitter). The `MatrixClient` itself
+implements `EventEmitter`, as do many of the high-level abstractions such as `Room` and `RoomMember`.
 
 ```javascript
 // Listen for low-level MatrixEvents
-client.on("event", function (event) {
+client.on(ClientEvent.Event, function (event) {
     console.log(event.getType());
 });
 
 // Listen for typing changes
-client.on("RoomMember.typing", function (event, member) {
+client.on(RoomMemberEvent.Typing, function (event, member) {
     if (member.typing) {
         console.log(member.name + " is typing...");
     } else {
@@ -173,41 +149,21 @@ client.on("RoomMember.typing", function (event, member) {
 client.startClient();
 ```
 
-### Promises and Callbacks
+## Entry points
 
-Most of the methods in the SDK are asynchronous: they do not directly return a
-result, but instead return a [Promise](http://documentup.com/kriskowal/q/)
-which will be fulfilled in the future.
+As well as the primary entry point (`matrix-js-sdk`), there are several other entry points which may be useful:
 
-The typical usage is something like:
-
-```javascript
-  matrixClient.someMethod(arg1, arg2).then(function(result) {
-    ...
-  });
-```
-
-Alternatively, if you have a Node.js-style `callback(err, result)` function,
-you can pass the result of the promise into it with something like:
-
-```javascript
-matrixClient.someMethod(arg1, arg2).nodeify(callback);
-```
-
-The main thing to note is that it is problematic to discard the result of a
-promise-returning function, as that will cause exceptions to go unobserved.
-
-Methods which return a promise show this in their documentation.
-
-Many methods in the SDK support _both_ Node.js-style callbacks _and_ Promises,
-via an optional `callback` argument. The callback support is now deprecated:
-new methods do not include a `callback` argument, and in the future it may be
-removed from existing methods.
+| Entry point                    | Description                                                                                         |
+| ------------------------------ | --------------------------------------------------------------------------------------------------- |
+| `matrix-js-sdk`                | Primary entry point. High-level functionality, and lots of historical clutter in need of a cleanup. |
+| `matrix-js-sdk/lib/crypto-api` | Cryptography functionality.                                                                         |
+| `matrix-js-sdk/lib/types`      | Low-level types, reflecting data structures defined in the Matrix spec.                             |
+| `matrix-js-sdk/lib/testing`    | Test utilities, which may be useful in test code but should not be used in production code.         |
 
 ## Examples
 
 This section provides some useful code snippets which demonstrate the
-core functionality of the SDK. These examples assume the SDK is setup like this:
+core functionality of the SDK. These examples assume the SDK is set up like this:
 
 ```javascript
 import * as sdk from "matrix-js-sdk";
@@ -223,10 +179,10 @@ const matrixClient = sdk.createClient({
 ### Automatically join rooms when invited
 
 ```javascript
-matrixClient.on("RoomMember.membership", function (event, member) {
-    if (member.membership === "invite" && member.userId === myUserId) {
-        matrixClient.joinRoom(member.roomId).then(function () {
-            console.log("Auto-joined %s", member.roomId);
+matrixClient.on(RoomEvent.MyMembership, function (room, membership, prevMembership) {
+    if (membership === KnownMembership.Invite) {
+        matrixClient.joinRoom(room.roomId).then(function () {
+            console.log("Auto-joined %s", room.roomId);
         });
     }
 });
@@ -237,7 +193,7 @@ matrixClient.startClient();
 ### Print out messages for all rooms
 
 ```javascript
-matrixClient.on("Room.timeline", function (event, room, toStartOfTimeline) {
+matrixClient.on(RoomEvent.Timeline, function (event, room, toStartOfTimeline) {
     if (toStartOfTimeline) {
         return; // don't print paginated results
     }
@@ -269,7 +225,7 @@ Output:
 ### Print out membership lists whenever they are changed
 
 ```javascript
-matrixClient.on("RoomState.members", function (event, state, member) {
+matrixClient.on(RoomStateEvent.Members, function (event, state, member) {
     const room = matrixClient.getRoom(state.roomId);
     if (!room) {
         return;
@@ -306,7 +262,7 @@ host the API reference from the source files like this:
 
 ```
   $ yarn gendoc
-  $ cd _docs
+  $ cd docs
   $ python -m http.server 8005
 ```
 
@@ -314,6 +270,9 @@ Then visit `http://localhost:8005` to see the API docs.
 
 # End-to-end encryption support
 
+**This section is outdated.** Use of `libolm` is deprecated and we are replacing it with support
+from the matrix-rust-sdk (https://github.com/element-hq/element-web/issues/21972).
+
 The SDK supports end-to-end encryption via the Olm and Megolm protocols, using
 [libolm](https://gitlab.matrix.org/matrix-org/olm). It is left up to the
 application to make libolm available, via the `Olm` global.
@@ -363,7 +322,7 @@ First, you need to pull in the right build tools:
 
 ## Building
 
-To build a browser version from scratch when developing::
+To build a browser version from scratch when developing:
 
 ```
  $ yarn build
@@ -375,9 +334,6 @@ To run tests (Jest):
  $ yarn test
 ```
 
-> **Note**
-> The `sync-browserify.spec.ts` requires a browser build (`yarn build`) in order to pass
-
 To run linting:
 
 ```

docs/SUMMARY.md

@@ -0,0 +1,8 @@
+# Summary
+
+-   [Introduction](../README.md)
+
+# Deep dive
+
+-   [Storage notes](storage-notes.md)
+-   [Unverified devices](warning-on-unverified-devices.md)

docs/warning-on-unverified-devices.md

@@ -1,31 +1,29 @@
 Random notes from Matthew on the two possible approaches for warning users about unexpected
 unverified devices popping up in their rooms....
 
-Original idea...
-================
+# Original idea...
 
 Warn when an existing user adds an unknown device to a room.
 
 Warn when a user joins the room with unverified or unknown devices.
 
 Warn when you initial sync if the room has any unverified devices in it.
- ^ this is good enough if we're doing local storage.
- OR, better:
+^ this is good enough if we're doing local storage.
+OR, better:
 Warn when you initial sync if the room has any new undefined devices since you were last there.
- => This means persisting the rooms that devices are in, across initial syncs.
+=> This means persisting the rooms that devices are in, across initial syncs.
 
-
-Updated idea...
-===============
+# Updated idea...
 
 Warn when the user tries to send a message:
-  - If the room has unverified devices which the user has not yet been told about in the context of this room
-    ...or in the context of this user?  currently all verification is per-user, not per-room.
+
+-   If the room has unverified devices which the user has not yet been told about in the context of this room
+    ...or in the context of this user? currently all verification is per-user, not per-room.
     ...this should be good enough.
 
-  - so track whether we have warned the user or not about unverified devices - blocked, unverified, verified, unverified_warned.
+-   so track whether we have warned the user or not about unverified devices - blocked, unverified, verified, unverified_warned.
     throw an error when trying to encrypt if there are pure unverified devices there
     app will have to search for the devices which are pure unverified to warn about them - have to do this from MembersList anyway?
-      - or megolm could warn which devices are causing the problems.
+    -   or megolm could warn which devices are causing the problems.
 
-Why do we wait to establish outbound sessions?  It just makes a horrible pause when we first try to send a message... but could otherwise unnecessarily consume resources?
+Why do we wait to establish outbound sessions? It just makes a horrible pause when we first try to send a message... but could otherwise unnecessarily consume resources?

examples/browser/README.md

@@ -1,10 +0,0 @@
-To try it out, **you must build the SDK first** and then host this folder:
-
-```
- $ yarn install
- $ yarn build
- $ cd examples/browser
- $ python -m http.server 8003
-```
-
-Then visit `http://localhost:8003`.

examples/browser/browserTest.js

@@ -1,9 +0,0 @@
-console.log("Loading browser sdk");
-
-var client = matrixcs.createClient({ baseUrl: "https://matrix.org" });
-client.publicRooms().then(function (data) {
-    console.log("data %s [...]", JSON.stringify(data).substring(0, 100));
-    console.log("Congratulations! The SDK is working on the browser!");
-    var result = document.getElementById("result");
-    result.innerHTML = "<p>The SDK appears to be working correctly.</p>";
-});

examples/browser/index.html

@@ -1,17 +0,0 @@
-<html lang="en">
-    <head>
-        <title>Test</title>
-        <meta charset="utf-8" />
-        <link rel="icon" href="data:," />
-        <script src="lib/matrix.js"></script>
-        <script src="browserTest.js"></script>
-    </head>
-    <body>
-        Sanity Testing (check the console) : This example is here to make sure that the SDK works inside a browser. It
-        simply does a GET /publicRooms on matrix.org
-        <br />
-        You should see a message confirming that the SDK works below:
-        <br />
-        <div id="result"></div>
-    </body>
-</html>

examples/browser/lib/matrix.js

@@ -1 +0,0 @@
-../../../dist/browser-matrix.js

examples/crypto-browser/lib/.gitignore

@@ -1,2 +0,0 @@
-olm.js
-olm.wasm

examples/crypto-browser/lib/matrix.js

@@ -1 +0,0 @@
-../../../dist/browser-matrix.js

examples/crypto-browser/olm-device-export-import.html

@@ -1,60 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-    <head>
-        <meta charset="UTF-8" />
-        <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-        <meta http-equiv="X-UA-Compatible" content="ie=edge" />
-        <title>Test Crypto in Browser</title>
-        <script src="lib/olm.js"></script>
-        <script src="lib/matrix.js"></script>
-    </head>
-    <body>
-        <h1>Testing export/import of Olm devices in the browser</h1>
-        <ul>
-            <li>Make sure you built the current version of the Matrix JS SDK (<code>yarn build</code>)</li>
-            <li>
-                copy <code>olm.js</code> and <code>olm.wasm</code> from a recent release of Olm (was tested with version
-                3.1.4) in directory <code>lib/</code>
-            </li>
-            <li>start a local Matrix homeserver (on port 8008, or change the port in the code)</li>
-            <li>Serve this HTML file (e.g. <code>python3 -m http.server</code>) and go to it through your browser</li>
-            <li>
-                in the JS console, do:
-                <pre>
-aliceMatrixClient = await newMatrixClient("alice-"+randomHex());
-await aliceMatrixClient.exportDevice();
-await aliceMatrixClient.getAccessToken();
-            </pre
-                >
-            </li>
-            <li>
-                copy the result of <code>exportDevice</code> and <code>getAccessToken</code> somewhere (<strong
-                    >not</strong
-                >
-                in a JS variable as it will be destroyed when you refresh the page)
-            </li>
-            <li><strong>refresh the page (F5)</strong> to make sure the client is destroyed</li>
-            <li>
-                Do the following, replacing <code>ALICE_ID</code>
-                with the user ID of Alice (you can find it in the exported data)
-                <pre>
-bobMatrixClient = await newMatrixClient("bob-"+randomHex());
-roomId = await bobMatrixClient.createEncryptedRoom([ALICE_ID]);
-await bobMatrixClient.sendTextMessage('Hi Alice!', roomId);
-            </pre
-                >
-            </li>
-            <li>Again, <strong>refresh the page (F5)</strong>. You may want to clear your console as well.</li>
-            <li>
-                Now do the following, using the exported data and the access token you saved previously:
-                <pre>
-aliceMatrixClient = await importMatrixClient(EXPORTED_DATA, ACCESS_TOKEN);
-            </pre
-                >
-            </li>
-            <li>You should see the message sent by Bob printed in the console.</li>
-        </ul>
-
-        <script src="olm-device-export-import.js"></script>
-    </body>
-</html>

examples/crypto-browser/olm-device-export-import.js

@@ -1,105 +0,0 @@
-if (!Olm) {
-    console.error("global.Olm does not seem to be present." + " Did you forget to add olm in the lib/ directory?");
-}
-
-const BASE_URL = "http://localhost:8008";
-const ROOM_CRYPTO_CONFIG = { algorithm: "m.megolm.v1.aes-sha2" };
-const PASSWORD = "password";
-
-// useful to create new usernames
-window.randomHex = () => Math.floor(Math.random() * 10 ** 6).toString(16);
-
-window.newMatrixClient = async function (username) {
-    const registrationClient = matrixcs.createClient(BASE_URL);
-
-    const userRegisterResult = await registrationClient.register(username, PASSWORD, null, { type: "m.login.dummy" });
-
-    const matrixClient = matrixcs.createClient({
-        baseUrl: BASE_URL,
-        userId: userRegisterResult.user_id,
-        accessToken: userRegisterResult.access_token,
-        deviceId: userRegisterResult.device_id,
-        sessionStore: new matrixcs.WebStorageSessionStore(window.localStorage),
-        cryptoStore: new matrixcs.MemoryCryptoStore(),
-    });
-
-    extendMatrixClient(matrixClient);
-
-    await matrixClient.initCrypto();
-    await matrixClient.startClient();
-    return matrixClient;
-};
-
-window.importMatrixClient = async function (exportedDevice, accessToken) {
-    const matrixClient = matrixcs.createClient({
-        baseUrl: BASE_URL,
-        deviceToImport: exportedDevice,
-        accessToken,
-        sessionStore: new matrixcs.WebStorageSessionStore(window.localStorage),
-        cryptoStore: new matrixcs.MemoryCryptoStore(),
-    });
-
-    extendMatrixClient(matrixClient);
-
-    await matrixClient.initCrypto();
-    await matrixClient.startClient();
-    return matrixClient;
-};
-
-function extendMatrixClient(matrixClient) {
-    // automatic join
-    matrixClient.on("RoomMember.membership", async (event, member) => {
-        if (member.membership === "invite" && member.userId === matrixClient.getUserId()) {
-            await matrixClient.joinRoom(member.roomId);
-            // setting up of room encryption seems to be triggered automatically
-            // but if we don't wait for it the first messages we send are unencrypted
-            await matrixClient.setRoomEncryption(member.roomId, { algorithm: "m.megolm.v1.aes-sha2" });
-        }
-    });
-
-    matrixClient.onDecryptedMessage = (message) => {
-        console.log("Got encrypted message: ", message);
-    };
-
-    matrixClient.on("Event.decrypted", (event) => {
-        if (event.getType() === "m.room.message") {
-            matrixClient.onDecryptedMessage(event.getContent().body);
-        } else {
-            console.log("decrypted an event of type", event.getType());
-            console.log(event);
-        }
-    });
-
-    matrixClient.createEncryptedRoom = async function (usersToInvite) {
-        const { room_id: roomId } = await this.createRoom({
-            visibility: "private",
-            invite: usersToInvite,
-        });
-
-        // matrixClient.setRoomEncryption() only updates local state
-        // but does not send anything to the server
-        // (see https://github.com/matrix-org/matrix-js-sdk/issues/905)
-        // so we do it ourselves with 'sendStateEvent'
-        await this.sendStateEvent(roomId, "m.room.encryption", ROOM_CRYPTO_CONFIG);
-        await this.setRoomEncryption(roomId, ROOM_CRYPTO_CONFIG);
-
-        // Marking all devices as verified
-        let room = this.getRoom(roomId);
-        let members = (await room.getEncryptionTargetMembers()).map((x) => x["userId"]);
-        let memberkeys = await this.downloadKeys(members);
-        for (const userId in memberkeys) {
-            for (const deviceId in memberkeys[userId]) {
-                await this.setDeviceVerified(userId, deviceId);
-            }
-        }
-
-        return roomId;
-    };
-
-    matrixClient.sendTextMessage = async function (message, roomId) {
-        return matrixClient.sendMessage(roomId, {
-            body: message,
-            msgtype: "m.text",
-        });
-    };
-}

examples/node/app.js

@@ -115,7 +115,7 @@ rl.on("line", function (line) {
         if (line.indexOf("/join ") === 0) {
             var roomIndex = line.split(" ")[1];
             viewingRoom = roomList[roomIndex];
-            if (viewingRoom.getMember(myUserId).membership === "invite") {
+            if (viewingRoom.getMember(myUserId).membership === KnownMembership.Invite) {
                 // join the room first
                 matrixClient.joinRoom(viewingRoom.roomId).then(
                     function (room) {

knip.ts

@@ -0,0 +1,36 @@
+import { KnipConfig } from "knip";
+
+export default {
+    entry: [
+        "src/index.ts",
+        "src/types.ts",
+        "src/browser-index.ts",
+        "src/indexeddb-worker.ts",
+        "scripts/**",
+        "spec/**",
+        "release.sh",
+        // For now, we include all source files as entrypoints as we have been bad about gutwrenched imports
+        "src/**",
+    ],
+    project: ["**/*.{js,ts}"],
+    ignore: ["examples/**"],
+    ignoreDependencies: [
+        // Required for `action-validator`
+        "@action-validator/*",
+        // Used for git pre-commit hooks
+        "husky",
+        // Used in script which only runs in environment with `@octokit/rest` installed
+        "@octokit/rest",
+        // Used by jest
+        "jest-environment-jsdom",
+        "babel-jest",
+        "ts-node",
+        // Used by `@babel/plugin-transform-runtime`
+        "@babel/runtime",
+    ],
+    ignoreBinaries: [
+        // Used when available by reusable workflow `.github/workflows/release-make.yml`
+        "dist",
+    ],
+    ignoreExportsUsedInFile: true,
+} satisfies KnipConfig;

package.json

@@ -1,26 +1,25 @@
 {
     "name": "matrix-js-sdk",
-    "version": "28.0.0",
+    "version": "32.2.0",
     "description": "Matrix Client-Server SDK for Javascript",
     "engines": {
         "node": ">=18.0.0"
     },
     "scripts": {
-        "prepublishOnly": "yarn build",
+        "prepack": "yarn build",
         "start": "echo THIS IS FOR LEGACY PURPOSES ONLY. && babel src -w -s -d lib --verbose --extensions \".ts,.js\"",
-        "dist": "echo 'This is for the release script so it can make assets (browser bundle).' && yarn build",
-        "clean": "rimraf lib dist",
-        "build": "yarn build:dev && yarn build:compile-browser && yarn build:minify-browser",
+        "clean": "rimraf lib",
+        "build": "yarn build:dev",
         "build:dev": "yarn clean && git rev-parse HEAD > git-revision.txt && yarn build:compile && yarn build:types",
         "build:types": "tsc -p tsconfig-build.json --emitDeclarationOnly",
         "build:compile": "babel -d lib --verbose --extensions \".ts,.js\" src",
-        "build:compile-browser": "mkdir dist && BROWSERIFYSWAP_ENV='no-rust-crypto' browserify -d src/browser-index.ts -p [ tsify -p ./tsconfig-build.json ] | exorcist dist/browser-matrix.js.map > dist/browser-matrix.js",
-        "build:minify-browser": "terser dist/browser-matrix.js --compress --mangle --source-map --output dist/browser-matrix.min.js",
         "gendoc": "typedoc",
-        "lint": "yarn lint:types && yarn lint:js",
+        "lint": "yarn lint:types && yarn lint:js && yarn lint:workflows",
         "lint:js": "eslint --max-warnings 0 src spec && prettier --check .",
-        "lint:js-fix": "prettier --loglevel=warn --write . && eslint --fix src spec",
+        "lint:js-fix": "prettier --log-level=warn --write . && eslint --fix src spec",
         "lint:types": "tsc --noEmit",
+        "lint:workflows": "find .github/workflows -type f \\( -iname '*.yaml' -o -iname '*.yml' \\) | xargs -I {} sh -c 'echo \"Linting {}\"; action-validator \"{}\"'",
+        "lint:knip": "knip",
         "test": "jest",
         "test:watch": "jest --watch",
         "coverage": "yarn test --coverage"
@@ -42,7 +41,6 @@
     "author": "matrix.org",
     "license": "Apache-2.0",
     "files": [
-        "dist",
         "lib",
         "src",
         "git-revision.txt",
@@ -55,21 +53,23 @@
     ],
     "dependencies": {
         "@babel/runtime": "^7.12.5",
-        "@matrix-org/matrix-sdk-crypto-wasm": "^1.2.1",
+        "@matrix-org/matrix-sdk-crypto-wasm": "^4.9.0",
         "another-json": "^0.2.0",
         "bs58": "^5.0.0",
         "content-type": "^1.0.4",
-        "jwt-decode": "^3.1.2",
+        "jwt-decode": "^4.0.0",
         "loglevel": "^1.7.1",
         "matrix-events-sdk": "0.0.1",
-        "matrix-widget-api": "^1.5.0",
-        "oidc-client-ts": "^2.2.4",
+        "matrix-widget-api": "^1.6.0",
+        "oidc-client-ts": "^3.0.1",
         "p-retry": "4",
         "sdp-transform": "^2.14.1",
         "unhomoglyph": "^1.0.6",
         "uuid": "9"
     },
     "devDependencies": {
+        "@action-validator/cli": "^0.6.0",
+        "@action-validator/core": "^0.6.0",
         "@babel/cli": "^7.12.10",
         "@babel/core": "^7.12.10",
         "@babel/eslint-parser": "^7.12.10",
@@ -81,9 +81,9 @@
         "@babel/plugin-transform-runtime": "^7.12.10",
         "@babel/preset-env": "^7.12.11",
         "@babel/preset-typescript": "^7.12.7",
-        "@babel/register": "^7.12.10",
-        "@casualbot/jest-sonar-reporter": "^2.2.5",
-        "@matrix-org/olm": "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.14.tgz",
+        "@casualbot/jest-sonar-reporter": "2.2.7",
+        "@matrix-org/olm": "3.2.15",
+        "@peculiar/webcrypto": "^1.4.5",
         "@types/bs58": "^4.0.1",
         "@types/content-type": "^1.1.5",
         "@types/debug": "^4.1.7",
@@ -92,70 +92,46 @@
         "@types/node": "18",
         "@types/sdp-transform": "^2.4.5",
         "@types/uuid": "9",
-        "@typescript-eslint/eslint-plugin": "^5.45.0",
-        "@typescript-eslint/parser": "^5.45.0",
-        "allchange": "^1.0.6",
+        "@typescript-eslint/eslint-plugin": "^7.0.0",
+        "@typescript-eslint/parser": "^7.0.0",
         "babel-jest": "^29.0.0",
-        "babelify": "^10.0.0",
-        "browserify": "^17.0.0",
-        "browserify-swap": "^0.2.2",
         "debug": "^4.3.4",
         "domexception": "^4.0.0",
-        "eslint": "8.46.0",
+        "eslint": "8.57.0",
         "eslint-config-google": "^0.14.0",
         "eslint-config-prettier": "^9.0.0",
         "eslint-import-resolver-typescript": "^3.5.1",
         "eslint-plugin-import": "^2.26.0",
-        "eslint-plugin-jest": "^27.1.6",
-        "eslint-plugin-jsdoc": "^46.0.0",
+        "eslint-plugin-jest": "^28.0.0",
+        "eslint-plugin-jsdoc": "^48.0.0",
         "eslint-plugin-matrix-org": "^1.0.0",
         "eslint-plugin-tsdoc": "^0.2.17",
-        "eslint-plugin-unicorn": "^48.0.0",
-        "exorcist": "^2.0.0",
-        "fake-indexeddb": "^4.0.0",
+        "eslint-plugin-unicorn": "^52.0.0",
+        "fake-indexeddb": "^5.0.2",
+        "fetch-mock": "9.11.0",
         "fetch-mock-jest": "^1.5.1",
+        "husky": "^9.0.0",
         "jest": "^29.0.0",
         "jest-environment-jsdom": "^29.0.0",
         "jest-localstorage-mock": "^2.4.6",
         "jest-mock": "^29.0.0",
+        "knip": "^5.0.0",
+        "lint-staged": "^15.0.2",
         "matrix-mock-request": "^2.5.0",
-        "prettier": "2.8.8",
+        "node-fetch": "^2.7.0",
+        "prettier": "3.2.5",
         "rimraf": "^5.0.0",
-        "terser": "^5.5.1",
-        "ts-node": "^10.9.1",
-        "tsify": "^5.0.2",
-        "typedoc": "^0.24.0",
-        "typedoc-plugin-coverage": "^2.1.0",
+        "ts-node": "^10.9.2",
+        "typedoc": "^0.25.10",
+        "typedoc-plugin-coverage": "^3.0.0",
         "typedoc-plugin-mdn-links": "^3.0.3",
         "typedoc-plugin-missing-exports": "^2.0.0",
-        "typedoc-plugin-versions": "^0.2.3",
-        "typedoc-plugin-versions-cli": "^0.1.12",
-        "typescript": "^5.0.0"
+        "typescript": "^5.3.3"
     },
     "@casualbot/jest-sonar-reporter": {
         "outputDirectory": "coverage",
         "outputName": "jest-sonar-report.xml",
         "relativePaths": true
     },
-    "browserify": {
-        "transform": [
-            "browserify-swap",
-            [
-                "babelify",
-                {
-                    "sourceMaps": "inline",
-                    "presets": [
-                        "@babel/preset-env",
-                        "@babel/preset-typescript"
-                    ]
-                }
-            ]
-        ]
-    },
-    "browserify-swap": {
-        "no-rust-crypto": {
-            "src/rust-crypto/index.ts$": "./src/rust-crypto/browserify-index.ts"
-        }
-    },
     "typings": "./lib/index.d.ts"
 }

post-release.sh

@@ -1,37 +0,0 @@
-#!/bin/bash
-#
-# Script to perform a post-release steps of matrix-js-sdk.
-#
-# Requires:
-#   jq; install from your distribution's package manager (https://stedolan.github.io/jq/)
-
-set -e
-
-jq --version > /dev/null || (echo "jq is required: please install it"; kill $$)
-
-if [ "$(git branch -lr | grep origin/develop -c)" -ge 1 ]; then
-    # When merging to develop, we need revert the `main` and `typings` fields if we adjusted them previously.
-    for i in main typings browser
-    do
-        # If a `lib` prefixed value is present, it means we adjusted the field
-        # earlier at publish time, so we should revert it now.
-        if [ "$(jq -r ".matrix_lib_$i" package.json)" != "null" ]; then
-            # If there's a `src` prefixed value, use that, otherwise delete.
-            # This is used to delete the `typings` field and reset `main` back
-            # to the TypeScript source.
-            src_value=$(jq -r ".matrix_src_$i" package.json)
-            if [ "$src_value" != "null" ]; then
-                jq ".$i = .matrix_src_$i" package.json > package.json.new && mv package.json.new package.json && yarn prettier --write package.json
-            else
-                jq "del(.$i)" package.json > package.json.new && mv package.json.new package.json && yarn prettier --write package.json
-            fi
-        fi
-    done
-
-    if [ -n "$(git ls-files --modified package.json)" ]; then
-        echo "Committing develop package.json"
-        git commit package.json -m "Resetting package fields for development"
-    fi
-
-    git push origin develop
-fi

release.sh

@@ -1,357 +0,0 @@
-#!/bin/bash
-#
-# Script to perform a release of matrix-js-sdk and downstream projects.
-#
-# Requires:
-#   jq; install from your distribution's package manager (https://stedolan.github.io/jq/)
-#   hub; install via brew (macOS) or source/pre-compiled binaries (debian) (https://github.com/github/hub) - Tested on v2.2.9
-#   yarn; install via brew (macOS) or similar (https://yarnpkg.com/docs/install/)
-#
-# Note: this script is also used to release matrix-react-sdk, element-web, and element-desktop.
-
-set -e
-
-jq --version > /dev/null || (echo "jq is required: please install it"; kill $$)
-if [[ $(command -v hub) ]] && [[ $(hub --version) =~ hub[[:space:]]version[[:space:]]([0-9]*).([0-9]*) ]]; then
-    HUB_VERSION_MAJOR=${BASH_REMATCH[1]}
-    HUB_VERSION_MINOR=${BASH_REMATCH[2]}
-    if [[ $HUB_VERSION_MAJOR -lt 2 ]] || [[ $HUB_VERSION_MAJOR -eq 2 && $HUB_VERSION_MINOR -lt 5 ]]; then
-        echo "hub version 2.5 is required, you have $HUB_VERSION_MAJOR.$HUB_VERSION_MINOR installed"
-        exit
-    fi
-else
-    echo "hub is required: please install it"
-    exit
-fi
-yarn --version > /dev/null || (echo "yarn is required: please install it"; kill $$)
-
-USAGE="$0 [-x] [-c changelog_file] vX.Y.Z"
-
-help() {
-    cat <<EOF
-$USAGE
-
-    -c changelog_file:  specify name of file containing changelog
-    -x:                 skip updating the changelog
-EOF
-}
-
-if ! git diff-index --quiet --cached HEAD; then
-    echo "this git checkout has staged (uncommitted) changes. Refusing to release."
-    exit
-fi
-
-if ! git diff-files --quiet; then
-    echo "this git checkout has uncommitted changes. Refusing to release."
-    exit
-fi
-
-skip_changelog=
-changelog_file="CHANGELOG.md"
-while getopts hc:x f; do
-    case $f in
-        h)
-            help
-            exit 0
-            ;;
-        c)
-            changelog_file="$OPTARG"
-            ;;
-        x)
-            skip_changelog=1
-            ;;
-    esac
-done
-shift $(expr $OPTIND - 1)
-
-if [ $# -ne 1 ]; then
-    echo "Usage: $USAGE" >&2
-    exit 1
-fi
-
-function check_dependency {
-    local depver=$(cat package.json | jq -r .dependencies[\"$1\"])
-    if [ "$depver" == "null" ]; then return 0; fi
-
-    echo "Checking version of $1..."
-    local latestver=$(yarn info -s "$1" dist-tags.next)
-    if [ "$depver" != "$latestver" ]
-    then
-        echo "The latest version of $1 is $latestver but package.json depends on $depver."
-        echo -n "Type 'u' to auto-upgrade, 'c' to continue anyway, or 'a' to abort:"
-        read resp
-        if [ "$resp" != "u" ] && [ "$resp" != "c" ]
-        then
-            echo "Aborting."
-            exit 1
-        fi
-        if [ "$resp" == "u" ]
-        then
-            echo "Upgrading $1 to $latestver..."
-            yarn add -E "$1@$latestver"
-            git add -u
-            git commit -m "Upgrade $1 to $latestver"
-        fi
-    fi
-}
-
-function reset_dependency {
-    local depver=$(cat package.json | jq -r .dependencies[\"$1\"])
-    if [ "$depver" == "null" ]; then return 0; fi
-
-    echo "Resetting $1 to develop branch..."
-    yarn add "github:matrix-org/$1#develop"
-    git add -u
-    git commit -m "Reset $1 back to develop branch"
-}
-
-has_subprojects=0
-if [ -f release_config.yaml ]; then
-    subprojects=$(cat release_config.yaml | python -c "import yaml; import sys; print(' '.join(list(yaml.load(sys.stdin)['subprojects'].keys())))" 2> /dev/null)
-    if [ "$?" -eq 0 ]; then
-        has_subprojects=1
-        echo "Checking subprojects for upgrades"
-        for proj in $subprojects; do
-            check_dependency "$proj"
-        done
-    fi
-fi
-
-ret=0
-cat package.json | jq '.dependencies[]' | grep -q '#develop' || ret=$?
-if [ "$ret" -eq 0 ]; then
-    echo "package.json contains develop dependencies. Refusing to release."
-    exit
-fi
-
-# We use Git branch / commit dependencies for some packages, and Yarn seems
-# to have a hard time getting that right. See also
-# https://github.com/yarnpkg/yarn/issues/4734. As a workaround, we clean the
-# global cache here to ensure we get the right thing.
-yarn cache clean
-# Ensure all dependencies are updated
-yarn install --ignore-scripts --frozen-lockfile
-
-# ignore leading v on release
-release="${1#v}"
-tag="v${release}"
-
-prerelease=0
-# We check if this build is a prerelease by looking to
-# see if the version has a hyphen in it. Crude,
-# but semver doesn't support postreleases so anything
-# with a hyphen is a prerelease.
-echo $release | grep -q '-' && prerelease=1
-
-if [ $prerelease -eq 1 ]; then
-    echo Making a PRE-RELEASE
-else
-    read -p "Making a FINAL RELEASE, press enter to continue " REPLY
-fi
-
-rel_branch=$(git symbolic-ref --short HEAD)
-
-if [ -z "$skip_changelog" ]; then
-    echo "Generating changelog"
-    yarn run allchange "$release"
-    read -p "Edit $changelog_file manually, or press enter to continue " REPLY
-
-    if [ -n "$(git ls-files --modified $changelog_file)" ]; then
-        echo "Committing updated changelog"
-        git commit "$changelog_file" -m "Prepare changelog for $tag"
-    fi
-fi
-latest_changes=$(mktemp)
-cat "${changelog_file}" | "$(dirname "$0")/scripts/changelog_head.py" > "${latest_changes}"
-
-set -x
-
-# Bump package.json and build the dist
-echo "yarn version"
-# yarn version will automatically commit its modification
-# and make a release tag. We don't want it to create the tag
-# because it can only sign with the default key, but we can
-# only turn off both of these behaviours, so we have to
-# manually commit the result.
-yarn version --no-git-tag-version --new-version "$release"
-
-# For the published and dist versions of the package, we copy the
-# `matrix_lib_main` and `matrix_lib_typings` fields to `main` and `typings` (if
-# they exist). This small bit of gymnastics allows us to use the TypeScript
-# source directly for development without needing to build before linting or
-# testing.
-for i in main typings browser
-do
-    lib_value=$(jq -r ".matrix_lib_$i" package.json)
-    if [ "$lib_value" != "null" ]; then
-        jq ".$i = .matrix_lib_$i" package.json > package.json.new && mv package.json.new package.json && yarn prettier --write package.json
-    fi
-done
-
-# commit yarn.lock if it exists, is versioned, and is modified
-if [[ -f yarn.lock && $(git status --porcelain yarn.lock | grep '^ M') ]];
-then
-    pkglock='yarn.lock'
-else
-    pkglock=''
-fi
-git commit package.json $pkglock -m "$tag"
-
-
-# figure out if we should be signing this release
-signing_id=
-if [ -f release_config.yaml ]; then
-    result=$(cat release_config.yaml | python -c "import yaml; import sys; print(yaml.load(sys.stdin)['signing_id'])" 2> /dev/null || true)
-    if [ "$?" -eq 0 ]; then
-        signing_id=$result
-    fi
-fi
-
-
-# If there is a 'dist' script in the package.json,
-# run it in a separate checkout of the project, then
-# upload any files in the 'dist' directory as release
-# assets.
-# We make a completely separate checkout to be sure
-# we're using released versions of the dependencies
-# (rather than whatever we're pulling in from yarn link)
-assets=''
-dodist=0
-jq -e .scripts.dist package.json 2> /dev/null || dodist=$?
-if [ $dodist -eq 0 ]; then
-    projdir=$(pwd)
-    builddir=$(mktemp -d 2>/dev/null || mktemp -d -t 'mytmpdir')
-    echo "Building distribution copy in $builddir"
-    pushd "$builddir"
-    git clone "$projdir" .
-    git checkout "$rel_branch"
-    yarn install --frozen-lockfile
-    # We haven't tagged yet, so tell the dist script what version
-    # it's building
-    DIST_VERSION="$tag" yarn dist
-
-    popd
-
-    for i in "$builddir"/dist/*; do
-        assets="$assets -a $i"
-        if [ -n "$signing_id" ]
-        then
-            gpg -u "$signing_id" --armor --output "$i".asc --detach-sig "$i"
-            assets="$assets -a $i.asc"
-        fi
-    done
-fi
-
-if [ -n "$signing_id" ]; then
-    # make a signed tag
-    # gnupg seems to fail to get the right tty device unless we set it here
-    GIT_COMMITTER_EMAIL="$signing_id" GPG_TTY=$(tty) git tag -u "$signing_id" -F "${latest_changes}" "$tag"
-else
-    git tag -a -F "${latest_changes}" "$tag"
-fi
-
-# push the tag and the release branch
-git push origin "$rel_branch" "$tag"
-
-if [ -n "$signing_id" ]; then
-    # make a signature for the source tarball.
-    #
-    # github will make us a tarball from the tag - we want to create a
-    # signature for it, which means that first of all we need to check that
-    # it's correct.
-    #
-    # we can't deterministically build exactly the same tarball, due to
-    # differences in gzip implementation - but we *can* build the same tar - so
-    # the easiest way to check the validity of the tarball from git is to unzip
-    # it and compare it with our own idea of what the tar should look like.
-
-    # This uses git archive which seems to be what github uses. Specifically,
-    # the header fields are set in the same way: same file mode, uid & gid
-    # both zero and mtime set to the timestamp of the commit that the tag
-    # references. Also note that this puts the commit into the tar headers
-    # and can be extracted with gunzip -c foo.tar.gz | git get-tar-commit-id
-
-    # the name of the sig file we want to create
-    source_sigfile="${tag}-src.tar.gz.asc"
-
-    tarfile="$tag.tar.gz"
-    gh_project_url=$(git remote get-url origin |
-                            sed -e 's#^git@github\.com:#https://github.com/#' \
-                                -e 's#^git\+ssh://git@github\.com/#https://github.com/#' \
-                                -e 's/\.git$//')
-    project_name="${gh_project_url##*/}"
-    curl -L "${gh_project_url}/archive/${tarfile}" -o "${tarfile}"
-
-    # unzip it and compare it with the tar we would generate
-    if ! cmp --silent <(gunzip -c $tarfile) \
-         <(git archive --format tar --prefix="${project_name}-${release}/" "$tag"); then
-
-        # we don't bail out here, because really it's more likely that our comparison
-        # screwed up and it's super annoying to abort the script at this point.
-        cat >&2 <<EOF
-!!!!!!!!!!!!!!!!!
-!!!! WARNING !!!!
-
-Mismatch between our own tarfile and that generated by github: not signing
-source tarball.
-
-To resolve, determine if $tarfile is correct, and if so sign it with gpg and
-attach it to the release as $source_sigfile.
-
-!!!!!!!!!!!!!!!!!
-EOF
-    else
-        gpg -u "$signing_id" --armor --output "$source_sigfile" --detach-sig "$tarfile"
-        assets="$assets -a $source_sigfile"
-    fi
-fi
-
-hubflags=''
-if [ $prerelease -eq 1 ]; then
-    hubflags='-p'
-fi
-
-release_text=$(mktemp)
-echo "$tag" > "${release_text}"
-echo >> "${release_text}"
-cat "${latest_changes}" >> "${release_text}"
-hub release create $hubflags $assets -F "${release_text}" "$tag"
-
-if [ $dodist -eq 0 ]; then
-    rm -rf "$builddir"
-fi
-rm "${release_text}"
-rm "${latest_changes}"
-
-# if it is a pre-release, leave it on the release branch for now.
-if [ $prerelease -eq 1 ]; then
-    git checkout "$rel_branch"
-    exit 0
-fi
-
-# merge release branch to master
-echo "updating master branch"
-git checkout master
-git pull
-git merge "$rel_branch" --no-edit
-
-# push master to github
-git push origin master
-
-# finally, merge master back onto develop (if it exists)
-if [ "$(git branch -lr | grep origin/develop -c)" -ge 1 ]; then
-    git checkout develop
-    git pull
-    git merge master --no-edit
-    git push origin develop
-fi
-
-[ -x ./post-release.sh ] && ./post-release.sh
-
-if [ $has_subprojects -eq 1 ] && [ $prerelease -eq 0 ]; then
-    echo "Resetting subprojects to develop"
-    for proj in $subprojects; do
-        reset_dependency "$proj"
-    done
-    git push origin develop
-fi

scripts/release/merge-release-notes.js

@@ -0,0 +1,164 @@
+#!/usr/bin/env node
+
+const fs = require("fs");
+
+async function listReleases(github, owner, repo) {
+    const response = await github.rest.repos.listReleases({
+        owner,
+        repo,
+        per_page: 100,
+    });
+    // Filters out draft releases
+    return response.data.filter((release) => !release.draft);
+}
+
+// Dependency can be a tuple of dependency, from version, to version, in which case a list of releases in that range (to inclusive) will be returned
+// Or it can be a string in the form accepted by `getRelease`
+async function getReleases(github, dependency) {
+    if (Array.isArray(dependency)) {
+        const [dep, fromVersion, toVersion] = dependency;
+        const upstreamPackageJson = getDependencyPackageJson(dep);
+        const [owner, repo] = upstreamPackageJson.repository.url.split("/").slice(-2);
+
+        const unfilteredReleases = await listReleases(github, owner, repo);
+        // Only include non-draft & non-prerelease releases, unless the to-release is a pre-release, include that one
+        const releases = unfilteredReleases.filter(
+            (release) => !release.prerelease || release.tag_name === `v${toVersion}`,
+        );
+
+        const fromVersionIndex = releases.findIndex((release) => release.tag_name === `v${fromVersion}`);
+        const toVersionIndex = releases.findIndex((release) => release.tag_name === `v${toVersion}`);
+
+        return releases.slice(toVersionIndex, fromVersionIndex);
+    }
+
+    return [await getRelease(github, dependency)];
+}
+
+// Dependency can be the name of an entry in package.json, in which case the owner, repo & version will be looked up in its own package.json
+// Or it can be a string in the form owner/repo@tag - in this case the tag is used exactly to find the release
+// Or it can be a string in the form owner/repo~tag - in this case the latest tag in the same major.minor.patch set is used to find the release
+async function getRelease(github, dependency) {
+    let owner;
+    let repo;
+    let tag;
+
+    if (dependency.includes("/")) {
+        let rest;
+        [owner, rest] = dependency.split("/");
+
+        if (dependency.includes("@")) {
+            [repo, tag] = rest.split("@");
+        } else if (dependency.includes("~")) {
+            [repo, tag] = rest.split("~");
+
+            if (tag.includes("-rc.")) {
+                // If the tag is an RC, find the latest matching RC in the set
+                try {
+                    const releases = await listReleases(github, owner, repo);
+                    const baseVersion = tag.split("-rc.")[0];
+                    const release = releases.find((release) => release.tag_name.startsWith(baseVersion));
+                    if (release) return release;
+                } catch (e) {
+                    // Fall back to getReleaseByTag
+                }
+            }
+        }
+    } else {
+        const upstreamPackageJson = getDependencyPackageJson(dependency);
+        [owner, repo] = upstreamPackageJson.repository.url.split("/").slice(-2);
+        tag = `v${upstreamPackageJson.version}`;
+    }
+
+    const response = await github.rest.repos.getReleaseByTag({
+        owner,
+        repo,
+        tag,
+    });
+    return response.data;
+}
+
+function getDependencyPackageJson(dependency) {
+    return JSON.parse(fs.readFileSync(`./node_modules/${dependency}/package.json`, "utf8"));
+}
+
+const HEADING_PREFIX = "## ";
+
+const categories = [
+    "🔒 SECURITY FIXES",
+    "🚨 BREAKING CHANGESd",
+    "🦖 Deprecations",
+    "✨ Features",
+    "🐛 Bug Fixes",
+    "🧰 Maintenance",
+];
+
+const parseReleaseNotes = (body, sections) => {
+    let heading = null;
+    for (const line of body.split("\n")) {
+        const trimmed = line.trim();
+        if (trimmed.startsWith(HEADING_PREFIX)) {
+            heading = trimmed.slice(HEADING_PREFIX.length);
+            if (!categories.includes(heading)) heading = null;
+            continue;
+        }
+        if (heading && trimmed) {
+            sections[heading].push(trimmed);
+        }
+    }
+};
+
+const main = async ({ github, releaseId, dependencies }) => {
+    const { GITHUB_REPOSITORY } = process.env;
+    const [owner, repo] = GITHUB_REPOSITORY.split("/");
+
+    const sections = Object.fromEntries(categories.map((cat) => [cat, []]));
+    for (const dependency of dependencies) {
+        const releases = await getReleases(github, dependency);
+        for (const release of releases) {
+            parseReleaseNotes(release.body, sections);
+        }
+    }
+
+    const { data: release } = await github.rest.repos.getRelease({
+        owner,
+        repo,
+        release_id: releaseId,
+    });
+
+    const intro = release.body.split(HEADING_PREFIX, 2)[0].trim();
+
+    let output = "";
+    if (intro) {
+        output = intro + "\n\n";
+    }
+
+    for (const section in sections) {
+        const lines = sections[section];
+        if (!lines.length) continue;
+        output += HEADING_PREFIX + section + "\n\n";
+        output += lines.join("\n");
+        output += "\n\n";
+    }
+
+    return output;
+};
+
+// This is just for testing locally
+// Needs environment variables GITHUB_TOKEN & GITHUB_REPOSITORY
+if (require.main === module) {
+    const { Octokit } = require("@octokit/rest");
+    const github = new Octokit({ auth: process.env.GITHUB_TOKEN });
+    if (process.argv.length < 4) {
+        // eslint-disable-next-line no-console
+        console.error("Usage: node merge-release-notes.js owner/repo:release_id npm-package-name ...");
+        process.exit(1);
+    }
+    const [releaseId, ...dependencies] = process.argv.slice(2);
+    main({ github, releaseId, dependencies }).then((output) => {
+        // eslint-disable-next-line no-console
+        console.log(output);
+    });
+}
+
+module.exports = main;

scripts/release/post-merge-master.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# When merging to develop, we need revert the `main` and `typings` fields if we adjusted them previously.
+for i in main typings browser
+do
+    # If a `lib` prefixed value is present, it means we adjusted the field earlier at publish time, so we should revert it now.
+    if [ "$(jq -r ".matrix_lib_$i" package.json)" != "null" ]; then
+        # If there's a `src` prefixed value, use that, otherwise delete.
+        # This is used to delete the `typings` field and reset `main` back to the TypeScript source.
+        src_value=$(jq -r ".matrix_src_$i" package.json)
+        if [ "$src_value" != "null" ]; then
+            jq ".$i = .matrix_src_$i" package.json > package.json.new && mv package.json.new package.json && yarn prettier --write package.json
+        else
+            jq "del(.$i)" package.json > package.json.new && mv package.json.new package.json && yarn prettier --write package.json
+        fi
+    fi
+done
+
+if [ -n "$(git ls-files --modified package.json)" ]; then
+    echo "Committing develop package.json"
+    git commit package.json -m "Resetting package fields for development"
+fi

scripts/release/pre-release.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# For the published and dist versions of the package,
+# we copy the `matrix_lib_main` and `matrix_lib_typings` fields to `main` and `typings` (if they exist).
+# This small bit of gymnastics allows us to use the TypeScript source directly for development without
+# needing to build before linting or testing.
+
+for i in main typings browser
+do
+    lib_value=$(jq -r ".matrix_lib_$i" package.json)
+    if [ "$lib_value" != "null" ]; then
+        jq ".$i = .matrix_lib_$i" package.json > package.json.new && mv package.json.new package.json && yarn prettier --write package.json
+    fi
+done

spec/browserify/setupTests.ts

@@ -1,34 +0,0 @@
-/*
-Copyright 2020 The Matrix.org Foundation C.I.C.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-import "../../dist/browser-matrix"; // uses browser-matrix instead of the src
-import type { default as BrowserMatrix } from "../../src/browser-index";
-
-// stub for browser-matrix browserify tests
-// @ts-ignore
-global.XMLHttpRequest = jest.fn();
-
-afterAll(() => {
-    // clean up XMLHttpRequest mock
-    // @ts-ignore
-    global.XMLHttpRequest = undefined;
-});
-
-// Akin to spec/setupTests.ts - but that won't affect the browser-matrix bundle
-global.matrixcs = {
-    ...global.matrixcs,
-    timeoutSignal: () => new AbortController().signal,
-} as typeof BrowserMatrix;

spec/browserify/sync-browserify.spec.ts

@@ -1,92 +0,0 @@
-/*
-Copyright 2020 The Matrix.org Foundation C.I.C.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-import HttpBackend from "matrix-mock-request";
-
-import "./setupTests"; // uses browser-matrix instead of the src
-import type { MatrixClient } from "../../src";
-
-const USER_ID = "@user:test.server";
-const DEVICE_ID = "device_id";
-const ACCESS_TOKEN = "access_token";
-const ROOM_ID = "!room_id:server.test";
-
-describe("Browserify Test", function () {
-    let client: MatrixClient;
-    let httpBackend: HttpBackend;
-
-    beforeEach(() => {
-        httpBackend = new HttpBackend();
-        client = new global.matrixcs.MatrixClient({
-            baseUrl: "http://test.server",
-            userId: USER_ID,
-            accessToken: ACCESS_TOKEN,
-            deviceId: DEVICE_ID,
-            fetchFn: httpBackend.fetchFn as typeof global.fetch,
-        });
-
-        httpBackend.when("GET", "/versions").respond(200, {});
-        httpBackend.when("GET", "/pushrules").respond(200, {});
-        httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
-    });
-
-    afterEach(async () => {
-        client.stopClient();
-        client.http.abort();
-        httpBackend.verifyNoOutstandingRequests();
-        httpBackend.verifyNoOutstandingExpectation();
-        await httpBackend.stop();
-    });
-
-    it("Sync", async () => {
-        const event = {
-            type: "m.room.member",
-            room_id: ROOM_ID,
-            content: {
-                membership: "join",
-                name: "Displayname",
-            },
-            event_id: "$foobar",
-        };
-
-        const syncData = {
-            next_batch: "batch1",
-            rooms: {
-                join: {
-                    [ROOM_ID]: {
-                        timeline: {
-                            events: [event],
-                            limited: false,
-                        },
-                    },
-                },
-            },
-        };
-
-        httpBackend.when("GET", "/sync").respond(200, syncData);
-        httpBackend.when("GET", "/sync").respond(200, syncData);
-
-        const syncPromise = new Promise((r) => client.once(global.matrixcs.ClientEvent.Sync, r));
-        const unexpectedErrorFn = jest.fn();
-        client.once(global.matrixcs.ClientEvent.SyncUnexpectedError, unexpectedErrorFn);
-
-        client.startClient();
-
-        await httpBackend.flushAllExpected();
-        await syncPromise;
-        expect(unexpectedErrorFn).not.toHaveBeenCalled();
-    }, 20000); // additional timeout as this test can take quite a while
-});

spec/integ/crypto/cross-signing.spec.ts

@@ -31,9 +31,12 @@ import {
     SELF_CROSS_SIGNING_PRIVATE_KEY_BASE64,
     SELF_CROSS_SIGNING_PUBLIC_KEY_BASE64,
     SIGNED_CROSS_SIGNING_KEYS_DATA,
+    SIGNED_TEST_DEVICE_DATA,
     USER_CROSS_SIGNING_PRIVATE_KEY_BASE64,
 } from "../../test-utils/test-data";
+import * as testData from "../../test-utils/test-data";
 import { E2EKeyResponder } from "../../test-utils/E2EKeyResponder";
+import { AccountDataAccumulator } from "../../test-utils/AccountDataAccumulator";
 
 afterEach(() => {
     // reset fake-indexeddb after each test, to make sure we don't leak connections
@@ -78,27 +81,37 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("cross-signing (%s)", (backend: s
         };
     }
 
-    beforeEach(async () => {
-        // anything that we don't have a specific matcher for silently returns a 404
-        fetchMock.catch(404);
-        fetchMock.config.warnOnFallback = false;
+    beforeEach(
+        async () => {
+            // anything that we don't have a specific matcher for silently returns a 404
+            fetchMock.catch(404);
+            fetchMock.config.warnOnFallback = false;
 
-        const homeserverUrl = "https://alice-server.com";
-        aliceClient = createClient({
-            baseUrl: homeserverUrl,
-            userId: TEST_USER_ID,
-            accessToken: "akjgkrgjs",
-            deviceId: TEST_DEVICE_ID,
-            cryptoCallbacks: createCryptoCallbacks(),
-        });
+            const homeserverUrl = "https://alice-server.com";
+            aliceClient = createClient({
+                baseUrl: homeserverUrl,
+                userId: TEST_USER_ID,
+                accessToken: "akjgkrgjs",
+                deviceId: TEST_DEVICE_ID,
+                cryptoCallbacks: createCryptoCallbacks(),
+            });
 
-        syncResponder = new SyncResponder(homeserverUrl);
-        e2eKeyResponder = new E2EKeyResponder(homeserverUrl);
-        /** an object which intercepts `/keys/upload` requests on the test homeserver */
-        new E2EKeyReceiver(homeserverUrl);
+            syncResponder = new SyncResponder(homeserverUrl);
+            e2eKeyResponder = new E2EKeyResponder(homeserverUrl);
+            /** an object which intercepts `/keys/upload` requests on the test homeserver */
+            new E2EKeyReceiver(homeserverUrl);
 
-        await initCrypto(aliceClient);
-    });
+            // Silence warnings from the backup manager
+            fetchMock.getOnce(new URL("/_matrix/client/v3/room_keys/version", homeserverUrl).toString(), {
+                status: 404,
+                body: { errcode: "M_NOT_FOUND" },
+            });
+
+            await initCrypto(aliceClient);
+        },
+        /* it can take a while to initialise the crypto library on the first pass, so bump up the timeout. */
+        10000,
+    );
 
     afterEach(async () => {
         await aliceClient.stopClient();
@@ -236,6 +249,53 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("cross-signing (%s)", (backend: s
                 `[${TEST_USER_ID}].[${TEST_DEVICE_ID}].signatures.[${TEST_USER_ID}].[ed25519:${SELF_CROSS_SIGNING_PUBLIC_KEY_BASE64}]`,
             );
         });
+
+        it("can bootstrapCrossSigning twice", async () => {
+            mockSetupCrossSigningRequests();
+
+            const authDict = { type: "test" };
+            await bootstrapCrossSigning(authDict);
+
+            // a second call should do nothing except GET requests
+            fetchMock.mockClear();
+            await bootstrapCrossSigning(authDict);
+            const calls = fetchMock.calls((url, opts) => opts.method != "GET");
+            expect(calls.length).toEqual(0);
+        });
+
+        newBackendOnly("will upload existing cross-signing keys to an established secret storage", async () => {
+            // This rather obscure codepath covers the case that:
+            //   - 4S is set up and working
+            //   - our device has private cross-signing keys, but has not published them to 4S
+            //
+            // To arrange that, we call `bootstrapCrossSigning` on our main device, and then (pretend to) set up 4S from
+            // a *different* device. Then, when we call `bootstrapCrossSigning` again, it should do the honours.
+
+            mockSetupCrossSigningRequests();
+            const accountDataAccumulator = new AccountDataAccumulator();
+            accountDataAccumulator.interceptGetAccountData();
+
+            const authDict = { type: "test" };
+            await bootstrapCrossSigning(authDict);
+
+            // Pretend that another device has uploaded a 4S key
+            accountDataAccumulator.accountDataEvents.set("m.secret_storage.default_key", { key: "key_id" });
+            accountDataAccumulator.accountDataEvents.set("m.secret_storage.key.key_id", {
+                key: "keykeykey",
+                algorithm: SECRET_STORAGE_ALGORITHM_V1_AES,
+            });
+
+            // Prepare for the cross-signing keys
+            const p = accountDataAccumulator.interceptSetAccountData(":type(m.cross_signing..*)");
+
+            await bootstrapCrossSigning(authDict);
+            await p;
+
+            // The cross-signing keys should have been uploaded
+            expect(accountDataAccumulator.accountDataEvents.has("m.cross_signing.master")).toBeTruthy();
+            expect(accountDataAccumulator.accountDataEvents.has("m.cross_signing.self_signing")).toBeTruthy();
+            expect(accountDataAccumulator.accountDataEvents.has("m.cross_signing.user_signing")).toBeTruthy();
+        });
     });
 
     describe("getCrossSigningStatus()", () => {
@@ -287,6 +347,67 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("cross-signing (%s)", (backend: s
 
             expect(isCrossSigningReady).toBeTruthy();
         });
+
+        it("should return false if identity is not trusted, even if the secrets are in 4S", async () => {
+            e2eKeyResponder.addCrossSigningData(SIGNED_CROSS_SIGNING_KEYS_DATA);
+
+            // Complete initial sync, to get the 4S account_data events stored
+            mockInitialApiRequests(aliceClient.getHomeserverUrl());
+
+            // For this test we need to have a well-formed 4S setup.
+            const mockSecretInfo = {
+                encrypted: {
+                    // Don't care about the actual values here, just need to be present for validation
+                    KeyId: {
+                        iv: "IVIVIVIVIVIVIV",
+                        ciphertext: "CIPHERTEXTB64",
+                        mac: "MACMACMAC",
+                    },
+                },
+            };
+            syncResponder.sendOrQueueSyncResponse({
+                next_batch: 1,
+                account_data: {
+                    events: [
+                        {
+                            type: "m.secret_storage.key.KeyId",
+                            content: {
+                                algorithm: "m.secret_storage.v1.aes-hmac-sha2",
+                                // iv and mac not relevant for this test
+                            },
+                        },
+                        {
+                            type: "m.secret_storage.default_key",
+                            content: {
+                                key: "KeyId",
+                            },
+                        },
+                        {
+                            type: "m.cross_signing.master",
+                            content: mockSecretInfo,
+                        },
+                        {
+                            type: "m.cross_signing.user_signing",
+                            content: mockSecretInfo,
+                        },
+                        {
+                            type: "m.cross_signing.self_signing",
+                            content: mockSecretInfo,
+                        },
+                    ],
+                },
+            });
+            await aliceClient.startClient();
+            await syncPromise(aliceClient);
+
+            // Sanity: ensure that the secrets are in 4S
+            const status = await aliceClient.getCrypto()!.getCrossSigningStatus();
+            expect(status.privateKeysInSecretStorage).toBeTruthy();
+
+            const isCrossSigningReady = await aliceClient.getCrypto()!.isCrossSigningReady();
+
+            expect(isCrossSigningReady).toBeFalsy();
+        });
     });
 
     describe("getCrossSigningKeyId", () => {
@@ -339,4 +460,49 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("cross-signing (%s)", (backend: s
             expect(userSigningKeyId).toBe(getPubKey(crossSigningKeys.user_signing_key));
         });
     });
+
+    describe("crossSignDevice", () => {
+        beforeEach(async () => {
+            // We want to use fake timers, but the wasm bindings of matrix-sdk-crypto rely on a working `queueMicrotask`.
+            jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
+
+            // make sure that there is another device which we can sign
+            e2eKeyResponder.addDeviceKeys(SIGNED_TEST_DEVICE_DATA);
+
+            // Complete initialsync, to get the outgoing requests going
+            mockInitialApiRequests(aliceClient.getHomeserverUrl());
+            syncResponder.sendOrQueueSyncResponse({ next_batch: 1 });
+            await aliceClient.startClient();
+            await syncPromise(aliceClient);
+
+            // Wait for legacy crypto to find the device
+            await jest.advanceTimersByTimeAsync(10);
+
+            const devices = await aliceClient.getCrypto()!.getUserDeviceInfo([aliceClient.getSafeUserId()]);
+            expect(devices.get(aliceClient.getSafeUserId())!.has(testData.TEST_DEVICE_ID)).toBeTruthy();
+        });
+
+        afterEach(async () => {
+            jest.useRealTimers();
+        });
+
+        it("fails for an unknown device", async () => {
+            await expect(aliceClient.getCrypto()!.crossSignDevice("unknown")).rejects.toThrow("Unknown device");
+        });
+
+        it("cross-signs the device", async () => {
+            mockSetupCrossSigningRequests();
+            await aliceClient.getCrypto()!.bootstrapCrossSigning({});
+
+            fetchMock.mockClear();
+            await aliceClient.getCrypto()!.crossSignDevice(testData.TEST_DEVICE_ID);
+
+            // check that a sig for the device was uploaded
+            const calls = fetchMock.calls("upload-sigs");
+            expect(calls.length).toEqual(1);
+            const body = JSON.parse(calls[0][1]!.body as string);
+            const deviceSig = body[aliceClient.getSafeUserId()][testData.TEST_DEVICE_ID];
+            expect(deviceSig).toHaveProperty("signatures");
+        });
+    });
 });

spec/integ/crypto/device-dehydration.spec.ts

@@ -0,0 +1,181 @@
+/*
+Copyright 2024 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import "fake-indexeddb/auto";
+import fetchMock from "fetch-mock-jest";
+
+import { createClient, ClientEvent, MatrixClient, MatrixEvent } from "../../../src";
+import { RustCrypto } from "../../../src/rust-crypto/rust-crypto";
+import { AddSecretStorageKeyOpts } from "../../../src/secret-storage";
+import { E2EKeyReceiver } from "../../test-utils/E2EKeyReceiver";
+import { E2EKeyResponder } from "../../test-utils/E2EKeyResponder";
+
+describe("Device dehydration", () => {
+    it("should rehydrate and dehydrate a device", async () => {
+        jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
+
+        const matrixClient = createClient({
+            baseUrl: "http://test.server",
+            userId: "@alice:localhost",
+            deviceId: "aliceDevice",
+            cryptoCallbacks: {
+                getSecretStorageKey: async (keys: any, name: string) => {
+                    return [[...Object.keys(keys.keys)][0], new Uint8Array(32)];
+                },
+            },
+        });
+
+        await initializeSecretStorage(matrixClient, "@alice:localhost", "http://test.server");
+
+        // count the number of times the dehydration key gets set
+        let setDehydrationCount = 0;
+        matrixClient.on(ClientEvent.AccountData, (event: MatrixEvent) => {
+            if (event.getType() === "org.matrix.msc3814") {
+                setDehydrationCount++;
+            }
+        });
+
+        const crypto = matrixClient.getCrypto()!;
+        fetchMock.config.overwriteRoutes = true;
+
+        // start dehydration -- we start with no dehydrated device, and we
+        // store the dehydrated device that we create
+        fetchMock.get("path:/_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device", {
+            status: 404,
+            body: {
+                errcode: "M_NOT_FOUND",
+                error: "Not found",
+            },
+        });
+        let dehydratedDeviceBody: any;
+        let dehydrationCount = 0;
+        let resolveDehydrationPromise: () => void;
+        fetchMock.put("path:/_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device", (_, opts) => {
+            dehydratedDeviceBody = JSON.parse(opts.body as string);
+            dehydrationCount++;
+            if (resolveDehydrationPromise) {
+                resolveDehydrationPromise();
+            }
+            return {};
+        });
+        await crypto.startDehydration();
+
+        expect(dehydrationCount).toEqual(1);
+
+        // a week later, we should have created another dehydrated device
+        const dehydrationPromise = new Promise<void>((resolve, reject) => {
+            resolveDehydrationPromise = resolve;
+        });
+        jest.advanceTimersByTime(7 * 24 * 60 * 60 * 1000);
+        await dehydrationPromise;
+        expect(dehydrationCount).toEqual(2);
+
+        // restart dehydration -- rehydrate the device that we created above,
+        // and create a new dehydrated device.  We also set `createNewKey`, so
+        // a new dehydration key will be set
+        fetchMock.get("path:/_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device", {
+            device_id: dehydratedDeviceBody.device_id,
+            device_data: dehydratedDeviceBody.device_data,
+        });
+        const eventsResponse = jest.fn((url, opts) => {
+            // rehydrating should make two calls to the /events endpoint.
+            // The first time will return a single event, and the second
+            // time will return no events (which will signal to the
+            // rehydration function that it can stop)
+            const body = JSON.parse(opts.body as string);
+            const nextBatch = body.next_batch ?? "0";
+            const events = nextBatch === "0" ? [{ sender: "@alice:localhost", type: "m.dummy", content: {} }] : [];
+            return {
+                events,
+                next_batch: nextBatch + "1",
+            };
+        });
+        fetchMock.post(
+            `path:/_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device/${encodeURIComponent(dehydratedDeviceBody.device_id)}/events`,
+            eventsResponse,
+        );
+        await crypto.startDehydration(true);
+        expect(dehydrationCount).toEqual(3);
+
+        expect(setDehydrationCount).toEqual(2);
+        expect(eventsResponse.mock.calls).toHaveLength(2);
+
+        matrixClient.stopClient();
+    });
+});
+
+/** create a new secret storage and cross-signing keys */
+async function initializeSecretStorage(
+    matrixClient: MatrixClient,
+    userId: string,
+    homeserverUrl: string,
+): Promise<void> {
+    fetchMock.get("path:/_matrix/client/v3/room_keys/version", {
+        status: 404,
+        body: {
+            errcode: "M_NOT_FOUND",
+            error: "Not found",
+        },
+    });
+    const e2eKeyReceiver = new E2EKeyReceiver(homeserverUrl);
+    const e2eKeyResponder = new E2EKeyResponder(homeserverUrl);
+    e2eKeyResponder.addKeyReceiver(userId, e2eKeyReceiver);
+    fetchMock.post("path:/_matrix/client/v3/keys/device_signing/upload", {});
+    fetchMock.post("path:/_matrix/client/v3/keys/signatures/upload", {});
+    const accountData: Map<string, object> = new Map();
+    fetchMock.get("glob:http://*/_matrix/client/v3/user/*/account_data/*", (url, opts) => {
+        const name = url.split("/").pop()!;
+        const value = accountData.get(name);
+        if (value) {
+            return value;
+        } else {
+            return {
+                status: 404,
+                body: {
+                    errcode: "M_NOT_FOUND",
+                    error: "Not found",
+                },
+            };
+        }
+    });
+    fetchMock.put("glob:http://*/_matrix/client/v3/user/*/account_data/*", (url, opts) => {
+        const name = url.split("/").pop()!;
+        const value = JSON.parse(opts.body as string);
+        accountData.set(name, value);
+        matrixClient.emit(ClientEvent.AccountData, new MatrixEvent({ type: name, content: value }));
+        return {};
+    });
+
+    await matrixClient.initRustCrypto();
+    const crypto = matrixClient.getCrypto()! as RustCrypto;
+    // we need to process a sync so that the OlmMachine will upload keys
+    await crypto.preprocessToDeviceMessages([]);
+    await crypto.onSyncCompleted({});
+
+    // create initial secret storage
+    async function createSecretStorageKey() {
+        return {
+            keyInfo: {} as AddSecretStorageKeyOpts,
+            privateKey: new Uint8Array(32),
+        };
+    }
+    await matrixClient.bootstrapCrossSigning({ setupNewCrossSigning: true });
+    await matrixClient.bootstrapSecretStorage({
+        createSecretStorageKey,
+        setupNewSecretStorage: true,
+        setupNewKeyBackup: false,
+    });
+}

spec/integ/crypto/megolm-backup.spec.ts

@@ -17,63 +17,41 @@ limitations under the License.
 import fetchMock from "fetch-mock-jest";
 import "fake-indexeddb/auto";
 import { IDBFactory } from "fake-indexeddb";
+import { Mocked } from "jest-mock";
 
-import { IKeyBackupSession } from "../../../src/crypto/keybackup";
-import { createClient, CryptoEvent, ICreateClientOpts, IEvent, MatrixClient, TypedEventEmitter } from "../../../src";
+import {
+    createClient,
+    CryptoApi,
+    CryptoEvent,
+    ICreateClientOpts,
+    IEvent,
+    IMegolmSessionData,
+    MatrixClient,
+    TypedEventEmitter,
+} from "../../../src";
 import { SyncResponder } from "../../test-utils/SyncResponder";
 import { E2EKeyReceiver } from "../../test-utils/E2EKeyReceiver";
 import { E2EKeyResponder } from "../../test-utils/E2EKeyResponder";
 import { mockInitialApiRequests } from "../../test-utils/mockEndpoints";
-import { awaitDecryption, CRYPTO_BACKENDS, InitCrypto, syncPromise } from "../../test-utils/test-utils";
+import {
+    advanceTimersUntil,
+    awaitDecryption,
+    CRYPTO_BACKENDS,
+    InitCrypto,
+    syncPromise,
+} from "../../test-utils/test-utils";
 import * as testData from "../../test-utils/test-data";
-import { KeyBackupInfo } from "../../../src/crypto-api/keybackup";
+import { KeyBackupInfo, KeyBackupSession } from "../../../src/crypto-api/keybackup";
 import { IKeyBackup } from "../../../src/crypto/backup";
+import { flushPromises } from "../../test-utils/flushPromises";
+import { defer, IDeferred } from "../../../src/utils";
+import { DecryptionFailureCode } from "../../../src/crypto-api";
 
-const ROOM_ID = "!ROOM:ID";
+const ROOM_ID = testData.TEST_ROOM_ID;
 
 /** The homeserver url that we give to the test client, and where we intercept /sync, /keys, etc requests. */
 const TEST_HOMESERVER_URL = "https://alice-server.com";
 
-const SESSION_ID = "o+21hSjP+mgEmcfdslPsQdvzWnkdt0Wyo00Kp++R8Kc";
-
-const ENCRYPTED_EVENT: Partial<IEvent> = {
-    type: "m.room.encrypted",
-    content: {
-        algorithm: "m.megolm.v1.aes-sha2",
-        sender_key: "SENDER_CURVE25519",
-        session_id: SESSION_ID,
-        ciphertext:
-            "AwgAEjD+VwXZ7PoGPRS/H4kwpAsMp/g+WPvJVtPEKE8fmM9IcT/N" +
-            "CiwPb8PehecDKP0cjm1XO88k6Bw3D17aGiBHr5iBoP7oSw8CXULXAMTkBl" +
-            "mkufRQq2+d0Giy1s4/Cg5n13jSVrSb2q7VTSv1ZHAFjUCsLSfR0gxqcQs",
-    },
-    room_id: "!ROOM:ID",
-    event_id: "$event1",
-    origin_server_ts: 1507753886000,
-};
-
-const CURVE25519_KEY_BACKUP_DATA: IKeyBackupSession = {
-    first_message_index: 0,
-    forwarded_count: 0,
-    is_verified: false,
-    session_data: {
-        ciphertext:
-            "2z2M7CZ+azAiTHN1oFzZ3smAFFt+LEOYY6h3QO3XXGdw" +
-            "6YpNn/gpHDO6I/rgj1zNd4FoTmzcQgvKdU8kN20u5BWRHxaHTZ" +
-            "Slne5RxE6vUdREsBgZePglBNyG0AogR/PVdcrv/v18Y6rLM5O9" +
-            "SELmwbV63uV9Kuu/misMxoqbuqEdG7uujyaEKtjlQsJ5MGPQOy" +
-            "Syw7XrnesSwF6XWRMxcPGRV0xZr3s9PI350Wve3EncjRgJ9IGF" +
-            "ru1bcptMqfXgPZkOyGvrphHoFfoK7nY3xMEHUiaTRfRIjq8HNV" +
-            "4o8QY1qmWGnxNBQgOlL8MZlykjg3ULmQ3DtFfQPj/YYGS3jzxv" +
-            "C+EBjaafmsg+52CTeK3Rswu72PX450BnSZ1i3If4xWAUKvjTpe" +
-            "Ug5aDLqttOv1pITolTJDw5W/SD+b5rjEKg1CFCHGEGE9wwV3Nf" +
-            "QHVCQL+dfpd7Or0poy4dqKMAi3g0o3Tg7edIF8d5rREmxaALPy" +
-            "iie8PHD8mj/5Y0GLqrac4CD6+Mop7eUTzVovprjg",
-        mac: "5lxYBHQU80M",
-        ephemeral: "/Bn0A4UMFwJaDDvh0aEk1XZj3k1IfgCxgFY9P9a0b14",
-    },
-};
-
 const TEST_USER_ID = "@alice:localhost";
 const TEST_DEVICE_ID = "xzcvb";
 
@@ -139,7 +117,8 @@ function mockUploadEmitter(
 describe.each(Object.entries(CRYPTO_BACKENDS))("megolm-keys backup (%s)", (backend: string, initCrypto: InitCrypto) => {
     // oldBackendOnly is an alternative to `it` or `test` which will skip the test if we are running against the
     // Rust backend. Once we have full support in the rust sdk, it will go away.
-    const oldBackendOnly = backend === "rust-sdk" ? test.skip : test;
+    // const oldBackendOnly = backend === "rust-sdk" ? test.skip : test;
+    // const newBackendOnly = backend === "libolm" ? test.skip : test;
 
     let aliceClient: MatrixClient;
     /** an object which intercepts `/sync` requests on the test homeserver */
@@ -150,9 +129,10 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("megolm-keys backup (%s)", (backe
     /** an object which intercepts `/keys/query` requests on the test homeserver */
     let e2eKeyResponder: E2EKeyResponder;
 
-    jest.useFakeTimers();
-
     beforeEach(async () => {
+        // We want to use fake timers, but the wasm bindings of matrix-sdk-crypto rely on a working `queueMicrotask`.
+        jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
+
         // anything that we don't have a specific matcher for silently returns a 404
         fetchMock.catch(404);
         fetchMock.config.warnOnFallback = false;
@@ -174,6 +154,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("megolm-keys backup (%s)", (backe
         await jest.runAllTimersAsync();
 
         fetchMock.mockReset();
+        jest.restoreAllMocks();
     });
 
     async function initTestClient(opts: Partial<ICreateClientOpts> = {}): Promise<MatrixClient> {
@@ -189,44 +170,420 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("megolm-keys backup (%s)", (backe
         return client;
     }
 
-    oldBackendOnly("Alice checks key backups when receiving a message she can't decrypt", async function () {
-        const syncResponse = {
+    describe("Key backup check on UTD message", () => {
+        // sync response which contains an encrypted event
+        const SYNC_RESPONSE = {
             next_batch: 1,
-            rooms: {
-                join: {
+            rooms: { join: { [ROOM_ID]: { timeline: { events: [testData.ENCRYPTED_EVENT] } } } },
+        };
+
+        const EXPECTED_URL =
+            [
+                "https://alice-server.com/_matrix/client/v3/room_keys/keys",
+                encodeURIComponent(testData.TEST_ROOM_ID),
+                encodeURIComponent(testData.MEGOLM_SESSION_DATA.session_id),
+            ].join("/") + "?version=1";
+
+        /** Flush promises enough times to get the crypto stacks to make the backup request */
+        async function flushBackupRequest() {
+            // we have to run flushPromises lots of times. It seems like each time the rust code touches indexeddb,
+            // it needs another round of flushPromises to progress, or something.
+            for (let i = 0; i < 10; i++) {
+                await flushPromises();
+            }
+        }
+
+        beforeEach(
+            async () => {
+                fetchMock.get("path:/_matrix/client/v3/room_keys/version", testData.SIGNED_BACKUP_DATA);
+
+                // ignore requests to send room key requests
+                fetchMock.put("express:/_matrix/client/v3/sendToDevice/m.room_key_request/:request_id", {});
+
+                aliceClient = await initTestClient();
+                const aliceCrypto = aliceClient.getCrypto()!;
+                await aliceCrypto.storeSessionBackupPrivateKey(
+                    Buffer.from(testData.BACKUP_DECRYPTION_KEY_BASE64, "base64"),
+                    testData.SIGNED_BACKUP_DATA.version!,
+                );
+
+                // start after saving the private key
+                await aliceClient.startClient();
+
+                // tell Alice to trust the dummy device that signed the backup, and re-check the backup.
+                // XXX: should we automatically re-check after a device becomes verified?
+                await waitForDeviceList();
+                await aliceClient.getCrypto()!.setDeviceVerified(testData.TEST_USER_ID, testData.TEST_DEVICE_ID);
+                await aliceClient.getCrypto()!.checkKeyBackupAndEnable();
+            } /* it can take a while to initialise the crypto library on the first pass, so bump up the timeout. */,
+            10000,
+        );
+
+        it("Alice checks key backups when receiving a message she can't decrypt", async () => {
+            fetchMock.get("express:/_matrix/client/v3/room_keys/keys/:room_id/:session_id", (url, request) => {
+                // check that the version is correct
+                const version = new URLSearchParams(new URL(url).search).get("version");
+                if (version == "1") {
+                    return testData.CURVE25519_KEY_BACKUP_DATA;
+                } else {
+                    return {
+                        status: 403,
+                        body: {
+                            current_version: "1",
+                            errcode: "M_WRONG_ROOM_KEYS_VERSION",
+                            error: "Wrong backup version.",
+                        },
+                    };
+                }
+            });
+
+            // Send Alice a message that she won't be able to decrypt, and check that she fetches the key from the backup.
+            syncResponder.sendOrQueueSyncResponse(SYNC_RESPONSE);
+            await syncPromise(aliceClient);
+
+            const room = aliceClient.getRoom(ROOM_ID)!;
+            const event = room.getLiveTimeline().getEvents()[0];
+
+            // On the first decryption attempt, decryption fails.
+            await awaitDecryption(event);
+            expect(event.decryptionFailureReason).toEqual(
+                backend === "libolm"
+                    ? DecryptionFailureCode.MEGOLM_UNKNOWN_INBOUND_SESSION_ID
+                    : DecryptionFailureCode.HISTORICAL_MESSAGE_WORKING_BACKUP,
+            );
+
+            // Eventually, decryption succeeds.
+            await awaitDecryption(event, { waitOnDecryptionFailure: true });
+            expect(event.getContent()).toEqual(testData.CLEAR_EVENT.content);
+        });
+
+        it("handles error on backup query gracefully", async () => {
+            jest.spyOn(console, "error").mockImplementation(() => {});
+
+            fetchMock.get(
+                "express:/_matrix/client/v3/room_keys/keys/:room_id/:session_id",
+                { status: 404, body: { errcode: "M_NOT_FOUND" } },
+                { name: "getKey" },
+            );
+
+            // Send Alice a message that she won't be able to decrypt
+            syncResponder.sendOrQueueSyncResponse(SYNC_RESPONSE);
+            await flushBackupRequest();
+
+            const calls = fetchMock.calls("getKey");
+            expect(calls.length).toEqual(1);
+            expect(calls[0][0]).toEqual(EXPECTED_URL);
+
+            await flushBackupRequest();
+
+            // we should not have logged an error.
+            // eslint-disable-next-line no-console
+            expect(console.error).not.toHaveBeenCalled();
+        });
+
+        it("Only queries once", async () => {
+            fetchMock.get(
+                "express:/_matrix/client/v3/room_keys/keys/:room_id/:session_id",
+                { status: 404, body: { errcode: "M_NOT_FOUND" } },
+                { name: "getKey" },
+            );
+
+            // Send Alice a message that she won't be able to decrypt
+            syncResponder.sendOrQueueSyncResponse(SYNC_RESPONSE);
+            await flushBackupRequest();
+            const calls = fetchMock.calls("getKey");
+            expect(calls.length).toEqual(1);
+            expect(calls[0][0]).toEqual(EXPECTED_URL);
+
+            fetchMock.resetHistory();
+
+            // another message
+            const event2 = { ...testData.ENCRYPTED_EVENT, event_id: "$event2" };
+            const syncResponse2 = {
+                next_batch: 1,
+                rooms: { join: { [ROOM_ID]: { timeline: { events: [event2] } } } },
+            };
+            syncResponder.sendOrQueueSyncResponse(syncResponse2);
+            await flushBackupRequest();
+            expect(fetchMock.calls("getKey").length).toEqual(0);
+        });
+    });
+
+    describe("recover from backup", () => {
+        let aliceCrypto: CryptoApi;
+
+        beforeEach(async () => {
+            fetchMock.get("path:/_matrix/client/v3/room_keys/version", testData.SIGNED_BACKUP_DATA);
+
+            aliceClient = await initTestClient();
+            aliceCrypto = aliceClient.getCrypto()!;
+            await aliceClient.startClient();
+
+            // tell Alice to trust the dummy device that signed the backup
+            await waitForDeviceList();
+            await aliceCrypto.setDeviceVerified(testData.TEST_USER_ID, testData.TEST_DEVICE_ID);
+        });
+
+        it("can restore from backup (Curve25519 version)", async function () {
+            const fullBackup = {
+                rooms: {
                     [ROOM_ID]: {
-                        timeline: {
-                            events: [ENCRYPTED_EVENT],
+                        sessions: {
+                            [testData.MEGOLM_SESSION_DATA.session_id]: testData.CURVE25519_KEY_BACKUP_DATA,
                         },
                     },
                 },
-            },
-        };
+            };
 
-        fetchMock.get("express:/_matrix/client/v3/room_keys/keys/:room_id/:session_id", CURVE25519_KEY_BACKUP_DATA);
-        fetchMock.get("path:/_matrix/client/v3/room_keys/version", testData.SIGNED_BACKUP_DATA);
+            fetchMock.get("express:/_matrix/client/v3/room_keys/keys", fullBackup);
 
-        aliceClient = await initTestClient();
-        const aliceCrypto = aliceClient.getCrypto()!;
-        await aliceCrypto.storeSessionBackupPrivateKey(Buffer.from(testData.BACKUP_DECRYPTION_KEY_BASE64, "base64"));
+            const check = await aliceCrypto.checkKeyBackupAndEnable();
 
-        // start after saving the private key
-        await aliceClient.startClient();
+            let onKeyCached: () => void;
+            const awaitKeyCached = new Promise<void>((resolve) => {
+                onKeyCached = resolve;
+            });
 
-        // tell Alice to trust the dummy device that signed the backup, and re-check the backup.
-        // XXX: should we automatically re-check after a device becomes verified?
-        await waitForDeviceList();
-        await aliceCrypto.setDeviceVerified(testData.TEST_USER_ID, testData.TEST_DEVICE_ID);
-        await aliceClient.checkKeyBackup();
+            const result = await advanceTimersUntil(
+                aliceClient.restoreKeyBackupWithRecoveryKey(
+                    testData.BACKUP_DECRYPTION_KEY_BASE58,
+                    undefined,
+                    undefined,
+                    check!.backupInfo!,
+                    {
+                        cacheCompleteCallback: () => onKeyCached(),
+                    },
+                ),
+            );
 
-        // Now, send Alice a message that she won't be able to decrypt, and check that she fetches the key from the backup.
-        syncResponder.sendOrQueueSyncResponse(syncResponse);
-        await syncPromise(aliceClient);
+            expect(result.imported).toStrictEqual(1);
 
-        const room = aliceClient.getRoom(ROOM_ID)!;
-        const event = room.getLiveTimeline().getEvents()[0];
-        await awaitDecryption(event, { waitOnDecryptionFailure: true });
-        expect(event.getContent()).toEqual("testytest");
+            await awaitKeyCached;
+
+            // The key should be now cached
+            const afterCache = await advanceTimersUntil(
+                aliceClient.restoreKeyBackupWithCache(undefined, undefined, check!.backupInfo!),
+            );
+
+            expect(afterCache.imported).toStrictEqual(1);
+        });
+
+        /**
+         * Creates a mock backup response of a GET `room_keys/keys` with a given number of keys per room.
+         * @param keysPerRoom The number of keys per room
+         */
+        function createBackupDownloadResponse(keysPerRoom: number[]) {
+            const response: {
+                rooms: {
+                    [roomId: string]: {
+                        sessions: {
+                            [sessionId: string]: KeyBackupSession;
+                        };
+                    };
+                };
+            } = { rooms: {} };
+
+            const expectedTotal = keysPerRoom.reduce((a, b) => a + b, 0);
+            for (let i = 0; i < keysPerRoom.length; i++) {
+                const roomId = `!room${i}:example.com`;
+                response.rooms[roomId] = { sessions: {} };
+                for (let j = 0; j < keysPerRoom[i]; j++) {
+                    const sessionId = `session${j}`;
+                    // Put the same fake session data, not important for that test
+                    response.rooms[roomId].sessions[sessionId] = testData.CURVE25519_KEY_BACKUP_DATA;
+                }
+            }
+            return { response, expectedTotal };
+        }
+
+        it("Should import full backup in chunks", async function () {
+            const importMockImpl = jest.fn();
+            // @ts-ignore - mock a private method for testing purpose
+            aliceCrypto.importBackedUpRoomKeys = importMockImpl;
+
+            // We need several rooms with several sessions to test chunking
+            const { response, expectedTotal } = createBackupDownloadResponse([45, 300, 345, 12, 130]);
+
+            fetchMock.get("express:/_matrix/client/v3/room_keys/keys", response);
+
+            const check = await aliceCrypto.checkKeyBackupAndEnable();
+
+            const progressCallback = jest.fn();
+            const result = await aliceClient.restoreKeyBackupWithRecoveryKey(
+                testData.BACKUP_DECRYPTION_KEY_BASE58,
+                undefined,
+                undefined,
+                check!.backupInfo!,
+                {
+                    progressCallback,
+                },
+            );
+
+            expect(result.imported).toStrictEqual(expectedTotal);
+            // Should be called 5 times: 200*4 plus one chunk with the remaining 32
+            expect(importMockImpl).toHaveBeenCalledTimes(5);
+            for (let i = 0; i < 4; i++) {
+                expect(importMockImpl.mock.calls[i][0].length).toEqual(200);
+            }
+            expect(importMockImpl.mock.calls[4][0].length).toEqual(32);
+
+            expect(progressCallback).toHaveBeenCalledWith({
+                stage: "fetch",
+            });
+
+            // Should be called 4 times and report 200/400/600/800
+            for (let i = 0; i < 4; i++) {
+                expect(progressCallback).toHaveBeenCalledWith({
+                    total: expectedTotal,
+                    successes: (i + 1) * 200,
+                    stage: "load_keys",
+                    failures: 0,
+                });
+            }
+
+            // The last chunk
+            expect(progressCallback).toHaveBeenCalledWith({
+                total: expectedTotal,
+                successes: 832,
+                stage: "load_keys",
+                failures: 0,
+            });
+        });
+
+        it("Should continue to process backup if a chunk import fails and report failures", async function () {
+            // @ts-ignore - mock a private method for testing purpose
+            aliceCrypto.importBackedUpRoomKeys = jest
+                .fn()
+                .mockImplementationOnce(() => {
+                    // Fail to import first chunk
+                    throw new Error("test error");
+                })
+                // Ok for other chunks
+                .mockResolvedValue(undefined);
+
+            const { response, expectedTotal } = createBackupDownloadResponse([100, 300]);
+
+            fetchMock.get("express:/_matrix/client/v3/room_keys/keys", response);
+
+            const check = await aliceCrypto.checkKeyBackupAndEnable();
+
+            const progressCallback = jest.fn();
+            const result = await aliceClient.restoreKeyBackupWithRecoveryKey(
+                testData.BACKUP_DECRYPTION_KEY_BASE58,
+                undefined,
+                undefined,
+                check!.backupInfo!,
+                {
+                    progressCallback,
+                },
+            );
+
+            expect(result.total).toStrictEqual(expectedTotal);
+            // A chunk failed to import
+            expect(result.imported).toStrictEqual(200);
+
+            expect(progressCallback).toHaveBeenCalledWith({
+                total: expectedTotal,
+                successes: 0,
+                stage: "load_keys",
+                failures: 200,
+            });
+
+            expect(progressCallback).toHaveBeenCalledWith({
+                total: expectedTotal,
+                successes: 200,
+                stage: "load_keys",
+                failures: 200,
+            });
+        });
+
+        it("Should continue if some keys fails to decrypt", async function () {
+            // @ts-ignore - mock a private method for testing purpose
+            aliceCrypto.importBackedUpRoomKeys = jest.fn();
+
+            const decryptionFailureCount = 2;
+
+            const mockDecryptor = {
+                // DecryptSessions does not reject on decryption failure, but just skip the key
+                decryptSessions: jest.fn().mockImplementation((sessions) => {
+                    // simulate fail to decrypt 2 keys out of all
+                    const decrypted = [];
+                    const keys = Object.keys(sessions);
+                    for (let i = 0; i < keys.length - decryptionFailureCount; i++) {
+                        decrypted.push({
+                            session_id: keys[i],
+                        } as unknown as Mocked<IMegolmSessionData>);
+                    }
+                    return decrypted;
+                }),
+                free: jest.fn(),
+            };
+
+            // @ts-ignore - mock a private method for testing purpose
+            aliceCrypto.getBackupDecryptor = jest.fn().mockResolvedValue(mockDecryptor);
+
+            const { response, expectedTotal } = createBackupDownloadResponse([100]);
+
+            fetchMock.get("express:/_matrix/client/v3/room_keys/keys", response);
+
+            const check = await aliceCrypto.checkKeyBackupAndEnable();
+
+            const result = await aliceClient.restoreKeyBackupWithRecoveryKey(
+                testData.BACKUP_DECRYPTION_KEY_BASE58,
+                undefined,
+                undefined,
+                check!.backupInfo!,
+            );
+
+            expect(result.total).toStrictEqual(expectedTotal);
+            // A chunk failed to import
+            expect(result.imported).toStrictEqual(expectedTotal - decryptionFailureCount);
+        });
+
+        it("recover specific session from backup", async function () {
+            fetchMock.get(
+                "express:/_matrix/client/v3/room_keys/keys/:room_id/:session_id",
+                testData.CURVE25519_KEY_BACKUP_DATA,
+            );
+
+            const check = await aliceCrypto.checkKeyBackupAndEnable();
+
+            const result = await advanceTimersUntil(
+                aliceClient.restoreKeyBackupWithRecoveryKey(
+                    testData.BACKUP_DECRYPTION_KEY_BASE58,
+                    ROOM_ID,
+                    testData.MEGOLM_SESSION_DATA.session_id,
+                    check!.backupInfo!,
+                ),
+            );
+
+            expect(result.imported).toStrictEqual(1);
+        });
+
+        it("Fails on bad recovery key", async function () {
+            const fullBackup = {
+                rooms: {
+                    [ROOM_ID]: {
+                        sessions: {
+                            [testData.MEGOLM_SESSION_DATA.session_id]: testData.CURVE25519_KEY_BACKUP_DATA,
+                        },
+                    },
+                },
+            };
+
+            fetchMock.get("express:/_matrix/client/v3/room_keys/keys", fullBackup);
+
+            const check = await aliceCrypto.checkKeyBackupAndEnable();
+
+            await expect(
+                aliceClient.restoreKeyBackupWithRecoveryKey(
+                    "EsTx A7Xn aNFF k3jH zpV3 MQoN LJEg mscC HecF 982L wC77 mYQD",
+                    undefined,
+                    undefined,
+                    check!.backupInfo!,
+                ),
+            ).rejects.toThrow();
+        });
     });
 
     describe("backupLoop", () => {
@@ -566,6 +923,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("megolm-keys backup (%s)", (backe
             await aliceClient.startClient();
             await aliceCrypto.storeSessionBackupPrivateKey(
                 Buffer.from(testData.BACKUP_DECRYPTION_KEY_BASE64, "base64"),
+                testData.SIGNED_BACKUP_DATA.version!,
             );
 
             const result = await aliceCrypto.isKeyBackupTrusted(testData.SIGNED_BACKUP_DATA);
@@ -579,6 +937,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("megolm-keys backup (%s)", (backe
             await aliceClient.startClient();
             await aliceCrypto.storeSessionBackupPrivateKey(
                 Buffer.from(testData.BACKUP_DECRYPTION_KEY_BASE64, "base64"),
+                testData.SIGNED_BACKUP_DATA.version!,
             );
 
             const backup: KeyBackupInfo = JSON.parse(JSON.stringify(testData.SIGNED_BACKUP_DATA));
@@ -710,6 +1069,146 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("megolm-keys backup (%s)", (backe
         });
     });
 
+    describe("Backup Changed from other sessions", () => {
+        beforeEach(async () => {
+            fetchMock.get("path:/_matrix/client/v3/room_keys/version", testData.SIGNED_BACKUP_DATA);
+
+            // ignore requests to send room key requests
+            fetchMock.put("express:/_matrix/client/v3/sendToDevice/m.room_key_request/:request_id", {});
+
+            aliceClient = await initTestClient();
+            const aliceCrypto = aliceClient.getCrypto()!;
+            await aliceCrypto.storeSessionBackupPrivateKey(
+                Buffer.from(testData.BACKUP_DECRYPTION_KEY_BASE64, "base64"),
+                testData.SIGNED_BACKUP_DATA.version!,
+            );
+
+            // start after saving the private key
+            await aliceClient.startClient();
+
+            // tell Alice to trust the dummy device that signed the backup, and re-check the backup.
+            // XXX: should we automatically re-check after a device becomes verified?
+            await waitForDeviceList();
+            await aliceClient.getCrypto()!.setDeviceVerified(testData.TEST_USER_ID, testData.TEST_DEVICE_ID);
+            await aliceClient.getCrypto()!.checkKeyBackupAndEnable();
+        });
+
+        // let aliceClient: MatrixClient;
+
+        const SYNC_RESPONSE = {
+            next_batch: 1,
+            rooms: { join: { [ROOM_ID]: { timeline: { events: [testData.ENCRYPTED_EVENT] } } } },
+        };
+
+        it("If current backup has changed, the manager should switch to the new one on UTD", async () => {
+            // =====
+            // First ensure that the client checks for keys using the backup version 1
+            /// =====
+
+            fetchMock.get(
+                "express:/_matrix/client/v3/room_keys/keys/:room_id/:session_id",
+                (url, request) => {
+                    // check that the version is correct
+                    const version = new URLSearchParams(new URL(url).search).get("version");
+                    if (version == "1") {
+                        return testData.CURVE25519_KEY_BACKUP_DATA;
+                    } else {
+                        return {
+                            status: 403,
+                            body: {
+                                current_version: "1",
+                                errcode: "M_WRONG_ROOM_KEYS_VERSION",
+                                error: "Wrong backup version.",
+                            },
+                        };
+                    }
+                },
+                { overwriteRoutes: true },
+            );
+
+            // Send Alice a message that she won't be able to decrypt, and check that she fetches the key from the backup.
+            syncResponder.sendOrQueueSyncResponse(SYNC_RESPONSE);
+            await syncPromise(aliceClient);
+
+            const room = aliceClient.getRoom(ROOM_ID)!;
+            const event = room.getLiveTimeline().getEvents()[0];
+            await advanceTimersUntil(awaitDecryption(event, { waitOnDecryptionFailure: true }));
+
+            expect(event.getContent()).toEqual(testData.CLEAR_EVENT.content);
+
+            // =====
+            // Second suppose now that the backup has changed to version 2
+            /// =====
+
+            const newBackup = {
+                ...testData.SIGNED_BACKUP_DATA,
+                version: "2",
+            };
+
+            fetchMock.get("path:/_matrix/client/v3/room_keys/version", newBackup, { overwriteRoutes: true });
+            // suppose the new key is now known
+            const aliceCrypto = aliceClient.getCrypto()!;
+            await aliceCrypto.storeSessionBackupPrivateKey(
+                Buffer.from(testData.BACKUP_DECRYPTION_KEY_BASE64, "base64"),
+                newBackup.version,
+            );
+
+            // A check backup should happen at some point
+            await aliceCrypto.checkKeyBackupAndEnable();
+
+            const awaitHasQueriedNewBackup: IDeferred<void> = defer<void>();
+
+            fetchMock.get(
+                "express:/_matrix/client/v3/room_keys/keys/:room_id/:session_id",
+                (url, request) => {
+                    // check that the version is correct
+                    const version = new URLSearchParams(new URL(url).search).get("version");
+                    if (version == newBackup.version) {
+                        awaitHasQueriedNewBackup.resolve();
+                        return testData.CURVE25519_KEY_BACKUP_DATA;
+                    } else {
+                        // awaitHasQueriedOldBackup.resolve();
+                        return {
+                            status: 403,
+                            body: {
+                                current_version: "2",
+                                errcode: "M_WRONG_ROOM_KEYS_VERSION",
+                                error: "Wrong backup version.",
+                            },
+                        };
+                    }
+                },
+                { overwriteRoutes: true },
+            );
+
+            // Send Alice a message that she won't be able to decrypt, and check that she fetches the key from the new backup.
+            const newMessage: Partial<IEvent> = {
+                type: "m.room.encrypted",
+                room_id: "!room:id",
+                sender: "@alice:localhost",
+                content: {
+                    algorithm: "m.megolm.v1.aes-sha2",
+                    ciphertext:
+                        "AwgAEpABKvf9FqPW52zeHfeVTn90a3jlBLlx7g6VDEkc2089RQUJoWpSJRiK13E83rN41wgGFJccyfoCr7ZDGJeuGYMGETTrgnLQhLs6JmyPf37JYkzxW8uS8rGUKEqTFQriKhibHVLvVacOlSIObUiKU/V3r176XuixqZF/4eyK9A22JNpInbgI10ZUT6LnApH9LR3FpZbE2zImf1uNPuvp7r0xQbW7CcJjqpH+qTPBD5zFdFnMkc2SnbXCsIOaX11Dm0krWfQz7iA26ZnI1nyZnyh7XPrCnJCRsuQH",
+                    device_id: "WVMJGTSSVB",
+                    sender_key: "E5RiY/YCIrHWaF4u416CqvblC6udK2jt9SJ/h1QeLS0",
+                    session_id: "ybnW+LGdUhoS4fHm1DAEphukO3sZ1GCqZD7UQz7L+GA",
+                },
+                event_id: "$event2",
+                origin_server_ts: 1507753887000,
+            };
+
+            const nextSyncResponse = {
+                next_batch: 2,
+                rooms: { join: { [ROOM_ID]: { timeline: { events: [newMessage] } } } },
+            };
+            syncResponder.sendOrQueueSyncResponse(nextSyncResponse);
+            await syncPromise(aliceClient);
+
+            await awaitHasQueriedNewBackup.promise;
+        });
+    });
+
     /** make sure that the client knows about the dummy device */
     async function waitForDeviceList(): Promise<void> {
         // Completing the initial sync will make the device list download outdated device lists (of which our own

spec/integ/crypto/olm-encryption-spec.ts

@@ -34,8 +34,9 @@ import { logger } from "../../../src/logger";
 import * as testUtils from "../../test-utils/test-utils";
 import { TestClient } from "../../TestClient";
 import { CRYPTO_ENABLED, IClaimKeysRequest, IQueryKeysRequest, IUploadKeysRequest } from "../../../src/client";
-import { ClientEvent, IContent, ISendEventResponse, MatrixClient, MatrixEvent } from "../../../src/matrix";
+import { ClientEvent, IContent, ISendEventResponse, MatrixClient, MatrixEvent, MsgType } from "../../../src/matrix";
 import { DeviceInfo } from "../../../src/crypto/deviceinfo";
+import { KnownMembership } from "../../../src/@types/membership";
 
 let aliTestClient: TestClient;
 const roomId = "!room:localhost";
@@ -216,7 +217,7 @@ async function expectBobSendMessageRequest(): Promise<OlmPayload> {
 }
 
 function sendMessage(client: MatrixClient): Promise<ISendEventResponse> {
-    return client.sendMessage(roomId, { msgtype: "m.text", body: "Hello, World" });
+    return client.sendMessage(roomId, { msgtype: MsgType.Text, body: "Hello, World" });
 }
 
 async function expectSendMessageRequest(httpBackend: TestClient["httpBackend"]): Promise<IContent> {
@@ -316,11 +317,11 @@ function firstSync(testClient: TestClient): Promise<void> {
                     state: {
                         events: [
                             testUtils.mkMembership({
-                                mship: "join",
+                                mship: KnownMembership.Join,
                                 user: aliUserId,
                             }),
                             testUtils.mkMembership({
-                                mship: "join",
+                                mship: KnownMembership.Join,
                                 user: bobUserId,
                             }),
                         ],

spec/integ/crypto/olm-utils.ts

@@ -0,0 +1,408 @@
+/*
+Copyright 2016-2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import Olm from "@matrix-org/olm";
+import anotherjson from "another-json";
+
+import { IContent, IDeviceKeys, IDownloadKeyResult, IEvent, Keys, MatrixClient, SigningKeys } from "../../../src";
+import { IE2EKeyReceiver } from "../../test-utils/E2EKeyReceiver";
+import { ISyncResponder } from "../../test-utils/SyncResponder";
+import { syncPromise } from "../../test-utils/test-utils";
+import { KeyBackupInfo } from "../../../src/crypto-api";
+
+/**
+ * @module
+ *
+ * A set of utilities for creating Olm accounts and sessions, and encrypting/decrypting with Olm/Megolm.
+ */
+
+/** Create an Olm Account object */
+export async function createOlmAccount(): Promise<Olm.Account> {
+    await Olm.init();
+    const testOlmAccount = new Olm.Account();
+    testOlmAccount.create();
+    return testOlmAccount;
+}
+
+/**
+ * Get the device keys for the test Olm Account
+ *
+ * @param olmAccount - Test olm account
+ * @param userId - The user ID to present the keys as belonging to
+ */
+export function getTestOlmAccountKeys(olmAccount: Olm.Account, userId: string, deviceId: string): IDeviceKeys {
+    const testE2eKeys = JSON.parse(olmAccount.identity_keys());
+    const testDeviceKeys: IDeviceKeys = {
+        algorithms: ["m.olm.v1.curve25519-aes-sha2", "m.megolm.v1.aes-sha2"],
+        device_id: deviceId,
+        keys: {
+            [`curve25519:${deviceId}`]: testE2eKeys.curve25519,
+            [`ed25519:${deviceId}`]: testE2eKeys.ed25519,
+        },
+        user_id: userId,
+    };
+
+    const j = anotherjson.stringify(testDeviceKeys);
+    const sig = olmAccount.sign(j);
+    testDeviceKeys.signatures = { [userId]: { [`ed25519:${deviceId}`]: sig } };
+    return testDeviceKeys;
+}
+
+/**
+ * Bootstrap cross signing for the given Olm account.
+ *
+ * Will generate the cross signing keys and sign them with the master key, and returns the `IDownloadKeyResult`
+ * that can be directly fed into a test e2eKeyResponder.
+ *
+ * The cross-signing keys are randomly generated, similar to how the olm account keys are generated. There may not
+ * be any value in using static vectors, as the device keys change at every test run.
+ *
+ * If some `KeyBackupInfo` are provided, the `auth_data` of each backup info will be signed with the
+ * master key, meaning the backups will be then trusted after verification.
+ *
+ * @param olmAccount - The Olm account object to use for signing the device keys.
+ * @param userId - The user ID to associate with the device keys.
+ * @param deviceId - The device ID to associate with the device keys.
+ * @param keyBackupInfo - Optional key backup infos to sign with the master key.
+ * @returns A valid keys/query response that can be fed into a test e2eKeyResponder.
+ */
+export function bootstrapCrossSigningTestOlmAccount(
+    olmAccount: Olm.Account,
+    userId: string,
+    deviceId: string,
+    keyBackupInfo: KeyBackupInfo[] = [],
+): Partial<IDownloadKeyResult> {
+    const olmAliceMSK = new global.Olm.PkSigning();
+    const masterPrivkey = olmAliceMSK.generate_seed();
+    const masterPubkey = olmAliceMSK.init_with_seed(masterPrivkey);
+
+    const olmAliceUSK = new global.Olm.PkSigning();
+    const userPrivkey = olmAliceUSK.generate_seed();
+    const userPubkey = olmAliceUSK.init_with_seed(userPrivkey);
+
+    const olmAliceSSK = new global.Olm.PkSigning();
+    const sskPrivkey = olmAliceSSK.generate_seed();
+    const sskPubkey = olmAliceSSK.init_with_seed(sskPrivkey);
+
+    const mskInfo: Keys = {
+        user_id: userId,
+        usage: ["master"],
+        keys: {
+            ["ed25519:" + masterPubkey]: masterPubkey,
+        },
+    };
+
+    const sskInfo: Partial<SigningKeys> = {
+        user_id: userId,
+        usage: ["self_signing"],
+        keys: {
+            ["ed25519:" + sskPubkey]: sskPubkey,
+        },
+    };
+    // sign the ssk with the msk
+    const sskSig = olmAliceMSK.sign(anotherjson.stringify(sskInfo));
+    sskInfo.signatures = {
+        [userId]: {
+            ["ed25519:" + masterPubkey]: sskSig,
+        },
+    };
+
+    const uskInfo: Partial<SigningKeys> = {
+        user_id: userId,
+        usage: ["user_signing"],
+        keys: {
+            ["ed25519:" + userPubkey]: userPubkey,
+        },
+    };
+
+    // sign the usk with the msk
+    const uskSig = olmAliceMSK.sign(anotherjson.stringify(uskInfo));
+    uskInfo.signatures = {
+        [userId]: {
+            ["ed25519:" + masterPubkey]: uskSig,
+        },
+    };
+
+    // get the device keys and sign them with the ssk (the device is then cross signed)
+    const deviceKeys = getTestOlmAccountKeys(olmAccount, userId, deviceId);
+
+    const copy = Object.assign({}, deviceKeys);
+    delete copy.signatures;
+    const crossSignature = olmAliceSSK.sign(anotherjson.stringify(copy));
+
+    // add the signature
+    deviceKeys.signatures![userId]["ed25519:" + sskPubkey] = crossSignature;
+
+    // if we have some key backup info, sign them with the msk
+    keyBackupInfo.forEach((info) => {
+        const unsignedAuthData = Object.assign({}, info.auth_data);
+        delete unsignedAuthData.signatures;
+        const backupSignature = olmAliceMSK.sign(anotherjson.stringify(unsignedAuthData));
+
+        info.auth_data.signatures = {
+            [userId]: {
+                ["ed25519:" + masterPubkey]: backupSignature,
+            },
+        };
+    });
+
+    // clean the olm resources as we don't need them anymore
+    olmAliceMSK.free();
+    olmAliceSSK.free();
+    olmAliceUSK.free();
+
+    return {
+        master_keys: { [userId]: mskInfo },
+        user_signing_keys: { [userId]: uskInfo as SigningKeys },
+        self_signing_keys: { [userId]: sskInfo as SigningKeys },
+        device_keys: { [userId]: { [deviceId]: deviceKeys } },
+    };
+}
+
+/** start an Olm session with a given recipient */
+export async function createOlmSession(
+    olmAccount: Olm.Account,
+    recipientTestClient: IE2EKeyReceiver,
+): Promise<Olm.Session> {
+    const keys = await recipientTestClient.awaitOneTimeKeyUpload();
+    const otkId = Object.keys(keys)[0];
+    const otk = keys[otkId];
+
+    const session = new global.Olm.Session();
+    session.create_outbound(olmAccount, recipientTestClient.getDeviceKey(), otk.key);
+    return session;
+}
+
+// IToDeviceEvent isn't exported by src/sync-accumulator.ts
+export interface ToDeviceEvent {
+    content: IContent;
+    sender: string;
+    type: string;
+}
+
+/** encrypt an event with an existing olm session */
+export function encryptOlmEvent(opts: {
+    /** the sender's user id */
+    sender?: string;
+    /** the sender's curve25519 key */
+    senderKey: string;
+    /** the sender's ed25519 key */
+    senderSigningKey: string;
+    /** the olm session to use for encryption */
+    p2pSession: Olm.Session;
+    /** the recipient's user id */
+    recipient: string;
+    /** the recipient's curve25519 key */
+    recipientCurve25519Key: string;
+    /** the recipient's ed25519 key */
+    recipientEd25519Key: string;
+    /** the payload of the message */
+    plaincontent?: object;
+    /** the event type of the payload */
+    plaintype?: string;
+}): ToDeviceEvent {
+    expect(opts.senderKey).toBeTruthy();
+    expect(opts.p2pSession).toBeTruthy();
+    expect(opts.recipient).toBeTruthy();
+
+    const plaintext = {
+        content: opts.plaincontent || {},
+        recipient: opts.recipient,
+        recipient_keys: {
+            ed25519: opts.recipientEd25519Key,
+        },
+        keys: {
+            ed25519: opts.senderSigningKey,
+        },
+        sender: opts.sender || "@bob:xyz",
+        type: opts.plaintype || "m.test",
+    };
+
+    return {
+        content: {
+            algorithm: "m.olm.v1.curve25519-aes-sha2",
+            ciphertext: {
+                [opts.recipientCurve25519Key]: opts.p2pSession.encrypt(JSON.stringify(plaintext)),
+            },
+            sender_key: opts.senderKey,
+        },
+        sender: opts.sender || "@bob:xyz",
+        type: "m.room.encrypted",
+    };
+}
+
+// encrypt an event with megolm
+export function encryptMegolmEvent(opts: {
+    senderKey: string;
+    groupSession: Olm.OutboundGroupSession;
+    plaintext?: Partial<IEvent>;
+    room_id?: string;
+}): IEvent {
+    expect(opts.senderKey).toBeTruthy();
+    expect(opts.groupSession).toBeTruthy();
+
+    const plaintext = opts.plaintext || {};
+    if (!plaintext.content) {
+        plaintext.content = {
+            body: "42",
+            msgtype: "m.text",
+        };
+    }
+    if (!plaintext.type) {
+        plaintext.type = "m.room.message";
+    }
+    if (!plaintext.room_id) {
+        expect(opts.room_id).toBeTruthy();
+        plaintext.room_id = opts.room_id;
+    }
+    return encryptMegolmEventRawPlainText({
+        senderKey: opts.senderKey,
+        groupSession: opts.groupSession,
+        plaintext,
+    });
+}
+
+export function encryptMegolmEventRawPlainText(opts: {
+    senderKey: string;
+    groupSession: Olm.OutboundGroupSession;
+    plaintext: Partial<IEvent>;
+    origin_server_ts?: number;
+}): IEvent {
+    return {
+        event_id: "$test_megolm_event_" + Math.random(),
+        sender: opts.plaintext.sender ?? "@not_the_real_sender:example.com",
+        origin_server_ts: opts.plaintext.origin_server_ts ?? 1672944778000,
+        content: {
+            algorithm: "m.megolm.v1.aes-sha2",
+            ciphertext: opts.groupSession.encrypt(JSON.stringify(opts.plaintext)),
+            device_id: "testDevice",
+            sender_key: opts.senderKey,
+            session_id: opts.groupSession.session_id(),
+        },
+        type: "m.room.encrypted",
+        unsigned: {},
+    };
+}
+
+/** build an encrypted room_key event to share a group session, using an existing olm session */
+export function encryptGroupSessionKey(opts: {
+    /** recipient's user id */
+    recipient: string;
+    /** the recipient's curve25519 key */
+    recipientCurve25519Key: string;
+    /** the recipient's ed25519 key */
+    recipientEd25519Key: string;
+    /** sender's olm account */
+    olmAccount: Olm.Account;
+    /** sender's olm session with the recipient */
+    p2pSession: Olm.Session;
+    groupSession: Olm.OutboundGroupSession;
+    room_id?: string;
+}): ToDeviceEvent {
+    const senderKeys = JSON.parse(opts.olmAccount.identity_keys());
+    return encryptOlmEvent({
+        senderKey: senderKeys.curve25519,
+        senderSigningKey: senderKeys.ed25519,
+        recipient: opts.recipient,
+        recipientCurve25519Key: opts.recipientCurve25519Key,
+        recipientEd25519Key: opts.recipientEd25519Key,
+        p2pSession: opts.p2pSession,
+        plaincontent: {
+            algorithm: "m.megolm.v1.aes-sha2",
+            room_id: opts.room_id,
+            session_id: opts.groupSession.session_id(),
+            session_key: opts.groupSession.session_key(),
+        },
+        plaintype: "m.room_key",
+    });
+}
+
+/**
+ * Test utility to correctly encrypt a secret send event to a test device using the provided p2p session.
+ *
+ * @param opts - the options for the secret send event
+ * @returns the to-device event, ready to be returned in a sync response for the test device.
+ */
+export function encryptSecretSend(opts: {
+    /** the sender's user id */
+    sender: string;
+    /** recipient's user id */
+    recipient: string;
+    /** the recipient's curve25519 key */
+    recipientCurve25519Key: string;
+    /** the recipient's ed25519 key */
+    recipientEd25519Key: string;
+    /** sender's olm account */
+    olmAccount: Olm.Account;
+    /** sender's olm session with the recipient */
+    p2pSession: Olm.Session;
+    /** The requestId of the secret request that this secret send is replying. */
+    requestId: string;
+    /** The secret value */
+    secret: string;
+}): ToDeviceEvent {
+    const senderKeys = JSON.parse(opts.olmAccount.identity_keys());
+    return encryptOlmEvent({
+        sender: opts.sender,
+        senderKey: senderKeys.curve25519,
+        senderSigningKey: senderKeys.ed25519,
+        recipient: opts.recipient,
+        recipientCurve25519Key: opts.recipientCurve25519Key,
+        recipientEd25519Key: opts.recipientEd25519Key,
+        p2pSession: opts.p2pSession,
+        plaincontent: {
+            request_id: opts.requestId,
+            secret: opts.secret,
+        },
+        plaintype: "m.secret.send",
+    });
+}
+
+/**
+ * Establish an Olm Session with the test user
+ *
+ * Waits for the test user to upload their keys, then sends a /sync response with a to-device message which will
+ * establish an Olm session.
+ *
+ * @param testClient - the MatrixClient under test, which we expect to upload account keys, and to make a
+ *    /sync request which we will respond to.
+ * @param keyReceiver - an IE2EKeyReceiver which will intercept the /keys/upload request from the client under test
+ * @param syncResponder - an ISyncResponder which will intercept /sync requests from the client under test
+ * @param peerOlmAccount: an OlmAccount which will be used to initiate the Olm session.
+ */
+export async function establishOlmSession(
+    testClient: MatrixClient,
+    keyReceiver: IE2EKeyReceiver,
+    syncResponder: ISyncResponder,
+    peerOlmAccount: Olm.Account,
+): Promise<Olm.Session> {
+    const peerE2EKeys = JSON.parse(peerOlmAccount.identity_keys());
+    const p2pSession = await createOlmSession(peerOlmAccount, keyReceiver);
+    const olmEvent = encryptOlmEvent({
+        senderKey: peerE2EKeys.curve25519,
+        senderSigningKey: peerE2EKeys.ed25519,
+        recipient: testClient.getUserId()!,
+        recipientCurve25519Key: keyReceiver.getDeviceKey(),
+        recipientEd25519Key: keyReceiver.getSigningKey(),
+        p2pSession: p2pSession,
+    });
+    syncResponder.sendOrQueueSyncResponse({
+        next_batch: 1,
+        to_device: { events: [olmEvent] },
+    });
+    await syncPromise(testClient);
+    return p2pSession;
+}

spec/integ/crypto/rust-crypto.spec.ts

@@ -16,8 +16,15 @@ limitations under the License.
 
 import "fake-indexeddb/auto";
 import { IDBFactory } from "fake-indexeddb";
+import fetchMock from "fetch-mock-jest";
 
-import { createClient } from "../../../src";
+import { createClient, CryptoEvent, IndexedDBCryptoStore } from "../../../src";
+import { populateStore } from "../../test-utils/test_indexeddb_cryptostore_dump";
+import { MSK_NOT_CACHED_DATASET } from "../../test-utils/test_indexeddb_cryptostore_dump/no_cached_msk_dump";
+import { IDENTITY_NOT_TRUSTED_DATASET } from "../../test-utils/test_indexeddb_cryptostore_dump/unverified";
+import { FULL_ACCOUNT_DATASET } from "../../test-utils/test_indexeddb_cryptostore_dump/full_account";
+
+jest.setTimeout(15000);
 
 afterEach(() => {
     // reset fake-indexeddb after each test, to make sure we don't leak connections
@@ -41,7 +48,7 @@ describe("MatrixClient.initRustCrypto", () => {
         await expect(() => unknownDeviceClient.initRustCrypto()).rejects.toThrow("unknown deviceId");
     });
 
-    it("should create the indexed dbs", async () => {
+    it("should create the indexed db", async () => {
         const matrixClient = createClient({
             baseUrl: "http://test.server",
             userId: "@alice:localhost",
@@ -53,7 +60,25 @@ describe("MatrixClient.initRustCrypto", () => {
 
         await matrixClient.initRustCrypto();
 
-        // should have two dbs now
+        // should have an indexed db now
+        const databaseNames = (await indexedDB.databases()).map((db) => db.name);
+        expect(databaseNames).toEqual(expect.arrayContaining(["matrix-js-sdk::matrix-sdk-crypto"]));
+    });
+
+    it("should create the meta db if given a pickleKey", async () => {
+        const matrixClient = createClient({
+            baseUrl: "http://test.server",
+            userId: "@alice:localhost",
+            deviceId: "aliceDevice",
+            pickleKey: "testKey",
+        });
+
+        // No databases.
+        expect(await indexedDB.databases()).toHaveLength(0);
+
+        await matrixClient.initRustCrypto();
+
+        // should have two indexed dbs now
         const databaseNames = (await indexedDB.databases()).map((db) => db.name);
         expect(databaseNames).toEqual(
             expect.arrayContaining(["matrix-js-sdk::matrix-sdk-crypto", "matrix-js-sdk::matrix-sdk-crypto-meta"]),
@@ -70,6 +95,302 @@ describe("MatrixClient.initRustCrypto", () => {
         await matrixClient.initRustCrypto();
         await matrixClient.initRustCrypto();
     });
+
+    describe("Libolm Migration", () => {
+        beforeEach(() => {
+            fetchMock.reset();
+        });
+
+        it("should migrate from libolm", async () => {
+            fetchMock.get("path:/_matrix/client/v3/room_keys/version", FULL_ACCOUNT_DATASET.backupResponse);
+
+            fetchMock.post("path:/_matrix/client/v3/keys/query", FULL_ACCOUNT_DATASET.keyQueryResponse);
+
+            const testStoreName = "test-store";
+            await populateStore(testStoreName, FULL_ACCOUNT_DATASET.dumpPath);
+            const cryptoStore = new IndexedDBCryptoStore(indexedDB, testStoreName);
+
+            const matrixClient = createClient({
+                baseUrl: "http://test.server",
+                userId: FULL_ACCOUNT_DATASET.userId,
+                deviceId: FULL_ACCOUNT_DATASET.deviceId,
+                cryptoStore,
+                pickleKey: FULL_ACCOUNT_DATASET.pickleKey,
+            });
+
+            const progressListener = jest.fn();
+            matrixClient.addListener(CryptoEvent.LegacyCryptoStoreMigrationProgress, progressListener);
+
+            await matrixClient.initRustCrypto();
+
+            const verificationStatus = await matrixClient
+                .getCrypto()!
+                .getDeviceVerificationStatus(FULL_ACCOUNT_DATASET.userId, FULL_ACCOUNT_DATASET.deviceId);
+
+            // Check that the current device and identity trust is migrated correctly just after migration
+            expect(verificationStatus).toBeDefined();
+            expect(verificationStatus!.crossSigningVerified).toEqual(true);
+            expect(verificationStatus!.signedByOwner).toEqual(true);
+
+            // Do some basic checks on the imported data
+            const deviceKeys = await matrixClient.getCrypto()!.getOwnDeviceKeys();
+            expect(deviceKeys.curve25519).toEqual("LKv0bKbc0EC4h0jknbemv3QalEkeYvuNeUXVRgVVTTU");
+            expect(deviceKeys.ed25519).toEqual("qK70DEqIXq7T+UU3v/al47Ab4JkMEBLpNrTBMbS5rrw");
+
+            expect(await matrixClient.getCrypto()!.getActiveSessionBackupVersion()).toEqual("7");
+
+            expect(await matrixClient.getCrypto()!.isEncryptionEnabledInRoom("!CWLUCoEWXSFyTCOtfL:matrix.org")).toBe(
+                true,
+            );
+
+            // check the progress callback
+            expect(progressListener.mock.calls.length).toBeGreaterThan(50);
+
+            // The first call should have progress == 0
+            const [firstProgress, totalSteps] = progressListener.mock.calls[0];
+            expect(totalSteps).toBeGreaterThan(3000);
+            expect(firstProgress).toEqual(0);
+
+            for (let i = 1; i < progressListener.mock.calls.length - 1; i++) {
+                const [progress, total] = progressListener.mock.calls[i];
+                expect(total).toEqual(totalSteps);
+                expect(progress).toBeGreaterThan(progressListener.mock.calls[i - 1][0]);
+                expect(progress).toBeLessThanOrEqual(totalSteps);
+            }
+
+            // The final call should have progress == total == -1
+            expect(progressListener).toHaveBeenLastCalledWith(-1, -1);
+        }, 60000);
+
+        describe("Private key backup migration", () => {
+            it("should not migrate the backup private key if backup has changed", async () => {
+                // Here we have a new backup server side, and the migrated account has the previous backup key.
+                fetchMock.get("path:/_matrix/client/v3/room_keys/version", MSK_NOT_CACHED_DATASET.newBackupResponse);
+
+                fetchMock.post("path:/_matrix/client/v3/keys/query", MSK_NOT_CACHED_DATASET.keyQueryResponse);
+
+                await populateStore("test-store", MSK_NOT_CACHED_DATASET.dumpPath);
+                const cryptoStore = new IndexedDBCryptoStore(indexedDB, "test-store");
+
+                const matrixClient = createClient({
+                    baseUrl: "http://test.server",
+                    userId: MSK_NOT_CACHED_DATASET.userId,
+                    deviceId: MSK_NOT_CACHED_DATASET.deviceId,
+                    cryptoStore,
+                    pickleKey: MSK_NOT_CACHED_DATASET.pickleKey,
+                });
+
+                await matrixClient.initRustCrypto();
+
+                const privateBackupKey = await matrixClient.getCrypto()?.getSessionBackupPrivateKey();
+                expect(privateBackupKey).toBeNull();
+            });
+
+            it("should not migrate the backup private key if backup has unknown algorithm", async () => {
+                // Here we have a new backup server side, and the migrated account has the previous backup key.
+                const backupResponse = {
+                    ...MSK_NOT_CACHED_DATASET.backupResponse,
+                    algorithm: "m.megolm_backup.v8",
+                };
+                fetchMock.get("path:/_matrix/client/v3/room_keys/version", backupResponse);
+
+                fetchMock.post("path:/_matrix/client/v3/keys/query", MSK_NOT_CACHED_DATASET.keyQueryResponse);
+
+                await populateStore("test-store", MSK_NOT_CACHED_DATASET.dumpPath);
+                const cryptoStore = new IndexedDBCryptoStore(indexedDB, "test-store");
+
+                const matrixClient = createClient({
+                    baseUrl: "http://test.server",
+                    userId: MSK_NOT_CACHED_DATASET.userId,
+                    deviceId: MSK_NOT_CACHED_DATASET.deviceId,
+                    cryptoStore,
+                    pickleKey: MSK_NOT_CACHED_DATASET.pickleKey,
+                });
+
+                await matrixClient.initRustCrypto();
+
+                const privateBackupKey = await matrixClient.getCrypto()?.getSessionBackupPrivateKey();
+                expect(privateBackupKey).toBeNull();
+            });
+
+            it("should not migrate the backup private key if the backup has been deleted", async () => {
+                // The old backup has been deleted server side.
+                fetchMock.get("path:/_matrix/client/v3/room_keys/version", {
+                    status: 404,
+                    body: {
+                        errcode: "M_NOT_FOUND",
+                        error: "No backup found",
+                    },
+                });
+
+                fetchMock.post("path:/_matrix/client/v3/keys/query", MSK_NOT_CACHED_DATASET.keyQueryResponse);
+
+                await populateStore("test-store", MSK_NOT_CACHED_DATASET.dumpPath);
+                const cryptoStore = new IndexedDBCryptoStore(indexedDB, "test-store");
+
+                const matrixClient = createClient({
+                    baseUrl: "http://test.server",
+                    userId: MSK_NOT_CACHED_DATASET.userId,
+                    deviceId: MSK_NOT_CACHED_DATASET.deviceId,
+                    cryptoStore,
+                    pickleKey: MSK_NOT_CACHED_DATASET.pickleKey,
+                });
+
+                await matrixClient.initRustCrypto();
+
+                const privateBackupKey = await matrixClient.getCrypto()?.getSessionBackupPrivateKey();
+                expect(privateBackupKey).toBeNull();
+            });
+
+            it("should migrate the backup private key if the backup matches", async () => {
+                // The old backup has been deleted server side.
+                fetchMock.get("path:/_matrix/client/v3/room_keys/version", MSK_NOT_CACHED_DATASET.backupResponse);
+
+                fetchMock.post("path:/_matrix/client/v3/keys/query", MSK_NOT_CACHED_DATASET.keyQueryResponse);
+
+                await populateStore("test-store", MSK_NOT_CACHED_DATASET.dumpPath);
+                const cryptoStore = new IndexedDBCryptoStore(indexedDB, "test-store");
+
+                const matrixClient = createClient({
+                    baseUrl: "http://test.server",
+                    userId: MSK_NOT_CACHED_DATASET.userId,
+                    deviceId: MSK_NOT_CACHED_DATASET.deviceId,
+                    cryptoStore,
+                    pickleKey: MSK_NOT_CACHED_DATASET.pickleKey,
+                });
+
+                await matrixClient.initRustCrypto();
+
+                const privateBackupKey = await matrixClient.getCrypto()?.getSessionBackupPrivateKey();
+                expect(privateBackupKey).toBeDefined();
+            });
+        });
+
+        describe("Legacy trust migration", () => {
+            async function populateAndStartLegacyCryptoStore(dumpPath: string): Promise<IndexedDBCryptoStore> {
+                const testStoreName = "test-store";
+                await populateStore(testStoreName, dumpPath);
+                const cryptoStore = new IndexedDBCryptoStore(indexedDB, testStoreName);
+                await cryptoStore.startup();
+                return cryptoStore;
+            }
+
+            it("should not revert to untrusted if legacy was trusted but msk not in cache, big account", async () => {
+                fetchMock.get("path:/_matrix/client/v3/room_keys/version", {
+                    status: 404,
+                    body: {
+                        errcode: "M_NOT_FOUND",
+                        error: "No backup found",
+                    },
+                });
+
+                fetchMock.post("path:/_matrix/client/v3/keys/query", FULL_ACCOUNT_DATASET.keyQueryResponse);
+
+                const cryptoStore = await populateAndStartLegacyCryptoStore(FULL_ACCOUNT_DATASET.dumpPath);
+
+                // Remove the master key from the cache
+                await cryptoStore.doTxn("readwrite", [IndexedDBCryptoStore.STORE_ACCOUNT], (txn) => {
+                    const objectStore = txn.objectStore("account");
+                    objectStore.delete(`ssss_cache:master`);
+                });
+
+                const matrixClient = createClient({
+                    baseUrl: "http://test.server",
+                    userId: FULL_ACCOUNT_DATASET.userId,
+                    deviceId: FULL_ACCOUNT_DATASET.deviceId,
+                    cryptoStore,
+                    pickleKey: FULL_ACCOUNT_DATASET.pickleKey,
+                });
+
+                await matrixClient.initRustCrypto();
+
+                const verificationStatus = await matrixClient
+                    .getCrypto()!
+                    .getUserVerificationStatus(FULL_ACCOUNT_DATASET.userId);
+
+                expect(verificationStatus.isCrossSigningVerified()).toBe(true);
+            }, 60000);
+
+            it("should not revert to untrusted if legacy was trusted but msk not in cache", async () => {
+                fetchMock.get("path:/_matrix/client/v3/room_keys/version", MSK_NOT_CACHED_DATASET.backupResponse);
+
+                fetchMock.post("path:/_matrix/client/v3/keys/query", MSK_NOT_CACHED_DATASET.keyQueryResponse);
+
+                const cryptoStore = await populateAndStartLegacyCryptoStore(MSK_NOT_CACHED_DATASET.dumpPath);
+
+                const matrixClient = createClient({
+                    baseUrl: "http://test.server",
+                    userId: MSK_NOT_CACHED_DATASET.userId,
+                    deviceId: MSK_NOT_CACHED_DATASET.deviceId,
+                    cryptoStore,
+                    pickleKey: MSK_NOT_CACHED_DATASET.pickleKey,
+                });
+
+                await matrixClient.initRustCrypto();
+
+                const verificationStatus = await matrixClient
+                    .getCrypto()!
+                    .getUserVerificationStatus("@migration:localhost");
+
+                expect(verificationStatus.isCrossSigningVerified()).toBe(true);
+            });
+
+            it("should not migrate local trust if key has changed", async () => {
+                fetchMock.get("path:/_matrix/client/v3/room_keys/version", MSK_NOT_CACHED_DATASET.backupResponse);
+
+                fetchMock.post("path:/_matrix/client/v3/keys/query", MSK_NOT_CACHED_DATASET.rotatedKeyQueryResponse);
+
+                const cryptoStore = await populateAndStartLegacyCryptoStore(MSK_NOT_CACHED_DATASET.dumpPath);
+
+                const matrixClient = createClient({
+                    baseUrl: "http://test.server",
+                    userId: MSK_NOT_CACHED_DATASET.userId,
+                    deviceId: MSK_NOT_CACHED_DATASET.deviceId,
+                    cryptoStore,
+                    pickleKey: MSK_NOT_CACHED_DATASET.pickleKey,
+                });
+
+                await matrixClient.initRustCrypto();
+
+                const verificationStatus = await matrixClient
+                    .getCrypto()!
+                    .getUserVerificationStatus("@migration:localhost");
+
+                expect(verificationStatus.isCrossSigningVerified()).toBe(false);
+            });
+
+            it("should not migrate local trust if was not trusted in legacy", async () => {
+                // Just 404 here for the test
+                fetchMock.get("path:/_matrix/client/v3/room_keys/version", {
+                    status: 404,
+                    body: {
+                        errcode: "M_NOT_FOUND",
+                        error: "No backup found",
+                    },
+                });
+
+                fetchMock.post("path:/_matrix/client/v3/keys/query", IDENTITY_NOT_TRUSTED_DATASET.keyQueryResponse);
+
+                const cryptoStore = await populateAndStartLegacyCryptoStore(IDENTITY_NOT_TRUSTED_DATASET.dumpPath);
+
+                const matrixClient = createClient({
+                    baseUrl: "http://test.server",
+                    userId: IDENTITY_NOT_TRUSTED_DATASET.userId,
+                    deviceId: IDENTITY_NOT_TRUSTED_DATASET.deviceId,
+                    cryptoStore,
+                    pickleKey: IDENTITY_NOT_TRUSTED_DATASET.pickleKey,
+                });
+
+                await matrixClient.initRustCrypto();
+
+                const verificationStatus = await matrixClient
+                    .getCrypto()!
+                    .getUserVerificationStatus("@untrusted:localhost");
+
+                expect(verificationStatus.isCrossSigningVerified()).toBe(false);
+            });
+        });
+    });
 });
 
 describe("MatrixClient.clearStores", () => {
@@ -78,6 +399,7 @@ describe("MatrixClient.clearStores", () => {
             baseUrl: "http://test.server",
             userId: "@alice:localhost",
             deviceId: "aliceDevice",
+            pickleKey: "testKey",
         });
 
         await matrixClient.initRustCrypto();
@@ -87,4 +409,19 @@ describe("MatrixClient.clearStores", () => {
         await matrixClient.clearStores();
         expect(await indexedDB.databases()).toHaveLength(0);
     });
+
+    it("should not fail in environments without indexedDB", async () => {
+        // eslint-disable-next-line no-global-assign
+        indexedDB = undefined!;
+        const matrixClient = createClient({
+            baseUrl: "http://test.server",
+            userId: "@alice:localhost",
+            deviceId: "aliceDevice",
+        });
+
+        await matrixClient.stopClient();
+
+        await matrixClient.clearStores();
+        // No error thrown in clearStores
+    });
 });

spec/integ/crypto/verification.spec.ts

@@ -21,12 +21,15 @@ import { MockResponse } from "fetch-mock";
 import fetchMock from "fetch-mock-jest";
 import { IDBFactory } from "fake-indexeddb";
 import { createHash } from "crypto";
+import Olm from "@matrix-org/olm";
 
 import {
     createClient,
     CryptoEvent,
+    DeviceVerification,
     IContent,
     ICreateClientOpts,
+    IEvent,
     MatrixClient,
     MatrixEvent,
     MatrixEventEvent,
@@ -41,13 +44,23 @@ import {
     Verifier,
     VerifierEvent,
 } from "../../../src/crypto-api/verification";
-import { escapeRegExp } from "../../../src/utils";
-import { CRYPTO_BACKENDS, emitPromise, getSyncResponse, InitCrypto, syncPromise } from "../../test-utils/test-utils";
+import { defer, escapeRegExp } from "../../../src/utils";
+import {
+    awaitDecryption,
+    CRYPTO_BACKENDS,
+    emitPromise,
+    getSyncResponse,
+    InitCrypto,
+    syncPromise,
+} from "../../test-utils/test-utils";
 import { SyncResponder } from "../../test-utils/SyncResponder";
 import {
+    BACKUP_DECRYPTION_KEY_BASE64,
+    BOB_ONE_TIME_KEYS,
     BOB_SIGNED_CROSS_SIGNING_KEYS_DATA,
     BOB_SIGNED_TEST_DEVICE_DATA,
     BOB_TEST_USER_ID,
+    CURVE25519_KEY_BACKUP_DATA,
     MASTER_CROSS_SIGNING_PUBLIC_KEY_BASE64,
     SIGNED_CROSS_SIGNING_KEYS_DATA,
     SIGNED_TEST_DEVICE_DATA,
@@ -55,15 +68,25 @@ import {
     TEST_DEVICE_PUBLIC_ED25519_KEY_BASE64,
     TEST_ROOM_ID,
     TEST_USER_ID,
-    BOB_ONE_TIME_KEYS,
 } from "../../test-utils/test-data";
 import { mockInitialApiRequests } from "../../test-utils/mockEndpoints";
 import { E2EKeyResponder } from "../../test-utils/E2EKeyResponder";
 import { E2EKeyReceiver } from "../../test-utils/E2EKeyReceiver";
+import {
+    bootstrapCrossSigningTestOlmAccount,
+    createOlmSession,
+    encryptGroupSessionKey,
+    encryptMegolmEvent,
+    encryptSecretSend,
+    ToDeviceEvent,
+} from "./olm-utils";
+import { KeyBackupInfo } from "../../../src/crypto-api";
+import { encodeBase64 } from "../../../src/base64";
 
 // The verification flows use javascript timers to set timeouts. We tell jest to use mock timer implementations
 // to ensure that we don't end up with dangling timeouts.
-jest.useFakeTimers();
+// But the wasm bindings of matrix-sdk-crypto rely on a working `queueMicrotask`.
+jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
 
 beforeAll(async () => {
     // we use the libolm primitives in the test, so init the Olm library
@@ -138,6 +161,12 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("verification (%s)", (backend: st
         beforeEach(async () => {
             // pretend that we have another device, which we will verify
             e2eKeyResponder.addDeviceKeys(SIGNED_TEST_DEVICE_DATA);
+
+            fetchMock.put(
+                new RegExp(`/_matrix/client/(r0|v3)/sendToDevice/${escapeRegExp("m.secret.request")}`),
+                { ok: false, status: 404 },
+                { overwriteRoutes: true },
+            );
         });
 
         // test with (1) the default verification method list, (2) a custom verification method list.
@@ -409,6 +438,15 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("verification (%s)", (backend: st
         });
 
         it("can verify another via QR code with an untrusted cross-signing key", async () => {
+            // This is a slightly weird thing to test; if we don't trust the cross-signing key, normally we would
+            // spam out a verification request to all devices rather than targeting a single device. Still, it's
+            // a thing both the Matrix protocol and the js-sdk API support, so we may as well test it.
+            //
+            // Since we don't yet trust the master key, this is a type 0x02 QR code:
+            //   "self-verifying in which the current device does not yet trust the master key"
+            //
+            // By the end of it, we should trust the master key.
+
             aliceClient = await startTestClient();
             // QRCode fails if we don't yet have the cross-signing keys, so make sure we have them now.
             e2eKeyResponder.addCrossSigningData(SIGNED_CROSS_SIGNING_KEYS_DATA);
@@ -480,12 +518,51 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("verification (%s)", (backend: st
             reciprocateQRCodeCallbacks.confirm();
             await sendToDevicePromise;
 
+            // at this point, on legacy crypto, the master key is already marked as trusted, and the request is "Done".
+            // Rust crypto, on the other hand, waits for the 'done' to arrive from the other side.
+            if (request.phase === VerificationPhase.Done) {
+                // legacy crypto: we're all done
+                const userVerificationStatus = await aliceClient.getCrypto()!.getUserVerificationStatus(TEST_USER_ID);
+                // eslint-disable-next-line jest/no-conditional-expect
+                expect(userVerificationStatus.isCrossSigningVerified()).toBeTruthy();
+                await verificationPromise;
+            } else {
+                // rust crypto: still in flight
+                // eslint-disable-next-line jest/no-conditional-expect
+                expect(request.phase).toEqual(VerificationPhase.Started);
+            }
+
             // the dummy device replies with its own 'done'
             returnToDeviceMessageFromSync(buildDoneMessage(transactionId));
 
-            // ... and the whole thing should be done!
+            // ... and now we're really done.
             await verificationPromise;
             expect(request.phase).toEqual(VerificationPhase.Done);
+            const userVerificationStatus = await aliceClient.getCrypto()!.getUserVerificationStatus(TEST_USER_ID);
+            expect(userVerificationStatus.isCrossSigningVerified()).toBeTruthy();
+        });
+
+        it("can try to generate a QR code when QR code is not supported", async () => {
+            aliceClient = await startTestClient();
+            // we need cross-signing keys for a QR code verification
+            e2eKeyResponder.addCrossSigningData(SIGNED_CROSS_SIGNING_KEYS_DATA);
+            await waitForDeviceList();
+
+            // Alice sends a m.key.verification.request
+            const [, request] = await Promise.all([
+                expectSendToDeviceMessage("m.key.verification.request"),
+                aliceClient.getCrypto()!.requestDeviceVerification(TEST_USER_ID, TEST_DEVICE_ID),
+            ]);
+            const transactionId = request.transactionId!;
+
+            // The dummy device replies with an m.key.verification.ready, indicating it can only use SaS
+            returnToDeviceMessageFromSync(buildReadyMessage(transactionId, ["m.sas.v1"]));
+            await waitForVerificationRequestChanged(request);
+            expect(request.phase).toEqual(VerificationPhase.Ready);
+
+            // Alice tries to generate a QR Code but it's unavailable
+            const qrCodeBuffer = await request.generateQRCode();
+            expect(qrCodeBuffer).toBeUndefined();
         });
 
         newBackendOnly("can verify another by scanning their QR code", async () => {
@@ -667,6 +744,8 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("verification (%s)", (backend: st
             expect(toDeviceMessage.transaction_id).toEqual(transactionId);
             expect(toDeviceMessage.code).toEqual("m.user");
             expect(request.phase).toEqual(VerificationPhase.Cancelled);
+            expect(request.cancellationCode).toEqual("m.user");
+            expect(request.cancellingUserId).toEqual("@alice:localhost");
         });
 
         it("can cancel during the SAS phase", async () => {
@@ -897,6 +976,488 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("verification (%s)", (backend: st
         });
     });
 
+    describe("Incoming verification in a DM", () => {
+        let testOlmAccount: Olm.Account;
+
+        beforeEach(async () => {
+            // create a test olm device which we will use to communicate with alice. We use libolm to implement this.
+            await Olm.init();
+            testOlmAccount = new Olm.Account();
+            testOlmAccount.create();
+
+            aliceClient = await startTestClient();
+            aliceClient.setGlobalErrorOnUnknownDevices(false);
+            syncResponder.sendOrQueueSyncResponse(getSyncResponse([BOB_TEST_USER_ID]));
+            await syncPromise(aliceClient);
+        });
+
+        /**
+         * Return a plaintext verification request event from Bob to Alice
+         * @see https://spec.matrix.org/v1.7/client-server-api/#mkeyverificationrequest
+         */
+        function createVerificationRequestEvent(): IEvent {
+            return {
+                content: {
+                    body: "Verification request from Bob to Alice",
+                    from_device: "BobDevice",
+                    methods: ["m.sas.v1"],
+                    msgtype: "m.key.verification.request",
+                    to: aliceClient.getUserId()!,
+                },
+                event_id: "$143273582443PhrSn:example.org",
+                origin_server_ts: Date.now(),
+                room_id: TEST_ROOM_ID,
+                sender: "@bob:xyz",
+                type: "m.room.message",
+                unsigned: {
+                    age: 1234,
+                },
+            };
+        }
+
+        /**
+         * Create a to-device event from Bob to Alice, sharing the group session key
+         * @param groupSession - group session key to share
+         * @param p2pSession - test Olm session to encrypt the key with
+         */
+        function encryptGroupSessionKeyForAlice(
+            groupSession: Olm.OutboundGroupSession,
+            p2pSession: Olm.Session,
+        ): ToDeviceEvent {
+            return encryptGroupSessionKey({
+                recipient: aliceClient.getUserId()!,
+                recipientCurve25519Key: e2eKeyReceiver.getDeviceKey(),
+                recipientEd25519Key: e2eKeyReceiver.getSigningKey(),
+                olmAccount: testOlmAccount,
+                p2pSession: p2pSession,
+                groupSession: groupSession,
+                room_id: TEST_ROOM_ID,
+            });
+        }
+
+        /**
+         * Create and encrypt a verification request event
+         * @param groupSession
+         */
+        function createEncryptedVerificationRequest(groupSession: Olm.OutboundGroupSession): IEvent {
+            const testOlmAccountKeys = JSON.parse(testOlmAccount.identity_keys());
+            return encryptMegolmEvent({
+                senderKey: testOlmAccountKeys.curve25519,
+                groupSession: groupSession,
+                room_id: TEST_ROOM_ID,
+                plaintext: createVerificationRequestEvent(),
+            });
+        }
+
+        it("Verification request not found", async () => {
+            // Expect to not find any verification request
+            const request = aliceClient.getCrypto()!.findVerificationRequestDMInProgress(TEST_ROOM_ID, "@bob:xyz");
+            expect(request).toBeUndefined();
+        });
+
+        it("ignores old verification requests", async () => {
+            const eventHandler = jest.fn();
+            aliceClient.on(CryptoEvent.VerificationRequestReceived, eventHandler);
+
+            const verificationRequestEvent = createVerificationRequestEvent();
+            verificationRequestEvent.origin_server_ts -= 1000000;
+            returnRoomMessageFromSync(TEST_ROOM_ID, verificationRequestEvent);
+
+            await syncPromise(aliceClient);
+
+            // make sure the event has arrived
+            const room = aliceClient.getRoom(TEST_ROOM_ID)!;
+            const matrixEvent = room.getLiveTimeline().getEvents()[0];
+            expect(matrixEvent.getId()).toEqual(verificationRequestEvent.event_id);
+
+            // check that an event has not been raised, and that the request is not found
+            expect(eventHandler).not.toHaveBeenCalled();
+            expect(
+                aliceClient.getCrypto()!.findVerificationRequestDMInProgress(TEST_ROOM_ID, "@bob:xyz"),
+            ).not.toBeDefined();
+        });
+
+        it("Plaintext verification request from Bob to Alice", async () => {
+            // Add verification request from Bob to Alice in the DM between them
+            returnRoomMessageFromSync(TEST_ROOM_ID, createVerificationRequestEvent());
+
+            // Wait for the request to be received
+            const request1 = await emitPromise(aliceClient, CryptoEvent.VerificationRequestReceived);
+            expect(request1.roomId).toBe(TEST_ROOM_ID);
+            expect(request1.isSelfVerification).toBe(false);
+            expect(request1.otherUserId).toBe("@bob:xyz");
+
+            const request = aliceClient.getCrypto()!.findVerificationRequestDMInProgress(TEST_ROOM_ID, "@bob:xyz");
+            // Expect to find the verification request received during the sync
+            expect(request?.roomId).toBe(TEST_ROOM_ID);
+            expect(request?.isSelfVerification).toBe(false);
+            expect(request?.otherUserId).toBe("@bob:xyz");
+        });
+
+        it("Encrypted verification request from Bob to Alice", async () => {
+            const p2pSession = await createOlmSession(testOlmAccount, e2eKeyReceiver);
+            const groupSession = new Olm.OutboundGroupSession();
+            groupSession.create();
+
+            // make the room_key event, but don't send it yet
+            const toDeviceEvent = encryptGroupSessionKeyForAlice(groupSession, p2pSession);
+
+            // Add verification request from Bob to Alice in the DM between them
+            returnRoomMessageFromSync(TEST_ROOM_ID, createEncryptedVerificationRequest(groupSession));
+
+            // Wait for the sync response to be processed
+            await syncPromise(aliceClient);
+
+            const room = aliceClient.getRoom(TEST_ROOM_ID)!;
+            const matrixEvent = room.getLiveTimeline().getEvents()[0];
+
+            // wait for a first attempt at decryption: should fail
+            await awaitDecryption(matrixEvent);
+            expect(matrixEvent.getContent().msgtype).toEqual("m.bad.encrypted");
+
+            const requestEventPromise = emitPromise(aliceClient, CryptoEvent.VerificationRequestReceived);
+
+            // Send Bob the room keys
+            returnToDeviceMessageFromSync(toDeviceEvent);
+
+            // advance the clock, because the devicelist likes to sleep for 5ms during key downloads
+            await jest.advanceTimersByTimeAsync(10);
+
+            // Wait for the request to be decrypted
+            const request1 = await requestEventPromise;
+            expect(request1.roomId).toBe(TEST_ROOM_ID);
+            expect(request1.isSelfVerification).toBe(false);
+            expect(request1.otherUserId).toBe("@bob:xyz");
+
+            const request = aliceClient.getCrypto()!.findVerificationRequestDMInProgress(TEST_ROOM_ID, "@bob:xyz");
+            // Expect to find the verification request received during the sync
+            expect(request?.roomId).toBe(TEST_ROOM_ID);
+            expect(request?.isSelfVerification).toBe(false);
+            expect(request?.otherUserId).toBe("@bob:xyz");
+        });
+
+        newBackendOnly(
+            "If the verification request is not decrypted within 5 minutes, the request is ignored",
+            async () => {
+                const p2pSession = await createOlmSession(testOlmAccount, e2eKeyReceiver);
+                const groupSession = new Olm.OutboundGroupSession();
+                groupSession.create();
+
+                // make the room_key event, but don't send it yet
+                const toDeviceEvent = encryptGroupSessionKeyForAlice(groupSession, p2pSession);
+
+                // Add verification request from Bob to Alice in the DM between them
+                returnRoomMessageFromSync(TEST_ROOM_ID, createEncryptedVerificationRequest(groupSession));
+
+                // Wait for the sync response to be processed
+                await syncPromise(aliceClient);
+
+                const room = aliceClient.getRoom(TEST_ROOM_ID)!;
+                const matrixEvent = room.getLiveTimeline().getEvents()[0];
+
+                // wait for a first attempt at decryption: should fail
+                await awaitDecryption(matrixEvent);
+                expect(matrixEvent.getContent().msgtype).toEqual("m.bad.encrypted");
+
+                // Advance time by 5mins, the verification request should be ignored after that
+                jest.advanceTimersByTime(5 * 60 * 1000);
+
+                // Send Bob the room keys
+                returnToDeviceMessageFromSync(toDeviceEvent);
+
+                // Wait for the message to be decrypted
+                await awaitDecryption(matrixEvent, { waitOnDecryptionFailure: true });
+
+                const request = aliceClient.getCrypto()!.findVerificationRequestDMInProgress(TEST_ROOM_ID, "@bob:xyz");
+                // the request should not be present
+                expect(request).not.toBeDefined();
+            },
+        );
+    });
+
+    describe("Secrets are gossiped after verification", () => {
+        // We use a legacy olm session as the existing session.
+        // This will give us access to low level olm functions in order to
+        // simulate a backup key request with proper olm encryption.
+        let testOlmAccount: Olm.Account;
+        const olmDeviceId = "OLM_DEVICE";
+        let usermasterPubKey: string;
+
+        const matchingBackupInfo: KeyBackupInfo = {
+            algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
+            version: "1",
+            auth_data: {
+                public_key: "hSDwCYkwp1R0i33ctD73Wg2/Og0mOBr066SpjqqbTmo",
+            },
+        };
+
+        const nonMatchingBackupInfo: KeyBackupInfo = {
+            algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
+            version: "1",
+            auth_data: {
+                public_key: "EjDwCYkwp1R0i33ctD73Wg2/Og0mOBr066Spjqqaqqo",
+            },
+        };
+
+        const unknownAlgorithmBackupInfo: KeyBackupInfo = {
+            algorithm: "m.megolm_backup.foo_bar",
+            version: "1",
+            auth_data: {
+                public_key: "EjDwCYkwp1R0i33ctD73Wg2/Og0mOBr066Spjqqaqqo",
+            },
+        };
+
+        beforeEach(async () => {
+            // create a test olm device which we will use to communicate with alice. We use libolm to implement this.
+            await Olm.init();
+            testOlmAccount = new Olm.Account();
+            testOlmAccount.create();
+
+            const bootstrapped = bootstrapCrossSigningTestOlmAccount(testOlmAccount, TEST_USER_ID, olmDeviceId, [
+                matchingBackupInfo,
+                nonMatchingBackupInfo,
+            ]);
+
+            e2eKeyResponder.addDeviceKeys(bootstrapped.device_keys![TEST_USER_ID]![olmDeviceId]);
+            e2eKeyResponder.addCrossSigningData(bootstrapped);
+
+            usermasterPubKey = Object.values(bootstrapped.master_keys![TEST_USER_ID].keys)[0];
+
+            aliceClient = await startTestClient();
+            syncResponder.sendOrQueueSyncResponse(getSyncResponse([TEST_USER_ID]));
+            await syncPromise(aliceClient);
+            // DeviceList has a sleep(5) which we need to make happen
+            await jest.advanceTimersByTimeAsync(10);
+
+            // The client should now know about the olm device
+            const devices = await aliceClient.getCrypto()!.getUserDeviceInfo([TEST_USER_ID]);
+            expect(devices.get(TEST_USER_ID)!.keys()).toContain(olmDeviceId);
+        });
+
+        afterEach(async () => {
+            aliceClient?.stopClient();
+            testOlmAccount?.free();
+
+            // Allow in-flight things to complete before we tear down the test
+            await jest.runAllTimersAsync();
+
+            fetchMock.mockReset();
+        });
+
+        newBackendOnly("Should request cross signing keys after verification", async () => {
+            const requestPromises = mockSecretRequestAndGetPromises();
+
+            await doInteractiveVerification();
+
+            // The secret must have been requested
+            await requestPromises.get("m.cross_signing.master");
+            await requestPromises.get("m.cross_signing.user_signing");
+            await requestPromises.get("m.cross_signing.self_signing");
+        });
+
+        newBackendOnly("Should accept the backup decryption key gossip if valid", async () => {
+            const requestPromises = mockSecretRequestAndGetPromises();
+
+            await doInteractiveVerification();
+
+            const requestId = await requestPromises.get("m.megolm_backup.v1");
+
+            const keyBackupIsCached = emitPromise(aliceClient, CryptoEvent.KeyBackupDecryptionKeyCached);
+
+            await sendBackupGossipAndExpectVersion(requestId!, BACKUP_DECRYPTION_KEY_BASE64, matchingBackupInfo);
+
+            await keyBackupIsCached;
+
+            // the backup secret should be cached
+            const cachedKey = await aliceClient.getCrypto()!.getSessionBackupPrivateKey();
+            expect(cachedKey).toBeTruthy();
+            expect(encodeBase64(cachedKey!)).toEqual(BACKUP_DECRYPTION_KEY_BASE64);
+        });
+
+        newBackendOnly("Should not accept the backup decryption key gossip if private key do not match", async () => {
+            const requestPromises = mockSecretRequestAndGetPromises();
+
+            await doInteractiveVerification();
+
+            const requestId = await requestPromises.get("m.megolm_backup.v1");
+
+            await sendBackupGossipAndExpectVersion(requestId!, BACKUP_DECRYPTION_KEY_BASE64, nonMatchingBackupInfo);
+
+            // We are lacking a way to signal that the secret has been received, so we wait a bit..
+            jest.useRealTimers();
+            await new Promise((resolve) => {
+                setTimeout(resolve, 500);
+            });
+            jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
+
+            // the backup secret should not be cached
+            const cachedKey = await aliceClient.getCrypto()!.getSessionBackupPrivateKey();
+            expect(cachedKey).toBeNull();
+        });
+
+        newBackendOnly("Should not accept the backup decryption key gossip if backup not trusted", async () => {
+            const requestPromises = mockSecretRequestAndGetPromises();
+
+            await doInteractiveVerification();
+
+            const requestId = await requestPromises.get("m.megolm_backup.v1");
+
+            const infoCopy = Object.assign({}, matchingBackupInfo);
+            delete infoCopy.auth_data.signatures;
+
+            await sendBackupGossipAndExpectVersion(requestId!, BACKUP_DECRYPTION_KEY_BASE64, infoCopy);
+
+            // We are lacking a way to signal that the secret has been received, so we wait a bit..
+            jest.useRealTimers();
+            await new Promise((resolve) => {
+                setTimeout(resolve, 500);
+            });
+            jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
+
+            // the backup secret should not be cached
+            const cachedKey = await aliceClient.getCrypto()!.getSessionBackupPrivateKey();
+            expect(cachedKey).toBeNull();
+        });
+
+        newBackendOnly("Should not accept the backup decryption key gossip if backup algorithm unknown", async () => {
+            const requestPromises = mockSecretRequestAndGetPromises();
+
+            await doInteractiveVerification();
+
+            const requestId = await requestPromises.get("m.megolm_backup.v1");
+
+            await sendBackupGossipAndExpectVersion(
+                requestId!,
+                BACKUP_DECRYPTION_KEY_BASE64,
+                unknownAlgorithmBackupInfo,
+            );
+
+            // We are lacking a way to signal that the secret has been received, so we wait a bit..
+            jest.useRealTimers();
+            await new Promise((resolve) => {
+                setTimeout(resolve, 500);
+            });
+            jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
+
+            // the backup secret should not be cached
+            const cachedKey = await aliceClient.getCrypto()!.getSessionBackupPrivateKey();
+            expect(cachedKey).toBeNull();
+        });
+
+        newBackendOnly("Should not accept an invalid backup decryption key", async () => {
+            const requestPromises = mockSecretRequestAndGetPromises();
+
+            await doInteractiveVerification();
+
+            const requestId = await requestPromises.get("m.megolm_backup.v1");
+
+            await sendBackupGossipAndExpectVersion(requestId!, "InvalidSecret", matchingBackupInfo);
+
+            // We are lacking a way to signal that the secret has been received, so we wait a bit..
+            jest.useRealTimers();
+            await new Promise((resolve) => {
+                setTimeout(resolve, 500);
+            });
+            jest.useFakeTimers({ doNotFake: ["queueMicrotask"] });
+
+            // the backup secret should not be cached
+            const cachedKey = await aliceClient.getCrypto()!.getSessionBackupPrivateKey();
+            expect(cachedKey).toBeNull();
+        });
+
+        /**
+         * Common test setup for gossiping secrets.
+         * Creates a peer to peer session, sends the secret, mockup the version API, send the secret back from sync, then await for the backup check.
+         */
+        async function sendBackupGossipAndExpectVersion(
+            requestId: string,
+            secret: string,
+            expectBackup: KeyBackupInfo,
+        ) {
+            const p2pSession = await createOlmSession(testOlmAccount, e2eKeyReceiver);
+
+            const toDeviceEvent = encryptSecretSend({
+                sender: aliceClient.getUserId()!,
+                recipient: aliceClient.getUserId()!,
+                recipientCurve25519Key: e2eKeyReceiver.getDeviceKey(),
+                recipientEd25519Key: e2eKeyReceiver.getSigningKey(),
+                p2pSession: p2pSession,
+                olmAccount: testOlmAccount,
+                requestId: requestId!,
+                secret: secret,
+            });
+
+            const expectBackupCheck = new Promise((resolve) => {
+                fetchMock.get(
+                    "express:/_matrix/client/v3/room_keys/version",
+                    (url, request) => {
+                        resolve(undefined);
+                        return expectBackup;
+                    },
+                    {
+                        overwriteRoutes: true,
+                    },
+                );
+            });
+
+            fetchMock.get("express:/_matrix/client/v3/room_keys/keys", CURVE25519_KEY_BACKUP_DATA);
+
+            // The dummy device sends the secret
+            returnToDeviceMessageFromSync(toDeviceEvent);
+
+            await expectBackupCheck;
+        }
+
+        /**
+         * Do an interactive verification between alice and the dummy device.
+         */
+        async function doInteractiveVerification(): Promise<void> {
+            // Do a QR code verification for simplicity
+
+            // Alice sends a m.key.verification.request
+            const [, request] = await Promise.all([
+                expectSendToDeviceMessage("m.key.verification.request"),
+                aliceClient.getCrypto()!.requestDeviceVerification(TEST_USER_ID, olmDeviceId),
+            ]);
+            const transactionId = request.transactionId!;
+
+            // The dummy device replies with an m.key.verification.ready, indicating it can show a QR code
+            returnToDeviceMessageFromSync(
+                buildReadyMessage(transactionId, ["m.qr_code.show.v1", "m.reciprocate.v1"], olmDeviceId),
+            );
+            await waitForVerificationRequestChanged(request);
+
+            const currentDeviceKey = e2eKeyReceiver.getSigningKey();
+            // the dummy device shows a QR code
+            const sharedSecret = "SUPERSEKRET";
+            // use mode 0x01, self-verifying in which the current device does trust the master key
+            const mode = 0x01;
+            const qrCodeBuffer = buildQRCode(transactionId, usermasterPubKey, currentDeviceKey, sharedSecret, mode);
+
+            // Alice scans the QR code
+            const sendToDevicePromise = expectSendToDeviceMessage("m.key.verification.start");
+            const verifier = await request.scanQRCode(qrCodeBuffer);
+
+            await sendToDevicePromise;
+
+            const verificationPromise = verifier.verify();
+            // the dummy device confirms that Alice scanned the QR code, by replying with a done
+            returnToDeviceMessageFromSync(buildDoneMessage(transactionId));
+
+            // Alice also replies with a 'done'
+            await expectSendToDeviceMessage("m.key.verification.done");
+
+            // ... and the whole thing should be done!
+            await verificationPromise;
+
+            // The other device should now be verified.
+            const otherDevice = (await aliceClient.getCrypto()!.getUserDeviceInfo([TEST_USER_ID]))
+                .get(TEST_USER_ID)!
+                .get(olmDeviceId);
+            expect(otherDevice?.verified).toEqual(DeviceVerification.Verified);
+        }
+    });
+
     async function startTestClient(opts: Partial<ICreateClientOpts> = {}): Promise<MatrixClient> {
         const client = createClient({
             baseUrl: TEST_HOMESERVER_URL,
@@ -927,6 +1488,17 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("verification (%s)", (backend: st
         ev.sender ??= TEST_USER_ID;
         syncResponder.sendOrQueueSyncResponse({ to_device: { events: [ev] } });
     }
+
+    function returnRoomMessageFromSync(roomId: string, ev: IEvent): void {
+        syncResponder.sendOrQueueSyncResponse({
+            next_batch: 1,
+            rooms: {
+                join: {
+                    [roomId]: { timeline: { events: [ev] } },
+                },
+            },
+        });
+    }
 });
 
 /**
@@ -947,6 +1519,52 @@ function expectSendToDeviceMessage(msgtype: string): Promise<{ messages: any }>
     });
 }
 
+/**
+ * Utility to add all needed mocks for secret requesting (to-device of type `m.secret.request`).
+ *
+ * The following secrets are mocked: `m.cross_signing.master`, `m.cross_signing.self_signing`,
+ * `m.cross_signing.user_signing`, `m.megolm_backup.v1`.
+ *
+ *  @returns a map of secret name to promise that will resolve (with the id of the secret request) when the secret is requested.
+ */
+function mockSecretRequestAndGetPromises(): Map<string, Promise<string>> {
+    const mskRequestDefer = defer<string>();
+    const sskRequestDefer = defer<string>();
+    const uskRequestDefer = defer<string>();
+    const backupKeyRequestDefer = defer<string>();
+
+    fetchMock.put(
+        new RegExp(`/_matrix/client/(r0|v3)/sendToDevice/m.secret.request`),
+        (url: string, opts: RequestInit): MockResponse => {
+            const messages = JSON.parse(opts.body as string).messages[TEST_USER_ID];
+            // rust crypto broadcasts to all devices, old crypto to a specific device, take the first one
+            const content = Object.values(messages)[0] as any;
+            if (content.action == "request") {
+                const name = content.name;
+                const requestId = content.request_id;
+                if (name == "m.cross_signing.user_signing") {
+                    uskRequestDefer.resolve(requestId);
+                } else if (name == "m.cross_signing.master") {
+                    mskRequestDefer.resolve(requestId);
+                } else if (name == "m.cross_signing.self_signing") {
+                    sskRequestDefer.resolve(requestId);
+                } else if (name == "m.megolm_backup.v1") {
+                    backupKeyRequestDefer.resolve(requestId);
+                }
+            }
+            return {};
+        },
+        { overwriteRoutes: true },
+    );
+
+    const promiseMap = new Map<string, Promise<string>>();
+    promiseMap.set("m.cross_signing.master", mskRequestDefer.promise);
+    promiseMap.set("m.cross_signing.self_signing", sskRequestDefer.promise);
+    promiseMap.set("m.cross_signing.user_signing", uskRequestDefer.promise);
+    promiseMap.set("m.megolm_backup.v1", backupKeyRequestDefer.promise);
+    return promiseMap;
+}
+
 /** wait for the verification request to emit a 'Change' event */
 function waitForVerificationRequestChanged(request: VerificationRequest): Promise<void> {
     return new Promise<void>((resolve) => {
@@ -991,12 +1609,16 @@ function buildRequestMessage(transactionId: string): { type: string; content: ob
     };
 }
 
-/** build an m.key.verification.ready to-device message originating from the dummy device */
-function buildReadyMessage(transactionId: string, methods: string[]): { type: string; content: object } {
+/** build an m.key.verification.ready to-device message originating from the given `fromDevice` (default to `TEST_DEVICE_ID` if not provided) */
+function buildReadyMessage(
+    transactionId: string,
+    methods: string[],
+    fromDevice?: string,
+): { type: string; content: object } {
     return {
         type: "m.key.verification.ready",
         content: {
-            from_device: TEST_DEVICE_ID,
+            from_device: fromDevice || TEST_DEVICE_ID,
             methods: methods,
             transaction_id: transactionId,
         },
@@ -1094,14 +1716,20 @@ function buildDoneMessage(transactionId: string) {
     };
 }
 
-function buildQRCode(transactionId: string, key1Base64: string, key2Base64: string, sharedSecret: string): Uint8Array {
+function buildQRCode(
+    transactionId: string,
+    key1Base64: string,
+    key2Base64: string,
+    sharedSecret: string,
+    mode = 0x02,
+): Uint8Array {
     // https://spec.matrix.org/v1.7/client-server-api/#qr-code-format
 
     const qrCodeBuffer = Buffer.alloc(150); // oversize
     let idx = 0;
     idx += qrCodeBuffer.write("MATRIX", idx, "ascii");
     idx = qrCodeBuffer.writeUInt8(0x02, idx); // version
-    idx = qrCodeBuffer.writeUInt8(0x02, idx); // mode
+    idx = qrCodeBuffer.writeUInt8(mode, idx); // mode
     idx = qrCodeBuffer.writeInt16BE(transactionId.length, idx);
     idx += qrCodeBuffer.write(transactionId, idx, "ascii");
 

spec/integ/devicelist-integ.spec.ts

@@ -19,6 +19,7 @@ limitations under the License.
 import { TestClient } from "../TestClient";
 import * as testUtils from "../test-utils/test-utils";
 import { logger } from "../../src/logger";
+import { KnownMembership } from "../../src/@types/membership";
 
 const ROOM_ID = "!room:id";
 
@@ -43,7 +44,7 @@ function getSyncResponse(roomMembers: string[]) {
         stateEvents,
         roomMembers.map((m) =>
             testUtils.mkMembership({
-                mship: "join",
+                mship: KnownMembership.Join,
                 sender: m,
             }),
         ),
@@ -323,7 +324,7 @@ describe("DeviceList management:", function () {
                             timeline: {
                                 events: [
                                     testUtils.mkMembership({
-                                        mship: "leave",
+                                        mship: KnownMembership.Leave,
                                         sender: "@bob:xyz",
                                     }),
                                 ],
@@ -357,7 +358,7 @@ describe("DeviceList management:", function () {
                             timeline: {
                                 events: [
                                     testUtils.mkMembership({
-                                        mship: "leave",
+                                        mship: KnownMembership.Leave,
                                         sender: "@bob:xyz",
                                     }),
                                 ],

spec/integ/matrix-client-event-emitter.spec.ts

@@ -28,6 +28,7 @@ import {
 } from "../../src";
 import * as utils from "../test-utils/test-utils";
 import { TestClient } from "../TestClient";
+import { KnownMembership } from "../../src/@types/membership";
 
 describe("MatrixClient events", function () {
     const selfUserId = "@alice:localhost";
@@ -85,16 +86,14 @@ describe("MatrixClient events", function () {
                             events: [
                                 utils.mkMembership({
                                     room: "!erufh:bar",
-                                    mship: "join",
+                                    mship: KnownMembership.Join,
                                     user: "@foo:bar",
                                 }),
                                 utils.mkEvent({
                                     type: "m.room.create",
                                     room: "!erufh:bar",
                                     user: "@foo:bar",
-                                    content: {
-                                        creator: "@foo:bar",
-                                    },
+                                    content: {},
                                 }),
                             ],
                         },
@@ -196,6 +195,37 @@ describe("MatrixClient events", function () {
             expect(fired).toBe(true);
         });
 
+        it("should emit User events when presence data is absent in first sync", async () => {
+            const MODIFIED_SYNC_DATA: any = structuredClone(SYNC_DATA);
+            delete MODIFIED_SYNC_DATA["presence"];
+            const MODIFIED_NEXT_SYNC_DATA: any = structuredClone(NEXT_SYNC_DATA);
+            MODIFIED_NEXT_SYNC_DATA.presence = {
+                events: [
+                    utils.mkPresence({
+                        user: "@foo:bar",
+                        name: "Foo Bar",
+                        presence: "online",
+                    }),
+                ],
+            };
+            httpBackend!.when("GET", "/sync").respond(200, MODIFIED_SYNC_DATA);
+            httpBackend!.when("GET", "/sync").respond(200, MODIFIED_NEXT_SYNC_DATA);
+            let fired = false;
+            client!.on(UserEvent.Presence, function (event, user) {
+                fired = true;
+                expect(user).toBeTruthy();
+                expect(event).toBeTruthy();
+                if (!user || !event) {
+                    return;
+                }
+                expect(event.event).toEqual(MODIFIED_NEXT_SYNC_DATA.presence.events[0]);
+                expect(user.presence).toEqual(MODIFIED_NEXT_SYNC_DATA.presence.events[0]?.content?.presence);
+            });
+            client!.startClient();
+            await httpBackend!.flushAllExpected();
+            expect(fired).toBe(true);
+        });
+
         it("should emit Room events", function () {
             httpBackend!.when("GET", "/sync").respond(200, SYNC_DATA);
             httpBackend!.when("GET", "/sync").respond(200, NEXT_SYNC_DATA);
@@ -243,7 +273,7 @@ describe("MatrixClient events", function () {
                 membersInvokeCount++;
                 expect(member.roomId).toEqual("!erufh:bar");
                 expect(member.userId).toEqual("@foo:bar");
-                expect(member.membership).toEqual("join");
+                expect(member.membership).toEqual(KnownMembership.Join);
             });
             client!.on(RoomStateEvent.NewMember, function (event, state, member) {
                 newMemberInvokeCount++;
@@ -281,7 +311,7 @@ describe("MatrixClient events", function () {
             });
             client!.on(RoomMemberEvent.Membership, function (event, member) {
                 membershipInvokeCount++;
-                expect(member.membership).toEqual("join");
+                expect(member.membership).toEqual(KnownMembership.Join);
             });
 
             client!.startClient();

spec/integ/matrix-client-event-timeline.spec.ts

@@ -33,9 +33,10 @@ import {
 import { logger } from "../../src/logger";
 import { encodeParams, encodeUri, QueryDict, replaceParam } from "../../src/utils";
 import { TestClient } from "../TestClient";
-import { FeatureSupport, Thread, THREAD_RELATION_TYPE, ThreadEvent } from "../../src/models/thread";
+import { FeatureSupport, Thread, ThreadEvent } from "../../src/models/thread";
 import { emitPromise } from "../test-utils/test-utils";
 import { Feature, ServerSupport } from "../../src/feature";
+import { KnownMembership } from "../../src/@types/membership";
 
 const userId = "@alice:localhost";
 const userName = "Alice";
@@ -63,7 +64,7 @@ const buildRelationPaginationQuery = (params: QueryDict): string => {
 
 const USER_MEMBERSHIP_EVENT = utils.mkMembership({
     room: roomId,
-    mship: "join",
+    mship: KnownMembership.Join,
     user: userId,
     name: userName,
     event: false,
@@ -98,7 +99,7 @@ const INITIAL_SYNC_DATA = {
                     events: [
                         withoutRoomId(ROOM_NAME_EVENT),
                         utils.mkMembership({
-                            mship: "join",
+                            mship: KnownMembership.Join,
                             user: otherUserId,
                             name: "Bob",
                             event: false,
@@ -107,9 +108,7 @@ const INITIAL_SYNC_DATA = {
                         utils.mkEvent({
                             type: "m.room.create",
                             user: userId,
-                            content: {
-                                creator: userId,
-                            },
+                            content: {},
                             event: false,
                         }),
                     ],
@@ -609,11 +608,6 @@ describe("MatrixClient event timelines", function () {
             await client.stopClient(); // we don't need the client to be syncing at this time
             const room = client.getRoom(roomId)!;
 
-            httpBackend
-                .when("GET", "/rooms/!foo%3Abar/event/" + encodeURIComponent(THREAD_ROOT.event_id!))
-                .respond(200, function () {
-                    return THREAD_ROOT;
-                });
             httpBackend
                 .when("GET", "/rooms/!foo%3Abar/event/" + encodeURIComponent(THREAD_ROOT.event_id!))
                 .respond(200, function () {
@@ -625,9 +619,7 @@ describe("MatrixClient event timelines", function () {
                     "GET",
                     "/rooms/!foo%3Abar/relations/" +
                         encodeURIComponent(THREAD_ROOT.event_id!) +
-                        "/" +
-                        encodeURIComponent(THREAD_RELATION_TYPE.name) +
-                        buildRelationPaginationQuery({ dir: Direction.Backward, limit: 1 }),
+                        buildRelationPaginationQuery({ dir: Direction.Backward }),
                 )
                 .respond(200, function () {
                     return {
@@ -1156,10 +1148,7 @@ describe("MatrixClient event timelines", function () {
         httpBackend
             .when(
                 "GET",
-                "/_matrix/client/v1/rooms/!foo%3Abar/relations/" +
-                    encodeURIComponent(THREAD_ROOT_UPDATED.event_id!) +
-                    "/" +
-                    encodeURIComponent(THREAD_RELATION_TYPE.name),
+                "/_matrix/client/v1/rooms/!foo%3Abar/relations/" + encodeURIComponent(THREAD_ROOT_UPDATED.event_id!),
             )
             .respond(200, {
                 chunk: [THREAD_REPLY3.event, THREAD_REPLY2.event, THREAD_REPLY],
@@ -1264,11 +1253,8 @@ describe("MatrixClient event timelines", function () {
                 "GET",
                 "/_matrix/client/v1/rooms/!foo%3Abar/relations/" +
                     encodeURIComponent(THREAD_ROOT_UPDATED.event_id!) +
-                    "/" +
-                    encodeURIComponent(THREAD_RELATION_TYPE.name) +
                     buildRelationPaginationQuery({
                         dir: Direction.Backward,
-                        limit: 3,
                         recurse: true,
                     }),
             )
@@ -1323,11 +1309,7 @@ describe("MatrixClient event timelines", function () {
         function respondToThread(root: Partial<IEvent>, replies: Partial<IEvent>[]): ExpectedHttpRequest {
             const request = httpBackend.when(
                 "GET",
-                "/_matrix/client/v1/rooms/!foo%3Abar/relations/" +
-                    encodeURIComponent(root.event_id!) +
-                    "/" +
-                    encodeURIComponent(THREAD_RELATION_TYPE.name) +
-                    "?dir=b&limit=1",
+                "/_matrix/client/v1/rooms/!foo%3Abar/relations/" + encodeURIComponent(root.event_id!) + "?dir=b",
             );
             request.respond(200, function () {
                 return {
@@ -1559,9 +1541,7 @@ describe("MatrixClient event timelines", function () {
                 expect(timelineSets).not.toBeNull();
                 respondToThreads(threadsResponse);
                 respondToThreads(threadsResponse);
-                respondToEvent(THREAD_ROOT);
                 respondToEvent(THREAD2_ROOT);
-                respondToThread(THREAD_ROOT, [THREAD_REPLY]);
                 respondToThread(THREAD2_ROOT, [THREAD2_REPLY]);
                 await flushHttp(room.fetchRoomThreads());
                 const threadIds = room.getThreads().map((thread) => thread.id);
@@ -1569,7 +1549,7 @@ describe("MatrixClient event timelines", function () {
                 expect(threadIds).toContain(THREAD2_ROOT.event_id);
                 const [allThreads] = timelineSets!;
                 const timeline = allThreads.getLiveTimeline()!;
-                // Test threads are in chronological order
+                // Test threads are in chronological order (first thread should be first because it has a more recent reply)
                 expect(timeline.getEvents().map((it) => it.event.event_id)).toEqual([
                     THREAD_ROOT.event_id,
                     THREAD2_ROOT.event_id,
@@ -1580,7 +1560,6 @@ describe("MatrixClient event timelines", function () {
                 thread.initialEventsFetched = true;
                 const prom = emitPromise(room, ThreadEvent.NewReply);
                 respondToEvent(THREAD_ROOT_UPDATED);
-                respondToEvent(THREAD2_ROOT);
                 await room.addLiveEvents([THREAD_REPLY2]);
                 await httpBackend.flushAllExpected();
                 await prom;
@@ -1657,7 +1636,7 @@ describe("MatrixClient event timelines", function () {
                             ...THREAD_ROOT.unsigned!["m.relations"],
                             "io.element.thread": {
                                 ...THREAD_ROOT.unsigned!["m.relations"]!["io.element.thread"],
-                                count: 2,
+                                count: 1,
                                 latest_event: THREAD_REPLY,
                             },
                         },
@@ -1706,7 +1685,6 @@ describe("MatrixClient event timelines", function () {
                 thread.initialEventsFetched = true;
                 const prom = emitPromise(room, ThreadEvent.Update);
                 respondToEvent(THREAD_ROOT_UPDATED);
-                respondToEvent(THREAD2_ROOT);
                 await room.addLiveEvents([THREAD_REPLY_REACTION]);
                 await httpBackend.flushAllExpected();
                 await prom;
@@ -1944,7 +1922,7 @@ describe("MatrixClient event timelines", function () {
 
         // a state event, followed by a redaction thereof
         const event = utils.mkMembership({
-            mship: "join",
+            mship: KnownMembership.Join,
             user: otherUserId,
         });
         const redaction = utils.mkEvent({
@@ -2021,11 +1999,6 @@ describe("MatrixClient event timelines", function () {
                     },
                 },
             });
-            httpBackend
-                .when("GET", "/rooms/!foo%3Abar/event/" + encodeURIComponent(THREAD_ROOT.event_id!))
-                .respond(200, function () {
-                    return THREAD_ROOT;
-                });
             httpBackend
                 .when("GET", "/rooms/!foo%3Abar/event/" + encodeURIComponent(THREAD_ROOT.event_id!))
                 .respond(200, function () {
@@ -2036,9 +2009,7 @@ describe("MatrixClient event timelines", function () {
                     "GET",
                     "/_matrix/client/v1/rooms/!foo%3Abar/relations/" +
                         encodeURIComponent(THREAD_ROOT.event_id!) +
-                        "/" +
-                        encodeURIComponent(THREAD_RELATION_TYPE.name) +
-                        buildRelationPaginationQuery({ dir: Direction.Backward, limit: 1 }),
+                        buildRelationPaginationQuery({ dir: Direction.Backward }),
                 )
                 .respond(200, function () {
                     return {

spec/integ/matrix-client-methods.spec.ts

@@ -19,7 +19,16 @@ import { Mocked } from "jest-mock";
 import * as utils from "../test-utils/test-utils";
 import { CRYPTO_ENABLED, IStoredClientOpts, MatrixClient } from "../../src/client";
 import { MatrixEvent } from "../../src/models/event";
-import { Filter, KnockRoomOpts, MemoryStore, Method, Room, SERVICE_TYPES } from "../../src/matrix";
+import {
+    Filter,
+    JoinRule,
+    KnockRoomOpts,
+    MemoryStore,
+    Method,
+    Room,
+    RoomSummary,
+    SERVICE_TYPES,
+} from "../../src/matrix";
 import { TestClient } from "../TestClient";
 import { THREAD_RELATION_TYPE } from "../../src/models/thread";
 import { IFilterDefinition } from "../../src/filter";
@@ -27,6 +36,7 @@ import { ISearchResults } from "../../src/@types/search";
 import { IStore } from "../../src/store";
 import { CryptoBackend } from "../../src/common-crypto/CryptoBackend";
 import { SetPresence } from "../../src/sync";
+import { KnownMembership } from "../../src/@types/membership";
 
 describe("MatrixClient", function () {
     const userId = "@alice:localhost";
@@ -150,7 +160,7 @@ describe("MatrixClient", function () {
     });
 
     describe("joinRoom", function () {
-        it("should no-op if you've already joined a room", function () {
+        it("should no-op given the ID of a room you've already joined", async () => {
             const roomId = "!foo:bar";
             const room = new Room(roomId, client, userId);
             client.fetchRoomEvent = () =>
@@ -162,14 +172,38 @@ describe("MatrixClient", function () {
                 utils.mkMembership({
                     user: userId,
                     room: roomId,
-                    mship: "join",
+                    mship: KnownMembership.Join,
                     event: true,
                 }),
             ]);
             httpBackend.verifyNoOutstandingRequests();
             store.storeRoom(room);
-            client.joinRoom(roomId);
+
+            const joinPromise = client.joinRoom(roomId);
             httpBackend.verifyNoOutstandingRequests();
+            expect(await joinPromise).toBe(room);
+        });
+
+        it("should no-op given the alias of a room you've already joined", async () => {
+            const roomId = "!roomId:server";
+            const roomAlias = "#my-fancy-room:server";
+            const room = new Room(roomId, client, userId);
+            room.addLiveEvents([
+                utils.mkMembership({
+                    user: userId,
+                    room: roomId,
+                    mship: KnownMembership.Join,
+                    event: true,
+                }),
+            ]);
+            store.storeRoom(room);
+
+            // The method makes a request to resolve the alias
+            httpBackend.when("POST", "/join/" + encodeURIComponent(roomAlias)).respond(200, { room_id: roomId });
+
+            const joinPromise = client.joinRoom(roomAlias);
+            await httpBackend.flushAllExpected();
+            expect(await joinPromise).toBe(room);
         });
 
         it("should send request to inviteSignUrl if specified", async () => {
@@ -245,7 +279,7 @@ describe("MatrixClient", function () {
                 utils.mkMembership({
                     user: userId,
                     room: roomId,
-                    mship: "knock",
+                    mship: KnownMembership.Knock,
                     event: true,
                 }),
             ]);
@@ -1180,51 +1214,20 @@ describe("MatrixClient", function () {
 
     describe("requestLoginToken", () => {
         it("should hit the expected API endpoint with UIA", async () => {
-            httpBackend
-                .when("GET", "/capabilities")
-                .respond(200, { capabilities: { "org.matrix.msc3882.get_login_token": { enabled: true } } });
             const response = {};
             const uiaData = {};
             const prom = client.requestLoginToken(uiaData);
-            httpBackend
-                .when("POST", "/unstable/org.matrix.msc3882/login/get_token", { auth: uiaData })
-                .respond(200, response);
+            httpBackend.when("POST", "/v1/login/get_token", { auth: uiaData }).respond(200, response);
             await httpBackend.flush("");
             expect(await prom).toStrictEqual(response);
         });
 
         it("should hit the expected API endpoint without UIA", async () => {
-            httpBackend
-                .when("GET", "/capabilities")
-                .respond(200, { capabilities: { "org.matrix.msc3882.get_login_token": { enabled: true } } });
             const response = { login_token: "xyz", expires_in_ms: 5000 };
             const prom = client.requestLoginToken();
-            httpBackend.when("POST", "/unstable/org.matrix.msc3882/login/get_token", {}).respond(200, response);
+            httpBackend.when("POST", "/v1/login/get_token", {}).respond(200, response);
             await httpBackend.flush("");
-            // check that expires_in has been populated for compatibility with r0
-            expect(await prom).toStrictEqual({ ...response, expires_in: 5 });
-        });
-
-        it("should hit the r1 endpoint when capability is disabled", async () => {
-            httpBackend
-                .when("GET", "/capabilities")
-                .respond(200, { capabilities: { "org.matrix.msc3882.get_login_token": { enabled: false } } });
-            const response = { login_token: "xyz", expires_in_ms: 5000 };
-            const prom = client.requestLoginToken();
-            httpBackend.when("POST", "/unstable/org.matrix.msc3882/login/get_token", {}).respond(200, response);
-            await httpBackend.flush("");
-            // check that expires_in has been populated for compatibility with r0
-            expect(await prom).toStrictEqual({ ...response, expires_in: 5 });
-        });
-
-        it("should hit the r0 endpoint for fallback", async () => {
-            httpBackend.when("GET", "/capabilities").respond(200, {});
-            const response = { login_token: "xyz", expires_in: 5 };
-            const prom = client.requestLoginToken();
-            httpBackend.when("POST", "/unstable/org.matrix.msc3882/login/token", {}).respond(200, response);
-            await httpBackend.flush("");
-            // check that expires_in has been populated for compatibility with r1
-            expect(await prom).toStrictEqual({ ...response, expires_in_ms: 5000 });
+            expect(await prom).toStrictEqual(response);
         });
     });
 
@@ -1716,6 +1719,102 @@ describe("MatrixClient", function () {
             await Promise.all([client.unbindThreePid("email", "alice@server.com"), httpBackend.flushAllExpected()]);
         });
     });
+
+    describe("getRoomSummary", () => {
+        const roomId = "!foo:bar";
+        const encodedRoomId = encodeURIComponent(roomId);
+
+        const roomSummary: RoomSummary = {
+            "room_id": roomId,
+            "name": "My Room",
+            "avatar_url": "",
+            "topic": "My room topic",
+            "world_readable": false,
+            "guest_can_join": false,
+            "num_joined_members": 1,
+            "room_type": "",
+            "join_rule": JoinRule.Public,
+            "membership": "leave",
+            "im.nheko.summary.room_version": "6",
+            "im.nheko.summary.encryption": "algo",
+        };
+
+        const prefix = "/_matrix/client/unstable/im.nheko.summary/";
+        const suffix = `summary/${encodedRoomId}`;
+        const deprecatedSuffix = `rooms/${encodedRoomId}/summary`;
+
+        const errorUnrecogStatus = 404;
+        const errorUnrecogBody = {
+            errcode: "M_UNRECOGNIZED",
+            error: "Unsupported endpoint",
+        };
+
+        const errorBadreqStatus = 400;
+        const errorBadreqBody = {
+            errcode: "M_UNKNOWN",
+            error: "Invalid request",
+        };
+
+        it("should respond with a valid room summary object", () => {
+            httpBackend.when("GET", prefix + suffix).respond(200, roomSummary);
+
+            const prom = client.getRoomSummary(roomId).then((response) => {
+                expect(response).toEqual(roomSummary);
+            });
+
+            httpBackend.flush("");
+            return prom;
+        });
+
+        it("should allow fallback to the deprecated endpoint", () => {
+            httpBackend.when("GET", prefix + suffix).respond(errorUnrecogStatus, errorUnrecogBody);
+            httpBackend.when("GET", prefix + deprecatedSuffix).respond(200, roomSummary);
+
+            const prom = client.getRoomSummary(roomId).then((response) => {
+                expect(response).toEqual(roomSummary);
+            });
+
+            httpBackend.flush("");
+            return prom;
+        });
+
+        it("should respond to unsupported path with error", () => {
+            httpBackend.when("GET", prefix + suffix).respond(errorUnrecogStatus, errorUnrecogBody);
+            httpBackend.when("GET", prefix + deprecatedSuffix).respond(errorUnrecogStatus, errorUnrecogBody);
+
+            const prom = client.getRoomSummary(roomId).then(
+                function (response) {
+                    throw Error("request not failed");
+                },
+                function (error) {
+                    expect(error.httpStatus).toEqual(errorUnrecogStatus);
+                    expect(error.errcode).toEqual(errorUnrecogBody.errcode);
+                    expect(error.message).toEqual(`MatrixError: [${errorUnrecogStatus}] ${errorUnrecogBody.error}`);
+                },
+            );
+
+            httpBackend.flush("");
+            return prom;
+        });
+
+        it("should respond to invalid path arguments with error", () => {
+            httpBackend.when("GET", prefix).respond(errorBadreqStatus, errorBadreqBody);
+
+            const prom = client.getRoomSummary("notAroom").then(
+                function (response) {
+                    throw Error("request not failed");
+                },
+                function (error) {
+                    expect(error.httpStatus).toEqual(errorBadreqStatus);
+                    expect(error.errcode).toEqual(errorBadreqBody.errcode);
+                    expect(error.message).toEqual(`MatrixError: [${errorBadreqStatus}] ${errorBadreqBody.error}`);
+                },
+            );
+
+            httpBackend.flush("");
+            return prom;
+        });
+    });
 });
 
 function withThreadId(event: MatrixEvent, newThreadId: string): MatrixEvent {
@@ -1919,7 +2018,7 @@ const buildEventJoinRules = () =>
     new MatrixEvent({
         age: 80123696,
         content: {
-            join_rule: "invite",
+            join_rule: KnownMembership.Invite,
         },
         event_id: "$6JDDeDp7fEc0F6YnTWMruNcKWFltR3e9wk7wWDDJrAU",
         origin_server_ts: 1643815441191,
@@ -1973,7 +2072,7 @@ const buildEventMember = () =>
         content: {
             avatar_url: "mxc://matrix.org/aNtbVcFfwotudypZcHsIcPOc",
             displayname: "andybalaam-test1",
-            membership: "join",
+            membership: KnownMembership.Join,
         },
         event_id: "$Ex5eVmMs_ti784mo8bgddynbwLvy6231lCycJr7Cl9M",
         origin_server_ts: 1643815439608,
@@ -1989,7 +2088,6 @@ const buildEventCreate = () =>
     new MatrixEvent({
         age: 80126105,
         content: {
-            creator: "@andybalaam-test1:matrix.org",
             room_version: "6",
         },
         event_id: "$e7j2Gt37k5NPwB6lz2N3V9lO5pUdNK8Ai7i2FPEK-oI",

spec/integ/matrix-client-opts.spec.ts

@@ -6,6 +6,7 @@ import { MatrixScheduler } from "../../src/scheduler";
 import { MemoryStore } from "../../src/store/memory";
 import { MatrixError } from "../../src/http-api";
 import { IStore } from "../../src/store";
+import { KnownMembership } from "../../src/@types/membership";
 
 describe("MatrixClient opts", function () {
     const baseUrl = "http://localhost.or.something";
@@ -43,13 +44,13 @@ describe("MatrixClient opts", function () {
                             }),
                             utils.mkMembership({
                                 room: roomId,
-                                mship: "join",
+                                mship: KnownMembership.Join,
                                 user: userB,
                                 name: "Bob",
                             }),
                             utils.mkMembership({
                                 room: roomId,
-                                mship: "join",
+                                mship: KnownMembership.Join,
                                 user: userId,
                                 name: "Alice",
                             }),
@@ -57,9 +58,7 @@ describe("MatrixClient opts", function () {
                                 type: "m.room.create",
                                 room: roomId,
                                 user: userId,
-                                content: {
-                                    creator: userId,
-                                },
+                                content: {},
                             }),
                         ],
                     },

spec/integ/matrix-client-retrying.spec.ts

@@ -16,7 +16,7 @@ limitations under the License.
 
 import HttpBackend from "matrix-mock-request";
 
-import { EventStatus, RoomEvent, MatrixClient, MatrixScheduler } from "../../src/matrix";
+import { EventStatus, MatrixClient, MatrixScheduler, MsgType, RoomEvent } from "../../src/matrix";
 import { Room } from "../../src/models/room";
 import { TestClient } from "../TestClient";
 
@@ -60,7 +60,7 @@ describe("MatrixClient retrying", function () {
         // send a couple of events; the second will be queued
         const p1 = client!
             .sendMessage(roomId, {
-                msgtype: "m.text",
+                msgtype: MsgType.Text,
                 body: "m1",
             })
             .then(
@@ -77,7 +77,7 @@ describe("MatrixClient retrying", function () {
         // never gets resolved.
         // https://github.com/matrix-org/matrix-js-sdk/issues/496
         client!.sendMessage(roomId, {
-            msgtype: "m.text",
+            msgtype: MsgType.Text,
             body: "m2",
         });
 

spec/integ/matrix-client-room-timeline.spec.ts

@@ -30,6 +30,7 @@ import {
     Room,
 } from "../../src";
 import { TestClient } from "../TestClient";
+import { KnownMembership } from "../../src/@types/membership";
 
 describe("MatrixClient room timelines", function () {
     const userId = "@alice:localhost";
@@ -42,7 +43,7 @@ describe("MatrixClient room timelines", function () {
 
     const USER_MEMBERSHIP_EVENT = utils.mkMembership({
         room: roomId,
-        mship: "join",
+        mship: KnownMembership.Join,
         user: userId,
         name: userName,
     });
@@ -76,7 +77,7 @@ describe("MatrixClient room timelines", function () {
                             ROOM_NAME_EVENT,
                             utils.mkMembership({
                                 room: roomId,
-                                mship: "join",
+                                mship: KnownMembership.Join,
                                 user: otherUserId,
                                 name: "Bob",
                             }),
@@ -85,9 +86,7 @@ describe("MatrixClient room timelines", function () {
                                 type: "m.room.create",
                                 room: roomId,
                                 user: userId,
-                                content: {
-                                    creator: userId,
-                                },
+                                content: {},
                             }),
                         ],
                     },
@@ -318,7 +317,7 @@ describe("MatrixClient room timelines", function () {
 
             // make an m.room.member event for alice's join
             const joinMshipEvent = utils.mkMembership({
-                mship: "join",
+                mship: KnownMembership.Join,
                 user: userId,
                 room: roomId,
                 name: "Old Alice",
@@ -328,7 +327,7 @@ describe("MatrixClient room timelines", function () {
             // make an m.room.member event with prev_content for alice's nick
             // change
             const oldMshipEvent = utils.mkMembership({
-                mship: "join",
+                mship: KnownMembership.Join,
                 user: userId,
                 room: roomId,
                 name: userName,
@@ -337,7 +336,7 @@ describe("MatrixClient room timelines", function () {
             oldMshipEvent.prev_content = {
                 displayname: "Old Alice",
                 avatar_url: undefined,
-                membership: "join",
+                membership: KnownMembership.Join,
             };
 
             // set the list of events to return on scrollback (/messages)
@@ -489,7 +488,7 @@ describe("MatrixClient room timelines", function () {
                 utils.mkMembership({
                     user: userId,
                     room: roomId,
-                    mship: "join",
+                    mship: KnownMembership.Join,
                     name: "New Name",
                 }),
                 utils.mkMessage({ user: userId, room: roomId }),
@@ -556,13 +555,13 @@ describe("MatrixClient room timelines", function () {
                 utils.mkMembership({
                     user: userC,
                     room: roomId,
-                    mship: "join",
+                    mship: KnownMembership.Join,
                     name: "C",
                 }),
                 utils.mkMembership({
                     user: userC,
                     room: roomId,
-                    mship: "invite",
+                    mship: KnownMembership.Invite,
                     skey: userD,
                 }),
             ];
@@ -573,9 +572,9 @@ describe("MatrixClient room timelines", function () {
                 return Promise.all([httpBackend!.flush("/sync", 1), utils.syncPromise(client!)]).then(function () {
                     expect(room.currentState.getMembers().length).toEqual(4);
                     expect(room.currentState.getMember(userC)!.name).toEqual("C");
-                    expect(room.currentState.getMember(userC)!.membership).toEqual("join");
+                    expect(room.currentState.getMember(userC)!.membership).toEqual(KnownMembership.Join);
                     expect(room.currentState.getMember(userD)!.name).toEqual(userD);
-                    expect(room.currentState.getMember(userD)!.membership).toEqual("invite");
+                    expect(room.currentState.getMember(userD)!.membership).toEqual(KnownMembership.Invite);
                 });
             });
         });
@@ -600,9 +599,9 @@ describe("MatrixClient room timelines", function () {
                     expect(room.timeline[0].event).toEqual(eventData[0]);
                     expect(room.currentState.getMembers().length).toEqual(2);
                     expect(room.currentState.getMember(userId)!.name).toEqual(userName);
-                    expect(room.currentState.getMember(userId)!.membership).toEqual("join");
+                    expect(room.currentState.getMember(userId)!.membership).toEqual(KnownMembership.Join);
                     expect(room.currentState.getMember(otherUserId)!.name).toEqual("Bob");
-                    expect(room.currentState.getMember(otherUserId)!.membership).toEqual("join");
+                    expect(room.currentState.getMember(otherUserId)!.membership).toEqual(KnownMembership.Join);
                 });
             });
         });

spec/integ/matrix-client-syncing-errors.spec.ts

@@ -0,0 +1,162 @@
+/*
+Copyright 2023 Holi Moli GmbH
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import "fake-indexeddb/auto";
+import fetchMock from "fetch-mock-jest";
+
+import { MatrixClient, ClientEvent, createClient, SyncState } from "../../src";
+
+const makeQueryablePromise = <T = void>(promise: Promise<T>) => {
+    let resolved = false;
+    let rejected = false;
+
+    // Observe the promise, saving the fulfillment in a closure scope.
+    const newPromise = promise.then(
+        (value) => {
+            resolved = true;
+            return value;
+        },
+        (error) => {
+            rejected = true;
+            throw error;
+        },
+    );
+    const isFulfilled = () => {
+        return resolved || rejected;
+    };
+    const isResolved = () => {
+        return resolved;
+    };
+    const isRejected = () => {
+        return rejected;
+    };
+    return { promise: newPromise, isFulfilled, isResolved, isRejected };
+};
+
+const queryablePromise = <T = void>() => {
+    let resolve!: (value: T | PromiseLike<T>) => void;
+    let reject!: (reason?: any) => void;
+
+    const promise = makeQueryablePromise<T>(
+        new Promise<T>((_resolve, _reject) => {
+            resolve = _resolve;
+            reject = _reject;
+        }),
+    );
+
+    return { resolve, reject, ...promise };
+};
+
+describe("MatrixClient syncing errors", () => {
+    const selfUserId = "@alice:localhost";
+    const selfAccessToken = "aseukfgwef";
+    const unknownTokenErrorData = {
+        status: 401,
+        body: {
+            errcode: "M_UNKNOWN_TOKEN",
+            error: "Invalid access token passed.",
+            soft_logout: false,
+        },
+    };
+    let client: MatrixClient | undefined;
+
+    beforeEach(() => {
+        client = createClient({
+            baseUrl: "http://tocal.test.server",
+            userId: selfUserId,
+            accessToken: selfAccessToken,
+            deviceId: "myDevice",
+        });
+    });
+
+    it("should retry, until errors are solved.", async () => {
+        jest.useFakeTimers();
+        fetchMock.config.overwriteRoutes = false;
+        fetchMock
+            .getOnce("end:versions", {}) // first version check without credentials needs to succeed
+            .getOnce("end:versions", 429) // second version check fails with 429 triggering another retry
+            .get("end:versions", {}) // further version checks succeed
+            .getOnce("end:pushrules/", 429) // first pushrules check fails starting retry
+            .get("end:pushrules/", {}) // further pushrules check succeed
+            .catch({}); // all other calls succeed
+
+        const syncEvents = Array.from({ length: 5 }, queryablePromise<SyncState>);
+
+        client!.on(ClientEvent.Sync, (state: SyncState, lastState: SyncState | null) => {
+            let i = 0;
+            for (; i < syncEvents.length && syncEvents[i].isFulfilled(); i++) {
+                // find index of first unfulfilled promise
+            }
+            syncEvents[i].resolve(state);
+        });
+
+        await client!.startClient();
+        expect(await syncEvents[0].promise).toBe(SyncState.Error);
+        jest.runAllTimers(); // this will skip forward to trigger the keepAlive/sync
+        expect(await syncEvents[1].promise).toBe(SyncState.Error);
+        jest.runAllTimers(); // this will skip forward to trigger the keepAlive/sync
+        expect(await syncEvents[2].promise).toBe(SyncState.Prepared);
+        jest.runAllTimers(); // this will skip forward to trigger the keepAlive/sync
+        expect(await syncEvents[3].promise).toBe(SyncState.Syncing);
+        jest.runAllTimers(); // this will skip forward to trigger the keepAlive/sync
+        expect(await syncEvents[4].promise).toBe(SyncState.Syncing);
+    });
+
+    it("should stop sync keep alive when client is stopped.", async () => {
+        jest.useFakeTimers();
+        fetchMock.config.overwriteRoutes = false;
+        fetchMock
+            .getOnce("end:versions", {}) // first version check without credentials needs to succeed
+            .get("end:versions", unknownTokenErrorData) // further version checks fails with 401
+            .get("end:pushrules/", 401) // fails with 401 without an error. This does happen in practice e.g. with Synapse
+            .post("end:logout", unknownTokenErrorData) // just to keep up a consistent scenario. Does not have a real effect for this testcase
+            .post("end:filter", 401); // just to keep up a consistent scenario. Does not have a real effect for this testcase
+
+        const firstSyncEvent = queryablePromise<SyncState>();
+        const secondSyncEvent = queryablePromise<SyncState>();
+        client!.on(ClientEvent.Sync, (state: SyncState, lastState: SyncState | null) => {
+            if (firstSyncEvent.isFulfilled()) secondSyncEvent.resolve(state);
+            firstSyncEvent.resolve(state);
+        });
+
+        await client!.startClient();
+        const logoutDone = queryablePromise();
+        client!
+            .logout(true)
+            .then(() => {
+                logoutDone.resolve();
+            })
+            .catch((e) => {
+                logoutDone.resolve();
+            });
+
+        const syntState = await firstSyncEvent.promise;
+        expect(syntState).toBe(SyncState.Error);
+        jest.runAllTimers(); // this will skip forward to trigger the keepAlive
+
+        jest.useRealTimers(); // we need real timer for the setTimout below to work
+
+        const timeoutPromise = makeQueryablePromise(new Promise<void>((res) => setTimeout(res, 1)));
+
+        await Promise.race([secondSyncEvent.promise, timeoutPromise.promise]);
+        // when syncing stopped, then the secondSyncEvent will never happen and the promise will not be resolved,
+        /// so the timeoutPromise will be resolved instead
+        expect(timeoutPromise.isFulfilled()).toBe(true);
+        expect(secondSyncEvent.isFulfilled()).toBe(false);
+
+        await logoutDone.promise; // wait for the logout to finish to prevent processing and logging after the test is done.
+    });
+});

spec/integ/matrix-client-syncing.spec.ts

@@ -39,6 +39,7 @@ import {
     IndexedDBStore,
     RelationType,
     EventType,
+    MatrixEventEvent,
 } from "../../src";
 import { ReceiptType } from "../../src/@types/read_receipts";
 import { UNREAD_THREAD_NOTIFICATIONS } from "../../src/@types/sync";
@@ -46,6 +47,8 @@ import * as utils from "../test-utils/test-utils";
 import { TestClient } from "../TestClient";
 import { emitPromise, mkEvent, mkMessage } from "../test-utils/test-utils";
 import { THREAD_RELATION_TYPE } from "../../src/models/thread";
+import { IActionsObject } from "../../src/pushprocessor";
+import { KnownMembership } from "../../src/@types/membership";
 
 describe("MatrixClient syncing", () => {
     const selfUserId = "@alice:localhost";
@@ -123,7 +126,7 @@ describe("MatrixClient syncing", () => {
                                     type: "m.room.member",
                                     state_key: selfUserId,
                                     content: {
-                                        membership: "invite",
+                                        membership: KnownMembership.Invite,
                                     },
                                 },
                             ],
@@ -151,10 +154,10 @@ describe("MatrixClient syncing", () => {
                                         type: "m.room.member",
                                         state_key: selfUserId,
                                         content: {
-                                            membership: "leave",
+                                            membership: KnownMembership.Leave,
                                         },
                                         prev_content: {
-                                            membership: "invite",
+                                            membership: KnownMembership.Invite,
                                         },
                                         // XXX: And other fields required on an event
                                     },
@@ -167,10 +170,10 @@ describe("MatrixClient syncing", () => {
                                         type: "m.room.member",
                                         state_key: selfUserId,
                                         content: {
-                                            membership: "leave",
+                                            membership: KnownMembership.Leave,
                                         },
                                         prev_content: {
-                                            membership: "invite",
+                                            membership: KnownMembership.Invite,
                                         },
                                         // XXX: And other fields required on an event
                                     },
@@ -193,22 +196,137 @@ describe("MatrixClient syncing", () => {
                 // Room, string, string
                 fires++;
                 expect(room.roomId).toBe(roomId);
-                expect(membership).toBe("invite");
+                expect(membership).toBe(KnownMembership.Invite);
                 expect(oldMembership).toBeFalsy();
 
                 // Second fire: a leave
                 client!.once(RoomEvent.MyMembership, (room, membership, oldMembership) => {
                     fires++;
                     expect(room.roomId).toBe(roomId);
-                    expect(membership).toBe("leave");
-                    expect(oldMembership).toBe("invite");
+                    expect(membership).toBe(KnownMembership.Leave);
+                    expect(oldMembership).toBe(KnownMembership.Invite);
 
                     // Third/final fire: a second invite
                     client!.once(RoomEvent.MyMembership, (room, membership, oldMembership) => {
                         fires++;
                         expect(room.roomId).toBe(roomId);
-                        expect(membership).toBe("invite");
-                        expect(oldMembership).toBe("leave");
+                        expect(membership).toBe(KnownMembership.Invite);
+                        expect(oldMembership).toBe(KnownMembership.Leave);
+                    });
+                });
+
+                // For maximum safety, "leave" the room after we register the handler
+                client!.leave(roomId);
+            });
+
+            // noinspection ES6MissingAwait
+            client!.startClient();
+            await httpBackend!.flushAllExpected();
+
+            expect(fires).toBe(3);
+        });
+
+        it("should emit RoomEvent.MyMembership for knock->leave->knock cycles", async () => {
+            await client!.initCrypto();
+
+            const roomId = "!cycles:example.org";
+
+            // First sync: an knock
+            const knockSyncRoomSection = {
+                knock: {
+                    [roomId]: {
+                        knock_state: {
+                            events: [
+                                {
+                                    type: "m.room.member",
+                                    state_key: selfUserId,
+                                    content: {
+                                        membership: KnownMembership.Knock,
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                },
+            };
+            httpBackend!.when("GET", "/sync").respond(200, {
+                ...syncData,
+                rooms: knockSyncRoomSection,
+            });
+
+            // Second sync: a leave (reject of some kind)
+            httpBackend!.when("POST", "/leave").respond(200, {});
+            httpBackend!.when("GET", "/sync").respond(200, {
+                ...syncData,
+                rooms: {
+                    leave: {
+                        [roomId]: {
+                            account_data: { events: [] },
+                            ephemeral: { events: [] },
+                            state: {
+                                events: [
+                                    {
+                                        type: "m.room.member",
+                                        state_key: selfUserId,
+                                        content: {
+                                            membership: KnownMembership.Leave,
+                                        },
+                                        prev_content: {
+                                            membership: KnownMembership.Knock,
+                                        },
+                                        // XXX: And other fields required on an event
+                                    },
+                                ],
+                            },
+                            timeline: {
+                                limited: false,
+                                events: [
+                                    {
+                                        type: "m.room.member",
+                                        state_key: selfUserId,
+                                        content: {
+                                            membership: KnownMembership.Leave,
+                                        },
+                                        prev_content: {
+                                            membership: KnownMembership.Knock,
+                                        },
+                                        // XXX: And other fields required on an event
+                                    },
+                                ],
+                            },
+                        },
+                    },
+                },
+            });
+
+            // Third sync: another knock
+            httpBackend!.when("GET", "/sync").respond(200, {
+                ...syncData,
+                rooms: knockSyncRoomSection,
+            });
+
+            // First fire: an initial knock
+            let fires = 0;
+            client!.once(RoomEvent.MyMembership, (room, membership, oldMembership) => {
+                // Room, string, string
+                fires++;
+                expect(room.roomId).toBe(roomId);
+                expect(membership).toBe(KnownMembership.Knock);
+                expect(oldMembership).toBeFalsy();
+
+                // Second fire: a leave
+                client!.once(RoomEvent.MyMembership, (room, membership, oldMembership) => {
+                    fires++;
+                    expect(room.roomId).toBe(roomId);
+                    expect(membership).toBe(KnownMembership.Leave);
+                    expect(oldMembership).toBe(KnownMembership.Knock);
+
+                    // Third/final fire: a second knock
+                    client!.once(RoomEvent.MyMembership, (room, membership, oldMembership) => {
+                        fires++;
+                        expect(room.roomId).toBe(roomId);
+                        expect(membership).toBe(KnownMembership.Knock);
+                        expect(oldMembership).toBe(KnownMembership.Leave);
                     });
                 });
 
@@ -266,7 +384,7 @@ describe("MatrixClient syncing", () => {
                                     type: "m.room.member",
                                     state_key: selfUserId,
                                     content: {
-                                        membership: "invite",
+                                        membership: KnownMembership.Invite,
                                     },
                                 },
                             ],
@@ -293,6 +411,46 @@ describe("MatrixClient syncing", () => {
             expect(fires).toBe(1);
         });
 
+        it("should emit ClientEvent.Room when knocked while crypto is disabled", async () => {
+            const roomId = "!knock:example.org";
+
+            // First sync: a knock
+            const knockSyncRoomSection = {
+                knock: {
+                    [roomId]: {
+                        knock_state: {
+                            events: [
+                                {
+                                    type: "m.room.member",
+                                    state_key: selfUserId,
+                                    content: {
+                                        membership: KnownMembership.Knock,
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                },
+            };
+            httpBackend!.when("GET", "/sync").respond(200, {
+                ...syncData,
+                rooms: knockSyncRoomSection,
+            });
+
+            // First fire: an initial knock
+            let fires = 0;
+            client!.once(ClientEvent.Room, (room) => {
+                fires++;
+                expect(room.roomId).toBe(roomId);
+            });
+
+            // noinspection ES6MissingAwait
+            client!.startClient();
+            await httpBackend!.flushAllExpected();
+
+            expect(fires).toBe(1);
+        });
+
         it("should work when all network calls fail", async () => {
             httpBackend!.expectedRequests = [];
             httpBackend!.when("GET", "").fail(0, new Error("CORS or something"));
@@ -358,6 +516,7 @@ describe("MatrixClient syncing", () => {
                 join: {},
                 invite: {},
                 leave: {},
+                knock: {},
             },
         };
 
@@ -377,21 +536,19 @@ describe("MatrixClient syncing", () => {
                     events: [
                         utils.mkMembership({
                             room: roomOne,
-                            mship: "join",
+                            mship: KnownMembership.Join,
                             user: otherUserId,
                         }),
                         utils.mkMembership({
                             room: roomOne,
-                            mship: "join",
+                            mship: KnownMembership.Join,
                             user: selfUserId,
                         }),
                         utils.mkEvent({
                             type: "m.room.create",
                             room: roomOne,
                             user: selfUserId,
-                            content: {
-                                creator: selfUserId,
-                            },
+                            content: {},
                         }),
                     ],
                 },
@@ -402,7 +559,7 @@ describe("MatrixClient syncing", () => {
             syncData.rooms.join[roomOne].state.events.push(
                 utils.mkMembership({
                     room: roomOne,
-                    mship: "invite",
+                    mship: KnownMembership.Invite,
                     user: userC,
                 }) as IStateEvent,
             );
@@ -435,7 +592,7 @@ describe("MatrixClient syncing", () => {
             syncData.rooms.join[roomOne].state.events.push(
                 utils.mkMembership({
                     room: roomOne,
-                    mship: "invite",
+                    mship: KnownMembership.Invite,
                     user: userC,
                 }) as IStateEvent,
             );
@@ -463,7 +620,7 @@ describe("MatrixClient syncing", () => {
             syncData.rooms.join[roomOne].state.events.push(
                 utils.mkMembership({
                     room: roomOne,
-                    mship: "invite",
+                    mship: KnownMembership.Invite,
                     user: userC,
                 }) as IStateEvent,
             );
@@ -490,7 +647,7 @@ describe("MatrixClient syncing", () => {
             syncData.rooms.join[roomOne].state.events.push(
                 utils.mkMembership({
                     room: roomOne,
-                    mship: "invite",
+                    mship: KnownMembership.Invite,
                     user: userC,
                 }) as IStateEvent,
             );
@@ -565,21 +722,19 @@ describe("MatrixClient syncing", () => {
                                 }),
                                 utils.mkMembership({
                                     room: roomOne,
-                                    mship: "join",
+                                    mship: KnownMembership.Join,
                                     user: otherUserId,
                                 }),
                                 utils.mkMembership({
                                     room: roomOne,
-                                    mship: "join",
+                                    mship: KnownMembership.Join,
                                     user: selfUserId,
                                 }),
                                 utils.mkEvent({
                                     type: "m.room.create",
                                     room: roomOne,
                                     user: selfUserId,
-                                    content: {
-                                        creator: selfUserId,
-                                    },
+                                    content: {},
                                 }),
                             ],
                         },
@@ -598,22 +753,20 @@ describe("MatrixClient syncing", () => {
                             events: [
                                 utils.mkMembership({
                                     room: roomTwo,
-                                    mship: "join",
+                                    mship: KnownMembership.Join,
                                     user: otherUserId,
                                     name: otherDisplayName,
                                 }),
                                 utils.mkMembership({
                                     room: roomTwo,
-                                    mship: "join",
+                                    mship: KnownMembership.Join,
                                     user: selfUserId,
                                 }),
                                 utils.mkEvent({
                                     type: "m.room.create",
                                     room: roomTwo,
                                     user: selfUserId,
-                                    content: {
-                                        creator: selfUserId,
-                                    },
+                                    content: {},
                                 }),
                             ],
                         },
@@ -758,7 +911,6 @@ describe("MatrixClient syncing", () => {
                         room: roomOne,
                         user: otherUserId,
                         content: {
-                            creator: otherUserId,
                             room_version: "9",
                         },
                     });
@@ -844,7 +996,6 @@ describe("MatrixClient syncing", () => {
                         room: roomOne,
                         user: otherUserId,
                         content: {
-                            creator: otherUserId,
                             room_version: testMeta.roomVersion,
                         },
                     });
@@ -1099,7 +1250,7 @@ describe("MatrixClient syncing", () => {
 
             const USER_MEMBERSHIP_EVENT = utils.mkMembership({
                 room: roomOne,
-                mship: "join",
+                mship: KnownMembership.Join,
                 user: userA,
             });
 
@@ -1360,21 +1511,19 @@ describe("MatrixClient syncing", () => {
                                 }),
                                 utils.mkMembership({
                                     room: roomOne,
-                                    mship: "join",
+                                    mship: KnownMembership.Join,
                                     user: otherUserId,
                                 }),
                                 utils.mkMembership({
                                     room: roomOne,
-                                    mship: "join",
+                                    mship: KnownMembership.Join,
                                     user: selfUserId,
                                 }),
                                 utils.mkEvent({
                                     type: "m.room.create",
                                     room: roomOne,
                                     user: selfUserId,
-                                    content: {
-                                        creator: selfUserId,
-                                    },
+                                    content: {},
                                 }),
                             ],
                         } as Partial<IJoinedRoom>,
@@ -1459,21 +1608,19 @@ describe("MatrixClient syncing", () => {
                                 }),
                                 utils.mkMembership({
                                     room: roomOne,
-                                    mship: "join",
+                                    mship: KnownMembership.Join,
                                     user: otherUserId,
                                 }),
                                 utils.mkMembership({
                                     room: roomOne,
-                                    mship: "join",
+                                    mship: KnownMembership.Join,
                                     user: selfUserId,
                                 }),
                                 utils.mkEvent({
                                     type: "m.room.create",
                                     room: roomOne,
                                     user: selfUserId,
-                                    content: {
-                                        creator: selfUserId,
-                                    },
+                                    content: {},
                                 }),
                             ],
                         },
@@ -1501,6 +1648,99 @@ describe("MatrixClient syncing", () => {
             });
         });
 
+        it("should zero total notifications for threads when absent from the notifications object", async () => {
+            syncData.rooms.join[roomOne][UNREAD_THREAD_NOTIFICATIONS.name] = {
+                [THREAD_ID]: {
+                    highlight_count: 2,
+                    notification_count: 5,
+                },
+            };
+
+            httpBackend!.when("GET", "/sync").respond(200, syncData);
+
+            client!.startClient();
+
+            await Promise.all([httpBackend!.flushAllExpected(), awaitSyncEvent()]);
+
+            const room = client!.getRoom(roomOne);
+
+            expect(room!.getThreadUnreadNotificationCount(THREAD_ID, NotificationCountType.Total)).toBe(5);
+
+            syncData.rooms.join[roomOne][UNREAD_THREAD_NOTIFICATIONS.name] = {};
+
+            httpBackend!.when("GET", "/sync").respond(200, syncData);
+
+            await Promise.all([httpBackend!.flushAllExpected(), awaitSyncEvent()]);
+
+            expect(room!.getThreadUnreadNotificationCount(THREAD_ID, NotificationCountType.Total)).toBe(0);
+        });
+
+        it("should zero highlight notifications for threads in encrypted rooms", async () => {
+            syncData.rooms.join[roomOne][UNREAD_THREAD_NOTIFICATIONS.name] = {
+                [THREAD_ID]: {
+                    highlight_count: 2,
+                    notification_count: 5,
+                },
+            };
+
+            httpBackend!.when("GET", "/sync").respond(200, syncData);
+
+            client!.startClient();
+
+            await Promise.all([httpBackend!.flushAllExpected(), awaitSyncEvent()]);
+
+            const room = client!.getRoom(roomOne);
+
+            expect(room!.getThreadUnreadNotificationCount(THREAD_ID, NotificationCountType.Total)).toBe(5);
+
+            syncData.rooms.join[roomOne][UNREAD_THREAD_NOTIFICATIONS.name] = {
+                [THREAD_ID]: {
+                    highlight_count: 0,
+                    notification_count: 0,
+                },
+            };
+
+            httpBackend!.when("GET", "/sync").respond(200, syncData);
+
+            await Promise.all([httpBackend!.flushAllExpected(), awaitSyncEvent()]);
+
+            expect(room!.getThreadUnreadNotificationCount(THREAD_ID, NotificationCountType.Highlight)).toBe(0);
+        });
+
+        it("should not zero highlight notifications for threads in encrypted rooms", async () => {
+            syncData.rooms.join[roomOne][UNREAD_THREAD_NOTIFICATIONS.name] = {
+                [THREAD_ID]: {
+                    highlight_count: 2,
+                    notification_count: 5,
+                },
+            };
+
+            httpBackend!.when("GET", "/sync").respond(200, syncData);
+
+            client!.startClient();
+
+            await Promise.all([httpBackend!.flushAllExpected(), awaitSyncEvent()]);
+
+            const room = client!.getRoom(roomOne);
+            room!.hasEncryptionStateEvent = jest.fn().mockReturnValue(true);
+
+            expect(room!.getThreadUnreadNotificationCount(THREAD_ID, NotificationCountType.Total)).toBe(5);
+
+            syncData.rooms.join[roomOne][UNREAD_THREAD_NOTIFICATIONS.name] = {
+                [THREAD_ID]: {
+                    highlight_count: 0,
+                    notification_count: 0,
+                },
+            };
+
+            httpBackend!.when("GET", "/sync").respond(200, syncData);
+
+            await Promise.all([httpBackend!.flushAllExpected(), awaitSyncEvent()]);
+
+            expect(room!.getThreadUnreadNotificationCount(THREAD_ID, NotificationCountType.Total)).toBe(0);
+            expect(room!.getThreadUnreadNotificationCount(THREAD_ID, NotificationCountType.Highlight)).toBe(2);
+        });
+
         it("caches unknown threads receipts and replay them when the thread is created", async () => {
             const THREAD_ID = "$unknownthread:localhost";
 
@@ -1588,66 +1828,351 @@ describe("MatrixClient syncing", () => {
             });
         });
 
-        it("should apply encrypted notification logic for events within the same sync blob", async () => {
-            const roomId = "!room123:server";
-            const syncData = {
-                rooms: {
-                    join: {
-                        [roomId]: {
-                            ephemeral: {
-                                events: [],
-                            },
-                            timeline: {
-                                events: [
-                                    utils.mkEvent({
-                                        room: roomId,
-                                        event: true,
-                                        skey: "",
-                                        type: EventType.RoomEncryption,
-                                        content: {},
-                                    }),
-                                    utils.mkMessage({
-                                        room: roomId,
-                                        user: otherUserId,
-                                        msg: "hello",
-                                    }),
-                                ],
-                            },
-                            state: {
-                                events: [
-                                    utils.mkMembership({
-                                        room: roomId,
-                                        mship: "join",
-                                        user: otherUserId,
-                                    }),
-                                    utils.mkMembership({
-                                        room: roomId,
-                                        mship: "join",
-                                        user: selfUserId,
-                                    }),
-                                    utils.mkEvent({
-                                        type: "m.room.create",
-                                        room: roomId,
-                                        user: selfUserId,
-                                        content: {
-                                            creator: selfUserId,
-                                        },
-                                    }),
-                                ],
+        describe("encrypted notification logic", () => {
+            let roomId: string;
+            let syncData: ISyncResponse;
+
+            beforeEach(() => {
+                roomId = "!room123:server";
+                syncData = {
+                    rooms: {
+                        join: {
+                            [roomId]: {
+                                ephemeral: {
+                                    events: [],
+                                },
+                                timeline: {
+                                    events: [
+                                        utils.mkEvent({
+                                            room: roomId,
+                                            event: true,
+                                            skey: "",
+                                            type: EventType.RoomEncryption,
+                                            content: {},
+                                        }),
+                                        utils.mkMessage({
+                                            room: roomId,
+                                            user: otherUserId,
+                                            msg: "hello",
+                                        }),
+                                    ],
+                                },
+                                state: {
+                                    events: [
+                                        utils.mkMembership({
+                                            room: roomId,
+                                            mship: KnownMembership.Join,
+                                            user: otherUserId,
+                                        }),
+                                        utils.mkMembership({
+                                            room: roomId,
+                                            mship: KnownMembership.Join,
+                                            user: selfUserId,
+                                        }),
+                                        utils.mkEvent({
+                                            type: "m.room.create",
+                                            room: roomId,
+                                            user: selfUserId,
+                                            content: {},
+                                        }),
+                                    ],
+                                },
                             },
                         },
                     },
-                },
-            } as unknown as ISyncResponse;
+                } as unknown as ISyncResponse;
+            });
 
-            httpBackend!.when("GET", "/sync").respond(200, syncData);
-            client!.startClient();
+            it("should apply encrypted notification logic for events within the same sync blob", async () => {
+                httpBackend!.when("GET", "/sync").respond(200, syncData);
+                client!.startClient();
 
-            await Promise.all([httpBackend!.flushAllExpected(), awaitSyncEvent()]);
+                await Promise.all([httpBackend!.flushAllExpected(), awaitSyncEvent()]);
 
-            const room = client!.getRoom(roomId)!;
-            expect(room).toBeInstanceOf(Room);
-            expect(room.getRoomUnreadNotificationCount(NotificationCountType.Total)).toBe(0);
+                const room = client!.getRoom(roomId)!;
+                expect(room).toBeInstanceOf(Room);
+                expect(room.getRoomUnreadNotificationCount(NotificationCountType.Total)).toBe(0);
+            });
+
+            it("should recalculate highlights on unthreaded receipt for encrypted rooms", async () => {
+                const myUserId = client!.getUserId()!;
+
+                const firstEventId = syncData.rooms.join[roomId].timeline.events[1].event_id;
+
+                // add a receipt for the first event in the room (let's say the user has already read that one)
+                syncData.rooms.join[roomId].ephemeral.events = [
+                    {
+                        content: {
+                            [firstEventId]: {
+                                "m.read": {
+                                    [myUserId]: { ts: 1 },
+                                },
+                            },
+                        },
+                        type: "m.receipt",
+                    },
+                ];
+
+                // Now add a highlighting event after that receipt
+                const pingEvent = utils.mkMessage({
+                    room: roomId,
+                    user: otherUserId,
+                    msg: client?.getUserId() + " ping",
+                }) as IRoomEvent;
+                syncData.rooms.join[roomId].timeline.events.push(pingEvent);
+
+                // fudge this to make it a highlight
+                client!.getPushActionsForEvent = (ev: MatrixEvent): IActionsObject | null => {
+                    if (ev.getId() === pingEvent.event_id) {
+                        return {
+                            notify: true,
+                            tweaks: {
+                                highlight: true,
+                            },
+                        };
+                    }
+                    return null;
+                };
+
+                httpBackend!.when("GET", "/sync").respond(200, syncData);
+                client!.startClient();
+
+                await Promise.all([httpBackend!.flushAllExpected(), awaitSyncEvent()]);
+
+                const room = client!.getRoom(roomId)!;
+                expect(room).toBeInstanceOf(Room);
+                // the room should now have one highlight since our receipt was before the ping message
+                expect(room.getRoomUnreadNotificationCount(NotificationCountType.Highlight)).toBe(1);
+            });
+
+            it("should recalculate highlights on main thread receipt for encrypted rooms", async () => {
+                const myUserId = client!.getUserId()!;
+
+                const firstEventId = syncData.rooms.join[roomId].timeline.events[1].event_id;
+
+                // add a receipt for the first event in the room (let's say the user has already read that one)
+                syncData.rooms.join[roomId].ephemeral.events = [
+                    {
+                        content: {
+                            [firstEventId]: {
+                                "m.read": {
+                                    [myUserId]: { ts: 1, thread_id: "main" },
+                                },
+                            },
+                        },
+                        type: "m.receipt",
+                    },
+                ];
+
+                // Now add a highlighting event after that receipt
+                const pingEvent = utils.mkMessage({
+                    room: roomId,
+                    user: otherUserId,
+                    msg: client?.getUserId() + " ping",
+                }) as IRoomEvent;
+                syncData.rooms.join[roomId].timeline.events.push(pingEvent);
+
+                // fudge this to make it a highlight
+                client!.getPushActionsForEvent = (ev: MatrixEvent): IActionsObject | null => {
+                    if (ev.getId() === pingEvent.event_id) {
+                        return {
+                            notify: true,
+                            tweaks: {
+                                highlight: true,
+                            },
+                        };
+                    }
+                    return null;
+                };
+
+                httpBackend!.when("GET", "/sync").respond(200, syncData);
+                client!.startClient();
+
+                await Promise.all([httpBackend!.flushAllExpected(), awaitSyncEvent()]);
+
+                const room = client!.getRoom(roomId)!;
+                expect(room).toBeInstanceOf(Room);
+                // the room should now have one highlight since our receipt was before the ping message
+                expect(room.getRoomUnreadNotificationCount(NotificationCountType.Highlight)).toBe(1);
+            });
+
+            describe("notification processing in threads", () => {
+                let threadEvent1: IRoomEvent;
+                let threadEvent2: IRoomEvent;
+                let firstEventId: string;
+
+                beforeEach(() => {
+                    firstEventId = syncData.rooms.join[roomId].timeline.events[1].event_id;
+
+                    // Add a threaded event off of the first event
+                    threadEvent1 = utils.mkEvent({
+                        type: EventType.RoomMessage,
+                        user: otherUserId,
+                        room: roomId,
+                        ts: 500,
+                        content: {
+                            "body": "first thread response",
+                            "m.relates_to": {
+                                "event_id": firstEventId,
+                                "m.in_reply_to": {
+                                    event_id: firstEventId,
+                                },
+                                "rel_type": "io.element.thread",
+                            },
+                        },
+                    }) as IRoomEvent;
+                    syncData.rooms.join[roomId].timeline.events.push(threadEvent1);
+
+                    // ...and another
+                    threadEvent2 = utils.mkEvent({
+                        type: EventType.RoomMessage,
+                        user: otherUserId,
+                        room: roomId,
+                        ts: 1500,
+                        content: {
+                            "body": "second thread response",
+                            "m.relates_to": {
+                                "event_id": firstEventId,
+                                "m.in_reply_to": {
+                                    event_id: firstEventId,
+                                },
+                                "rel_type": "io.element.thread",
+                            },
+                        },
+                    }) as IRoomEvent;
+                    syncData.rooms.join[roomId].timeline.events.push(threadEvent2);
+
+                    // fudge to make these highlights
+                    client!.getPushActionsForEvent = (ev: MatrixEvent): IActionsObject | null => {
+                        if ([threadEvent1.event_id, threadEvent2.event_id].includes(ev.getId()!)) {
+                            return {
+                                notify: true,
+                                tweaks: {
+                                    highlight: true,
+                                },
+                            };
+                        }
+                        return null;
+                    };
+                });
+
+                it("checks threads with notifications on unthreaded receipts", async () => {
+                    const myUserId = client!.getUserId()!;
+
+                    // add a receipt for a random, ficticious thread, otherwise the client will
+                    // think that the thread is before any threaded receipts and ignore it.
+                    syncData.rooms.join[roomId].ephemeral.events = [
+                        {
+                            content: {
+                                [firstEventId]: {
+                                    "m.read": {
+                                        [myUserId]: { ts: 1, thread_id: "some_other_thread" },
+                                    },
+                                },
+                            },
+                            type: "m.receipt",
+                        },
+                    ];
+
+                    httpBackend!.when("GET", "/sync").respond(200, syncData);
+                    client!.startClient({ threadSupport: true });
+
+                    await Promise.all([httpBackend!.flushAllExpected(), awaitSyncEvent()]);
+
+                    const room = client!.getRoom(roomId)!;
+
+                    // pretend that the client has decrypted an event to trigger it to compute
+                    // local notifications
+                    client?.emit(MatrixEventEvent.Decrypted, room.findEventById(firstEventId)!);
+                    client?.emit(MatrixEventEvent.Decrypted, room.findEventById(threadEvent1.event_id)!);
+                    client?.emit(MatrixEventEvent.Decrypted, room.findEventById(threadEvent2.event_id)!);
+
+                    expect(room).toBeInstanceOf(Room);
+
+                    // we should now have one highlight: the unread message that pings
+                    expect(
+                        room.getThreadUnreadNotificationCount(firstEventId, NotificationCountType.Highlight),
+                    ).toEqual(2);
+
+                    const syncData2 = {
+                        rooms: {
+                            join: {
+                                [roomId]: {
+                                    ephemeral: {
+                                        events: [
+                                            {
+                                                content: {
+                                                    [firstEventId]: {
+                                                        "m.read": {
+                                                            [myUserId]: { ts: 1 },
+                                                        },
+                                                    },
+                                                },
+                                                type: "m.receipt",
+                                            },
+                                        ],
+                                    },
+                                },
+                            },
+                        },
+                    } as unknown as ISyncResponse;
+
+                    httpBackend!.when("GET", "/sync").respond(200, syncData2);
+
+                    await Promise.all([httpBackend!.flush("/sync", 1), utils.syncPromise(client!)]);
+
+                    expect(room.getRoomUnreadNotificationCount(NotificationCountType.Highlight)).toBe(0);
+                });
+
+                it("should recalculate highlights on threaded receipt for encrypted rooms", async () => {
+                    const myUserId = client!.getUserId()!;
+
+                    // add a receipt for the first message in the threadm leaving the second one unread
+                    syncData.rooms.join[roomId].ephemeral.events = [
+                        {
+                            content: {
+                                [threadEvent1.event_id]: {
+                                    "m.read": {
+                                        [myUserId]: { ts: 1, thread_id: firstEventId },
+                                    },
+                                },
+                            },
+                            type: "m.receipt",
+                        },
+                    ];
+
+                    // fudge to make both thread replies highlights
+                    client!.getPushActionsForEvent = (ev: MatrixEvent): IActionsObject | null => {
+                        if ([threadEvent1.event_id, threadEvent2.event_id].includes(ev.getId()!)) {
+                            return {
+                                notify: true,
+                                tweaks: {
+                                    highlight: true,
+                                },
+                            };
+                        }
+                        return null;
+                    };
+
+                    httpBackend!.when("GET", "/sync").respond(200, syncData);
+                    client!.startClient({ threadSupport: true });
+
+                    await Promise.all([httpBackend!.flushAllExpected(), awaitSyncEvent()]);
+
+                    const room = client!.getRoom(roomId)!;
+                    expect(room).toBeInstanceOf(Room);
+
+                    // pretend that the client has decrypted an event to trigger it to compute
+                    // local notifications
+                    client?.emit(MatrixEventEvent.Decrypted, room.findEventById(firstEventId)!);
+
+                    // the room should now have one highlight: the second thread message
+
+                    expect(room.getThreadUnreadNotificationCount(firstEventId, NotificationCountType.Highlight)).toBe(
+                        1,
+                    );
+                });
+            });
         });
     });
 
@@ -1759,7 +2284,7 @@ describe("MatrixClient syncing", () => {
         it("should return a room based on the room initialSync API", async () => {
             httpBackend!.when("GET", `/rooms/${encodeURIComponent(roomOne)}/initialSync`).respond(200, {
                 room_id: roomOne,
-                membership: "leave",
+                membership: KnownMembership.Leave,
                 messages: {
                     start: "start",
                     end: "end",
@@ -1808,7 +2333,7 @@ describe("MatrixClient syncing", () => {
             const room = await prom;
 
             expect(room.roomId).toBe(roomOne);
-            expect(room.getMyMembership()).toBe("leave");
+            expect(room.getMyMembership()).toBe(KnownMembership.Leave);
             expect(room.name).toBe("Room Name");
             expect(room.currentState.getStateEvents("m.room.name", "")?.getId()).toBe("$eventId");
             expect(room.timeline[0].getContent().body).toBe("Message 1");
@@ -1900,7 +2425,7 @@ describe("MatrixClient syncing (IndexedDB version)", () => {
                                 type: "m.room.member",
                                 state_key: selfUserId,
                                 content: {
-                                    membership: "invite",
+                                    membership: KnownMembership.Invite,
                                 },
                             },
                         ],

spec/integ/matrix-client-unread-notifications.spec.ts

@@ -28,32 +28,71 @@ import {
     NotificationCountType,
     RelationType,
     Room,
+    fixNotificationCountOnDecryption,
 } from "../../src";
 import { TestClient } from "../TestClient";
 import { ReceiptType } from "../../src/@types/read_receipts";
 import { mkThread } from "../test-utils/thread";
 import { SyncState } from "../../src/sync";
+import { KnownMembership } from "../../src/@types/membership";
+
+const userA = "@alice:localhost";
+const userB = "@bob:localhost";
+const selfUserId = userA;
+const selfAccessToken = "aseukfgwef";
+
+function setupTestClient(): [MatrixClient, HttpBackend] {
+    const testClient = new TestClient(selfUserId, "DEVICE", selfAccessToken);
+    const httpBackend = testClient.httpBackend;
+    const client = testClient.client;
+    httpBackend!.when("GET", "/versions").respond(200, {});
+    httpBackend!.when("GET", "/pushrules").respond(200, {});
+    httpBackend!.when("POST", "/filter").respond(200, { filter_id: "a filter id" });
+    return [client, httpBackend];
+}
+
+describe("Notification count fixing", () => {
+    let client: MatrixClient | undefined;
+
+    beforeEach(() => {
+        [client] = setupTestClient();
+    });
+
+    it("doesn't increment notification count for events that can't be found in a room", async () => {
+        const roomId = "!room:localhost";
+
+        client!.startClient({ threadSupport: true });
+        const room = new Room(roomId, client!, selfUserId);
+        jest.spyOn(client!, "getRoom").mockImplementation((id) => (id === roomId ? room : null));
+
+        const event = new MatrixEvent({
+            room_id: roomId,
+            type: "m.reaction",
+            event_id: "$foo",
+            content: {
+                "m.relates_to": {
+                    rel_type: RelationType.Annotation,
+                    event_id: "$foo",
+                    key: "x",
+                },
+            },
+        });
+
+        jest.spyOn(event, "getPushActions").mockReturnValue({
+            notify: true,
+            tweaks: {},
+        });
+
+        fixNotificationCountOnDecryption(client!, event);
+
+        expect(room.getUnreadNotificationCount(NotificationCountType.Total)).toBe(0);
+    });
+});
 
 describe("MatrixClient syncing", () => {
-    const userA = "@alice:localhost";
-    const userB = "@bob:localhost";
-
-    const selfUserId = userA;
-    const selfAccessToken = "aseukfgwef";
-
     let client: MatrixClient | undefined;
     let httpBackend: HttpBackend | undefined;
 
-    const setupTestClient = (): [MatrixClient, HttpBackend] => {
-        const testClient = new TestClient(selfUserId, "DEVICE", selfAccessToken);
-        const httpBackend = testClient.httpBackend;
-        const client = testClient.client;
-        httpBackend!.when("GET", "/versions").respond(200, {});
-        httpBackend!.when("GET", "/pushrules").respond(200, {});
-        httpBackend!.when("POST", "/filter").respond(200, { filter_id: "a filter id" });
-        return [client, httpBackend];
-    };
-
     beforeEach(() => {
         [client, httpBackend] = setupTestClient();
     });
@@ -113,7 +152,7 @@ describe("MatrixClient syncing", () => {
         await client!.sendEvent(roomId, EventType.Reaction, {
             "m.relates_to": {
                 rel_type: RelationType.Annotation,
-                event_id: threadReply.getId(),
+                event_id: threadReply.getId()!,
                 key: "",
             },
         });
@@ -179,7 +218,6 @@ describe("MatrixClient syncing", () => {
                             events: [
                                 {
                                     content: {
-                                        creator: userB,
                                         room_version: "9",
                                     },
                                     origin_server_ts: 1,
@@ -192,7 +230,7 @@ describe("MatrixClient syncing", () => {
                                     content: {
                                         avatar_url: "",
                                         displayname: userB,
-                                        membership: "join",
+                                        membership: KnownMembership.Join,
                                     },
                                     origin_server_ts: 2,
                                     sender: userB,
@@ -233,7 +271,7 @@ describe("MatrixClient syncing", () => {
                                 },
                                 {
                                     content: {
-                                        join_rule: "invite",
+                                        join_rule: KnownMembership.Invite,
                                     },
                                     origin_server_ts: 4,
                                     sender: userB,
@@ -279,7 +317,7 @@ describe("MatrixClient syncing", () => {
                                         avatar_url: "",
                                         displayname: userA,
                                         is_direct: true,
-                                        membership: "invite",
+                                        membership: KnownMembership.Invite,
                                     },
                                     origin_server_ts: 8,
                                     sender: userB,
@@ -301,7 +339,7 @@ describe("MatrixClient syncing", () => {
                                     content: {
                                         avatar_url: "",
                                         displayname: userA,
-                                        membership: "join",
+                                        membership: KnownMembership.Join,
                                     },
                                     origin_server_ts: 10,
                                     sender: userA,
@@ -377,6 +415,7 @@ describe("MatrixClient syncing", () => {
                 },
                 [Category.Leave]: {},
                 [Category.Invite]: {},
+                [Category.Knock]: {},
             },
         };
     }

spec/integ/sliding-sync-sdk.spec.ts

@@ -43,6 +43,7 @@ import { IStoredClientOpts } from "../../src";
 import { logger } from "../../src/logger";
 import { emitPromise } from "../test-utils/test-utils";
 import { defer } from "../../src/utils";
+import { KnownMembership } from "../../src/@types/membership";
 
 describe("SlidingSyncSdk", () => {
     let client: MatrixClient | undefined;
@@ -188,8 +189,8 @@ describe("SlidingSyncSdk", () => {
                 [roomA]: {
                     name: "A",
                     required_state: [
-                        mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
-                        mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
+                        mkOwnStateEvent(EventType.RoomCreate, {}, ""),
+                        mkOwnStateEvent(EventType.RoomMember, { membership: KnownMembership.Join }, selfUserId),
                         mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
                         mkOwnStateEvent(EventType.RoomName, { name: "A" }, ""),
                     ],
@@ -203,8 +204,8 @@ describe("SlidingSyncSdk", () => {
                     name: "B",
                     required_state: [],
                     timeline: [
-                        mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
-                        mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
+                        mkOwnStateEvent(EventType.RoomCreate, {}, ""),
+                        mkOwnStateEvent(EventType.RoomMember, { membership: KnownMembership.Join }, selfUserId),
                         mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
                         mkOwnEvent(EventType.RoomMessage, { body: "hello B" }),
                         mkOwnEvent(EventType.RoomMessage, { body: "world B" }),
@@ -215,8 +216,8 @@ describe("SlidingSyncSdk", () => {
                     name: "C",
                     required_state: [],
                     timeline: [
-                        mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
-                        mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
+                        mkOwnStateEvent(EventType.RoomCreate, {}, ""),
+                        mkOwnStateEvent(EventType.RoomMember, { membership: KnownMembership.Join }, selfUserId),
                         mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
                         mkOwnEvent(EventType.RoomMessage, { body: "hello C" }),
                         mkOwnEvent(EventType.RoomMessage, { body: "world C" }),
@@ -228,8 +229,8 @@ describe("SlidingSyncSdk", () => {
                     name: "D",
                     required_state: [],
                     timeline: [
-                        mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
-                        mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
+                        mkOwnStateEvent(EventType.RoomCreate, {}, ""),
+                        mkOwnStateEvent(EventType.RoomMember, { membership: KnownMembership.Join }, selfUserId),
                         mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
                         mkOwnEvent(EventType.RoomMessage, { body: "hello D" }),
                         mkOwnEvent(EventType.RoomMessage, { body: "world D" }),
@@ -244,7 +245,7 @@ describe("SlidingSyncSdk", () => {
                     invite_state: [
                         {
                             type: EventType.RoomMember,
-                            content: { membership: "invite" },
+                            content: { membership: KnownMembership.Invite },
                             state_key: selfUserId,
                             sender: "@bob:localhost",
                             event_id: "$room_e_invite",
@@ -264,8 +265,8 @@ describe("SlidingSyncSdk", () => {
                 [roomF]: {
                     name: "#foo:localhost",
                     required_state: [
-                        mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
-                        mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
+                        mkOwnStateEvent(EventType.RoomCreate, {}, ""),
+                        mkOwnStateEvent(EventType.RoomMember, { membership: KnownMembership.Join }, selfUserId),
                         mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
                         mkOwnStateEvent(EventType.RoomCanonicalAlias, { alias: "#foo:localhost" }, ""),
                         mkOwnStateEvent(EventType.RoomName, { name: "This should be ignored" }, ""),
@@ -280,8 +281,8 @@ describe("SlidingSyncSdk", () => {
                     name: "G",
                     required_state: [],
                     timeline: [
-                        mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
-                        mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
+                        mkOwnStateEvent(EventType.RoomCreate, {}, ""),
+                        mkOwnStateEvent(EventType.RoomMember, { membership: KnownMembership.Join }, selfUserId),
                         mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
                     ],
                     joined_count: 5,
@@ -292,8 +293,8 @@ describe("SlidingSyncSdk", () => {
                     name: "H",
                     required_state: [],
                     timeline: [
-                        mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
-                        mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
+                        mkOwnStateEvent(EventType.RoomCreate, {}, ""),
+                        mkOwnStateEvent(EventType.RoomMember, { membership: KnownMembership.Join }, selfUserId),
                         mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
                         mkOwnEvent(EventType.RoomMessage, { body: "live event" }),
                     ],
@@ -308,7 +309,7 @@ describe("SlidingSyncSdk", () => {
                 const gotRoom = client!.getRoom(roomA);
                 expect(gotRoom).toBeTruthy();
                 expect(gotRoom!.name).toEqual(data[roomA].name);
-                expect(gotRoom!.getMyMembership()).toEqual("join");
+                expect(gotRoom!.getMyMembership()).toEqual(KnownMembership.Join);
                 assertTimelineEvents(gotRoom!.getLiveTimeline().getEvents().slice(-2), data[roomA].timeline);
             });
 
@@ -318,7 +319,7 @@ describe("SlidingSyncSdk", () => {
                 const gotRoom = client!.getRoom(roomB);
                 expect(gotRoom).toBeTruthy();
                 expect(gotRoom!.name).toEqual(data[roomB].name);
-                expect(gotRoom!.getMyMembership()).toEqual("join");
+                expect(gotRoom!.getMyMembership()).toEqual(KnownMembership.Join);
                 assertTimelineEvents(gotRoom!.getLiveTimeline().getEvents().slice(-5), data[roomB].timeline);
             });
 
@@ -372,7 +373,7 @@ describe("SlidingSyncSdk", () => {
                 const gotRoom = client!.getRoom(roomH);
                 expect(gotRoom).toBeTruthy();
                 expect(gotRoom!.name).toEqual(data[roomH].name);
-                expect(gotRoom!.getMyMembership()).toEqual("join");
+                expect(gotRoom!.getMyMembership()).toEqual(KnownMembership.Join);
                 // check the entire timeline is correct
                 assertTimelineEvents(gotRoom!.getLiveTimeline().getEvents(), data[roomH].timeline);
                 await expect(seenLiveEventDeferred.promise).resolves.toBeTruthy();
@@ -383,7 +384,7 @@ describe("SlidingSyncSdk", () => {
                 await emitPromise(client!, ClientEvent.Room);
                 const gotRoom = client!.getRoom(roomE);
                 expect(gotRoom).toBeTruthy();
-                expect(gotRoom!.getMyMembership()).toEqual("invite");
+                expect(gotRoom!.getMyMembership()).toEqual(KnownMembership.Invite);
                 expect(gotRoom!.currentState.getJoinRule()).toEqual(JoinRule.Invite);
             });
 
@@ -602,10 +603,10 @@ describe("SlidingSyncSdk", () => {
                 name: "Room with Invite",
                 required_state: [],
                 timeline: [
-                    mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
-                    mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
+                    mkOwnStateEvent(EventType.RoomCreate, {}, ""),
+                    mkOwnStateEvent(EventType.RoomMember, { membership: KnownMembership.Join }, selfUserId),
                     mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
-                    mkOwnStateEvent(EventType.RoomMember, { membership: "invite" }, invitee),
+                    mkOwnStateEvent(EventType.RoomMember, { membership: KnownMembership.Invite }, invitee),
                 ],
             });
             await httpBackend!.flush("/profile", 1, 1000);
@@ -718,8 +719,8 @@ describe("SlidingSyncSdk", () => {
                 name: "Room with account data",
                 required_state: [],
                 timeline: [
-                    mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
-                    mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
+                    mkOwnStateEvent(EventType.RoomCreate, {}, ""),
+                    mkOwnStateEvent(EventType.RoomMember, { membership: KnownMembership.Join }, selfUserId),
                     mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
                     mkOwnEvent(EventType.RoomMessage, { body: "hello" }),
                 ],
@@ -922,8 +923,8 @@ describe("SlidingSyncSdk", () => {
                 name: "Room with typing",
                 required_state: [],
                 timeline: [
-                    mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
-                    mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
+                    mkOwnStateEvent(EventType.RoomCreate, {}, ""),
+                    mkOwnStateEvent(EventType.RoomMember, { membership: KnownMembership.Join }, selfUserId),
                     mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
                     mkOwnEvent(EventType.RoomMessage, { body: "hello" }),
                 ],
@@ -963,8 +964,8 @@ describe("SlidingSyncSdk", () => {
                 name: "Room with typing",
                 required_state: [],
                 timeline: [
-                    mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
-                    mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
+                    mkOwnStateEvent(EventType.RoomCreate, {}, ""),
+                    mkOwnStateEvent(EventType.RoomMember, { membership: KnownMembership.Join }, selfUserId),
                     mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
                     mkOwnEvent(EventType.RoomMessage, { body: "hello" }),
                 ],
@@ -1049,13 +1050,13 @@ describe("SlidingSyncSdk", () => {
                 name: "Room with receipts",
                 required_state: [],
                 timeline: [
-                    mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
-                    mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
+                    mkOwnStateEvent(EventType.RoomCreate, {}, ""),
+                    mkOwnStateEvent(EventType.RoomMember, { membership: KnownMembership.Join }, selfUserId),
                     mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
                     {
                         type: EventType.RoomMember,
                         state_key: alice,
-                        content: { membership: "join" },
+                        content: { membership: KnownMembership.Join },
                         sender: alice,
                         origin_server_ts: Date.now(),
                         event_id: "$alice",

spec/integ/sliding-sync.spec.ts

@@ -107,8 +107,8 @@ describe("SlidingSync", () => {
                 onRequest: (initial) => {
                     return { initial: initial };
                 },
-                onResponse: (res) => {
-                    return {};
+                onResponse: async (res) => {
+                    return;
                 },
                 when: () => ExtensionState.PreProcess,
             };
@@ -1572,7 +1572,7 @@ describe("SlidingSync", () => {
             onPreExtensionRequest = () => {
                 return extReq;
             };
-            onPreExtensionResponse = (resp) => {
+            onPreExtensionResponse = async (resp) => {
                 extensionOnResponseCalled = true;
                 callbackOrder.push("onPreExtensionResponse");
                 expect(resp).toEqual(extResp);
@@ -1613,7 +1613,7 @@ describe("SlidingSync", () => {
                 return undefined;
             };
             let responseCalled = false;
-            onPreExtensionResponse = (resp) => {
+            onPreExtensionResponse = async (resp) => {
                 responseCalled = true;
             };
             httpBackend!
@@ -1649,7 +1649,7 @@ describe("SlidingSync", () => {
             };
             let responseCalled = false;
             const callbackOrder: string[] = [];
-            onPostExtensionResponse = (resp) => {
+            onPostExtensionResponse = async (resp) => {
                 expect(resp).toEqual(extResp);
                 responseCalled = true;
                 callbackOrder.push("onPostExtensionResponse");

spec/olm-loader.ts

@@ -20,7 +20,7 @@ import { logger } from "../src/logger";
 // try to load the olm library.
 try {
     // eslint-disable-next-line @typescript-eslint/no-var-requires
-    global.Olm = require("@matrix-org/olm");
+    globalThis.Olm = require("@matrix-org/olm");
     logger.log("loaded libolm");
 } catch (e) {
     logger.warn("unable to run crypto tests: libolm not available", e);

spec/test-utils/AccountDataAccumulator.ts

@@ -0,0 +1,108 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import fetchMock from "fetch-mock-jest";
+import { MockOptionsMethodPut } from "fetch-mock";
+
+import { ISyncResponder } from "./SyncResponder";
+
+/**
+ *  An object which intercepts `account_data` get and set requests via fetch-mock.
+ */
+export class AccountDataAccumulator {
+    /**
+     * The account data events to be returned by the sync.
+     * Will be updated when fetchMock intercepts calls to PUT `/_matrix/client/v3/user/:userId/account_data/`.
+     * Will be used by `sendSyncResponseWithUpdatedAccountData`
+     */
+    public accountDataEvents: Map<String, any> = new Map();
+
+    /**
+     * Intercept requests to set a particular type of account data.
+     *
+     * Once it is set, its data is stored (for future return by `interceptGetAccountData` etc) and the resolved promise is
+     * resolved.
+     *
+     * @param accountDataType - type of account data to be intercepted
+     * @param opts - options to pass to fetchMock
+     * @returns a Promise which will resolve (with the content of the account data) once it is set.
+     */
+    public interceptSetAccountData(accountDataType: string, opts?: MockOptionsMethodPut): Promise<any> {
+        return new Promise((resolve) => {
+            // Called when the cross signing key is uploaded
+            fetchMock.put(
+                `express:/_matrix/client/v3/user/:userId/account_data/${accountDataType}`,
+                (url: string, options: RequestInit) => {
+                    const content = JSON.parse(options.body as string);
+                    const type = url.split("/").pop();
+                    // update account data for sync response
+                    this.accountDataEvents.set(type!, content);
+                    resolve(content);
+                    return {};
+                },
+                opts,
+            );
+        });
+    }
+
+    /**
+     * Intercept all requests to get account data
+     */
+    public interceptGetAccountData(): void {
+        fetchMock.get(
+            `express:/_matrix/client/v3/user/:userId/account_data/:type`,
+            (url) => {
+                const type = url.split("/").pop();
+                const existing = this.accountDataEvents.get(type!);
+                if (existing) {
+                    // return it
+                    return {
+                        status: 200,
+                        body: existing,
+                    };
+                } else {
+                    // 404
+                    return {
+                        status: 404,
+                        body: { errcode: "M_NOT_FOUND", error: "Account data not found." },
+                    };
+                }
+            },
+            { overwriteRoutes: true },
+        );
+    }
+
+    /**
+     * Send a sync response the current account data events.
+     */
+    public sendSyncResponseWithUpdatedAccountData(syncResponder: ISyncResponder): void {
+        try {
+            syncResponder.sendOrQueueSyncResponse({
+                next_batch: 1,
+                account_data: {
+                    events: Array.from(this.accountDataEvents, ([type, content]) => ({
+                        type: type,
+                        content: content,
+                    })),
+                },
+            });
+        } catch (err) {
+            // Might fail with "Cannot queue more than one /sync response" if called too often.
+            // It's ok if it fails here, the sync response is cumulative and will contain
+            // the latest account data.
+        }
+    }
+}

spec/test-utils/mockEndpoints.ts

@@ -16,17 +16,29 @@ limitations under the License.
 
 import fetchMock from "fetch-mock-jest";
 
+import { KeyBackupInfo } from "../../src/crypto-api";
+
 /**
  * Mock out the endpoints that the js-sdk calls when we call `MatrixClient.start()`.
  *
  * @param homeserverUrl - the homeserver url for the client under test
  */
 export function mockInitialApiRequests(homeserverUrl: string) {
-    fetchMock.getOnce(new URL("/_matrix/client/versions", homeserverUrl).toString(), { versions: ["v1.1"] });
-    fetchMock.getOnce(new URL("/_matrix/client/v3/pushrules/", homeserverUrl).toString(), {});
-    fetchMock.postOnce(new URL("/_matrix/client/v3/user/%40alice%3Alocalhost/filter", homeserverUrl).toString(), {
-        filter_id: "fid",
-    });
+    fetchMock.getOnce(
+        new URL("/_matrix/client/versions", homeserverUrl).toString(),
+        { versions: ["v1.1"] },
+        { overwriteRoutes: true },
+    );
+    fetchMock.getOnce(
+        new URL("/_matrix/client/v3/pushrules/", homeserverUrl).toString(),
+        {},
+        { overwriteRoutes: true },
+    );
+    fetchMock.postOnce(
+        new URL("/_matrix/client/v3/user/%40alice%3Alocalhost/filter", homeserverUrl).toString(),
+        { filter_id: "fid" },
+        { overwriteRoutes: true },
+    );
 }
 
 /**
@@ -56,3 +68,35 @@ export function mockSetupCrossSigningRequests(): void {
         {},
     );
 }
+
+/**
+ * Mock out requests to `/room_keys/version`.
+ *
+ * Returns `404 M_NOT_FOUND` for GET requests until `POST room_keys/version` is called.
+ * Once the POST is done, `GET /room_keys/version` will return the posted backup
+ * instead of 404.
+ *
+ * @param backupVersion - The backup version that will be returned by `POST room_keys/version`.
+ */
+export function mockSetupMegolmBackupRequests(backupVersion: string): void {
+    fetchMock.get("path:/_matrix/client/v3/room_keys/version", {
+        status: 404,
+        body: {
+            errcode: "M_NOT_FOUND",
+            error: "No current backup version",
+        },
+    });
+
+    fetchMock.post("path:/_matrix/client/v3/room_keys/version", (url, request) => {
+        const backupData: KeyBackupInfo = JSON.parse(request.body?.toString() ?? "{}");
+        backupData.version = backupVersion;
+        backupData.count = 0;
+        backupData.etag = "zer";
+        fetchMock.get("path:/_matrix/client/v3/room_keys/version", backupData, {
+            overwriteRoutes: true,
+        });
+        return {
+            version: backupVersion,
+        };
+    });
+}

spec/test-utils/oidc.ts

@@ -14,8 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import { OidcClientConfig } from "../../src";
-import { ValidatedIssuerMetadata } from "../../src/oidc/validate";
+import { OidcClientConfig, ValidatedIssuerMetadata } from "../../src";
 
 /**
  * Makes a valid OidcClientConfig with minimum valid values
@@ -26,8 +25,7 @@ export const makeDelegatedAuthConfig = (issuer = "https://auth.org/"): OidcClien
     const metadata = mockOpenIdConfiguration(issuer);
 
     return {
-        issuer,
-        account: issuer + "account",
+        accountManagementEndpoint: issuer + "account",
         registrationEndpoint: metadata.registration_endpoint,
         authorizationEndpoint: metadata.authorization_endpoint,
         tokenEndpoint: metadata.token_endpoint,
@@ -46,6 +44,7 @@ export const mockOpenIdConfiguration = (issuer = "https://auth.org/"): Validated
     token_endpoint: issuer + "token",
     authorization_endpoint: issuer + "auth",
     registration_endpoint: issuer + "registration",
+    device_authorization_endpoint: issuer + "device",
     jwks_uri: issuer + "jwks",
     response_types_supported: ["code"],
     grant_types_supported: ["authorization_code", "refresh_token"],

spec/test-utils/test-data/generate-test-data.py

@@ -26,10 +26,15 @@ python -m venv env
 
 import base64
 import json
+import base58
 
 from canonicaljson import encode_canonical_json
 from cryptography.hazmat.primitives.asymmetric import ed25519, x25519
 from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat
+from cryptography.hazmat.primitives import hashes, padding, hmac
+from cryptography.hazmat.primitives.kdf.hkdf import HKDF
+from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
+
 from random import randbytes, seed
 
 ALICE_DATA = {
@@ -38,6 +43,8 @@ ALICE_DATA = {
     "TEST_ROOM_ID": "!room:id",
     # any 32-byte string can be an ed25519 private key.
     "TEST_DEVICE_PRIVATE_KEY_BYTES": b"deadbeefdeadbeefdeadbeefdeadbeef",
+    # any 32-byte string can be an curve25519 private key.
+    "TEST_DEVICE_CURVE_PRIVATE_KEY_BYTES": b"deadmuledeadmuledeadmuledeadmule",
 
     "MASTER_CROSS_SIGNING_PRIVATE_KEY_BYTES": b"doyouspeakwhaaaaaaaaaaaaaaaaaale",
     "USER_CROSS_SIGNING_PRIVATE_KEY_BYTES": b"useruseruseruseruseruseruseruser",
@@ -55,6 +62,8 @@ BOB_DATA = {
     "TEST_ROOM_ID": "!room:id",
     # any 32-byte string can be an ed25519 private key.
     "TEST_DEVICE_PRIVATE_KEY_BYTES": b"Deadbeefdeadbeefdeadbeefdeadbeef",
+    # any 32-byte string can be an curve25519 private key.
+    "TEST_DEVICE_CURVE_PRIVATE_KEY_BYTES": b"Deadmuledeadmuledeadmuledeadmule",
 
     "MASTER_CROSS_SIGNING_PRIVATE_KEY_BYTES": b"Doyouspeakwhaaaaaaaaaaaaaaaaaale",
     "USER_CROSS_SIGNING_PRIVATE_KEY_BYTES": b"Useruseruseruseruseruseruseruser",
@@ -75,8 +84,8 @@ def main() -> None:
  */
 
 import {{ IDeviceKeys, IMegolmSessionData }} from "../../../src/@types/crypto";
-import {{ IDownloadKeyResult }} from "../../../src";
-import {{ KeyBackupInfo }} from "../../../src/crypto-api";
+import {{ IDownloadKeyResult, IEvent }} from "../../../src";
+import {{ KeyBackupSession, KeyBackupInfo }} from "../../../src/crypto-api/keybackup";
 
 /* eslint-disable comma-dangle */
 
@@ -97,6 +106,11 @@ def build_test_data(user_data, prefix = "") -> str:
     private_key = ed25519.Ed25519PrivateKey.from_private_bytes(
              user_data["TEST_DEVICE_PRIVATE_KEY_BYTES"]
         )
+
+    device_curve_key = x25519.X25519PrivateKey.from_private_bytes(
+             user_data["TEST_DEVICE_CURVE_PRIVATE_KEY_BYTES"]
+        )
+
     b64_public_key = encode_base64(
         private_key.public_key().public_bytes(Encoding.Raw, PublicFormat.Raw)
     )
@@ -164,9 +178,10 @@ def build_test_data(user_data, prefix = "") -> str:
         user_data["TEST_USER_ID"]: {f"ed25519:{user_data['TEST_DEVICE_ID']}": sig}
     }
 
-    set_of_exported_room_keys = [build_exported_megolm_key(), build_exported_megolm_key()]
+    set_of_exported_room_keys = [build_exported_megolm_key(device_curve_key)[0], build_exported_megolm_key(device_curve_key)[0]]
 
-    additional_exported_room_key = build_exported_megolm_key()
+    additional_exported_room_key, additional_exported_ed_key = build_exported_megolm_key(device_curve_key)
+    ratcheted_exported_room_key = symetric_ratchet_step_of_megolm_key(additional_exported_room_key, additional_exported_ed_key)
 
     otk_to_sign = {
         "key": user_data['OTK']
@@ -186,6 +201,13 @@ def build_test_data(user_data, prefix = "") -> str:
         }
     }
 
+
+    backed_up_room_key = encrypt_megolm_key_for_backup(additional_exported_room_key, backup_decryption_key.public_key())
+
+    clear_event, encrypted_event = generate_encrypted_event_content(additional_exported_room_key, additional_exported_ed_key, device_curve_key)
+
+    backup_recovery_key = export_recovery_key(user_data["B64_BACKUP_DECRYPTION_KEY"])
+
     return f"""\
 export const {prefix}TEST_USER_ID = "{user_data['TEST_USER_ID']}";
 export const {prefix}TEST_DEVICE_ID = "{user_data['TEST_DEVICE_ID']}";
@@ -220,9 +242,15 @@ export const {prefix}SIGNED_CROSS_SIGNING_KEYS_DATA: Partial<IDownloadKeyResult>
         json.dumps(build_cross_signing_keys_data(user_data), indent=4)
 };
 
+/** Signed OTKs, returned by `POST /keys/claim` */
+export const {prefix}ONE_TIME_KEYS = { json.dumps(otks, indent=4) };
+
 /** base64-encoded backup decryption (private) key */
 export const {prefix}BACKUP_DECRYPTION_KEY_BASE64 = "{ user_data['B64_BACKUP_DECRYPTION_KEY'] }";
 
+/** Backup decryption key in export format */
+export const {prefix}BACKUP_DECRYPTION_KEY_BASE58 = "{ backup_recovery_key }";
+
 /** Signed backup data, suitable for return from `GET /_matrix/client/v3/room_keys/keys/{{roomId}}/{{sessionId}}` */
 export const {prefix}SIGNED_BACKUP_DATA: KeyBackupInfo = { json.dumps(backup_data, indent=4) };
 
@@ -236,8 +264,19 @@ export const {prefix}MEGOLM_SESSION_DATA: IMegolmSessionData = {
         json.dumps(additional_exported_room_key, indent=4)
 };
 
-/** Signed OTKs, returned by `POST /keys/claim` */
-export const {prefix}ONE_TIME_KEYS = { json.dumps(otks, indent=4) };
+/** A ratcheted version of {prefix}MEGOLM_SESSION_DATA */
+export const {prefix}RATCHTED_MEGOLM_SESSION_DATA: IMegolmSessionData = {
+        json.dumps(ratcheted_exported_room_key, indent=4)
+};
+
+/** The key from {prefix}MEGOLM_SESSION_DATA, encrypted for backup using `m.megolm_backup.v1.curve25519-aes-sha2` algorithm*/
+export const {prefix}CURVE25519_KEY_BACKUP_DATA: KeyBackupSession = {json.dumps(backed_up_room_key, indent=4)};
+
+/** A test clear event */
+export const {prefix}CLEAR_EVENT: Partial<IEvent> = {json.dumps(clear_event, indent=4)};
+
+/** The encrypted CLEAR_EVENT by MEGOLM_SESSION_DATA */
+export const {prefix}ENCRYPTED_EVENT: Partial<IEvent> = {json.dumps(encrypted_event, indent=4)};
 """
 
 
@@ -334,10 +373,11 @@ def sign_json(json_object: dict, private_key: ed25519.Ed25519PrivateKey) -> str:
 
     return signature_base64
 
-def build_exported_megolm_key() -> dict:
+def build_exported_megolm_key(device_curve_key: x25519.X25519PrivateKey) -> tuple[dict, ed25519.Ed25519PrivateKey]:
     """
     Creates an exported megolm room key, as per https://gitlab.matrix.org/matrix-org/olm/blob/master/docs/megolm.md#session-export-format
     that can be imported via importRoomKeys API.
+    Returns the exported key, the matching privat edKey (needed to encrypt)
     """
     index = 0
     private_key = ed25519.Ed25519PrivateKey.from_private_bytes(randbytes(32))
@@ -353,8 +393,10 @@ def build_exported_megolm_key() -> dict:
 
     megolm_export = {
         "algorithm": "m.megolm.v1.aes-sha2",
-        "room_id": "!roomA:example.org",
-        "sender_key": "/Bu9e34hUClhddpf4E5gu5qEAdMY31+1A9HbiAeeQgo",
+        "room_id": "!room:id",
+        "sender_key": encode_base64(
+            device_curve_key.public_key().public_bytes(Encoding.Raw, PublicFormat.Raw)
+        ),
         "session_id": encode_base64(
             private_key.public_key().public_bytes(Encoding.Raw, PublicFormat.Raw)
         ),
@@ -365,8 +407,248 @@ def build_exported_megolm_key() -> dict:
         "forwarding_curve25519_key_chain": [],
     }
 
+    return megolm_export, private_key
+
+def symetric_ratchet_step_of_megolm_key(previous: dict , megolm_private_key: ed25519.Ed25519PrivateKey) -> dict:
+
+    """
+    Very simple ratchet step from 0 to 1
+    Used to generate a ratcheted key to test unknown message index.
+    """
+    session_key: str = previous["session_key"]
+
+    # Get the megolm R0 from the export format
+    decoded = base64.b64decode(session_key.encode("ascii"))
+    ri = decoded[5:133]
+
+    ri0 = ri[0:32]
+    ri1 = ri[32:64]
+    ri2 = ri[64:96]
+    ri3 = ri[96:128]
+
+    h = hmac.HMAC(ri3, hashes.SHA256())
+    h.update(b'x\03')
+    ri1_3 = h.finalize()
+
+    index = 1
+    private_key = megolm_private_key
+
+    # exported key, start with version byte
+    exported_key = bytearray(b'\x01')
+    exported_key += index.to_bytes(4, 'big')
+    exported_key += ri0
+    exported_key += ri1
+    exported_key += ri2
+    exported_key += ri1_3
+    # KPub
+    exported_key += private_key.public_key().public_bytes(Encoding.Raw, PublicFormat.Raw)
+
+
+    megolm_export = {
+        "algorithm": "m.megolm.v1.aes-sha2",
+        "room_id": "!room:id",
+        "sender_key": previous["sender_key"],
+        "session_id": previous["session_id"],
+        "session_key": encode_base64(exported_key),
+        "sender_claimed_keys": previous["sender_claimed_keys"],
+        "forwarding_curve25519_key_chain": [],
+    }
+
     return megolm_export
 
+def encrypt_megolm_key_for_backup(session_data: dict, backup_public_key: x25519.X25519PublicKey) -> dict:
+
+    """
+    Encrypts an exported megolm key for key backup, using the m.megolm_backup.v1.curve25519-aes-sha2 algorithm.
+    """
+    data = encode_canonical_json(session_data)
+
+    # Generate an ephemeral curve25519 key, and perform an ECDH with the ephemeral key
+    # and the backup’s public key to generate a shared secret.
+    # The public half of the ephemeral key, encoded using unpadded base64,
+    # becomes the ephemeral property of the session_data.
+    ephemeral_keypair = x25519.X25519PrivateKey.from_private_bytes(randbytes(32))
+    shared_secret = ephemeral_keypair.exchange(backup_public_key)
+    ephemeral = encode_base64(ephemeral_keypair.public_key().public_bytes(Encoding.Raw, PublicFormat.Raw))
+
+    # Using the shared secret, generate 80 bytes by performing an HKDF using SHA-256 as the hash,
+    # with a salt of 32 bytes of 0, and with the empty string as the info.
+    # The first 32 bytes are used as the AES key, the next 32 bytes are used as the MAC key,
+    #  and the last 16 bytes are used as the AES initialization vector.
+    salt = bytes(32)
+    info = b""
+
+    hkdf = HKDF(
+        algorithm=hashes.SHA256(),
+        length=80,
+        salt=salt,
+        info=info,
+    )
+
+    raw_key = hkdf.derive(shared_secret)
+    aes_key = raw_key[:32]
+    mac = raw_key[32:64]
+    iv = raw_key[64:80]
+
+    # Stringify the JSON object, and encrypt it using AES-CBC-256 with PKCS#7 padding.
+    # This encrypted data, encoded using unpadded base64, becomes the ciphertext property of the session_data.
+    cipher = Cipher(algorithms.AES(aes_key), modes.CBC(iv))
+    encryptor = cipher.encryptor()
+    padder = padding.PKCS7(128).padder()
+    padded_data = padder.update(data) + padder.finalize()
+    ct = encryptor.update(padded_data) + encryptor.finalize()
+    cipher_text = encode_base64(ct)
+
+    # Pass the raw encrypted data (prior to base64 encoding) through HMAC-SHA-256 using the MAC key generated above.
+    # The first 8 bytes of the resulting MAC are base64-encoded, and become the mac property of the session_data.
+    h = hmac.HMAC(mac, hashes.SHA256())
+    # h.update(ct)
+    signature = h.finalize()
+    mac = encode_base64(signature[:8])
+
+    encrypted_key = {
+        "first_message_index": 1,
+        "forwarded_count": 0,
+        "is_verified": False,
+        "session_data": {
+            "ciphertext": cipher_text,
+            "ephemeral": ephemeral,
+            "mac": mac
+        }
+
+    }
+
+    return encrypted_key
+
+def generate_encrypted_event_content(exported_key: dict, ed_key: ed25519.Ed25519PrivateKey, curve_key: x25519.X25519PrivateKey) -> tuple[dict, dict]:
+    """
+        Encrypts an event using the given key in session export format.
+        Will not do any ratcheting, just encrypt at index 0.
+    """
+
+    clear_event = {
+        "type": "m.room.message",
+        "room_id": "!room:id",
+        "sender": "@alice:localhost",
+        "content": {
+            "msgtype": "m.text",
+            "body": "Hello world"
+        }
+    }
+
+    session_key: str = exported_key["session_key"]
+
+    # Get the megolm R0 from the export format
+    decoded = base64.b64decode(session_key.encode("ascii"))
+    r0 = decoded[5:133]
+
+    hkdf = HKDF(
+        algorithm=hashes.SHA256(),
+        length=80,
+        salt=bytes(32),
+        info=b"MEGOLM_KEYS",
+    )
+
+    raw_key = hkdf.derive(r0)
+    aes_key = raw_key[:32]
+    mac = raw_key[32:64]
+    aes_iv = raw_key[64:80]
+
+    payload_json = {
+        "room_id": clear_event["room_id"],
+        "type": clear_event["type"],
+        "content": clear_event["content"]
+    }
+
+    payload_string = encode_canonical_json(payload_json)
+
+    cipher = Cipher(algorithms.AES(aes_key), modes.CBC(aes_iv))
+    encryptor = cipher.encryptor()
+    padder = padding.PKCS7(128).padder()
+
+    padded_data = padder.update(payload_string)
+    padded_data += padder.finalize()
+
+    ct = encryptor.update(padded_data) + encryptor.finalize()
+
+    # The ratchet index i, and the cipher-text, are then packed
+    # into a message as described in Message format. Then the entire message
+    # (including the version bytes and all payload bytes) are passed through
+    # HMAC-SHA-256. The first 8 bytes of the MAC are appended to the message.
+    message = bytearray()
+    message += b'\x03'
+    # int tag for index
+    message += b'\x08'
+    # index is 0
+    message += b'\x00'
+    message += b'\x12'
+    # probably works only for short messages
+    message += len(ct).to_bytes(1, 'big')
+    # encrypted data
+    message += ct
+
+    h = hmac.HMAC(mac, hashes.SHA256())
+    h.update(message)
+    signature = h.finalize()
+    mac = signature[:8]
+
+    message += mac
+
+    # Finally, the authenticated message is signed using the Ed25519 keypair;
+    # the 64 byte signature is appended to the message
+    signature = ed_key.sign(bytes(message))
+
+    message += signature
+
+    cipher_text = encode_base64(message)
+
+    encrypted_payload = {
+        "algorithm" : "m.megolm.v1.aes-sha2",
+        "sender_key" : encode_base64(curve_key.public_key().public_bytes(Encoding.Raw, PublicFormat.Raw)),
+        "ciphertext" : cipher_text,
+        "session_id" : exported_key["session_id"],
+        "device_id" : "TEST_DEVICE"
+    }
+
+    encrypted_event = {
+        "type": "m.room.encrypted",
+        "room_id": "!room:id",
+        "sender": "@alice:localhost",
+        "content": encrypted_payload,
+        "event_id": "$event1",
+        "origin_server_ts": 1507753886000,
+    }
+
+    return clear_event, encrypted_event
+
+
+def export_recovery_key(key_b64: str) -> str:
+    """
+        Export a private recovery key as a recovery key that can be presented to users.
+        As per spec https://spec.matrix.org/v1.8/client-server-api/#recovery-key
+    """
+    private_key_bytes = base64.b64decode(key_b64)
+
+    # The 256-bit curve25519 private key is prepended by the bytes 0x8B and 0x01
+    export_bytes = bytearray()
+    export_bytes += b'\x8b'
+    export_bytes += b'\x01'
+
+    export_bytes += private_key_bytes
+
+    # All the bytes in the string above, including the two header bytes,
+    # are XORed together to form a parity byte. This parity byte is appended to the byte string.
+    parity_byte = 0 #b'\x8b' ^ b'\x01'
+    [parity_byte := parity_byte ^ x for x in export_bytes]
+
+    export_bytes += parity_byte.to_bytes(1, 'big')
+
+    # The byte string is encoded using base58
+    recovery_key = base58.b58encode(export_bytes).decode('utf-8')
+
+    split = [recovery_key[i:i + 4] for i in range(0, len(recovery_key), 4)]
+    return ' '.join(split)
+
 
 if __name__ == "__main__":
     main()

spec/test-utils/test-data/index.ts

@@ -4,8 +4,8 @@
  */
 
 import { IDeviceKeys, IMegolmSessionData } from "../../../src/@types/crypto";
-import { IDownloadKeyResult } from "../../../src";
-import { KeyBackupInfo } from "../../../src/crypto-api";
+import { IDownloadKeyResult, IEvent } from "../../../src";
+import { KeyBackupSession, KeyBackupInfo } from "../../../src/crypto-api/keybackup";
 
 /* eslint-disable comma-dangle */
 
@@ -102,9 +102,28 @@ export const SIGNED_CROSS_SIGNING_KEYS_DATA: Partial<IDownloadKeyResult> = {
     }
 };
 
+/** Signed OTKs, returned by `POST /keys/claim` */
+export const ONE_TIME_KEYS = {
+    "@alice:localhost": {
+        "test_device": {
+            "signed_curve25519:AAAAHQ": {
+                "key": "j3fR3HemM16M7CWhoI4Sk5ZsdmdfQHsKL1xuSft6MSw",
+                "signatures": {
+                    "@alice:localhost": {
+                        "ed25519:test_device": "25djC6Rk6gIgFBMVawY9X9LnY8XMMziey6lKqL8Q5Bbp7T1vw9uk0RE7eKO2a/jNLcYroO2xRztGhBrKz5sOCQ"
+                    }
+                }
+            }
+        }
+    }
+};
+
 /** base64-encoded backup decryption (private) key */
 export const BACKUP_DECRYPTION_KEY_BASE64 = "dwdtCnMYpX08FsFyUbJmRd9ML4frwJkqsXf7pR25LCo=";
 
+/** Backup decryption key in export format */
+export const BACKUP_DECRYPTION_KEY_BASE58 = "EsTc LW2K PGiF wKEA 3As5 g5c4 BXwk qeeJ ZJV8 Q9fu gUMN UE4d";
+
 /** Signed backup data, suitable for return from `GET /_matrix/client/v3/room_keys/keys/{roomId}/{sessionId}` */
 export const SIGNED_BACKUP_DATA: KeyBackupInfo = {
     "algorithm": "m.megolm_backup.v1.curve25519-aes-sha2",
@@ -123,8 +142,8 @@ export const SIGNED_BACKUP_DATA: KeyBackupInfo = {
 export const MEGOLM_SESSION_DATA_ARRAY: IMegolmSessionData[] = [
     {
         "algorithm": "m.megolm.v1.aes-sha2",
-        "room_id": "!roomA:example.org",
-        "sender_key": "/Bu9e34hUClhddpf4E5gu5qEAdMY31+1A9HbiAeeQgo",
+        "room_id": "!room:id",
+        "sender_key": "WimPd2udAU/1S/+YBpPbmr9L+0H5H+BnAVHSwDxlPGc",
         "session_id": "FYOoKQSwe4d9jhTZ/LQCZFJINjPEqZ7Or4Z08reP92M",
         "session_key": "AQAAAABZ0jXQOprFfXe41tIFmAtHxflJp4O2hM/vzQQpOazOCFeWSoW5P3Z9Q+voU3eXehMwyP8/hm/Q8xLP6/PmJdy+71se/17kdFwcDGgLxBWfa4ODM9zlI4EjKbNqmiii5loJ7rBhA/XXaw80m0hfU6zTDX/KrO55J0Pt4vJ0LDa3LBWDqCkEsHuHfY4U2fy0AmRSSDYzxKmezq+GdPK3j/dj",
         "sender_claimed_keys": {
@@ -134,8 +153,8 @@ export const MEGOLM_SESSION_DATA_ARRAY: IMegolmSessionData[] = [
     },
     {
         "algorithm": "m.megolm.v1.aes-sha2",
-        "room_id": "!roomA:example.org",
-        "sender_key": "/Bu9e34hUClhddpf4E5gu5qEAdMY31+1A9HbiAeeQgo",
+        "room_id": "!room:id",
+        "sender_key": "WimPd2udAU/1S/+YBpPbmr9L+0H5H+BnAVHSwDxlPGc",
         "session_id": "mPYSGA2l1tOQiipEDEVYhDSdTSFh2lDW1qpGKYZRxTc",
         "session_key": "AQAAAAAHwgkB49BTPAEGTCK6degxUIbl8GPG2ugPRYhNtOpNic63u11+baXFfjDw5fmVfD1gJXpQQjGsqrIYioxrB1xzl7mfb942UHhYdaMQZowpp1fSpJVsxR5TddUU2EWifYD9EQsoz8mY1zqoazm4vUP4v9yxaTcUBj2c6HMJCY0gCJj2EhgNpdbTkIoqRAxFWIQ0nU0hYdpQ1taqRimGUcU3",
         "sender_claimed_keys": {
@@ -148,8 +167,8 @@ export const MEGOLM_SESSION_DATA_ARRAY: IMegolmSessionData[] = [
 /** An exported megolm session */
 export const MEGOLM_SESSION_DATA: IMegolmSessionData = {
     "algorithm": "m.megolm.v1.aes-sha2",
-    "room_id": "!roomA:example.org",
-    "sender_key": "/Bu9e34hUClhddpf4E5gu5qEAdMY31+1A9HbiAeeQgo",
+    "room_id": "!room:id",
+    "sender_key": "WimPd2udAU/1S/+YBpPbmr9L+0H5H+BnAVHSwDxlPGc",
     "session_id": "ipdI6Zs/7DzFTEhiA2iGaMDfHkIYCleqXT6L+5e1/co",
     "session_key": "AQAAAABXGO+Z9jlQJhIL6ByhXrv2BwCIxkhh7MXpKLsYmXkJcWrQlirmXmD79ga1zo+I4DCtEZzyGSpDWXBC6G7ez3H4gDMBam1RE3Jm5tc+oTlIri32UkYgSL0kBkcEnttqmIXBlK8tAfJo3cJnlh7F4ltEOAqrdME6dU0zXTkqXmURqYqXSOmbP+w8xUxIYgNohmjA3x5CGApXql0+i/uXtf3K",
     "sender_claimed_keys": {
@@ -158,22 +177,58 @@ export const MEGOLM_SESSION_DATA: IMegolmSessionData = {
     "forwarding_curve25519_key_chain": []
 };
 
-/** Signed OTKs, returned by `POST /keys/claim` */
-export const ONE_TIME_KEYS = {
-    "@alice:localhost": {
-        "test_device": {
-            "signed_curve25519:AAAAHQ": {
-                "key": "j3fR3HemM16M7CWhoI4Sk5ZsdmdfQHsKL1xuSft6MSw",
-                "signatures": {
-                    "@alice:localhost": {
-                        "ed25519:test_device": "25djC6Rk6gIgFBMVawY9X9LnY8XMMziey6lKqL8Q5Bbp7T1vw9uk0RE7eKO2a/jNLcYroO2xRztGhBrKz5sOCQ"
-                    }
-                }
-            }
-        }
+/** A ratcheted version of MEGOLM_SESSION_DATA */
+export const RATCHTED_MEGOLM_SESSION_DATA: IMegolmSessionData = {
+    "algorithm": "m.megolm.v1.aes-sha2",
+    "room_id": "!room:id",
+    "sender_key": "WimPd2udAU/1S/+YBpPbmr9L+0H5H+BnAVHSwDxlPGc",
+    "session_id": "ipdI6Zs/7DzFTEhiA2iGaMDfHkIYCleqXT6L+5e1/co",
+    "session_key": "AQAAAAFXGO+Z9jlQJhIL6ByhXrv2BwCIxkhh7MXpKLsYmXkJcWrQlirmXmD79ga1zo+I4DCtEZzyGSpDWXBC6G7ez3H4gDMBam1RE3Jm5tc+oTlIri32UkYgSL0kBkcEnttqmIUWvpwC7by/yg231+gyzu9lDHAU4ivCj48pt7WGiORWmIqXSOmbP+w8xUxIYgNohmjA3x5CGApXql0+i/uXtf3K",
+    "sender_claimed_keys": {
+        "ed25519": "Bhbpt6hqMZlSH4sJV7xiEEEiPVeTWz4Vkujl1EMdIPI"
+    },
+    "forwarding_curve25519_key_chain": []
+};
+
+/** The key from MEGOLM_SESSION_DATA, encrypted for backup using `m.megolm_backup.v1.curve25519-aes-sha2` algorithm*/
+export const CURVE25519_KEY_BACKUP_DATA: KeyBackupSession = {
+    "first_message_index": 1,
+    "forwarded_count": 0,
+    "is_verified": false,
+    "session_data": {
+        "ciphertext": "r6HRk2/Im2yJe5cLP8R81aVjFWjYWPHpw7TVxphiSK1cdIDZTTK57r6MfU+0i/mTPn+/PosT74OvYwCnehy2d1BPGxhDl8AhPcBu3//Kzlq2o5CssPsw+88gRehkAsPg9Zp5G9sL9to6giltvTWTbsaQpmvv3HLmBOYSFIxvyZrOT/Ffqu325f0IEsKcyV2BdIkw8Ob9Xt+VWoe4MYEGG6y1T8W125zeFgKWI4Ow76uput64H9zZjIo+Cc+hCTO9Ea4EnosSjizCotevkNck7C/zGgfhBikiohROb6SbaZgxicSsEDZ+f7brnri9yP3iXS3PMDHHpa1+XzG2VOG/Y9OQZpkPq+pbLrCC+NWJeJPslDAK5i+RURwzjnPmaHKCRHTq86CwhFyiCDf61MGwCY3xjrmBJg44BCdxWqCx0YJvwsvVqqnl4vTieUfrwThNPsQ81aVkDHvlmrgrTt8icDa8jTJhu34jem+pbRSEM5aJikV4B+zYiLz+dH/v6UpYA2eG8ReOvwpPXp6CAcIlplRPpWbMBeLFVcPkT4KAXTp9exFpB4on4pf8OsaDomlt4qAA0rhAZmhPWPKcU/A0Tz4gyMu54OivVtw1SPj+5Iq+YDQ8jB6Po3ApzMf6fwF9x/FjevbboFB05X2Jr0NrbFqXMOUwXHMgDAGiIWX8+gkmmbaiNWqg2etjN94pobQSGZelb18XGN7kuwMk+Zwk7A",
+        "ephemeral": "q+P1WdRtEiPIEtNuuGrRcueZxUbLnSKdsuTAkxewXgU",
+        "mac": "OibmACbORhI"
     }
 };
 
+/** A test clear event */
+export const CLEAR_EVENT: Partial<IEvent> = {
+    "type": "m.room.message",
+    "room_id": "!room:id",
+    "sender": "@alice:localhost",
+    "content": {
+        "msgtype": "m.text",
+        "body": "Hello world"
+    }
+};
+
+/** The encrypted CLEAR_EVENT by MEGOLM_SESSION_DATA */
+export const ENCRYPTED_EVENT: Partial<IEvent> = {
+    "type": "m.room.encrypted",
+    "room_id": "!room:id",
+    "sender": "@alice:localhost",
+    "content": {
+        "algorithm": "m.megolm.v1.aes-sha2",
+        "sender_key": "WimPd2udAU/1S/+YBpPbmr9L+0H5H+BnAVHSwDxlPGc",
+        "ciphertext": "AwgAEnAkBmciEAyhh1j6DCk29UXJ7kv/kvayUNfuNT0iAioLxcXjFXOZ5ho3jF1/wrytlt0Lb298uMM67OxdVMi+/mMfYpwlvi07P9cIH6CMSj8tyhYoWl0SrKY6tkPf5GWOlRSRRKbziXa96FHXvnA3V2FCAIGtAe3G4ei5RPbhkmKAFBLAen33/D6MjJVqU8Ojr5vTkgls5eyirarlVpsmnH06alDaxO8avrU0NL+Vsw26xvlUQgEMOnUJ",
+        "session_id": "ipdI6Zs/7DzFTEhiA2iGaMDfHkIYCleqXT6L+5e1/co",
+        "device_id": "TEST_DEVICE"
+    },
+    "event_id": "$event1",
+    "origin_server_ts": 1507753886000
+};
+
 // Bob data
 
 export const BOB_TEST_USER_ID = "@bob:xyz";
@@ -267,9 +322,28 @@ export const BOB_SIGNED_CROSS_SIGNING_KEYS_DATA: Partial<IDownloadKeyResult> = {
     }
 };
 
+/** Signed OTKs, returned by `POST /keys/claim` */
+export const BOB_ONE_TIME_KEYS = {
+    "@bob:xyz": {
+        "bob_device": {
+            "signed_curve25519:AAAAHQ": {
+                "key": "j3fR3HemM16M7CWhoI4Sk5ZsdmdfQHsKL1xuSft6MSw",
+                "signatures": {
+                    "@bob:xyz": {
+                        "ed25519:bob_device": "dlZc9VA/hP980Mxvu9qwi0qJx8VK7sADGOM48CE01YM7K/Mbty9lis/QjtQAWqDg371QyynVRjEzt9qj7eSFCg"
+                    }
+                }
+            }
+        }
+    }
+};
+
 /** base64-encoded backup decryption (private) key */
 export const BOB_BACKUP_DECRYPTION_KEY_BASE64 = "DwdtCnMYpX08FsFyUbJmRd9ML4frwJkqsXf7pR25LCo=";
 
+/** Backup decryption key in export format */
+export const BOB_BACKUP_DECRYPTION_KEY_BASE58 = "EsT5 Sd5m mEXs NQYE ibRe 3q9E 4aXW rHih 5f9J 6rU6 AfwY mASR";
+
 /** Signed backup data, suitable for return from `GET /_matrix/client/v3/room_keys/keys/{roomId}/{sessionId}` */
 export const BOB_SIGNED_BACKUP_DATA: KeyBackupInfo = {
     "algorithm": "m.megolm_backup.v1.curve25519-aes-sha2",
@@ -288,23 +362,23 @@ export const BOB_SIGNED_BACKUP_DATA: KeyBackupInfo = {
 export const BOB_MEGOLM_SESSION_DATA_ARRAY: IMegolmSessionData[] = [
     {
         "algorithm": "m.megolm.v1.aes-sha2",
-        "room_id": "!roomA:example.org",
-        "sender_key": "/Bu9e34hUClhddpf4E5gu5qEAdMY31+1A9HbiAeeQgo",
-        "session_id": "X9SK5JceUdvUr9gZkmdfS78qmKztoL60oORJ9JA5XD8",
-        "session_key": "AQAAAADbfOdUj/ec5bK4xVEhhRw+jfd1FD4uA0MLy7NyVHugZxyXNZUx5YEof4H9YyjBretviZreMSXqflbgYKz257rkKu7MPeKFf7zmln2GxX0F/p++GOnvpY1FqOglhfRQi3tqiyOa7SL4f7TuERDTOpMqlWhIfTKQnqy0AyF2vpDi5V/UiuSXHlHb1K/YGZJnX0u/Kpis7aC+tKDkSfSQOVw/",
+        "room_id": "!room:id",
+        "sender_key": "FOvlmz18LLI3k/llCpqRoKT90+gFF8YhuL+v1YBXHlw",
+        "session_id": "/2K+V777vipCxPZ0gpY9qcpz1DYaXwuMRIu0UEP0Wa0",
+        "session_key": "AQAAAAAclzWVMeWBKH+B/WMowa3rb4ma3jEl6n5W4GCs9ue65CruzD3ihX+85pZ9hsV9Bf6fvhjp76WNRajoJYX0UIt7aosjmu0i+H+07hEQ0zqTKpVoSH0ykJ6stAMhdr6Q4uW5crBmdTTBIsqmoWsNJZKKoE2+ldYrZ1lrFeaJbjBIY/9ivle++74qQsT2dIKWPanKc9Q2Gl8LjESLtFBD9Fmt",
         "sender_claimed_keys": {
-            "ed25519": "ZG6lrfATe+958wN1xaGf3dKG/CThEfkmNdp1jcu4zok"
+            "ed25519": "F4P7f1Z0RjbiZMgHk1xBCG3KC4/Ng9PmxLJ4hQ13sHA"
         },
         "forwarding_curve25519_key_chain": []
     },
     {
         "algorithm": "m.megolm.v1.aes-sha2",
-        "room_id": "!roomA:example.org",
-        "sender_key": "/Bu9e34hUClhddpf4E5gu5qEAdMY31+1A9HbiAeeQgo",
-        "session_id": "F4P7f1Z0RjbiZMgHk1xBCG3KC4/Ng9PmxLJ4hQ13sHA",
-        "session_key": "AQAAAACv0khqPrQ91MmWCgm0RTzfpn65AGCrRnAKLxGJdfSfECNZ8gyj34FZLwi+F+xC6ibFddcbLXW0mzR6PnTnHF3VHM4g/h+2rcxtlix8fySpIwFzaXViba7cOSy/b+dHTMZB40iA7F4y7AdTdHLv4N1XUj3puU/KVUIKf9/lEDLqyReD+39WdEY24mTIB5NcQQhtyguPzYPT5sSyeIUNd7Bw",
+        "room_id": "!room:id",
+        "sender_key": "FOvlmz18LLI3k/llCpqRoKT90+gFF8YhuL+v1YBXHlw",
+        "session_id": "+07YOpSgdZ1X9le3n3NMByw0V1B0H0Djnbm76jgmWoo",
+        "session_key": "AQAAAAAjWfIMo9+BWS8IvhfsQuomxXXXGy11tJs0ej505xxd1RzOIP4ftq3MbZYsfH8kqSMBc2l1Ym2u3Dksv2/nR0zGQeNIgOxeMuwHU3Ry7+DdV1I96blPylVCCn/f5RAy6smKoaeylptPdXgVXmw3YBBUVYpHpm+xCIUUp9foAdb8hftO2DqUoHWdV/ZXt59zTAcsNFdQdB9A4525u+o4JlqK",
         "sender_claimed_keys": {
-            "ed25519": "HxUKnGfeUu0fF3cLyCFSDXYtVCQHy/+33q9RkzKfsiU"
+            "ed25519": "OsZMdC1gQ5nPr+L9tuT6xXsaFJkVPkgxP2FexHF1/QM"
         },
         "forwarding_curve25519_key_chain": []
     }
@@ -313,29 +387,65 @@ export const BOB_MEGOLM_SESSION_DATA_ARRAY: IMegolmSessionData[] = [
 /** An exported megolm session */
 export const BOB_MEGOLM_SESSION_DATA: IMegolmSessionData = {
     "algorithm": "m.megolm.v1.aes-sha2",
-    "room_id": "!roomA:example.org",
-    "sender_key": "/Bu9e34hUClhddpf4E5gu5qEAdMY31+1A9HbiAeeQgo",
-    "session_id": "OsZMdC1gQ5nPr+L9tuT6xXsaFJkVPkgxP2FexHF1/QM",
-    "session_key": "AQAAAACvcoGk7mOY59fOqZaxFUiTCBRV1Ia94KBjAZx6kgdgBtkkvs50z8od8/Nc9ncK2UsEiXNvCTTp2dlN3du+Rx0/m7vet2ZOEEp2oYDjHMLLFmwd1gtlGuWYPdXA6Y1+9Yyph0/EDVfS+zd3XvbL0QgbyL43+yQnFNHKlxVJX1eiKTrGTHQtYEOZz6/i/bbk+sV7GhSZFT5IMT9hXsRxdf0D",
+    "room_id": "!room:id",
+    "sender_key": "FOvlmz18LLI3k/llCpqRoKT90+gFF8YhuL+v1YBXHlw",
+    "session_id": "gywydBrIJcJWktC/ic3tunKZM1XZm1MpYiYtdbj8Rpc",
+    "session_key": "AQAAAADZJL7OdM/KHfPzXPZ3CtlLBIlzbwk06dnZTd3bvkcdP5u73rdmThBKdqGA4xzCyxZsHdYLZRrlmD3VwOmNfvWMqYdPxA1X0vs3d172y9EIG8i+N/skJxTRypcVSV9XoinBNIWr/gkyepuAKiQqemlc8J5amD9OkmbVkmnrxP1uyYMsMnQayCXCVpLQv4nN7bpymTNV2ZtTKWImLXW4/EaX",
     "sender_claimed_keys": {
-        "ed25519": "dV0TIhhkToXpL+gZLo+zXDHJfw7MWYxpg80cynIQDv0"
+        "ed25519": "zBdpQwWYyz1MkZuEUhXqcdMfUNN/B9psLFDDDTJOg64"
     },
     "forwarding_curve25519_key_chain": []
 };
 
-/** Signed OTKs, returned by `POST /keys/claim` */
-export const BOB_ONE_TIME_KEYS = {
-    "@bob:xyz": {
-        "bob_device": {
-            "signed_curve25519:AAAAHQ": {
-                "key": "j3fR3HemM16M7CWhoI4Sk5ZsdmdfQHsKL1xuSft6MSw",
-                "signatures": {
-                    "@bob:xyz": {
-                        "ed25519:bob_device": "dlZc9VA/hP980Mxvu9qwi0qJx8VK7sADGOM48CE01YM7K/Mbty9lis/QjtQAWqDg371QyynVRjEzt9qj7eSFCg"
-                    }
-                }
-            }
-        }
+/** A ratcheted version of BOB_MEGOLM_SESSION_DATA */
+export const BOB_RATCHTED_MEGOLM_SESSION_DATA: IMegolmSessionData = {
+    "algorithm": "m.megolm.v1.aes-sha2",
+    "room_id": "!room:id",
+    "sender_key": "FOvlmz18LLI3k/llCpqRoKT90+gFF8YhuL+v1YBXHlw",
+    "session_id": "gywydBrIJcJWktC/ic3tunKZM1XZm1MpYiYtdbj8Rpc",
+    "session_key": "AQAAAAHZJL7OdM/KHfPzXPZ3CtlLBIlzbwk06dnZTd3bvkcdP5u73rdmThBKdqGA4xzCyxZsHdYLZRrlmD3VwOmNfvWMqYdPxA1X0vs3d172y9EIG8i+N/skJxTRypcVSV9Xoil2JdGx9oPqR0dFVh661Aqs86rJRbQ4IeRiuEm35VMxboMsMnQayCXCVpLQv4nN7bpymTNV2ZtTKWImLXW4/EaX",
+    "sender_claimed_keys": {
+        "ed25519": "zBdpQwWYyz1MkZuEUhXqcdMfUNN/B9psLFDDDTJOg64"
+    },
+    "forwarding_curve25519_key_chain": []
+};
+
+/** The key from BOB_MEGOLM_SESSION_DATA, encrypted for backup using `m.megolm_backup.v1.curve25519-aes-sha2` algorithm*/
+export const BOB_CURVE25519_KEY_BACKUP_DATA: KeyBackupSession = {
+    "first_message_index": 1,
+    "forwarded_count": 0,
+    "is_verified": false,
+    "session_data": {
+        "ciphertext": "d7UVOK17WEVky/8hK0h3HsTQrFMEbKbfqMcl2KtyTWcI9S5gGFWK9Git5BzVRxRggvxQ0c8PDfqL+dr3zHytAMW+71BJqIPQW910vV7SX3IcGylnoUcS3doVkJZiprXytXMP89AKcgv5Dj7mS2ZdvNGE+Atro74bzZ5yot5BrE0ZE5SjoUBPLaLMMu9HopLIV+qx01Rc3F0wmkocSPo51N0nv6wvO5Cst0FiOGHDK6r1pFlgDEJLmBkOyC4e8oMVbKTJzsSQVbJ8tJ37xuhI+T5P0ZlmiqKDqYRp8uh50w+txLEixYhEUunFgCTt1DAmiS9pLNYhLyl1ggwuQjzZe+AV6timbRxNJy18/AEcPomJw7z/pxYIiNLHRKOC13Wp8kGWx9cOgfMQ5KmBuLS8psGiLTBkfWPLOfNYqjbeqAR+OGZQoS6hUjbBYU7QuFa4FOYBHkNB2UqNsdsMb9qB/qs7QGTSb8Lok5YjW1c81BUpmIyKvuqnKma0MZskrpTYGQD2eJDABFCZwLFm+LgDyUTeSiV5xguYztLrHOk8LHKo9M8dIZgoBjeFVJxyjbcXKsVS3aQkMXKCrRlKLqhZTws/ZJwVfW9DbktZ9dT+tRZQvI7tjJofojcLX61AGJDnqUf5+2Gv1tEnmUI953gIzc8NlcFabPOsDsZEODt7MdOCTPT3w29umyhKbCsslpb64LoS/AB2QRPRCgkJS7snRA",
+        "ephemeral": "oO0VX84OUIzm2i/12zAhTWOZT5IFRH5mXaKZ8fXkCgU",
+        "mac": "lEfHlqfJQwU"
     }
 };
 
+/** A test clear event */
+export const BOB_CLEAR_EVENT: Partial<IEvent> = {
+    "type": "m.room.message",
+    "room_id": "!room:id",
+    "sender": "@alice:localhost",
+    "content": {
+        "msgtype": "m.text",
+        "body": "Hello world"
+    }
+};
+
+/** The encrypted CLEAR_EVENT by MEGOLM_SESSION_DATA */
+export const BOB_ENCRYPTED_EVENT: Partial<IEvent> = {
+    "type": "m.room.encrypted",
+    "room_id": "!room:id",
+    "sender": "@alice:localhost",
+    "content": {
+        "algorithm": "m.megolm.v1.aes-sha2",
+        "sender_key": "FOvlmz18LLI3k/llCpqRoKT90+gFF8YhuL+v1YBXHlw",
+        "ciphertext": "AwgAEnA/mEqZm2lSrhoG11OpDqsohGSBJWsudbuoItLlivmpFZQHrKMbE6z/dhCTwUi76vwfRCtf4tyPMD845cqZH1nL0bowq3/awyzZ8Q263Y3WrLfkUTFBU6oPF/IULUFZZuw6kLdfd5g5+uigvqUhFFpICoj7KNHznv4sFNssd00/WgJquZ6PRt6e1v6ANFNiZPAwghIL+kBc6pb8i6MUWt9JnXilJhTqFDHdXiY4qkaKBWbwebC26PYM",
+        "session_id": "gywydBrIJcJWktC/ic3tunKZM1XZm1MpYiYtdbj8Rpc",
+        "device_id": "TEST_DEVICE"
+    },
+    "event_id": "$event1",
+    "origin_server_ts": 1507753886000
+};
+

spec/test-utils/test-utils.ts

@@ -19,6 +19,7 @@ import {
 import { SyncState } from "../../src/sync";
 import { eventMapperFor } from "../../src/event-mapper";
 import { TEST_ROOM_ID } from "./test-data";
+import { KnownMembership, Membership } from "../../src/@types/membership";
 
 /**
  * Return a promise that is resolved when the client next emits a
@@ -87,7 +88,7 @@ export function getSyncResponse(roomMembers: string[], roomId = TEST_ROOM_ID): I
     for (let i = 0; i < roomMembers.length; i++) {
         roomResponse.state.events.push(
             mkMembershipCustom({
-                membership: "join",
+                membership: KnownMembership.Join,
                 sender: roomMembers[i],
             }),
         );
@@ -99,6 +100,7 @@ export function getSyncResponse(roomMembers: string[], roomId = TEST_ROOM_ID): I
             join: { [roomId]: roomResponse },
             invite: {},
             leave: {},
+            knock: {},
         },
         account_data: { events: [] },
     };
@@ -250,7 +252,7 @@ export function mkPresence(opts: IPresenceOpts & { event?: boolean }): Partial<I
 
 interface IMembershipOpts {
     room?: string;
-    mship: string;
+    mship: Membership;
     sender?: string;
     user?: string;
     skey?: string;
@@ -296,7 +298,7 @@ export function mkMembership(opts: IMembershipOpts & { event?: boolean }): Parti
 }
 
 export function mkMembershipCustom<T>(
-    base: T & { membership: string; sender: string; content?: IContent },
+    base: T & { membership: Membership; sender: string; content?: IContent },
 ): T & { type: EventType; sender: string; state_key: string; content: IContent } & GeneratedMetadata {
     const content = base.content || {};
     return mkEventCustom({
@@ -314,6 +316,7 @@ export interface IMessageOpts {
     event?: boolean;
     relatesTo?: IEventRelation;
     ts?: number;
+    unsigned?: IUnsigned;
 }
 
 /**
@@ -520,15 +523,22 @@ export async function awaitDecryption(
     }
 
     return new Promise((resolve) => {
-        event.once(MatrixEventEvent.Decrypted, (ev, err) => {
-            logger.log(`${Date.now()}: MatrixEventEvent.Decrypted for event ${event.getId()}: ${err ?? "success"}`);
-            resolve(ev);
-        });
+        if (waitOnDecryptionFailure) {
+            event.on(MatrixEventEvent.Decrypted, (ev, err) => {
+                logger.log(`${Date.now()}: MatrixEventEvent.Decrypted for event ${event.getId()}: ${err ?? "success"}`);
+                if (!err) {
+                    resolve(ev);
+                }
+            });
+        } else {
+            event.once(MatrixEventEvent.Decrypted, (ev, err) => {
+                logger.log(`${Date.now()}: MatrixEventEvent.Decrypted for event ${event.getId()}: ${err ?? "success"}`);
+                resolve(ev);
+            });
+        }
     });
 }
 
-export const emitPromise = (e: EventEmitter, k: string): Promise<any> => new Promise((r) => e.once(k, r));
-
 export const mkPusher = (extra: Partial<IPusher> = {}): IPusher => ({
     app_display_name: "app",
     app_id: "123",
@@ -551,3 +561,25 @@ CRYPTO_BACKENDS["rust-sdk"] = (client: MatrixClient) => client.initRustCrypto();
 if (global.Olm) {
     CRYPTO_BACKENDS["libolm"] = (client: MatrixClient) => client.initCrypto();
 }
+
+export const emitPromise = (e: EventEmitter, k: string): Promise<any> => new Promise((r) => e.once(k, r));
+
+/**
+ * Advance the fake timers in a loop until the given promise resolves or rejects.
+ *
+ * Returns the result of the promise.
+ *
+ * This can be useful when there are multiple steps in the code which require an iteration of the event loop.
+ */
+export async function advanceTimersUntil<T>(promise: Promise<T>): Promise<T> {
+    let resolved = false;
+    promise.finally(() => {
+        resolved = true;
+    });
+
+    while (!resolved) {
+        await jest.advanceTimersByTimeAsync(1);
+    }
+
+    return await promise;
+}

spec/test-utils/test_indexeddb_cryptostore_dump/README.md

@@ -0,0 +1,55 @@
+## Dumps of libolm indexeddb cryptostore
+
+This directory contains several dumps of real indexeddb stores from a session using
+libolm crypto.
+
+Each directory contains, in dump.json, a dump of data created by pasting the following
+code into the browser console; and in index.ts, details of the user, pickle key,
+and corresponding key query and backup responses (`DumpDataSetInfo`).
+
+The dump is created by pasting the following into the browser console:
+
+```javascript
+async function exportIndexedDb(name) {
+    const db = await new Promise((resolve, reject) => {
+        const dbReq = indexedDB.open(name);
+        dbReq.onerror = reject;
+        dbReq.onsuccess = () => resolve(dbReq.result);
+    });
+
+    const storeNames = db.objectStoreNames;
+    const exports = {};
+    for (const store of storeNames) {
+        exports[store] = [];
+        const txn = db.transaction(store, "readonly");
+        const objectStore = txn.objectStore(store);
+        await new Promise((resolve, reject) => {
+            const cursorReq = objectStore.openCursor();
+            cursorReq.onerror = reject;
+            cursorReq.onsuccess = (event) => {
+                const cursor = event.target.result;
+                if (cursor) {
+                    const entry = { value: cursor.value };
+                    if (!objectStore.keyPath) {
+                        entry.key = cursor.key;
+                    }
+                    exports[store].push(entry);
+                    cursor.continue();
+                } else {
+                    resolve();
+                }
+            };
+        });
+    }
+    return exports;
+}
+
+window.saveAs(
+    new Blob([JSON.stringify(await exportIndexedDb("matrix-js-sdk:crypto"), null, 2)], {
+        type: "application/json;charset=utf-8",
+    }),
+    "dump.json",
+);
+```
+
+The pickle key is extracted via `mxMatrixClientPeg.get().crypto.olmDevice.pickleKey`.

spec/test-utils/test_indexeddb_cryptostore_dump/full_account/README.md

@@ -0,0 +1,4 @@
+## Dump of a libolm indexeddb cryptostore to test migration of a full account
+
+A dump of an account containing a complete set of data to migrate.
+The data set is substantial enough to allow for testing of chunking mechanisms and progress reporting during the migration process.

spec/test-utils/test_indexeddb_cryptostore_dump/full_account/index.ts

@@ -0,0 +1,109 @@
+import { DumpDataSetInfo } from "../index";
+
+/**
+ * A key query response containing the current keys of the tested user.
+ * To be used during tests with fetchmock.
+ */
+const KEYS_QUERY_RESPONSE: any = {
+    device_keys: {
+        "@vdhtest200713:matrix.org": {
+            KMFSTJSMLB: {
+                algorithms: ["m.olm.v1.curve25519-aes-sha2", "m.megolm.v1.aes-sha2"],
+                device_id: "KMFSTJSMLB",
+                keys: {
+                    "curve25519:KMFSTJSMLB": "LKv0bKbc0EC4h0jknbemv3QalEkeYvuNeUXVRgVVTTU",
+                    "ed25519:KMFSTJSMLB": "qK70DEqIXq7T+UU3v/al47Ab4JkMEBLpNrTBMbS5rrw",
+                },
+                user_id: "@vdhtest200713:matrix.org",
+                signatures: {
+                    "@vdhtest200713:matrix.org": {
+                        "ed25519:KMFSTJSMLB":
+                            "aE+PdxLAdwQ/xfJwLmqebvt/lrT97fZas2SQFFrM+dPmHxQtjyS8csm88BLfGRjJKK1B/vWev3AaKqQZwLTUAw",
+                        "ed25519:lDvg6vi3P80L9XFNpUSU+5Y87m3p6yHcC83jhSU4Q5k":
+                            "lCd4SA/JT1nnxsgN9yQaLJQhH5hkLMVVx6ba5JAjL1wpWVqyPxzMJHImX6vTztk6S8rybcdfYkea5W/Ii+4HCQ",
+                    },
+                },
+            },
+        },
+    },
+    master_keys: {
+        "@vdhtest200713:matrix.org": {
+            user_id: "@vdhtest200713:matrix.org",
+            usage: ["master"],
+            keys: {
+                "ed25519:gh9fGr39eNZUdWynEMJ/q/WZq/Pk/foFxHXFBFm18ZI": "gh9fGr39eNZUdWynEMJ/q/WZq/Pk/foFxHXFBFm18ZI",
+            },
+            signatures: {
+                "@vdhtest200713:matrix.org": {
+                    "ed25519:MWOGVUTXZN":
+                        "stOu1aHbhsWB/Aj5M/HqBR83QzME+682C995Uc8JxSmmyrlWmgG8QrnoUDG2OFR1t6zNQ+QLEilU4WNEOV73DQ",
+                },
+            },
+        },
+    },
+    self_signing_keys: {
+        "@vdhtest200713:matrix.org": {
+            user_id: "@vdhtest200713:matrix.org",
+            usage: ["self_signing"],
+            keys: {
+                "ed25519:lDvg6vi3P80L9XFNpUSU+5Y87m3p6yHcC83jhSU4Q5k": "lDvg6vi3P80L9XFNpUSU+5Y87m3p6yHcC83jhSU4Q5k",
+            },
+            signatures: {
+                "@vdhtest200713:matrix.org": {
+                    "ed25519:gh9fGr39eNZUdWynEMJ/q/WZq/Pk/foFxHXFBFm18ZI":
+                        "HKTC7NoBhAkfJtmemmkn/HvCCgBQViWZ0uH7aGPRaWMDFgD8T7Q+y1j3FKZv4mhSopR85Fq3FRyXsG8OVvGeBA",
+                },
+            },
+        },
+    },
+    user_signing_keys: {
+        "@vdhtest200713:matrix.org": {
+            user_id: "@vdhtest200713:matrix.org",
+            usage: ["user_signing"],
+            keys: {
+                "ed25519:YShqO/3u5vQ0uucojraWrtoLrek0CYrurN/vH/YPMg8": "YShqO/3u5vQ0uucojraWrtoLrek0CYrurN/vH/YPMg8",
+            },
+            signatures: {
+                "@vdhtest200713:matrix.org": {
+                    "ed25519:gh9fGr39eNZUdWynEMJ/q/WZq/Pk/foFxHXFBFm18ZI":
+                        "u8VOi4IaeRJwDgy2ftK02NJQPdBijy8f/0+WnHGG72yfOvMthwWzEw8SrRSNG8glBNrfHinKwCyJJzAJwyepCQ",
+                },
+            },
+        },
+    },
+};
+
+/**
+ * A `/room_keys/version` response containing the current server-side backup info.
+ * To be used during tests with fetchmock.
+ */
+const BACKUP_RESPONSE: any = {
+    auth_data: {
+        public_key: "q+HZiJdHl2Yopv9GGvv7EYSzDMrAiRknK4glSdoaomI",
+        signatures: {
+            "@vdhtest200713:matrix.org": {
+                "ed25519:gh9fGr39eNZUdWynEMJ/q/WZq/Pk/foFxHXFBFm18ZI":
+                    "reDp6Mu+j+tfUL3/T6f5OBT3N825Lzpc43vvG+RvjX6V+KxXzodBQArgCoeEHLtL9OgSBmNrhTkSOX87MWCKAw",
+                "ed25519:KMFSTJSMLB":
+                    "F8tyV5W6wNi0GXTdSg+gxSCULQi0EYxdAAqfkyNq58KzssZMw5i+PRA0aI2b+D7NH/aZaJrtiYNHJ0gWLSQvAw",
+            },
+        },
+    },
+    version: "7",
+    algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
+    etag: "1",
+    count: 79,
+};
+
+/**
+ * A dataset containing the information for the tested user.
+ * To be used during tests.
+ */
+export const FULL_ACCOUNT_DATASET: DumpDataSetInfo = {
+    userId: "@vdhtest200713:matrix.org",
+    deviceId: "KMFSTJSMLB",
+    pickleKey: "+1k2Ppd7HIisUY824v7JtV3/oEE4yX0TqtmNPyhaD7o",
+    backupResponse: BACKUP_RESPONSE,
+    keyQueryResponse: KEYS_QUERY_RESPONSE,
+    dumpPath: "spec/test-utils/test_indexeddb_cryptostore_dump/full_account/dump.json",
+};

spec/test-utils/test_indexeddb_cryptostore_dump/index.ts

@@ -0,0 +1,154 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { readFile } from "node:fs/promises";
+import { resolve } from "node:path";
+
+/**
+ * Populate an IndexedDB store with a set of test data.
+ *
+ * @param name - Name of the IndexedDB database to create.
+ * @param dumpPath - The path to the dump file to import.
+ */
+export async function populateStore(name: string, dumpPath: string): Promise<IDBDatabase> {
+    const req = indexedDB.open(name, 11);
+
+    const db = await new Promise<IDBDatabase>((resolve, reject) => {
+        req.onupgradeneeded = (ev): void => {
+            const db = req.result;
+            const oldVersion = ev.oldVersion;
+            upgradeDatabase(oldVersion, db);
+        };
+
+        req.onerror = (ev): void => {
+            reject(req.error);
+        };
+
+        req.onsuccess = (): void => {
+            const db = req.result;
+            resolve(db);
+        };
+    });
+
+    await importData(db, dumpPath);
+
+    return db;
+}
+
+/** Create the schema for the indexed db store */
+function upgradeDatabase(oldVersion: number, db: IDBDatabase) {
+    if (oldVersion < 1) {
+        const outgoingRoomKeyRequestsStore = db.createObjectStore("outgoingRoomKeyRequests", { keyPath: "requestId" });
+        outgoingRoomKeyRequestsStore.createIndex("session", ["requestBody.room_id", "requestBody.session_id"]);
+        outgoingRoomKeyRequestsStore.createIndex("state", "state");
+    }
+
+    if (oldVersion < 2) {
+        db.createObjectStore("account");
+    }
+
+    if (oldVersion < 3) {
+        const sessionsStore = db.createObjectStore("sessions", { keyPath: ["deviceKey", "sessionId"] });
+        sessionsStore.createIndex("deviceKey", "deviceKey");
+    }
+
+    if (oldVersion < 4) {
+        db.createObjectStore("inbound_group_sessions", { keyPath: ["senderCurve25519Key", "sessionId"] });
+    }
+
+    if (oldVersion < 5) {
+        db.createObjectStore("device_data");
+    }
+
+    if (oldVersion < 6) {
+        db.createObjectStore("rooms");
+    }
+
+    if (oldVersion < 7) {
+        db.createObjectStore("sessions_needing_backup", { keyPath: ["senderCurve25519Key", "sessionId"] });
+    }
+
+    if (oldVersion < 8) {
+        db.createObjectStore("inbound_group_sessions_withheld", { keyPath: ["senderCurve25519Key", "sessionId"] });
+    }
+
+    if (oldVersion < 9) {
+        const problemsStore = db.createObjectStore("session_problems", { keyPath: ["deviceKey", "time"] });
+        problemsStore.createIndex("deviceKey", "deviceKey");
+
+        db.createObjectStore("notified_error_devices", { keyPath: ["userId", "deviceId"] });
+    }
+
+    if (oldVersion < 10) {
+        db.createObjectStore("shared_history_inbound_group_sessions", { keyPath: ["roomId"] });
+    }
+
+    if (oldVersion < 11) {
+        db.createObjectStore("parked_shared_history", { keyPath: ["roomId"] });
+    }
+}
+
+async function importData(db: IDBDatabase, dumpPath: string) {
+    const path = resolve(dumpPath);
+    const json: Record<string, Array<{ key?: any; value: any }>> = JSON.parse(
+        await readFile(path, { encoding: "utf8" }),
+    );
+
+    for (const [storeName, data] of Object.entries(json)) {
+        await new Promise((resolve, reject) => {
+            const store = db.transaction(storeName, "readwrite").objectStore(storeName);
+
+            function putEntry(idx: number) {
+                if (idx >= data.length) {
+                    resolve(undefined);
+                    return;
+                }
+
+                const { key, value } = data[idx];
+                try {
+                    const putReq = store.put(value, key);
+                    putReq.onsuccess = (_) => putEntry(idx + 1);
+                    putReq.onerror = (_) => reject(putReq.error);
+                } catch (e) {
+                    throw new Error(
+                        `Error populating '${storeName}' with key ${JSON.stringify(key)}, value ${JSON.stringify(
+                            value,
+                        )}: ${e}`,
+                    );
+                }
+            }
+
+            putEntry(0);
+        });
+    }
+}
+
+export interface DumpDataSetInfo {
+    /** The user ID to use for the test.*/
+    userId: string;
+    /** The device ID to use for the test.*/
+    deviceId: string;
+    /** The path to the dump file to import via {@link populateStore}.*/
+    dumpPath: string;
+    /** The pickle key to use for the dumped account.*/
+    pickleKey: string;
+    /** The response to use for the keys query. */
+    keyQueryResponse: any;
+    /** The response to use for the backup query.*/
+    backupResponse?: any;
+    /** Additional dump info specific for some tests.*/
+    [key: string]: any;
+}

spec/test-utils/test_indexeddb_cryptostore_dump/no_cached_msk_dump/README.md

@@ -0,0 +1,4 @@
+## Dump of a libolm indexeddb cryptostore where the msk is not cached
+
+A dump simulating an account where the identity was verified, but the msk was not in cache.
+Used to test that the owner identity local trust is migrated correctly.

spec/test-utils/test_indexeddb_cryptostore_dump/no_cached_msk_dump/index.ts

@@ -0,0 +1,283 @@
+import { KeyBackupInfo } from "../../../../src/crypto-api/keybackup";
+import { DumpDataSetInfo } from "../index";
+
+/**
+ * A key query response containing the current keys of the tested user.
+ * To be used during tests with fetchmock.
+ */
+const KEY_QUERY_RESPONSE: any = {
+    device_keys: {
+        "@migration:localhost": {
+            CBGTADUILV: {
+                algorithms: ["m.olm.v1.curve25519-aes-sha2", "m.megolm.v1.aes-sha2"],
+                device_id: "CBGTADUILV",
+                keys: {
+                    "curve25519:CBGTADUILV": "gqhFlc7Wzc1wmmmAu3ySIEe4LtDcBK/bdzrtZg+mMSg",
+                    "ed25519:CBGTADUILV": "q1q3L1Il4l61c/6TmI4fYWMsseNMJJYE2Y0r+5ajKQI",
+                },
+                signatures: {
+                    "@migration:localhost": {
+                        "ed25519:CBGTADUILV":
+                            "ppSmA0slyQ7RJOFn+qZSLCGeHN6/jAmqKvUZo5Q1hWk0ugkKycRoSUi9TOfbfAVSf8xvFirXy2VGXQbEVPJqAA",
+                        "ed25519:d+4HhsodR2Zqv4Z5V0VxPfy8zbjLjUCdCyv5qme5Ygc":
+                            "cFLWl1fjehLrzrEn3UnmZMIgy3C23WMgGRsn4e6Z/55vmen4KMs8bLpgZaDoWhIdn/8siHRWafA5sFdzK2NsBQ",
+                        "ed25519:bmFmNcVPvaqrlNzmyKn9uU+QRHyx2QRbn/bUAlTH760":
+                            "C6EeqNPcaQyuZgo8+HOUywc/TMkW5IMjg7aoxyu93X//KcNNXKRfj1banYP6XqyPuQITLamBYc1089Jpt9g4Cw",
+                        "ed25519:RkQzi0+aKIL9Y+GzsN23xMz3i3QRkH03G5aqqEbbuy4":
+                            "YwBN/SbCxO8hPgv1B9JY2WVFK4LNK9vq1UNVrkF2j0ZDw9LrvaOws72mbmzZ0nbD3ohcEZ8rXsEosxEVr5r7AQ",
+                    },
+                },
+                user_id: "@migration:localhost",
+                unsigned: {
+                    device_display_name: "localhost:8080: Chrome on macOS",
+                },
+            },
+            TMWBMDZPFT: {
+                algorithms: ["m.olm.v1.curve25519-aes-sha2", "m.megolm.v1.aes-sha2"],
+                device_id: "TMWBMDZPFT",
+                keys: {
+                    "curve25519:TMWBMDZPFT": "oYP9EXvHMbliFdfk8jPvUw0KhAd0+PBqdMslJAt/ZGQ",
+                    "ed25519:TMWBMDZPFT": "IyfPT67JutFWJsUxrxSqEWxgRjKn9B/w78uKU4OBj1E",
+                },
+                signatures: {
+                    "@migration:localhost": {
+                        "ed25519:TMWBMDZPFT":
+                            "IWIuuDag4ZMDhMObYV63X7dBYEUYNHYR0Yu/bwLvQh5ieDjQSrZSLOzDrgCyPCM4hkc4JlhneQpJsYo1lUH7DA",
+                        "ed25519:d+4HhsodR2Zqv4Z5V0VxPfy8zbjLjUCdCyv5qme5Ygc":
+                            "iEcTKElQu4CAsQIXmBaZmXwfB6Diut+4ZXakP1ob7OIDMrCYBcgXsBFYg6GuxwL0LCTVcUgbUw7VuPKSvM8UAA",
+                        "ed25519:MYgcP5P7P6KucWjLvTRofY5PWxsf+WDj2BiXtqOO5Gw":
+                            "KcBLDWkCwZyIzlBkC29PNzHxx7Br14TYlhBfREEEQo/Rd34ZZUYwbQ8iPhB8S1GVq3YwgAV6piYIcxpQin+dBA",
+                        "ed25519:HGN9m99VprMuQBDA3o+KZKcEYTaGmiaujrkygjScMnY":
+                            "VqrvA148Uxib9TNFI1rc9r8qpwTojCkqLofEz9dMLc/XV3U14WD5/LDEhMuCwNu6wsu/uO+dS4AmJlJnN/iAAg",
+                        "ed25519:Nt0L/p+UVHMx603sYHXwXja+VyQIUVFvu0vDBYn56Zk":
+                            "D1COHzROOTNlCn8b1zI9+6phUtF0OVqWxLfOLnX5t14H2oENYV2ASgaxsdmXcSZPrGzaJkmSOginHHzsabe5CA",
+                        "ed25519:bmFmNcVPvaqrlNzmyKn9uU+QRHyx2QRbn/bUAlTH760":
+                            "SFSDrsi3GQ9jjBYUc2aUSzf777/0NfQWrOBi2CK+v5VQY3FkyHBln3K4YzvxIKSVIhOaQtBlEDtfQb33kwTgDg",
+                        "ed25519:RkQzi0+aKIL9Y+GzsN23xMz3i3QRkH03G5aqqEbbuy4":
+                            "BtJkzQe0YFAa8gJiYXYtzGtktl9vZMNYl5jd4DA8Toi4VxgosJNZQE7lT5qpYU0BrlFn46QIs/38X8JhSt+wAQ",
+                    },
+                },
+                user_id: "@migration:localhost",
+                unsigned: {
+                    device_display_name: "localhost:8080: Chrome on macOS",
+                },
+            },
+        },
+    },
+    failures: {},
+    master_keys: {
+        "@migration:localhost": {
+            keys: {
+                "ed25519:cFjUBAhAZ2tjYF1TpQtYNA3x9XRzTiIdP2N2EvRaOH4": "cFjUBAhAZ2tjYF1TpQtYNA3x9XRzTiIdP2N2EvRaOH4",
+            },
+            signatures: {
+                "@migration:localhost": {
+                    "ed25519:TMWBMDZPFT":
+                        "RrPUnYoekK7wZGrLNXshgoupF8v53S/vJyvkBJi+q9THh4Qrf3CieuVJFx8mwtmEZgGoA2tSroAVnRqvEQ+IBQ",
+                    "ed25519:cFjUBAhAZ2tjYF1TpQtYNA3x9XRzTiIdP2N2EvRaOH4":
+                        "o4CbtdU3IqJK90UXAEBtxps2m4XBYvWJI2nbVlzBaGRr+Xt/3vtwDMlc5G970kPQWBbs/koYJh8MSaE7Fm1mAg",
+                    "ed25519:CBGTADUILV":
+                        "AgZoG+ix8aW3FAW6v+/Xu+QJpxzvsx5itbB8RyqMet9YlNqX90vYIbBV7IoV2WFY2WdANYEffX2CE0FpR6NnCg",
+                },
+            },
+            usage: ["master"],
+            user_id: "@migration:localhost",
+        },
+    },
+    self_signing_keys: {
+        "@migration:localhost": {
+            keys: {
+                "ed25519:RkQzi0+aKIL9Y+GzsN23xMz3i3QRkH03G5aqqEbbuy4": "RkQzi0+aKIL9Y+GzsN23xMz3i3QRkH03G5aqqEbbuy4",
+            },
+            signatures: {
+                "@migration:localhost": {
+                    "ed25519:cFjUBAhAZ2tjYF1TpQtYNA3x9XRzTiIdP2N2EvRaOH4":
+                        "hs8VqoTfipDjC2pzFdmzb1aENhDjVV+gc86fuYftczaCcsXUWop/NPwoF51Ie6Nb3YL0N7ZZAUrycuJP5hFbDg",
+                },
+            },
+            usage: ["self_signing"],
+            user_id: "@migration:localhost",
+        },
+    },
+    user_signing_keys: {
+        "@migration:localhost": {
+            keys: {
+                "ed25519:WNJ2G3Ig5EdC4wYiRKcK7bhLP2+I4wI6V7SKgJTXdw8": "WNJ2G3Ig5EdC4wYiRKcK7bhLP2+I4wI6V7SKgJTXdw8",
+            },
+            signatures: {
+                "@migration:localhost": {
+                    "ed25519:cFjUBAhAZ2tjYF1TpQtYNA3x9XRzTiIdP2N2EvRaOH4":
+                        "Vlba5rJQxG+ussVLoycvHcin7Ghv0uUeClDqDbM+RPF+jx9w4ozbcuEOTJdyzyPA+GxN9Kzh2lmVFMMQGyvNAw",
+                },
+            },
+            usage: ["user_signing"],
+            user_id: "@migration:localhost",
+        },
+    },
+};
+
+/**
+ * A new  key query response for the same user simulating a cross-signing key reset.
+ * To be used during tests with fetchmock.
+ */
+const ROTATED_KEY_QUERY_RESPONSE: any = {
+    device_keys: {
+        "@migration:localhost": {
+            TMWBMDZPFT: {
+                algorithms: ["m.olm.v1.curve25519-aes-sha2", "m.megolm.v1.aes-sha2"],
+                device_id: "TMWBMDZPFT",
+                keys: {
+                    "curve25519:TMWBMDZPFT": "oYP9EXvHMbliFdfk8jPvUw0KhAd0+PBqdMslJAt/ZGQ",
+                    "ed25519:TMWBMDZPFT": "IyfPT67JutFWJsUxrxSqEWxgRjKn9B/w78uKU4OBj1E",
+                },
+                signatures: {
+                    "@migration:localhost": {
+                        "ed25519:TMWBMDZPFT":
+                            "IWIuuDag4ZMDhMObYV63X7dBYEUYNHYR0Yu/bwLvQh5ieDjQSrZSLOzDrgCyPCM4hkc4JlhneQpJsYo1lUH7DA",
+                        "ed25519:d+4HhsodR2Zqv4Z5V0VxPfy8zbjLjUCdCyv5qme5Ygc":
+                            "iEcTKElQu4CAsQIXmBaZmXwfB6Diut+4ZXakP1ob7OIDMrCYBcgXsBFYg6GuxwL0LCTVcUgbUw7VuPKSvM8UAA",
+                        "ed25519:MYgcP5P7P6KucWjLvTRofY5PWxsf+WDj2BiXtqOO5Gw":
+                            "KcBLDWkCwZyIzlBkC29PNzHxx7Br14TYlhBfREEEQo/Rd34ZZUYwbQ8iPhB8S1GVq3YwgAV6piYIcxpQin+dBA",
+                        "ed25519:HGN9m99VprMuQBDA3o+KZKcEYTaGmiaujrkygjScMnY":
+                            "VqrvA148Uxib9TNFI1rc9r8qpwTojCkqLofEz9dMLc/XV3U14WD5/LDEhMuCwNu6wsu/uO+dS4AmJlJnN/iAAg",
+                        "ed25519:Nt0L/p+UVHMx603sYHXwXja+VyQIUVFvu0vDBYn56Zk":
+                            "D1COHzROOTNlCn8b1zI9+6phUtF0OVqWxLfOLnX5t14H2oENYV2ASgaxsdmXcSZPrGzaJkmSOginHHzsabe5CA",
+                        "ed25519:bmFmNcVPvaqrlNzmyKn9uU+QRHyx2QRbn/bUAlTH760":
+                            "SFSDrsi3GQ9jjBYUc2aUSzf777/0NfQWrOBi2CK+v5VQY3FkyHBln3K4YzvxIKSVIhOaQtBlEDtfQb33kwTgDg",
+                        "ed25519:RkQzi0+aKIL9Y+GzsN23xMz3i3QRkH03G5aqqEbbuy4":
+                            "BtJkzQe0YFAa8gJiYXYtzGtktl9vZMNYl5jd4DA8Toi4VxgosJNZQE7lT5qpYU0BrlFn46QIs/38X8JhSt+wAQ",
+                    },
+                },
+                user_id: "@migration:localhost",
+                unsigned: {
+                    device_display_name: "localhost:8080: Chrome on macOS",
+                },
+            },
+            XFZFSCUOFL: {
+                algorithms: ["m.olm.v1.curve25519-aes-sha2", "m.megolm.v1.aes-sha2"],
+                device_id: "XFZFSCUOFL",
+                keys: {
+                    "curve25519:XFZFSCUOFL": "aN2Ty+0rutNkrRtxhV+ciI8GhF4epSxzL7bAOr8zfkc",
+                    "ed25519:XFZFSCUOFL": "V7CPhXdfLFk+qAOFivrpFskmunVTeuM+EOM3DMlDxkI",
+                },
+                signatures: {
+                    "@migration:localhost": {
+                        "ed25519:XFZFSCUOFL":
+                            "4Pqc2FWJ5p/L/tSlfUBIlcQzLmN5CksJriAibY8LSDAXdGYiQJ7hvKqneEuVhrMYwqyIxb4bAad+r6wnY0/7Cg",
+                        "ed25519:RkQzi0+aKIL9Y+GzsN23xMz3i3QRkH03G5aqqEbbuy4":
+                            "yH8pKnD+E8YaawS+1NCjwy0cf2WzBRff9BBNX4YnAuTyc6s5b1QqNfu9DP5qblw8TZ7hZmaziePZKsjRiqJLBg",
+                        "ed25519:OEv0wHLusJx7zTCc0h3HbNIHLIxlGZKh63tc2ptKb+Y":
+                            "M8SfAiEUzd7AsWp8InS7BxV3cRqV3MjMxks4DwSxsVxvkCco2JWybKgev+vTZyM6XDg930o0FObQOxWm4+CkBw",
+                    },
+                },
+                user_id: "@migration:localhost",
+                unsigned: {
+                    device_display_name: "localhost:8080: Chrome on macOS",
+                },
+            },
+        },
+    },
+    failures: {},
+    master_keys: {
+        "@migration:localhost": {
+            user_id: "@migration:localhost",
+            usage: ["master"],
+            keys: {
+                "ed25519:rXCrBin/+xyh+yW//vWte+2UV0et1ZHTWfalp/Ekack": "rXCrBin/+xyh+yW//vWte+2UV0et1ZHTWfalp/Ekack",
+            },
+            signatures: {
+                "@migration:localhost": {
+                    "ed25519:XFZFSCUOFL":
+                        "C8aswtyUABWvj2DInehVoh2P/EDbwRhlIk51LtV3L71POUCh7pZuyXRMMWKZeyRvHRmEllXBtRkH1iol/p56Bg",
+                },
+            },
+        },
+    },
+    self_signing_keys: {
+        "@migration:localhost": {
+            user_id: "@migration:localhost",
+            usage: ["self_signing"],
+            keys: {
+                "ed25519:OEv0wHLusJx7zTCc0h3HbNIHLIxlGZKh63tc2ptKb+Y": "OEv0wHLusJx7zTCc0h3HbNIHLIxlGZKh63tc2ptKb+Y",
+            },
+            signatures: {
+                "@migration:localhost": {
+                    "ed25519:rXCrBin/+xyh+yW//vWte+2UV0et1ZHTWfalp/Ekack":
+                        "dH596pGp8+f8dlwd81UrKDWoRDd24yAqqMSLqR4fJHyfszbn7qCvQA6LYZ023TLmk33FKcJqRtd2v/ykTmS3Bg",
+                },
+            },
+        },
+    },
+    user_signing_keys: {
+        "@migration:localhost": {
+            user_id: "@migration:localhost",
+            usage: ["user_signing"],
+            keys: {
+                "ed25519:8XHpC3MeMReIfYneWIRX8c4ANgJuQ1+oFrktBcLka4o": "8XHpC3MeMReIfYneWIRX8c4ANgJuQ1+oFrktBcLka4o",
+            },
+            signatures: {
+                "@migration:localhost": {
+                    "ed25519:rXCrBin/+xyh+yW//vWte+2UV0et1ZHTWfalp/Ekack":
+                        "FX6ylagvx3IG1zMf/ayYgDb/1+x0/F28pHQqzQMGGssAmc15nat/R6AF0QO7Qg7uqTAf04ohuZtWax3dTwjNDQ",
+                },
+            },
+        },
+    },
+};
+
+/**
+ * A `/room_keys/version` response containing the current server-side backup info.
+ * To be used during tests with fetchmock.
+ */
+const BACKUP_RESPONSE: KeyBackupInfo = {
+    auth_data: {
+        public_key: "2ffIfIB4oryqZpsJQjQNUaxgCzxliC6A4PJvnrN+XAA",
+        signatures: {
+            "@migration:localhost": {
+                "ed25519:TMWBMDZPFT":
+                    "qBvalid/G4hnSF3hAeX4TtRN6/BqprgiYnLEtDuatyQ5WxWr0s4uSOyvHSglsRdpoo32FDBHfTIZkCOVxSLwAA",
+            },
+        },
+    },
+    version: "2",
+    algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
+    etag: "0",
+    count: 0,
+};
+
+/**
+ * This was generated by doing a backup reset on the account.
+ * This is a new valid backup for this account.
+ */
+const NEW_BACKUP_RESPONSE: KeyBackupInfo = {
+    auth_data: {
+        public_key: "CkDxWALi3lcChgjEZFEM6clYq5x768XBwsL++eaOzTI",
+        signatures: {
+            "@migration:localhost": {
+                "ed25519:YVEGEYPYWX":
+                    "ZSYuQDdwgB9WKXQ+z5aWWfqSolBCGRw53kur1Vy956gFefgzCBkMbw5M0I2UgfU2Cukri7jZ4ig201zmLNmaAA",
+                "ed25519:rXCrBin/+xyh+yW//vWte+2UV0et1ZHTWfalp/Ekack":
+                    "+UQ8EA507LoIqgK9rPsqPoGrj+iRBJeY2Oz0mMtXmVf8c1y8G0KWJNUWqvOysnOhsoJf1bt8ey48CxjjtSQ2AA",
+            },
+        },
+    },
+    version: "3",
+    algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
+    etag: "0",
+    count: 0,
+};
+
+/**
+ * A dataset containing the information for the tested user.
+ * To be used during tests.
+ */
+export const MSK_NOT_CACHED_DATASET: DumpDataSetInfo = {
+    userId: "@migration:localhost",
+    deviceId: "CBGTADUILV",
+    pickleKey: "qEURMepfkMvoBQGaWlI9MZKYnDMsSAiW8aFTKXaeDV0",
+    keyQueryResponse: KEY_QUERY_RESPONSE,
+    rotatedKeyQueryResponse: ROTATED_KEY_QUERY_RESPONSE,
+    backupResponse: BACKUP_RESPONSE,
+    newBackupResponse: NEW_BACKUP_RESPONSE,
+    dumpPath: "spec/test-utils/test_indexeddb_cryptostore_dump/no_cached_msk_dump/dump.json",
+};

spec/test-utils/test_indexeddb_cryptostore_dump/unverified/README.md

@@ -0,0 +1,4 @@
+## Dump of a libolm indexeddb cryptostore where the identity is not trusted.
+
+A dump of an account where the identity was not verified.
+Used as a test case for migration of the identity local trust.

spec/test-utils/test_indexeddb_cryptostore_dump/unverified/index.ts

@@ -0,0 +1,110 @@
+import { DumpDataSetInfo } from "../index";
+
+/**
+ * A key query response containing the current keys of the tested user.
+ * To be used during tests with fetchmock.
+ */
+const KEY_QUERY_RESPONSE = {
+    device_keys: {
+        "@untrusted:localhost": {
+            IXNYALOZWU: {
+                algorithms: ["m.olm.v1.curve25519-aes-sha2", "m.megolm.v1.aes-sha2"],
+                device_id: "IXNYALOZWU",
+                keys: {
+                    "curve25519:IXNYALOZWU": "EHMQEtJd9INJg28HwKK8Te1EX8obR3VTtyNwf/rcczM",
+                    "ed25519:IXNYALOZWU": "OxMfZHsYJvroTp1RtjUOejpWbRBryN6VsojC5dKR74U",
+                },
+                signatures: {
+                    "@untrusted:localhost": {
+                        "ed25519:IXNYALOZWU":
+                            "tWaTiRKc95ZCqM2qrKTdq1sQ3DPFgw3vdrOVmWIHQwj92DCgJtnQ9uymLMOq+MSb80bdBBjXwrNeOufgaL/6CQ",
+                        "ed25519:+ik0n/QnBPq8H/48wAT+54slKk1SL7NIk/HtiN/cNEg":
+                            "+QXZFLiAv+k7UXgAP6AXLk/PdZ3TlJ77M23m73v8qvavAlnkLBAjKNA3BG39JTQET5UhW5DnCohwsbGP+aY1Cw",
+                    },
+                },
+                user_id: "@untrusted:localhost",
+                unsigned: {
+                    device_display_name: "localhost:8080: Chrome on macOS",
+                },
+            },
+            VJPSPVPWZT: {
+                algorithms: ["m.olm.v1.curve25519-aes-sha2", "m.megolm.v1.aes-sha2"],
+                device_id: "VJPSPVPWZT",
+                keys: {
+                    "curve25519:VJPSPVPWZT": "+RxCNRFPqBZJm6PLjEJsSdFixGWQJygD5Os11/+6PC0",
+                    "ed25519:VJPSPVPWZT": "wqH7xK/DQya8m05Vy4rnacjugGNBiY+7Ml6wyRVkM9U",
+                },
+                signatures: {
+                    "@untrusted:localhost": {
+                        "ed25519:VJPSPVPWZT":
+                            "XC+RoKL/zVZOIwk/bGEQJlJu49QicY1v6vSDMHA2y0/fpX/MD4KiWGD7+W5DFD54E8FrFVTsIgkzat561qdTBQ",
+                    },
+                },
+                user_id: "@untrusted:localhost",
+                unsigned: {
+                    device_display_name: "localhost:8080: Chrome on macOS",
+                },
+            },
+        },
+    },
+    failures: {},
+    master_keys: {
+        "@untrusted:localhost": {
+            keys: {
+                "ed25519:Uahbc3+Rk65y0ku6T2RL/29fEA9Bum6+OaqptG6df3g": "Uahbc3+Rk65y0ku6T2RL/29fEA9Bum6+OaqptG6df3g",
+            },
+            signatures: {
+                "@untrusted:localhost": {
+                    "ed25519:IXNYALOZWU":
+                        "KdAdyKO2sb3Di3bdK+oxf+gjMSmW/sisRNvpKZORPKwmy2SGaKGYkecBtslunoFjnb+hjIESgweQu6cHoNX4AA",
+                    "ed25519:Uahbc3+Rk65y0ku6T2RL/29fEA9Bum6+OaqptG6df3g":
+                        "b0R9Id5HxHYo+MA22Vlq0OckTrWnSWhgHLvF8Wr4e154JdtOyK7N0aXPQPkrLB0fmyVmGdbDa9xs9jsfINGmDw",
+                },
+            },
+            usage: ["master"],
+            user_id: "@untrusted:localhost",
+        },
+    },
+    self_signing_keys: {
+        "@untrusted:localhost": {
+            keys: {
+                "ed25519:+ik0n/QnBPq8H/48wAT+54slKk1SL7NIk/HtiN/cNEg": "+ik0n/QnBPq8H/48wAT+54slKk1SL7NIk/HtiN/cNEg",
+            },
+            signatures: {
+                "@untrusted:localhost": {
+                    "ed25519:Uahbc3+Rk65y0ku6T2RL/29fEA9Bum6+OaqptG6df3g":
+                        "z/5z51jbRpyDQhYnfUHhhb5fUbzRDlfjD8mZA2ZGStpE/F41lDyxjlvF2W/E2CJ27bmJFdk7nC+ZCwriYfYxBw",
+                },
+            },
+            usage: ["self_signing"],
+            user_id: "@untrusted:localhost",
+        },
+    },
+    user_signing_keys: {
+        "@untrusted:localhost": {
+            keys: {
+                "ed25519:L/8HbQWnK9OidAcDVB+Az9b0Mx3OdBtIMFsUjV6qgSQ": "L/8HbQWnK9OidAcDVB+Az9b0Mx3OdBtIMFsUjV6qgSQ",
+            },
+            signatures: {
+                "@untrusted:localhost": {
+                    "ed25519:Uahbc3+Rk65y0ku6T2RL/29fEA9Bum6+OaqptG6df3g":
+                        "UuNvzebLQn31LYGbx+ADe60BB25kWy4SVVyd9BXlY/tAZMoA8Tmq1e2R2tJJtPdJxC/Oogktj2+iikZV/YMjAQ",
+                },
+            },
+            usage: ["user_signing"],
+            user_id: "@untrusted:localhost",
+        },
+    },
+};
+
+/**
+ * A dataset containing the information for the tested user.
+ * To be used during tests.
+ */
+export const IDENTITY_NOT_TRUSTED_DATASET: DumpDataSetInfo = {
+    userId: "@untrusted:localhost",
+    deviceId: "VJPSPVPWZT",
+    pickleKey: "WVllQb4Lk/WwP4Q7iBfeTUHpgydZm9YqXI1B5bTvnIM",
+    keyQueryResponse: KEY_QUERY_RESPONSE,
+    dumpPath: "spec/test-utils/test_indexeddb_cryptostore_dump/unverified/dump.json",
+};

spec/test-utils/thread.ts

@@ -161,3 +161,23 @@ export const mkThread = ({
 
     return { thread, rootEvent, events };
 };
+
+/**
+ * Create a thread, and make sure the events are added to the thread and the
+ * room's timeline as if they came in via sync.
+ *
+ * Note that mkThread doesn't actually add the events properly to the room.
+ */
+export const populateThread = ({
+    room,
+    client,
+    authorId,
+    participantUserIds,
+    length = 2,
+    ts = 1,
+}: MakeThreadProps): MakeThreadResult => {
+    const ret = mkThread({ room, client, authorId, participantUserIds, length, ts });
+    ret.thread.initialEventsFetched = true;
+    room.addLiveEvents(ret.events);
+    return ret;
+};

spec/test-utils/webrtc.ts

@@ -269,7 +269,11 @@ export class MockRTCRtpTransceiver {
 }
 
 export class MockMediaStreamTrack {
-    constructor(public readonly id: string, public readonly kind: "audio" | "video", public enabled = true) {}
+    constructor(
+        public readonly id: string,
+        public readonly kind: "audio" | "video",
+        public enabled = true,
+    ) {}
 
     public stop = jest.fn<void, []>();
 
@@ -306,7 +310,10 @@ export class MockMediaStreamTrack {
 // XXX: Using EventTarget in jest doesn't seem to work, so we write our own
 // implementation
 export class MockMediaStream {
-    constructor(public id: string, private tracks: MockMediaStreamTrack[] = []) {}
+    constructor(
+        public id: string,
+        private tracks: MockMediaStreamTrack[] = [],
+    ) {}
 
     public listeners: [string, (...args: any[]) => any][] = [];
     public isStopped = false;
@@ -435,7 +442,11 @@ type EmittedEventMap = CallEventHandlerEventHandlerMap &
 export class MockCallMatrixClient extends TypedEventEmitter<EmittedEvents, EmittedEventMap> {
     public mediaHandler = new MockMediaHandler();
 
-    constructor(public userId: string, public deviceId: string, public sessionId: string) {
+    constructor(
+        public userId: string,
+        public deviceId: string,
+        public sessionId: string,
+    ) {
         super();
     }
 
@@ -485,6 +496,7 @@ export class MockCallMatrixClient extends TypedEventEmitter<EmittedEvents, Emitt
 
     public getRooms = jest.fn<Room[], []>().mockReturnValue([]);
     public getRoom = jest.fn();
+    public getFoci = jest.fn();
 
     public supportsThreads(): boolean {
         return true;
@@ -501,7 +513,10 @@ export class MockCallMatrixClient extends TypedEventEmitter<EmittedEvents, Emitt
 }
 
 export class MockMatrixCall extends TypedEventEmitter<CallEvent, CallEventHandlerMap> {
-    constructor(public roomId: string, public groupCallId?: string) {
+    constructor(
+        public roomId: string,
+        public groupCallId?: string,
+    ) {
         super();
     }
 
@@ -549,7 +564,11 @@ export class MockMatrixCall extends TypedEventEmitter<CallEvent, CallEventHandle
 }
 
 export class MockCallFeed {
-    constructor(public userId: string, public deviceId: string | undefined, public stream: MockMediaStream) {}
+    constructor(
+        public userId: string,
+        public deviceId: string | undefined,
+        public stream: MockMediaStream,
+    ) {}
 
     public measureVolumeActivity(val: boolean) {}
     public dispose() {}

spec/unit/autodiscovery.spec.ts

@@ -15,13 +15,10 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import fetchMock from "fetch-mock-jest";
 import MockHttpBackend from "matrix-mock-request";
 
-import { AutoDiscoveryAction, M_AUTHENTICATION } from "../../src";
+import { AutoDiscoveryAction } from "../../src";
 import { AutoDiscovery } from "../../src/autodiscovery";
-import { OidcError } from "../../src/oidc/error";
-import { makeDelegatedAuthConfig } from "../test-utils/oidc";
 
 // keep to reset the fetch function after using MockHttpBackend
 // @ts-ignore private property
@@ -351,7 +348,7 @@ describe("AutoDiscovery", function () {
         function () {
             const httpBackend = getHttpBackend();
             httpBackend.when("GET", "/_matrix/client/versions").respond(200, {
-                not_matrix_versions: ["v1.1"],
+                not_matrix_versions: ["v1.5"],
             });
             httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
                 "m.homeserver": {
@@ -388,7 +385,7 @@ describe("AutoDiscovery", function () {
                 expect(req.path).toEqual("https://example.org/_matrix/client/versions");
             })
             .respond(200, {
-                versions: ["v1.1"],
+                versions: ["v1.5"],
             });
         httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
             "m.homeserver": {
@@ -409,10 +406,6 @@ describe("AutoDiscovery", function () {
                         error: null,
                         base_url: null,
                     },
-                    "m.authentication": {
-                        state: "IGNORE",
-                        error: OidcError.NotSupported,
-                    },
                 };
 
                 expect(conf).toEqual(expected);
@@ -428,7 +421,7 @@ describe("AutoDiscovery", function () {
                 expect(req.path).toEqual("https://chat.example.org/_matrix/client/versions");
             })
             .respond(200, {
-                versions: ["v1.1"],
+                versions: ["v1.5"],
             });
         httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
             "m.homeserver": {
@@ -450,10 +443,6 @@ describe("AutoDiscovery", function () {
                         error: null,
                         base_url: null,
                     },
-                    "m.authentication": {
-                        state: "IGNORE",
-                        error: OidcError.NotSupported,
-                    },
                 };
 
                 expect(conf).toEqual(expected);
@@ -469,16 +458,13 @@ describe("AutoDiscovery", function () {
                 expect(req.path).toEqual("https://chat.example.org/_matrix/client/versions");
             })
             .respond(200, {
-                versions: ["v1.1"],
+                versions: ["v1.5"],
             });
         httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
             "m.homeserver": {
                 // Note: we also expect this test to trim the trailing slash
                 base_url: "https://chat.example.org/",
             },
-            "m.authentication": {
-                invalid: true,
-            },
         });
         return Promise.all([
             httpBackend.flushAllExpected(),
@@ -494,10 +480,6 @@ describe("AutoDiscovery", function () {
                         error: null,
                         base_url: null,
                     },
-                    "m.authentication": {
-                        state: "FAIL_ERROR",
-                        error: OidcError.Misconfigured,
-                    },
                 };
 
                 expect(conf).toEqual(expected);
@@ -515,7 +497,7 @@ describe("AutoDiscovery", function () {
                     expect(req.path).toEqual("https://chat.example.org/_matrix/client/versions");
                 })
                 .respond(200, {
-                    versions: ["v1.1"],
+                    versions: ["v1.5"],
                 });
             httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
                 "m.homeserver": {
@@ -560,7 +542,7 @@ describe("AutoDiscovery", function () {
                     expect(req.path).toEqual("https://chat.example.org/_matrix/client/versions");
                 })
                 .respond(200, {
-                    versions: ["v1.1"],
+                    versions: ["v1.5"],
                 });
             httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
                 "m.homeserver": {
@@ -606,7 +588,7 @@ describe("AutoDiscovery", function () {
                     expect(req.path).toEqual("https://chat.example.org/_matrix/client/versions");
                 })
                 .respond(200, {
-                    versions: ["v1.1"],
+                    versions: ["v1.5"],
                 });
             httpBackend.when("GET", "/_matrix/identity/v2").respond(404, {});
             httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
@@ -653,7 +635,7 @@ describe("AutoDiscovery", function () {
                     expect(req.path).toEqual("https://chat.example.org/_matrix/client/versions");
                 })
                 .respond(200, {
-                    versions: ["v1.1"],
+                    versions: ["v1.5"],
                 });
             httpBackend.when("GET", "/_matrix/identity/v2").respond(500, {});
             httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
@@ -697,7 +679,7 @@ describe("AutoDiscovery", function () {
                 expect(req.path).toEqual("https://chat.example.org/_matrix/client/versions");
             })
             .respond(200, {
-                versions: ["v1.1"],
+                versions: ["v1.5"],
             });
         httpBackend
             .when("GET", "/_matrix/identity/v2")
@@ -728,10 +710,6 @@ describe("AutoDiscovery", function () {
                         error: null,
                         base_url: "https://identity.example.org",
                     },
-                    "m.authentication": {
-                        state: "IGNORE",
-                        error: OidcError.NotSupported,
-                    },
                 };
 
                 expect(conf).toEqual(expected);
@@ -747,7 +725,7 @@ describe("AutoDiscovery", function () {
                 expect(req.path).toEqual("https://chat.example.org/_matrix/client/versions");
             })
             .respond(200, {
-                versions: ["v1.1"],
+                versions: ["v1.5"],
             });
         httpBackend
             .when("GET", "/_matrix/identity/v2")
@@ -784,10 +762,6 @@ describe("AutoDiscovery", function () {
                     "org.example.custom.property": {
                         cupcakes: "yes",
                     },
-                    "m.authentication": {
-                        state: "IGNORE",
-                        error: OidcError.NotSupported,
-                    },
                 };
 
                 expect(conf).toEqual(expected);
@@ -897,75 +871,4 @@ describe("AutoDiscovery", function () {
             }),
         ]);
     });
-
-    describe("m.authentication", () => {
-        const homeserverName = "example.org";
-        const homeserverUrl = "https://chat.example.org/";
-        const issuer = "https://auth.org/";
-
-        beforeAll(() => {
-            // make these tests independent from fetch mocking above
-            AutoDiscovery.setFetchFn(realAutoDiscoveryFetch);
-        });
-
-        beforeEach(() => {
-            fetchMock.resetBehavior();
-            fetchMock.get(`${homeserverUrl}_matrix/client/versions`, { versions: ["v1.1"] });
-
-            fetchMock.get("https://example.org/.well-known/matrix/client", {
-                "m.homeserver": {
-                    // Note: we also expect this test to trim the trailing slash
-                    base_url: "https://chat.example.org/",
-                },
-                "m.authentication": {
-                    issuer,
-                },
-            });
-        });
-
-        it("should return valid authentication configuration", async () => {
-            const config = makeDelegatedAuthConfig(issuer);
-
-            fetchMock.get(`${config.metadata.issuer}.well-known/openid-configuration`, config.metadata);
-            fetchMock.get(`${config.metadata.issuer}jwks`, {
-                status: 200,
-                headers: {
-                    "Content-Type": "application/json",
-                },
-                keys: [],
-            });
-
-            const result = await AutoDiscovery.findClientConfig(homeserverName);
-
-            expect(result[M_AUTHENTICATION.stable!]).toEqual({
-                state: AutoDiscovery.SUCCESS,
-                ...config,
-                signingKeys: [],
-                account: undefined,
-                error: null,
-            });
-        });
-
-        it("should set state to error for invalid authentication configuration", async () => {
-            const config = makeDelegatedAuthConfig(issuer);
-            // authorization_code is required
-            config.metadata.grant_types_supported = ["openid"];
-
-            fetchMock.get(`${config.metadata.issuer}.well-known/openid-configuration`, config.metadata);
-            fetchMock.get(`${config.metadata.issuer}jwks`, {
-                status: 200,
-                headers: {
-                    "Content-Type": "application/json",
-                },
-                keys: [],
-            });
-
-            const result = await AutoDiscovery.findClientConfig(homeserverName);
-
-            expect(result[M_AUTHENTICATION.stable!]).toEqual({
-                state: AutoDiscovery.FAIL_ERROR,
-                error: OidcError.OpSupport,
-            });
-        });
-    });
 });

spec/unit/base64.spec.ts

@@ -0,0 +1,88 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { TextEncoder, TextDecoder } from "util";
+import NodeBuffer from "node:buffer";
+
+import { decodeBase64, encodeBase64, encodeUnpaddedBase64, encodeUnpaddedBase64Url } from "../../src/base64";
+
+describe.each(["browser", "node"])("Base64 encoding (%s)", (env) => {
+    let origBuffer = Buffer;
+
+    beforeAll(() => {
+        if (env === "browser") {
+            origBuffer = Buffer;
+            // @ts-ignore
+            // eslint-disable-next-line no-global-assign
+            Buffer = undefined;
+
+            global.atob = NodeBuffer.atob;
+            global.btoa = NodeBuffer.btoa;
+        }
+    });
+
+    afterAll(() => {
+        // eslint-disable-next-line no-global-assign
+        Buffer = origBuffer;
+        // @ts-ignore
+        global.atob = undefined;
+        // @ts-ignore
+        global.btoa = undefined;
+    });
+
+    it("Should decode properly encoded data", () => {
+        const decoded = new TextDecoder().decode(decodeBase64("ZW5jb2RpbmcgaGVsbG8gd29ybGQ="));
+
+        expect(decoded).toStrictEqual("encoding hello world");
+    });
+
+    it("Should encode unpadded URL-safe base64", () => {
+        const toEncode = "?????";
+        const data = new TextEncoder().encode(toEncode);
+
+        const encoded = encodeUnpaddedBase64Url(data);
+        expect(encoded).toEqual("Pz8_Pz8");
+    });
+
+    it("Should decode URL-safe base64", () => {
+        const decoded = new TextDecoder().decode(decodeBase64("Pz8_Pz8="));
+
+        expect(decoded).toStrictEqual("?????");
+    });
+
+    it("Encode unpadded should not have padding", () => {
+        const toEncode = "encoding hello world";
+        const data = new TextEncoder().encode(toEncode);
+
+        const paddedEncoded = encodeBase64(data);
+        const unpaddedEncoded = encodeUnpaddedBase64(data);
+
+        expect(paddedEncoded).not.toEqual(unpaddedEncoded);
+
+        const padding = paddedEncoded.charAt(paddedEncoded.length - 1);
+        expect(padding).toStrictEqual("=");
+    });
+
+    it("Decode should be indifferent to padding", () => {
+        const withPadding = "ZW5jb2RpbmcgaGVsbG8gd29ybGQ=";
+        const withoutPadding = "ZW5jb2RpbmcgaGVsbG8gd29ybGQ";
+
+        const decodedPad = decodeBase64(withPadding);
+        const decodedNoPad = decodeBase64(withoutPadding);
+
+        expect(decodedPad).toStrictEqual(decodedNoPad);
+    });
+});

spec/unit/content-repo.spec.ts

@@ -37,6 +37,21 @@ describe("ContentRepo", function () {
             );
         });
 
+        it("should allow redirects when requested on download URLs", function () {
+            const mxcUri = "mxc://server.name/resourceid";
+            expect(getHttpUriForMxc(baseUrl, mxcUri, undefined, undefined, undefined, false, true)).toEqual(
+                baseUrl + "/_matrix/media/v3/download/server.name/resourceid?allow_redirect=true",
+            );
+        });
+
+        it("should allow redirects when requested on thumbnail URLs", function () {
+            const mxcUri = "mxc://server.name/resourceid";
+            expect(getHttpUriForMxc(baseUrl, mxcUri, 32, 32, "scale", false, true)).toEqual(
+                baseUrl +
+                    "/_matrix/media/v3/thumbnail/server.name/resourceid?width=32&height=32&method=scale&allow_redirect=true",
+            );
+        });
+
         it("should return the empty string for null input", function () {
             expect(getHttpUriForMxc(null as any, "")).toEqual("");
         });

spec/unit/crypto.spec.ts

@@ -15,11 +15,17 @@ import { sleep } from "../../src/utils";
 import { CRYPTO_ENABLED } from "../../src/client";
 import { DeviceInfo } from "../../src/crypto/deviceinfo";
 import { logger } from "../../src/logger";
-import { MemoryStore } from "../../src";
+import { DeviceVerification, MemoryStore } from "../../src";
 import { RoomKeyRequestState } from "../../src/crypto/OutgoingRoomKeyRequestManager";
 import { RoomMember } from "../../src/models/room-member";
 import { IStore } from "../../src/store";
 import { IRoomEncryption, RoomList } from "../../src/crypto/RoomList";
+import { EventShieldColour, EventShieldReason } from "../../src/crypto-api";
+import { UserTrustLevel } from "../../src/crypto/CrossSigning";
+import { CryptoBackend } from "../../src/common-crypto/CryptoBackend";
+import { EventDecryptionResult } from "../../src/common-crypto/CryptoBackend";
+import * as testData from "../test-utils/test-data";
+import { KnownMembership } from "../../src/@types/membership";
 
 const Olm = global.Olm;
 
@@ -110,14 +116,25 @@ describe("Crypto", function () {
         expect(Crypto.getOlmVersion()[0]).toEqual(3);
     });
 
+    it("getVersion() should return the current version of the olm library", async () => {
+        const client = new TestClient("@alice:example.com", "deviceid").client;
+        await client.initCrypto();
+
+        const olmVersionTuple = Crypto.getOlmVersion();
+        expect(client.getCrypto()?.getVersion()).toBe(
+            `Olm ${olmVersionTuple[0]}.${olmVersionTuple[1]}.${olmVersionTuple[2]}`,
+        );
+    });
+
     describe("encrypted events", function () {
-        it("provides encryption information", async function () {
+        it("provides encryption information for events from unverified senders", async function () {
             const client = new TestClient("@alice:example.com", "deviceid").client;
             await client.initCrypto();
 
             // unencrypted event
             const event = {
                 getId: () => "$event_id",
+                getSender: () => "@bob:example.com",
                 getSenderKey: () => null,
                 getWireContent: () => {
                     return {};
@@ -127,6 +144,8 @@ describe("Crypto", function () {
             let encryptionInfo = client.getEventEncryptionInfo(event);
             expect(encryptionInfo.encrypted).toBeFalsy();
 
+            expect(await client.getCrypto()!.getEncryptionInfoForEvent(event)).toBe(null);
+
             // unknown sender (e.g. deleted device), forwarded megolm key (untrusted)
             event.getSenderKey = () => "YmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmI";
             event.getWireContent = () => {
@@ -141,6 +160,11 @@ describe("Crypto", function () {
             expect(encryptionInfo.authenticated).toBeFalsy();
             expect(encryptionInfo.sender).toBeFalsy();
 
+            expect(await client.getCrypto()!.getEncryptionInfoForEvent(event)).toEqual({
+                shieldColour: EventShieldColour.GREY,
+                shieldReason: EventShieldReason.AUTHENTICITY_NOT_GUARANTEED,
+            });
+
             // known sender, megolm key from backup
             event.getForwardingCurve25519KeyChain = () => [];
             event.isKeySourceUntrusted = () => true;
@@ -155,6 +179,11 @@ describe("Crypto", function () {
             expect(encryptionInfo.sender).toBeTruthy();
             expect(encryptionInfo.mismatchedSender).toBeFalsy();
 
+            expect(await client.getCrypto()!.getEncryptionInfoForEvent(event)).toEqual({
+                shieldColour: EventShieldColour.GREY,
+                shieldReason: EventShieldReason.AUTHENTICITY_NOT_GUARANTEED,
+            });
+
             // known sender, trusted megolm key, but bad ed25519key
             event.isKeySourceUntrusted = () => false;
             device.keys["ed25519:FLIBBLE"] = "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB";
@@ -165,9 +194,115 @@ describe("Crypto", function () {
             expect(encryptionInfo.sender).toBeTruthy();
             expect(encryptionInfo.mismatchedSender).toBeTruthy();
 
+            expect(await client.getCrypto()!.getEncryptionInfoForEvent(event)).toEqual({
+                shieldColour: EventShieldColour.RED,
+                shieldReason: EventShieldReason.MISMATCHED_SENDER_KEY,
+            });
+
             client.stopClient();
         });
 
+        describe("provides encryption information for events from verified senders", function () {
+            const testDeviceId = testData.BOB_TEST_DEVICE_ID;
+            const testDevice = testData.BOB_SIGNED_TEST_DEVICE_DATA;
+
+            let client: MatrixClient;
+            beforeEach(async () => {
+                client = new TestClient("@alice:example.com", "deviceid").client;
+                await client.initCrypto();
+
+                // mock out the verification check
+                client.crypto!.checkUserTrust = (userId) => new UserTrustLevel(true, false, false);
+            });
+
+            afterEach(() => {
+                client.stopClient();
+            });
+
+            async function buildEncryptedEvent(
+                decryptionResult: Partial<EventDecryptionResult> = {},
+            ): Promise<MatrixEvent> {
+                const mockCryptoBackend = {
+                    decryptEvent: async (event: MatrixEvent): Promise<EventDecryptionResult> => {
+                        return {
+                            claimedEd25519Key: testDevice.keys["ed25519:" + testDeviceId],
+                            clearEvent: {
+                                room_id: "!room_id",
+                                type: "m.room.message",
+                                content: { body: "test" },
+                            },
+                            forwardingCurve25519KeyChain: [],
+                            senderCurve25519Key: testDevice.keys["curve25519:" + testDeviceId],
+                            ...decryptionResult,
+                        };
+                    },
+                } as unknown as CryptoBackend;
+
+                const event = new MatrixEvent({
+                    event_id: "$event_id",
+                    sender: testData.BOB_TEST_USER_ID,
+                    type: "m.room.encrypted",
+                    content: { algorithm: "m.megolm.v1.aes-sha2" },
+                });
+                await event.attemptDecryption(mockCryptoBackend);
+                return event;
+            }
+
+            it("unknown device", async () => {
+                const event = await buildEncryptedEvent();
+                expect(await client.getCrypto()!.getEncryptionInfoForEvent(event)).toEqual({
+                    shieldColour: EventShieldColour.GREY,
+                    shieldReason: EventShieldReason.UNKNOWN_DEVICE,
+                });
+            });
+
+            it("known but unsigned device", async () => {
+                client.crypto!.deviceList.storeDevicesForUser(testData.BOB_TEST_USER_ID, {
+                    [testDeviceId]: {
+                        keys: testDevice.keys,
+                        algorithms: testDevice.algorithms,
+                        verified: DeviceVerification.Unverified,
+                        known: true,
+                    },
+                });
+
+                const event = await buildEncryptedEvent();
+                expect(await client.getCrypto()!.getEncryptionInfoForEvent(event)).toEqual({
+                    shieldColour: EventShieldColour.RED,
+                    shieldReason: EventShieldReason.UNSIGNED_DEVICE,
+                });
+            });
+
+            describe("known and verified device", () => {
+                beforeEach(() => {
+                    client.crypto!.deviceList.storeDevicesForUser(testData.BOB_TEST_USER_ID, {
+                        [testDeviceId]: {
+                            keys: testDevice.keys,
+                            algorithms: testDevice.algorithms,
+                            verified: DeviceVerification.Verified,
+                            known: true,
+                        },
+                    });
+                });
+
+                it("regular key", async () => {
+                    const event = await buildEncryptedEvent();
+                    expect(await client.getCrypto()!.getEncryptionInfoForEvent(event)).toEqual({
+                        shieldColour: EventShieldColour.NONE,
+                        shieldReason: null,
+                    });
+                });
+
+                it("unauthenticated key", async () => {
+                    const event = await buildEncryptedEvent({ untrusted: true });
+                    expect(await client.getCrypto()!.getEncryptionInfoForEvent(event)).toEqual({
+                        shieldColour: EventShieldColour.GREY,
+                        shieldReason: EventShieldReason.AUTHENTICITY_NOT_GUARANTEED,
+                    });
+                });
+            });
+        });
+
         it("doesn't throw an error when attempting to decrypt a redacted event", async () => {
             const client = new TestClient("@alice:example.com", "deviceid").client;
             await client.initCrypto();
@@ -222,7 +357,6 @@ describe("Crypto", function () {
 
         let crypto: Crypto;
         let mockBaseApis: MatrixClient;
-        let mockRoomList: RoomList;
 
         let fakeEmitter: EventEmitter;
 
@@ -256,19 +390,10 @@ describe("Crypto", function () {
                 isGuest: jest.fn(),
                 emit: jest.fn(),
             } as unknown as MatrixClient;
-            mockRoomList = {} as unknown as RoomList;
 
             fakeEmitter = new EventEmitter();
 
-            crypto = new Crypto(
-                mockBaseApis,
-                "@alice:home.server",
-                "FLIBBLE",
-                clientStore,
-                cryptoStore,
-                mockRoomList,
-                [],
-            );
+            crypto = new Crypto(mockBaseApis, "@alice:home.server", "FLIBBLE", clientStore, cryptoStore, []);
             crypto.registerEventHandlers(fakeEmitter as any);
             await crypto.init();
         });
@@ -339,7 +464,7 @@ describe("Crypto", function () {
                     type: "m.room.member",
                     sender: "@alice:example.com",
                     room_id: roomId,
-                    content: { membership: "invite" },
+                    content: { membership: KnownMembership.Invite },
                     state_key: "@bob:example.com",
                 }),
             ]);
@@ -671,7 +796,7 @@ describe("Crypto", function () {
                     type: "m.room.member",
                     sender: "@clara:example.com",
                     room_id: roomId,
-                    content: { membership: "invite" },
+                    content: { membership: KnownMembership.Invite },
                     state_key: "@bob:example.com",
                 }),
             ]);
@@ -982,7 +1107,7 @@ describe("Crypto", function () {
 
     describe("Secret storage", function () {
         it("creates secret storage even if there is no keyInfo", async function () {
-            jest.spyOn(logger, "log").mockImplementation(() => {});
+            jest.spyOn(logger, "debug").mockImplementation(() => {});
             jest.setTimeout(10000);
             const client = new TestClient("@a:example.com", "dev").client;
             await client.initCrypto();
@@ -1139,7 +1264,7 @@ describe("Crypto", function () {
                     ({
                         init_with_private_key: jest.fn(),
                         free,
-                    } as unknown as PkDecryption),
+                    }) as unknown as PkDecryption,
             );
             client.client.checkSecretStoragePrivateKey(new Uint8Array(), "");
             expect(free).toHaveBeenCalled();
@@ -1165,7 +1290,7 @@ describe("Crypto", function () {
                     ({
                         init_with_seed: jest.fn(),
                         free,
-                    } as unknown as PkSigning),
+                    }) as unknown as PkSigning,
             );
             client.client.checkCrossSigningPrivateKey(new Uint8Array(), "");
             expect(free).toHaveBeenCalled();
@@ -1207,15 +1332,9 @@ describe("Crypto", function () {
                 setRoomEncryption: jest.fn().mockResolvedValue(undefined),
             } as unknown as RoomList;
 
-            crypto = new Crypto(
-                mockClient,
-                "@alice:home.server",
-                "FLIBBLE",
-                clientStore,
-                cryptoStore,
-                mockRoomList,
-                [],
-            );
+            crypto = new Crypto(mockClient, "@alice:home.server", "FLIBBLE", clientStore, cryptoStore, []);
+            // @ts-ignore we are injecting a mock into a private property
+            crypto.roomList = mockRoomList;
         });
 
         it("should set the algorithm if called for a known room", async () => {

spec/unit/crypto/algorithms/megolm.spec.ts

@@ -36,6 +36,7 @@ import { DeviceTrustLevel } from "../../../../src/crypto/CrossSigning";
 import { MegolmEncryption as MegolmEncryptionClass } from "../../../../src/crypto/algorithms/megolm";
 import { recursiveMapToObject } from "../../../../src/utils";
 import { sleep } from "../../../../src/utils";
+import { KnownMembership } from "../../../../src/@types/membership";
 
 const MegolmDecryption = algorithms.DECRYPTION_CLASSES.get("m.megolm.v1.aes-sha2")!;
 const MegolmEncryption = algorithms.ENCRYPTION_CLASSES.get("m.megolm.v1.aes-sha2")!;
@@ -806,11 +807,11 @@ describe("MegolmDecryption", function () {
         aliceRoom.getEncryptionTargetMembers = jest.fn().mockResolvedValue([
             {
                 userId: "@alice:example.com",
-                membership: "join",
+                membership: KnownMembership.Join,
             },
             {
                 userId: "@bob:example.com",
-                membership: "join",
+                membership: KnownMembership.Join,
             },
         ]);
         const BOB_DEVICES = {

spec/unit/crypto/backup.spec.ts

@@ -215,6 +215,36 @@ describe("MegolmBackup", function () {
             jest.spyOn(global, "setTimeout").mockRestore();
         });
 
+        test("fail if crypto not enabled", async () => {
+            const client = makeTestClient(cryptoStore);
+            const data = {
+                algorithm: olmlib.MEGOLM_BACKUP_ALGORITHM,
+                version: "1",
+                auth_data: {
+                    public_key: "hSDwCYkwp1R0i33ctD73Wg2/Og0mOBr066SpjqqbTmo",
+                },
+            };
+            await expect(client.restoreKeyBackupWithSecretStorage(data)).rejects.toThrow(
+                "End-to-end encryption disabled",
+            );
+        });
+
+        test("fail if given backup has no version", async () => {
+            const client = makeTestClient(cryptoStore);
+            await client.initCrypto();
+            const data = {
+                algorithm: olmlib.MEGOLM_BACKUP_ALGORITHM,
+                auth_data: {
+                    public_key: "hSDwCYkwp1R0i33ctD73Wg2/Og0mOBr066SpjqqbTmo",
+                },
+            };
+            const key = Uint8Array.from([1, 2, 3, 4, 5, 6, 7, 8]);
+            await client.getCrypto()!.storeSessionBackupPrivateKey(key, "1");
+            await expect(client.restoreKeyBackupWithCache(undefined, undefined, data)).rejects.toThrow(
+                "Backup version must be defined",
+            );
+        });
+
         it("automatically calls the key back up", function () {
             const groupSession = new Olm.OutboundGroupSession();
             groupSession.create();

spec/unit/crypto/cross-signing.spec.ts

@@ -106,7 +106,7 @@ describe("Cross Signing", function () {
         });
         alice.uploadKeySignatures = async () => ({ failures: {} });
         alice.setAccountData = async () => ({});
-        alice.getAccountDataFromServer = async <T>() => ({} as T);
+        alice.getAccountDataFromServer = async <T>() => ({}) as T;
         // set Alice's cross-signing key
         await alice.bootstrapCrossSigning({
             authUploadDeviceSigningKeys: async (func) => {
@@ -146,7 +146,7 @@ describe("Cross Signing", function () {
         };
         alice.uploadKeySignatures = async () => ({ failures: {} });
         alice.setAccountData = async () => ({});
-        alice.getAccountDataFromServer = async <T extends { [k: string]: any }>(): Promise<T | null> => ({} as T);
+        alice.getAccountDataFromServer = async <T extends { [k: string]: any }>(): Promise<T | null> => ({}) as T;
         const authUploadDeviceSigningKeys: BootstrapCrossSigningOpts["authUploadDeviceSigningKeys"] = async (func) => {
             await func({});
         };

spec/unit/crypto/crypto-utils.ts

@@ -33,12 +33,10 @@ export async function resetCrossSigningKeys(
 
 export async function createSecretStorageKey(): Promise<IRecoveryKey> {
     const decryption = new global.Olm.PkDecryption();
-    const storagePublicKey = decryption.generate_key();
+    decryption.generate_key();
     const storagePrivateKey = decryption.get_private_key();
     decryption.free();
     return {
-        // `pubkey` not used anymore with symmetric 4S
-        keyInfo: { pubkey: storagePublicKey, key: undefined! },
         privateKey: storagePrivateKey,
     };
 }

spec/unit/crypto/secrets.spec.ts

@@ -28,6 +28,7 @@ import { DeviceInfo } from "../../../src/crypto/deviceinfo";
 import { ISignatures } from "../../../src/@types/signed";
 import { ICurve25519AuthData } from "../../../src/crypto/keybackup";
 import { SecretStorageKeyDescription, SECRET_STORAGE_ALGORITHM_V1_AES } from "../../../src/secret-storage";
+import { decodeBase64 } from "../../../src/base64";
 
 async function makeTestClient(
     userInfo: { userId: string; deviceId: string },
@@ -189,10 +190,7 @@ describe("Secrets", function () {
         };
         resetCrossSigningKeys(alice);
 
-        const { keyId: newKeyId } = await alice.addSecretStorageKey(SECRET_STORAGE_ALGORITHM_V1_AES, {
-            pubkey: undefined,
-            key: undefined,
-        });
+        const { keyId: newKeyId } = await alice.addSecretStorageKey(SECRET_STORAGE_ALGORITHM_V1_AES, { key });
         // we don't await on this because it waits for the event to come down the sync
         // which won't happen in the test setup
         alice.setDefaultSecretStorageKeyId(newKeyId);
@@ -275,13 +273,13 @@ describe("Secrets", function () {
 
     describe("bootstrap", function () {
         // keys used in some of the tests
-        const XSK = new Uint8Array(olmlib.decodeBase64("3lo2YdJugHjfE+Or7KJ47NuKbhE7AAGLgQ/dc19913Q="));
+        const XSK = new Uint8Array(decodeBase64("3lo2YdJugHjfE+Or7KJ47NuKbhE7AAGLgQ/dc19913Q="));
         const XSPubKey = "DRb8pFVJyEJ9OWvXeUoM0jq/C2Wt+NxzBZVuk2nRb+0";
-        const USK = new Uint8Array(olmlib.decodeBase64("lKWi3hJGUie5xxHgySoz8PHFnZv6wvNaud/p2shN9VU="));
+        const USK = new Uint8Array(decodeBase64("lKWi3hJGUie5xxHgySoz8PHFnZv6wvNaud/p2shN9VU="));
         const USPubKey = "CUpoiTtHiyXpUmd+3ohb7JVxAlUaOG1NYs9Jlx8soQU";
-        const SSK = new Uint8Array(olmlib.decodeBase64("1R6JVlXX99UcfUZzKuCDGQgJTw8ur1/ofgPD8pp+96M="));
+        const SSK = new Uint8Array(decodeBase64("1R6JVlXX99UcfUZzKuCDGQgJTw8ur1/ofgPD8pp+96M="));
         const SSPubKey = "0DfNsRDzEvkCLA0gD3m7VAGJ5VClhjEsewI35xq873Q";
-        const SSSSKey = new Uint8Array(olmlib.decodeBase64("XrmITOOdBhw6yY5Bh7trb/bgp1FRdIGyCUxxMP873R0="));
+        const SSSSKey = new Uint8Array(decodeBase64("XrmITOOdBhw6yY5Bh7trb/bgp1FRdIGyCUxxMP873R0="));
 
         it("bootstraps when no storage or cross-signing keys locally", async function () {
             const key = new Uint8Array(16);
@@ -312,6 +310,7 @@ describe("Secrets", function () {
                 this.emit(ClientEvent.AccountData, event);
                 return {};
             };
+            bob.getKeyBackupVersion = jest.fn().mockResolvedValue(null);
 
             await bob.bootstrapCrossSigning({
                 authUploadDeviceSigningKeys: async (func) => {
@@ -333,7 +332,6 @@ describe("Secrets", function () {
 
         it("bootstraps when cross-signing keys in secret storage", async function () {
             const decryption = new global.Olm.PkDecryption();
-            const storagePublicKey = decryption.generate_key();
             const storagePrivateKey = decryption.get_private_key();
 
             const bob: MatrixClient = await makeTestClient(
@@ -376,8 +374,6 @@ describe("Secrets", function () {
             });
             await bob.bootstrapSecretStorage({
                 createSecretStorageKey: async () => ({
-                    // `pubkey` not used anymore with symmetric 4S
-                    keyInfo: { pubkey: storagePublicKey },
                     privateKey: storagePrivateKey,
                 }),
             });