Merge branch 'master' into hs/proposal-appservice-login

Author: turt2live

.buildkite/pipeline.yaml

@@ -1,16 +1,39 @@
 steps:
-  - label: ":books: Build spec"
+  - label: ":snake: Build swagger definitions for matrix.org"
     command:
-      - python3 -m venv env
-      - env/bin/pip install -r scripts/requirements.txt
-      - ". env/bin/activate; scripts/generate-matrix-org-assets"
+      # Install the python dependencies necessary to build the spec
+      - python3 -m venv env && . env/bin/activate
+      - pip install -r scripts/requirements.txt
+      # Build the spec
+      - scripts/generate-matrix-org-assets
     artifact_paths:
       - assets.tar.gz
     plugins:
-      - docker#v3.0.1:
-          image: "python:3.6"
+      - docker#v3.7.0:
+          image: python:3.9
 
   - label: "rebuild matrix.org"
     trigger: "matrix-dot-org"
     async: true
     branches: "master"
+
+  - label: ":books: Build the spec"
+    command:
+      # Install package dependencies
+      - apk add nodejs npm git hugo
+      # Install the node dependencies necessary to build the spec
+      - npm i
+      # Pull all git submodules, required for the hugo theme
+      - git submodule update --init --recursive
+      # Pull current proposal information
+      - npm run get-proposals
+      # Build the spec, will build to './spec'
+      # Set the baseURL as we're deploying to https://spec.matrix.org/unstable
+      - hugo --baseURL "/unstable" -d "spec"
+      # Compress the result and make it available as an artifact
+      - tar -czf spec.tar.gz spec
+    artifact_paths:
+      - spec.tar.gz
+    plugins:
+      - docker#v3.7.0:
+          image: alpine

.circleci/config.yml

@@ -1,13 +1,19 @@
 gendoc: &gendoc
   name: Generate the docs
+  # Note: Node dependencies are required for the hugo build.
+  # Note: We use a custom config file for circleci due to some specifics with hosting the
+  #       site using CircleCI's artifacts platform. See config-circleci.toml for details.
   command: |
-    source /env/bin/activate
-    scripts/gendoc.py
+    apk add nodejs npm git hugo
+    npm i
+    cat config-circleci.toml config.toml > hugo-config.toml
+    hugo --config hugo-config.toml --baseURL "/${CIRCLE_NODE_INDEX}/public"
 
 genswagger: &genswagger
-  name: Generate the swagger
+  name: Validate sources and generate swagger json
   command: |
     source /env/bin/activate
+    scripts/check-swagger-sources.py
     scripts/dump-swagger.py
 
 buildswaggerui: &buildswaggerui
@@ -27,36 +33,21 @@ checkexamples: &checkexamples
   name: Check Event Examples
   command: |
     source /env/bin/activate
-    cd event-schemas
-    ./check_examples.py
-    cd ../api
-    ./check_examples.py
-
-genmatrixassets: &genmatrixassets
-  name: Generate/Verify matrix.org assets
-  command: |
-    source /env/bin/activate
-    ./scripts/generate-matrix-org-assets
+    scripts/check-event-schema-examples.py
 
 validateapi: &validateapi
   name: Validate OpenAPI specifications
   command: |
-    cd api
+    cd scripts
     npm install
-    node validator.js -s "client-server"
+    node validator.js -s "../data/api/client-server"
 
 buildspeculator: &buildspeculator
   name: Build Speculator
   command: |
     cd scripts/speculator
     go build -v
 
-buildcontinuserv: &buildcontinuserv
-  name: Build Continuserv
-  command: |
-    cd scripts/continuserv
-    go build -v
-
 version: 2
 jobs:
   validate-docs:
@@ -71,18 +62,21 @@ jobs:
     steps:
       - checkout
       - run: *checkexamples
-      - run: *genmatrixassets # We don't actually use the assets, but we do want to make sure they build
   build-docs:
     docker:
-      - image: uhoreg/matrix-doc-build
+      - image: alpine
     steps:
+      # Note: We install git in the image so we can pull git submodules. The hugo theme in use
+      # is a git submodule, which has its own submodules, and all need to be loaded.
+      - run: apk add git
       - checkout
+      - run: git submodule update --init --recursive
       - run: *gendoc
       - store_artifacts:
-          path: scripts/gen
+          path: public
       - run:
           name: "Doc build is available at:"
-          command: DOCS_URL="${CIRCLE_BUILD_URL}/artifacts/${CIRCLE_NODE_INDEX}/${CIRCLE_WORKING_DIRECTORY/#\~/$HOME}/scripts/gen/index.html"; echo $DOCS_URL
+          command: DOCS_URL="${CIRCLE_BUILD_URL}/artifacts/${CIRCLE_NODE_INDEX}/public/index.html"; echo $DOCS_URL
   build-swagger:
     docker:
       - image: uhoreg/matrix-doc-build
@@ -104,8 +98,6 @@ jobs:
           name: Install Dependencies
           command: |
             go get -v github.com/hashicorp/golang-lru
-            go get -v gopkg.in/fsnotify/fsnotify.v1
-      - run: *buildcontinuserv
       - run: *buildspeculator
 
 workflows:

.github/ISSUE_TEMPLATE/clarification.md

@@ -0,0 +1,13 @@
+---
+name: Clarity problem
+about: Report an area of the spec that is unclear.
+title: ''
+labels: 'clarification'
+assignees: ''
+
+---
+
+**Link to problem area**:
+
+**Issue**
+What is wrong? How can we improve?

.github/ISSUE_TEMPLATE/config.yaml

@@ -0,0 +1,8 @@
+blank_issues_enabled: true
+contact_links:
+ - name: Matrix Spec Discussion
+   url: "https://matrix.to/#/#matrix-spec:matrix.org"
+   about: Questions about the spec and proposal process can be asked here.
+ - name: Matrix Security Policy
+   url: https://www.matrix.org/security-disclosure-policy/
+   about: Learn more about our security disclosure policy.

.github/ISSUE_TEMPLATE/cosmetic-bug.md

@@ -0,0 +1,13 @@
+---
+name: Cosmetic issue
+about: Report an issue with how the spec looks.
+title: ''
+labels: 'aesthetic'
+assignees: ''
+
+---
+
+**Link to problem area**:
+
+**Issue**
+What is wrong? What can we do to improve?

.github/ISSUE_TEMPLATE/idea.md

@@ -0,0 +1,12 @@
+---
+name: Spec idea
+about: Suggest a future MSC idea.
+title: ''
+labels: 'improvement'
+assignees: ''
+
+---
+
+**Suggestion**
+What would you like to see in Matrix? If your idea is vaguely complete enough, we
+recommend submitting [an MSC](https://matrix.org/docs/spec/proposals) instead.

.github/ISSUE_TEMPLATE/spec-bug.md

@@ -0,0 +1,16 @@
+---
+name: Documentation error
+about: Report an issue with the spec itself (incorrect text).
+title: ''
+labels: 'spec-bug'
+assignees: ''
+
+---
+
+**Link to problem area**:
+
+**Issue**
+What is wrong?
+
+**Expected behaviour**
+How can the issue be fixed? Links to implementations/documents which prove the spec is wrong are appreciated.

.gitignore

@@ -1,15 +1,21 @@
 /api/node_modules
 /assets
 /assets.tar.gz
+/data/msc
 /env*
+/node_modules
+/resources
 /scripts/gen
 /scripts/continuserv/continuserv
 /scripts/speculator/speculator
 /scripts/swagger
 /scripts/tmp
 /templating/out
+/hugo-config.toml
+/public
 *.pyc
 *.swp
 _rendered.rst
 /.vscode/
 /.idea/
+/spec/

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "themes/docsy"]
+	path = themes/docsy
+	url = https://github.com/matrix-org/docsy.git
+	branch = master

CONTRIBUTING.rst

@@ -29,9 +29,7 @@ some time to complete.
 Changes to the protocol (new endpoints, ideas, etc) need to go through the
 `proposals process <https://matrix.org/docs/spec/proposals>`_. Other changes,
 such as fixing bugs, typos, or clarifying existing behaviour do not need a proposal.
-If you're not sure, visit us at `#matrix-spec:matrix.org`_
-and ask.
-
+If you're not sure, visit us at `#matrix-spec:matrix.org`_ and ask.
 
 Other changes
 ~~~~~~~~~~~~~
@@ -64,12 +62,17 @@ following:
   to fix. On the other hand, introducing new behaviour is best represented by a
   proposal.
 
+* Design or aesthetic changes, such as improving accessibility, colour schemes,
+  etc. Please check in with us at `#matrix-docs:matrix.org`_ with your proposed
+  design change before opening a PR so we can work with you on it.
+
 For such changes, please do just open a `pull request`_. If you're not sure if
 your change is covered by the above, please visit `#matrix-spec:matrix.org` and
 ask.
 
 .. _`pull request`: https://help.github.com/articles/about-pull-requests
 .. _`#matrix-spec:matrix.org`: https://matrix.to/#/#matrix-spec:matrix.org
+.. _`#matrix-docs:matrix.org`: https://matrix.to/#/#matrix-docs:matrix.org
 
 
 Adding to the changelog
@@ -86,8 +89,8 @@ To create a changelog entry, create a file named in the format ``prNumber.type``
 the ``newsfragments`` directory. The ``type`` can be one of the following:
 
 * ``new`` - Used when adding new endpoints. Please have the file contents be the
-  method and route being added, surrounded in RST code tags. For example: ``POST
-  /accounts/whoami``
+  method and route being added, surrounded in markdown code tags. For example: \`POST
+  /accounts/whoami\`.
 
 * ``feature`` - Used when adding backwards-compatible changes to the API.
 
@@ -100,8 +103,7 @@ the ``newsfragments`` directory. The ``type`` can be one of the following:
 
 All news fragments must have a brief summary explaining the change in the
 contents of the file. The summary must end in a full stop to be in line with
-the style guide and and formatting must be done using `Restructured Text
-<http://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html>`_.
+the style guide and formatting must be done using Markdown.
 
 Changes that do not change the spec, such as changes to the build script, formatting,
 CSS, etc should not get a news fragment.

README.md

@@ -0,0 +1,96 @@
+# Matrix Specification
+
+This repository contains the Matrix Specification, rendered at [spec.matrix.org](http://spec.matrix.org/).
+
+Developers looking to use Matrix should join [#matrix-dev:matrix.org](https://matrix.to/#/#matrix-dev:matrix.org)
+on Matrix for help.
+
+Spec authors and proposal writers are welcome to join [#matrix-spec:matrix.org](https://matrix.to/#/#matrix-spec:matrix.org).
+We welcome contributions! See [CONTRIBUTING.rst](./CONTRIBUTING.rst) for details.
+
+## Structure
+
+The Matrix spec is compiled with [Hugo](https://gohugo.io/) (a static site generator) with the following structure:
+
+* `/assets`: assets that need postprocessing using [Hugo Pipes](https://gohugo.io/hugo-pipes/introduction/).
+  For example, Sass files would go here.
+
+* `/content`: files that will become pages in the site go here. Typically these are Markdown files with some YAML front
+  matter indicating, [among other things](https://gohugo.io/content-management/front-matter/), what layout should be
+  applied to this page. The organization of files under `/content` determines the organization of pages in the built
+  site.
+
+* `/data`: this can contain TOML, YAML, or JSON files. Files kept here are directly available to template code as
+  [data objects](https://gohugo.io/templates/data-templates/), so templates don't need to load them from a file and
+  parse them. This is also where our Swagger/OpenAPI definitions and schemas are.
+
+* `/layouts`: this contains [Hugo templates](https://gohugo.io/templates/). Some templates define the overall layout of
+  a page: for example, whether it has header, footer, sidebar, and so on.
+    * `/layouts/partials`: these templates can be called from other templates, so they can be used to factor out
+      template code that's used in more than one template. An obvious example here is something like a sidebar, where
+      several different page layouts might all include the sidebar. But also, partial templates can return values: this
+      means they can be used like functions, that can be called by multiple templates to do some common processing.
+    * `/layouts/shortcodes`: these templates can be called directly from files in `/content`.
+
+* `/static`: static files which don't need preprocessing. JS or CSS files could live here.
+
+* `/themes`: you can use just Hugo or use it with a theme. Themes primarily provide additional templates, which are
+  supplied in a `/themes/$theme_name/layouts` directory. You can use a theme but customise it by providing your own
+  versions of any of the theme layouts in the base `/layouts` directory. That is, if a theme provides
+  `/themes/$theme_name/layouts/sidebar.html` and you provide `/layouts/sidebar.html`, then your version of the
+  template will be used.
+
+It also has the following top-level file:
+
+* `config.toml`: site-wide configuration settings. Some of these are built-in and you can add your own. Config settings
+  defined here are available in templates. All these directories above are configurable via `config.toml` settings.
+
+Additionally, the following directories may be of interest:
+
+* `/attic`: Here contains historical sections of specification and legacy drafts for the specification.
+* `/changelogs`: Various bits of changelog for the specification areas.
+* `/data-definitions`: Bits of structured data consumable by Matrix implementations.
+* `/meta`: Documentation relating to the spec's processes that are otherwise untracked (release instructions, etc).
+* `/scripts`: Various scripts for generating the spec and validating its contents.
+* `/proposals`: Matrix Spec Change (MSC) proposals. See <https://spec.matrix.org/unstable/proposals/>.
+
+## Authoring changes to the spec
+
+Please read [CONTRIBUTING.rst](./CONTRIBUTING.rst) before authoring a change to the spec. Note that spec authoring takes
+place after an MSC has been accepted, not as part of a proposal itself.
+
+1. Install the extended version (often the OS default) of Hugo: <https://gohugo.io/getting-started/installing>
+2. Run `git submodule update --init --recursive` for good measure.
+3. Run `npm i` to install the dependencies. Note that this will require NodeJS to be installed.
+4. Run `npm run get-proposals` to seed proposal data. This is merely for populating the content of the "Spec Change Proposals"
+   page and is not required.
+5. Run `hugo serve` to run a local webserver which builds whenever a file change is detected. If watching doesn't appear
+   to be working for you, try `hugo serve --disableFastRender` instead.
+6. Edit the specification 🙂
+
+We use a highly customized [Docsy](https://www.docsy.dev/) theme for our generated site, which uses Bootstrap and Font
+Awesome. If you're looking at making design-related changes to the spec site, please coordinate with us in
+[#matrix-docs:matrix.org](https://matrix.to/#/#matrix-docs:matrix.org) before opening a PR.
+
+## Building the specification
+
+If for some reason you're not a CI/CD system and want to render a static version of the spec for yourself, follow the above
+steps for authoring changes to the specification and instead of `hugo serve` run `hugo -d "spec"` - this will generate the
+spec to `/spec`. If you'd like to serve the spec off a path instead of a domain root (eg: `/unstable`), add `--baseURL "/unstable"`
+to the `hugo -d "spec"` command.
+
+For building the swagger definitions, create a python3 virtualenv and activate it. Then run `pip install -r ./scripts/requirements.txt`
+and finally `python ./scripts/dump-swagger.py` to generate it to `./scripts/swagger/api-docs.json`. To make use of the generated file,
+there are a number of options:
+
+* It can be uploaded from your filesystem to an online editor/viewer such as [on the swagger website](http://editor.swagger.io/).
+* You can run a local HTTP server by running `./scripts/swagger-http-server.py`, and then view the documentation via an
+  online viewer; for example, at <http://petstore.swagger.io/?url=http://localhost:8000/api-docs.json>.
+* You can host the swagger UI yourself. See <https://github.com/swagger-api/swagger-ui#how-to-run> for advice on how to
+  do so.
+
+## Issue tracking
+
+Specification issues are tracked on github at <https://github.com/matrix-org/matrix-doc/issues>.
+
+See [meta/github-labels.rst](./meta/github-labels.rst) for information on what the labels mean.