Add CI workflow to deploy a versioned MkDocs-based website to GitHub Pages

.github/workflows/publish-docs.yaml

@@ -0,0 +1,81 @@
+name: publish-docs
+
+on:
+  push:
+    branches:
+      # Branch to base "dev" website on. Set in build.py also.
+      - main
+      # release branches have names like 0.8.x, 0.9.x, ...
+      - "[0-9]+.[0-9]+.x"
+    paths:
+      - "docs/**"
+      - "docsgen/**"
+      - "cli/**"
+      - ".github/workflows/publish-docs.ya?ml"
+      - "Taskfile.ya?ml"
+      - "mkdocs.ya?ml"
+      - "poetry.lock"
+      - "pyproject.toml"
+      - "go.mod"
+      - "go.sum"
+  # Run on branch or tag creation (will be filtered by the publish-determination job)
+  create:
+
+jobs:
+  publish-determination:
+    runs-on: ubuntu-latest
+    outputs:
+      result: ${{ steps.determination.outputs.result }}
+    steps:
+      - name: Determine if documentation should be published on this workflow run
+        id: determination
+        run: |
+          RELEASE_BRANCH_REGEX="refs/heads/[0-9]+.[0-9]+.x"
+          if [[ "${{ github.event_name }}" == "push" || ( "${{ github.event_name }}" == "create" && "${{ github.ref }}" =~ $RELEASE_BRANCH_REGEX ) ]]; then
+            RESULT="true"
+          else
+            RESULT="false"
+          fi
+
+          echo "::set-output name=result::$RESULT"
+  publish:
+    runs-on: ubuntu-latest
+    needs: publish-determination
+    if: needs.publish-determination.outputs.result == 'true'
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Install Taskfile
+        uses: arduino/setup-task@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          version: 3.x
+
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: "1.16"
+
+      - name: Install Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.8"
+
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install poetry
+
+      - name: Install Python dependencies
+          poetry install --no-root
+
+      - name: Publish docs
+        # Determine docs version for the commit pushed and publish accordingly using Mike.
+        # Publishing implies creating a git commit on the gh-pages branch, we let @ArduinoBot own these commits.
+        run: |
+          git config --global user.email "[email protected]"
+          git config --global user.name "ArduinoBot"
+          git fetch --no-tags --prune --depth=1 origin +refs/heads/gh-pages:refs/remotes/origin/gh-pages
+          poetry run python docs/build/build.py

.github/workflows/validate-docs.yaml

@@ -22,23 +22,14 @@ jobs:
         uses: actions/setup-go@v2
 
       - name: Setup Python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v2
         with:
           python-version: "3.8"
-          architecture: "x64"
-
-      - name: Cache dependencies
-        uses: actions/cache@v2
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements_docs.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
 
       - name: Install Python dependencies
         run: |
-          python3 -m pip install --upgrade pip
-          python3 -m pip install -r ./requirements_docs.txt
+          python -m pip install --upgrade pip
+          python -m pip install poetry
 
       - name: Build docs website
         # Ensure the docs build is sane, these docs won't be published

.gitignore

@@ -8,3 +8,8 @@ __pycache__/
 
 # Misc.
 .DS_Store
+
+# Mkdocs
+/site/
+/docsgen/arduino-fwuploader
+/docs/commands/*.md

Taskfile.yml

@@ -4,6 +4,37 @@ includes:
   dist: ./DistTasks.yml
 
 tasks:
+  poetry:install-deps:
+    desc: Install dependencies managed by Poetry
+    cmds:
+      - poetry install --no-root
+
+  docs:serve:
+    desc: Run website locally
+    deps:
+      - task: docs:build
+    cmds:
+      - poetry run mkdocs serve
+
+  docs:publish:
+    desc: Use Mike to build and push versioned docs
+    deps:
+      - docs:gen:commands
+    cmds:
+      - poetry run mike deploy --update-aliases --push --remote {{.DOCS_REMOTE}} {{.DOCS_VERSION}} {{.DOCS_ALIAS}}
+
+  docs:gen:commands:
+    desc: Generate command reference files
+    dir: ./docsgen
+    cmds:
+      # docs will generate examples using os.Args[0] so we need to call
+      # the generator `arduino-fwuploader`
+      - go build -o {{.PROJECT_NAME}}{{exeExt}}
+      # we invoke `arduino-fwuploader` like this instead of `./arduino-fwuploader` to remove
+      # the `./` chars from the examples
+      - PATH=. {{.PROJECT_NAME}} ../docs/commands
+      - task: docs:format
+
   docs:check:
     desc: Run documentation linting
     cmds:
@@ -28,8 +59,11 @@ tasks:
 
   docs:build:
     desc: Build documentation website contents
+    deps:
+      - docs:gen:commands
+      - poetry:install-deps
     cmds:
-      - mkdocs build -s
+      - poetry run mkdocs build -s
 
   build:
     desc: Build the project
@@ -126,3 +160,6 @@ vars:
     sh: go list -f {{"{{"}}".Target{{"}}"}}" golang.org/x/lint/golint
   GOLINTFLAGS: "-min_confidence 0.8 -set_exit_status"
   PRETTIER: [email protected]
+  DOCS_VERSION: dev
+  DOCS_ALIAS: ""
+  DOCS_REMOTE: "origin"

docs/build/build.py

@@ -0,0 +1,109 @@
+# Source:
+# https://github.com/arduino/tooling-project-assets/blob/main/workflow-templates/assets/deploy-mkdocs-versioned/build/build.py
+
+# Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+
+# This software is released under the GNU General Public License version 3
+# The terms of this license can be found at:
+# https://www.gnu.org/licenses/gpl-3.0.en.html
+
+# You can be released from the requirements of the above licenses by purchasing
+# a commercial license. Buying such a license is mandatory if you want to
+# modify or otherwise use the software for commercial activities involving the
+# Arduino software without disclosing the source code of your own applications.
+# To purchase a commercial license, send an email to [email protected].
+import os
+import sys
+import re
+import subprocess
+
+import click
+from git import Repo
+
+# In order to provide support for multiple project releases, Documentation is versioned so that visitors can select
+# which version of the documentation website should be displayed. Unfortunately this feature isn't provided by GitHub
+# pages or MkDocs, so we had to implement it on top of the generation process.
+#
+# - A special version of the documentation called `dev` is provided to reflect the status of the project on the
+#   default branch - this includes unreleased features and bugfixes.
+# - Docs are versioned after the minor version of a release. For example, release version `0.99.1` and
+#   `0.99.2` will be both covered by documentation version `0.99`.
+#
+# The CI is responsible for guessing which version of the project we're building docs for, so that generated content
+# will be stored in the appropriate section of the documentation website. Because this guessing might be fairly complex,
+# the logic is implemented in this Python script. The script will determine the version of the project that was
+# modified in the current commit (either `dev` or an official, numbered release) and whether the redirect to the latest
+# version that happens on the landing page should be updated or not.
+
+
+DEV_BRANCHES = ["main"]  # Name of the branch used for the "dev" website source content
+
+
+def get_docs_version(ref_name, release_branches):
+    if ref_name in DEV_BRANCHES:
+        return "dev", ""
+
+    if ref_name in release_branches:
+        # if version is latest, add an alias
+        alias = "latest" if ref_name == release_branches[0] else ""
+        # strip `.x` suffix from the branch name to get the version: 0.3.x -> 0.3
+        return ref_name[:-2], alias
+
+    return None, None
+
+
+def get_rel_branch_names(blist):
+    """Get the names of the release branches, sorted from newest to older.
+    Only process remote refs so we're sure to get all of them and clean up the
+    name so that we have a list of strings like 0.6.x, 0.7.x, ...
+    """
+    pattern = re.compile(r"origin/(\d+\.\d+\.x)")
+    names = []
+    for b in blist:
+        res = pattern.search(b.name)
+        if res is not None:
+            names.append(res.group(1))
+
+    # Since sorting is stable, first sort by major...
+    names = sorted(names, key=lambda x: int(x.split(".")[0]), reverse=True)
+    # ...then by minor
+    return sorted(names, key=lambda x: int(x.split(".")[1]), reverse=True)
+
+
+@click.command()
+@click.option("--dry", is_flag=True)
+@click.option("--remote", default="origin", help="The git remote where to push.")
+def main(dry, remote):
+    # Detect repo root folder
+    here = os.path.dirname(os.path.realpath(__file__))
+    repo_dir = os.path.join(here, "..", "..")
+
+    # Get current repo
+    repo = Repo(repo_dir)
+
+    # Get the list of release branch names
+    rel_br_names = get_rel_branch_names(repo.refs)
+
+    # Deduce docs version from current branch. Use the 'latest' alias if
+    # version is the most recent
+    docs_version, alias = get_docs_version(repo.active_branch.name, rel_br_names)
+    if docs_version is None:
+        print(f"Can't get version from current branch '{repo.active_branch}', skip docs generation")
+        return 0
+
+    # Taskfile args aren't regular args so we put everything in one string
+    cmd = (f"task docs:publish DOCS_REMOTE={remote} DOCS_VERSION={docs_version} DOCS_ALIAS={alias}",)
+
+    if dry:
+        print(cmd)
+        return 0
+
+    subprocess.run(cmd, shell=True, check=True, cwd=repo_dir)
+
+
+# Usage:
+#     To run the script (must be run from within the repo tree):
+#         $python build.py
+#
+if __name__ == "__main__":
+    sys.exit(main())

docsgen/main.go

@@ -0,0 +1,41 @@
+/*
+  arduino-fwuploader
+  Copyright (c) 2021 Arduino LLC.  All right reserved.
+
+  This library is free software; you can redistribute it and/or
+  modify it under the terms of the GNU Lesser General Public
+  License as published by the Free Software Foundation; either
+  version 2.1 of the License, or (at your option) any later version.
+
+  This library is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  Lesser General Public License for more details.
+
+  You should have received a copy of the GNU Lesser General Public
+  License along with this library; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+*/
+
+package main
+
+import (
+	"log"
+	"os"
+
+	"github.com/arduino/arduino-fwuploader/cli"
+	"github.com/spf13/cobra/doc"
+)
+
+func main() {
+	if len(os.Args) < 2 {
+		log.Fatal("Please provide output folder")
+	}
+
+	cli := cli.NewCommand()
+	cli.DisableAutoGenTag = true // Disable addition of auto-generated date stamp
+	err := doc.GenMarkdownTree(cli, os.Args[1])
+	if err != nil {
+		log.Fatal(err)
+	}
+}

go.sum

@@ -62,6 +62,7 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/creack/goselect v0.1.1 h1:tiSSgKE1eJtxs1h/VgGQWuXUP0YS4CDIFMp6vaI1ls0=
 github.com/creack/goselect v0.1.1/go.mod h1:a/NhLweNvqIYMuxcMOuWY516Cimucms3DglDzQP3hKY=
@@ -253,6 +254,7 @@ github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 h1:mZHayPoR0lNmnH
 github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g=
@@ -262,6 +264,7 @@ github.com/segmentio/objconv v1.0.1/go.mod h1:auayaH5k3137Cl4SoXTgrzQcuQDmvuVtZg
 github.com/segmentio/stats/v4 v4.5.3/go.mod h1:LsaahUJR7iiSs8mnkvQvdQ/RLHAS5adGLxuntg0ydGo=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
+github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=

mkdocs.yml

@@ -57,3 +57,11 @@ markdown_extensions:
 # Navigation
 nav:
   - Documentation Home: README.md
+  - Command reference:
+      - arduino-fwuploader: commands/arduino-fwuploader.md
+      - certificates: commands/arduino-fwuploader_certificates.md
+      - certificates flash: commands/arduino-fwuploader_certificates_flash.md
+      - firmware: commands/arduino-fwuploader_firmware.md
+      - firmware flash: commands/arduino-fwuploader_firmware_flash.md
+      - firmware list: commands/arduino-fwuploader_firmware_list.md
+      - version: commands/arduino-fwuploader_version.md