Skip to content

Commit 71bcd2d

Browse files
mikemckiernanmikemckiernan
authored
Build docs with docs deps and make (#956)
* Build docs with docs deps and make * Update CI for publishing docs * Build docs only when docs change Signed-off-by: Mike McKiernan <[email protected]>
1 parent 715e88b commit 71bcd2d

15 files changed

+2772
-1878
lines changed

‎.github/workflows/docs-build.yaml

+147
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
name: docs-build
2+
3+
on:
4+
pull_request:
5+
branches: [develop]
6+
types: [opened, synchronize]
7+
paths:
8+
- "docs/**"
9+
- "**/*.md"
10+
- "**/*.rst"
11+
push:
12+
branches: [develop]
13+
tags:
14+
- docs-v*
15+
- v*
16+
workflow_dispatch:
17+
18+
defaults:
19+
run:
20+
shell: bash
21+
22+
env:
23+
PYTHON_VERSION: "3.10"
24+
POETRY_VERSION: "1.8.2"
25+
26+
concurrency:
27+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
28+
cancel-in-progress: true
29+
30+
jobs:
31+
build-docs:
32+
runs-on: ubuntu-latest
33+
steps:
34+
- name: Checkout
35+
uses: actions/checkout@v4
36+
- name: Set up Python ${{ env.PYTHON_VERSION }}
37+
uses: actions/setup-python@v5
38+
with:
39+
python-version: ${{ env.PYTHON_VERSION }}
40+
- name: Bootstrap poetry (Linux and macOS)
41+
run: |
42+
curl -sSL https://install.python-poetry.org | POETRY_VERSION=${{ env.POETRY_VERSION }} python -
43+
- name: Update PATH
44+
run: echo "$HOME/.local/bin" >> $GITHUB_PATH
45+
- name: Configure poetry
46+
run: poetry config virtualenvs.in-project true
47+
- name: Install dependencies
48+
run: poetry install --only docs
49+
- name: Build documentation
50+
run: make docs
51+
- name: Delete unnecessary files
52+
run: |
53+
sudo find _build -name .doctrees -prune -exec rm -rf {} \;
54+
sudo find _build -name .buildinfo -exec rm {} \;
55+
- name: Upload HTML
56+
uses: actions/upload-artifact@v4
57+
with:
58+
name: html-build-artifact
59+
path: _build/docs
60+
if-no-files-found: error
61+
retention-days: 1
62+
- name: Store PR information
63+
if: github.event_name == 'pull_request'
64+
run: |
65+
mkdir ./pr
66+
echo ${{ github.event.number }} > ./pr/pr.txt
67+
echo ${{ github.event.pull_request.merged }} > ./pr/merged.txt
68+
echo ${{ github.event.action }} > ./pr/action.txt
69+
- name: Upload PR information
70+
if: github.event_name == 'pull_request'
71+
uses: actions/upload-artifact@v4
72+
with:
73+
name: pr
74+
path: pr/
75+
76+
store-html:
77+
if: github.event_name == 'push'
78+
needs: [build-docs]
79+
runs-on: ubuntu-latest
80+
steps:
81+
- uses: actions/checkout@v4
82+
with:
83+
ref: "gh-pages"
84+
- name: Initialize Git configuration
85+
run: |
86+
git config user.name docs-build
87+
git config user.email [email protected]
88+
- name: Download artifacts
89+
uses: actions/download-artifact@v4
90+
with:
91+
name: html-build-artifact
92+
- name: Copy HTML directories
93+
run: |
94+
ls -asl
95+
for i in `ls -d *`
96+
do
97+
echo "Git adding ${i}"
98+
git add "${i}"
99+
done
100+
- name: Check or create dot-no-jekyll file
101+
run: |
102+
if [ -f ".nojekyll" ]; then
103+
echo "The dot-no-jekyll file already exists."
104+
exit 0
105+
fi
106+
touch .nojekyll
107+
git add .nojekyll
108+
- name: Check or create redirect page
109+
env:
110+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
111+
run: |
112+
resp=$(grep 'http-equiv="refresh"' index.html 2>/dev/null) || true
113+
if [ -n "${resp}" ]; then
114+
echo "The redirect file already exists."
115+
exit 0
116+
fi
117+
# If any of these commands fail, fail the build.
118+
def_branch=$(gh api "repos/${GITHUB_REPOSITORY}" --jq ".default_branch")
119+
html_url=$(gh api "repos/${GITHUB_REPOSITORY}/pages" --jq ".html_url")
120+
# Beware ugly quotation mark avoidance in the foll lines.
121+
echo '<!DOCTYPE html>' > index.html
122+
echo '<html>' >> index.html
123+
echo ' <head>' >> index.html
124+
echo ' <title>Redirect to documentation</title>' >> index.html
125+
echo ' <meta charset="utf-8">' >> index.html
126+
echo ' <meta http=equiv="refresh" content="3; URL='${html_url}${def_branch}'/index.html">' >> index.html
127+
echo ' <link rel="canonical" href="'${html_url}${def_branch}'/index.html">' >> index.html
128+
echo ' <script language="javascript">' >> index.html
129+
echo ' function redirect() {' >> index.html
130+
echo ' window.location.assign("'${html_url}${def_branch}'/index.html")' >> index.html
131+
echo ' }' >> index.html
132+
echo ' </script>' >> index.html
133+
echo ' </head>' >> index.html
134+
echo ' <body onload="redirect()">' >> index.html
135+
echo ' <p>Please follow the link to the <a href="'${html_url}${def_branch}'/index.html">' >> index.html
136+
echo ${def_branch}'</a> branch documentation.</p>' >> index.html
137+
echo ' </body>' >> index.html
138+
echo '</html>' >> index.html
139+
git add index.html
140+
- name: Commit changes to the GitHub Pages branch
141+
run: |
142+
git status
143+
if git commit -m 'Pushing changes to GitHub Pages.'; then
144+
git push -f
145+
else
146+
echo "Nothing changed."
147+
fi

‎.github/workflows/docs-preview-pr.yaml

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
name: docs-preview-pr
2+
3+
on:
4+
workflow_run:
5+
workflows: [docs-build]
6+
types: [completed]
7+
branches-ignore: [develop]
8+
9+
concurrency:
10+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
11+
cancel-in-progress: true
12+
13+
env:
14+
WF_ID: ${{ github.event.workflow_run.id }}
15+
16+
jobs:
17+
preview:
18+
uses: nvidia-merlin/.github/.github/workflows/docs-preview-pr-common.yaml@main

‎.github/workflows/docs-remove-stale-reviews.yaml

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
name: docs-remove-stale-reviews
2+
3+
on:
4+
schedule:
5+
# 42 minutes after 0:00 UTC on Sundays
6+
- cron: "42 0 * * 0"
7+
workflow_dispatch:
8+
9+
jobs:
10+
remove:
11+
uses: nvidia-merlin/.github/.github/workflows/docs-remove-stale-reviews-common.yaml@main

‎.gitlab-ci.yml

+63-59
Original file line numberDiff line numberDiff line change
@@ -6,89 +6,93 @@
66
# Note that environment variables can be set in several places
77
# See https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence
88
stages:
9-
- tests
10-
- build
11-
- docker-test
9+
- tests
10+
- build
11+
- docker-test
12+
- deploy
1213
sast:
13-
stage: tests
14+
stage: tests
1415
include:
15-
- template: Security/SAST.gitlab-ci.yml
16+
- template: Security/SAST.gitlab-ci.yml
17+
- project: "sw-gpu-doctools/docs-ci"
18+
ref: main
19+
file: "nemo-guardrails-toolkit/docs-ci.yaml"
1620

1721
variables:
18-
PIP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/pip"
19-
POETRY_VERSION: "1.8.2"
20-
IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
21-
LATEST_TAG: $CI_REGISTRY_IMAGE:latest
22+
PIP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/pip"
23+
POETRY_VERSION: "1.8.2"
24+
IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
25+
LATEST_TAG: $CI_REGISTRY_IMAGE:latest
2226

2327
cache:
24-
key: "${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}"
25-
paths:
26-
- .venv/
27-
- .cache/pip
28-
- poetry.lock
29-
- pyproject.toml
28+
key: "${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}"
29+
paths:
30+
- .venv/
31+
- .cache/pip
32+
- poetry.lock
33+
- pyproject.toml
3034

3135
# Jobs templates
3236

3337
.install-deps-template: &install-deps
34-
before_script:
35-
- pip install poetry==$POETRY_VERSION
36-
- poetry --version
37-
- poetry config virtualenvs.in-project true
38-
- poetry install --extras all --with dev
38+
before_script:
39+
- pip install poetry==$POETRY_VERSION
40+
- poetry --version
41+
- poetry config virtualenvs.in-project true
42+
- poetry install --extras all --with dev
3943

4044
.test-template: &test
41-
<<: *install-deps
42-
stage: tests
43-
coverage: /(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/
44-
script: make test
45+
<<: *install-deps
46+
stage: tests
47+
coverage: /(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/
48+
script: make test
4549

4650
# Tests jobs
4751

4852
python3.9:
49-
<<: *test
50-
image: python:3.9
53+
<<: *test
54+
image: python:3.9
5155

5256
python3.10:
53-
<<: *test
54-
image: python:3.10
57+
<<: *test
58+
image: python:3.10
5559

5660
python3.11:
57-
<<: *test
58-
image: python:3.11
61+
<<: *test
62+
image: python:3.11
5963

6064
# Build job
6165
build:
62-
stage: build
63-
image: docker:19.03.12
64-
services:
65-
- docker:19.03.12-dind
66-
variables:
67-
DOCKER_DRIVER: overlay2
68-
DOCKER_TLS_CERTDIR: "/certs"
69-
before_script:
70-
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
71-
script:
72-
- docker build -t $IMAGE_TAG -f ./qa/Dockerfile.qa .
73-
- docker push $IMAGE_TAG
74-
only:
75-
- tags
76-
tags:
77-
- gitlab-runner-bignlp-api
66+
stage: build
67+
image: docker:19.03.12
68+
services:
69+
- docker:19.03.12-dind
70+
variables:
71+
DOCKER_DRIVER: overlay2
72+
DOCKER_TLS_CERTDIR: "/certs"
73+
before_script:
74+
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
75+
script:
76+
- docker build -t $IMAGE_TAG -f ./qa/Dockerfile.qa .
77+
- docker push $IMAGE_TAG
78+
only:
79+
- tags
80+
tags:
81+
- gitlab-runner-bignlp-api
7882

7983
# Docker test job
8084
docker-test:
81-
stage: docker-test
82-
image: docker:19.03.12
83-
services:
84-
- docker:19.03.12-dind
85-
before_script:
86-
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
87-
script:
88-
- docker pull $IMAGE_TAG
89-
- docker run --rm $IMAGE_TAG pytest || (docker rmi $IMAGE_TAG && exit 1)
90-
only:
91-
- tags
85+
stage: docker-test
86+
image: docker:19.03.12
87+
services:
88+
- docker:19.03.12-dind
89+
before_script:
90+
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
91+
script:
92+
- docker pull $IMAGE_TAG
93+
- docker run --rm $IMAGE_TAG pytest || (docker rmi $IMAGE_TAG && exit 1)
94+
only:
95+
- tags
9296

93-
tags:
94-
- gitlab-runner-bignlp-api
97+
tags:
98+
- gitlab-runner-bignlp-api

‎CONTRIBUTING.md

+3-4
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ To get started quickly, follow the steps below.
135135
Ensure you have `poetry` installed:
136136

137137
```bash
138-
poetry -- version
138+
poetry --version
139139
```
140140

141141
6. Install the dev dependencies:
@@ -144,7 +144,8 @@ To get started quickly, follow the steps below.
144144
poetry install --with dev
145145
```
146146

147-
This will install pre-commit, pytest, and other development tools.
147+
The preceding command installs pre-commit, pytest, and other development tools.
148+
Specify `--with dev,docs` to add the dependencies for building the documentation.
148149

149150
7. If needed, you can install extra dependencies as below:
150151

@@ -159,7 +160,6 @@ To get started quickly, follow the steps below.
159160

160161
```bash
161162
poetry install --all-extras
162-
163163
```
164164

165165
> **Note**: `dev` is not part of the extras but it is an optional dependency group, so you need to install it as instructed above.
@@ -168,7 +168,6 @@ To get started quickly, follow the steps below.
168168

169169
```
170170
pre-commit install
171-
172171
```
173172
174173
This will ensure that the pre-commit checks, including Black, are run before each commit.

‎Makefile

+5-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
.PHONY: all test tests test_watch test_coverage test_profile pre_commit help
1+
.PHONY: all test tests test_watch test_coverage test_profile docs pre_commit help
22

33
# Default target executed when no specific target is provided to make.
44
all: help
@@ -21,6 +21,9 @@ test_coverage:
2121
test_profile:
2222
poetry run pytest -vv tests/ --profile-svg
2323

24+
docs:
25+
poetry run sphinx-build -b html docs _build/docs
26+
2427
pre_commit:
2528
pre-commit install
2629
pre-commit run --all-files
@@ -35,4 +38,5 @@ help:
3538
@echo 'test TEST_FILE=<test_file> - run all tests in given file'
3639
@echo 'test_watch - run unit tests in watch mode'
3740
@echo 'test_coverage - run unit tests with coverage'
41+
@echo 'docs - build docs, if you installed the docs dependencies'
3842
@echo 'pre_commit - run pre-commit hooks'

0 commit comments

Comments
 (0)