diff --git a/.gitignore b/.gitignore index f860dfdb2..f506fcb59 100644 --- a/.gitignore +++ b/.gitignore @@ -144,7 +144,7 @@ venv.bak/ .ropeproject # mkdocs documentation -/site +/docs/site # mypy .mypy_cache/ diff --git a/CONTRIBUTION.md b/CONTRIBUTION.md new file mode 100644 index 000000000..e69de29bb diff --git a/README.md b/README.md index 6975e4cf1..fe677fd95 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,6 @@ DataJoint (https://datajoint.com). - [DataJoint Elements](https://datajoint.com/docs/elements/) - Catalog of example pipelines for neuroscience experiments - Contribute - - [Development Environment](https://datajoint.com/docs/core/datajoint-python/latest/develop/) + - [Contribution Guidelines](https://datajoint.com/docs/about/contribute/) - - [Guidelines](https://datajoint.com/docs/about/contribute/) + - [Developer Guide](https://datajoint.com/docs/core/datajoint-python/latest/develop/) diff --git a/docs/.docker/Dockerfile b/docs/.docker/Dockerfile deleted file mode 100644 index 7fadd8b83..000000000 --- a/docs/.docker/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM datajoint/miniconda3:4.10.3-py3.9-alpine -ARG PACKAGE -WORKDIR /main -COPY --chown=anaconda:anaconda ./docs/.docker/apk_requirements.txt ${APK_REQUIREMENTS} -COPY --chown=anaconda:anaconda ./docs/.docker/pip_requirements.txt ${PIP_REQUIREMENTS} -RUN \ - /entrypoint.sh echo "Dependencies installed" && \ - git config --global user.name "GitHub Action" && \ - git config --global user.email "action@github.com"&& \ - git config --global pull.rebase false && \ - git init -COPY --chown=anaconda:anaconda ./${PACKAGE} /main/${PACKAGE} -COPY --chown=anaconda:anaconda ./docs/mkdocs.yaml /main/docs/mkdocs.yaml -COPY --chown=anaconda:anaconda ./docs/src /main/docs/src -COPY --chown=anaconda:anaconda ./CHANGELOG.md /main/docs/src/changelog.md diff --git a/docs/.docker/apk_requirements.txt b/docs/.docker/apk_requirements.txt deleted file mode 100644 index 5664e303b..000000000 --- a/docs/.docker/apk_requirements.txt +++ /dev/null @@ -1 +0,0 @@ -git diff --git a/docs/.markdownlint.yaml b/docs/.markdownlint.yaml index a045a4962..7229b06e8 100644 --- a/docs/.markdownlint.yaml +++ b/docs/.markdownlint.yaml @@ -1,12 +1,17 @@ # https://github.com/DavidAnson/markdownlint # https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md +MD007: false # Unordered list indentation MD009: false # permit trailing spaces MD013: - line_length: "88" # Line length limits + # previously we defined line_length to 88 which is better for python + # but not for markdown + line_length: + - strict: false tables: false # disable for tables headings: false # disable for headings - code_blocks: false # disable for code blocks +MD029: false # Ordered list item prefix MD030: false # Number of spaces after a list +MD032: false # Lists should be surrounded by blank lines MD033: # HTML elements allowed allowed_elements: - "div" diff --git a/docs/Dockerfile b/docs/Dockerfile new file mode 100644 index 000000000..10b1a9a05 --- /dev/null +++ b/docs/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3 + +WORKDIR /main +COPY ./docs/pip_requirements.txt /main/docs/pip_requirements.txt +COPY ./datajoint /main/datajoint/ +COPY ./pyproject.toml /main/pyproject.toml + +RUN \ + # Install docs dependencies + pip install --no-cache-dir -r /main/docs/pip_requirements.txt && \ + # Install datajoint + pip install --no-cache-dir -e /main/ + +# Install dependencies first and use docker cache +# modify docs content won't cause image rebuild +COPY ./docs/ /main/docs/ diff --git a/docs/README.md b/docs/README.md index a7da95426..df42fe764 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,16 +1,97 @@ -# Docs Contributions +# Contribute to DataJoint Documentation -Docs contributors should be aware of the following extensions, with the corresponding -settings files, that were used in developing these docs: +This is the home for DataJoint software documentation as hosted at https://datajoint.com/docs/core/datajoint-python/latest/ -- [MarkdownLinter](https://github.com/DavidAnson/markdownlint): +## VSCode Linter Extensions and Settings + +The following extensions were used in developing these docs, with the corresponding +settings files: + +- Recommended extensions are already specified in `.vscode/extensions.json`, it will ask you to install them when you open the project if you haven't installed them. +- settings in `.vscode/settings.json` +- [MarkdownLinter](https://marketplace.visualstudio.com/items?itemName=DavidAnson.vscode-markdownlint): - `.markdownlint.yaml` establishes settings for various [linter rules](https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md) - `.vscode/settings.json` formatting on save to fix linting -- [CSpell](https://github.com/streetsidesoftware/vscode-spell-checker): `cspell.json` +- [CSpell](https://marketplace.visualstudio.com/items?itemName=streetsidesoftware.code-spell-checker): `cspell.json` has various ignored words. -- [ReWrap](https://github.com/stkb/Rewrap/): `.vscode/settings.json` allows toggling +- [ReWrap](https://marketplace.visualstudio.com/items?itemName=stkb.rewrap): `.vscode/settings.json` allows toggling automated hard wrapping for files at 88 characters. This can also be keymapped to be performed on individual paragraphs, see documentation. + +## With Virtual Environment + +conda +```bash +conda create -n djdocs -y +conda activate djdocs +``` +venv +```bash +python -m venv .venv +source .venv/bin/activate +``` + +Then install the required packages: +```bash +# go to the repo's root directory to generate API docs +# cd ~/datajoint-python/ + +# install mkdocs related requirements +pip install -r ./docs/pip_requirements.txt +# install datajoint, since API docs are generated from the package +pip install -e .[dev] +``` + +Run mkdocs at: http://127.0.0.1:8000/ +```bash +# go to the repo's root directory to generate API docs +# cd ~/datajoint-python/ + +# It will automatically reload the docs when changes are made +mkdocs serve --config-file ./docs/mkdocs.yaml +``` + +## With Docker + +> We mostly use Docker to simplify docs deployment + +Ensure you have `Docker` and `Docker Compose` installed. + +Then run the following: +```bash +# It will automatically reload the docs when changes are made +MODE="LIVE" docker compose up --build +``` + +Navigate to http://127.0.0.1:8000/ to preview the changes. + +This setup supports live-reloading so all that is needed is to save the markdown files +and/or `mkdocs.yaml` file to trigger a reload. + +## Mkdocs Warning Explanation + +> TL;DR: We need to do it this way for hosting, please keep it as is. + +```log +INFO - Doc file 'index.md' contains an unrecognized relative link './develop', it was left as is. Did you mean + 'develop.md'? +``` + +- We use reverse proxy to proxy our docs sites, here is the proxy flow: + - You hit `datajoint.com/*` on your browser + - It'll bring you to the reverse proxy server first, that you wouldn't notice + - when your URL ends with: + - `/` is the company page + - `/docs/` is the landing/navigation page hosted by datajoint/datajoint-docs's github pages + - `/docs/core/datajoint-python/` is the actual docs site hosted by datajoint/datajoint-python's github pages + - `/docs/elements/element-*/` is the actual docs site hosted by each element's github pages + + +```log +WARNING - Doc file 'query/operators.md' contains a link '../../../images/concepts-operators-restriction.png', but + the target '../../images/concepts-operators-restriction.png' is not found among documentation files. +``` +- We use Github Pages to host our docs, the image references needs to follow the mkdocs's build directory structure, under `site/` directory once you run mkdocs. \ No newline at end of file diff --git a/docs/docker-compose.yaml b/docs/docker-compose.yaml index ba0ff3373..ccb2bac79 100644 --- a/docs/docker-compose.yaml +++ b/docs/docker-compose.yaml @@ -1,36 +1,34 @@ -# MODE="LIVE|QA|BUILD" PACKAGE=datajoint UPSTREAM_REPO=https://github.com/datajoint/datajoint-python.git HOST_UID=$(id -u) docker compose -f docs/docker-compose.yaml up --build -version: "2.4" +# MODE="LIVE|QA|BUILD" PACKAGE=datajoint UPSTREAM_REPO=https://github.com/datajoint/datajoint-python.git docker compose up --build services: docs: build: - dockerfile: docs/.docker/Dockerfile - context: ../ - args: - - PACKAGE - image: ${PACKAGE}_python-docs + # some docs need to be dynamically generated from the datajoint PACKAGE + context: .. + dockerfile: docs/Dockerfile + image: datajoint-python-docs environment: - - PACKAGE - - UPSTREAM_REPO - - MODE + MODE: ${MODE:-LIVE} # specify mode: LIVE, QA, BUILD + # specify package to generate API docs + PACKAGE: ${PACKAGE:-datajoint} + UPSTREAM_REPO: ${UPSTREAM_REPO:-https://github.com/datajoint/datajoint-python.git} volumes: - ..:/main - user: ${HOST_UID}:anaconda ports: - - 80:80 + - 8000:8000 command: - - sh + - bash - -c - | set -e if echo "$${MODE}" | grep -i live &>/dev/null; then - mkdocs serve --config-file ./docs/mkdocs.yaml -a 0.0.0.0:80 + mkdocs serve --config-file /main/docs/mkdocs.yaml -a 0.0.0.0:8000 elif echo "$${MODE}" | grep -iE "qa|build" &>/dev/null; then git branch -D gh-pages || true git fetch $${UPSTREAM_REPO} gh-pages:gh-pages || true - mike deploy --config-file ./docs/mkdocs.yaml -u $$(grep -oE '\d+\.\d+' /main/$${PACKAGE}/version.py) latest - mike set-default --config-file ./docs/mkdocs.yaml latest + mike deploy --ignore-remote-status --config-file /main/docs/mkdocs.yaml -u $$(grep -oP '\d+\.\d+' /main/$${PACKAGE}/version.py) latest + # mike set-default --config-file /main/docs/mkdocs.yaml latest if echo "$${MODE}" | grep -i qa &>/dev/null; then - mike serve --config-file ./docs/mkdocs.yaml -a 0.0.0.0:80 + mike serve --config-file /main/docs/mkdocs.yaml -a 0.0.0.0:8000 fi else echo "Unexpected mode..." diff --git a/docs/mkdocs.yaml b/docs/mkdocs.yaml index ecd1ec6e2..4de4f58e1 100644 --- a/docs/mkdocs.yaml +++ b/docs/mkdocs.yaml @@ -139,8 +139,8 @@ markdown_extensions: - toc: permalink: true - pymdownx.emoji: - emoji_index: !!python/name:materialx.emoji.twemoji - emoji_generator: !!python/name:materialx.emoji.to_svg + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg options: custom_icons: - .overrides/.icons diff --git a/docs/.docker/pip_requirements.txt b/docs/pip_requirements.txt similarity index 87% rename from docs/.docker/pip_requirements.txt rename to docs/pip_requirements.txt index 111ecb946..057cf585d 100644 --- a/docs/.docker/pip_requirements.txt +++ b/docs/pip_requirements.txt @@ -1,4 +1,4 @@ -mkdocs-material==9.1.17 +mkdocs-material mkdocs-redirects mkdocstrings mkdocstrings-python diff --git a/docs/src/.overrides/partials/nav.html b/docs/src/.overrides/partials/nav.html index 019d79726..6329b6b23 100644 --- a/docs/src/.overrides/partials/nav.html +++ b/docs/src/.overrides/partials/nav.html @@ -1,33 +1,53 @@ + + +{% import "partials/nav-item.html" as item with context %} + + {% set class = "md-nav md-nav--primary" %} {% if "navigation.tabs" in features %} -{% set class = class ~ " md-nav--lifted" %} + {% set class = class ~ " md-nav--lifted" %} {% endif %} {% if "toc.integrate" in features %} -{% set class = class ~ " md-nav--integrated" %} + {% set class = class ~ " md-nav--integrated" %} {% endif %} - \ No newline at end of file diff --git a/docs/src/api/make_pages.py b/docs/src/api/make_pages.py index 72c1fc326..3072cb46a 100644 --- a/docs/src/api/make_pages.py +++ b/docs/src/api/make_pages.py @@ -5,7 +5,7 @@ import mkdocs_gen_files -package = os.getenv("PACKAGE") +package = os.getenv("PACKAGE", "datajoint") nav = mkdocs_gen_files.Nav() for path in sorted(Path(package).glob("**/*.py")): with mkdocs_gen_files.open(f"api/{path.with_suffix('')}.md", "w") as f: diff --git a/docs/src/citation.md b/docs/src/citation.md index 358fcf90c..b5eb2d88b 100644 --- a/docs/src/citation.md +++ b/docs/src/citation.md @@ -4,4 +4,4 @@ If your work uses the DataJoint for Python, please cite the following manuscript - Yatsenko D, Reimer J, Ecker AS, Walker EY, Sinz F, Berens P, Hoenselaar A, Cotton RJ, Siapas AS, Tolias AS. DataJoint: managing big scientific data using MATLAB or Python. bioRxiv. 2015 Jan 1:031658. doi: https://doi.org/10.1101/031658 -- DataJoint for Python - [RRID:SCR_014543](https://scicrunch.org/resolver/SCR_014543) - Version `Enter version here` +- DataJoint for Python - [RRID:SCR_014543](https://scicrunch.org/resolver/SCR_014543) - Version `Enter datajoint-python version you are using here` diff --git a/docs/src/develop.md b/docs/src/develop.md index 4c66d52de..bc636cc20 100644 --- a/docs/src/develop.md +++ b/docs/src/develop.md @@ -1,8 +1,66 @@ -# Develop +# Developer Guide -Included with the codebase is the recommended development environment configured using [DevContainer](https://containers.dev/). +## Table of Contents -## Launch Environment +- [Contribute to DataJoint Python Documentation](#contribute-to-datajoint-python-documentation) +- [Setup Development Environment](#setup-development-environment) + - [Prerequisites](#prerequisites) + - [With Virtual Environment](#with-virtual-environment) + - [With DevContainer](#with-devcontainer) + - [Extra Efficiency, Optional But Recommended](#extra-efficiency-optional-but-recommended) + - [Pre-commit Hooks](#pre-commit-hooks) + - [Integration Tests](#integration-tests) + - [VSCode](#vscode) + - [Jupyter Extension](#jupyter-extension) + - [Debugger](#debugger) + - [MySQL CLI](#mysql-cli) + +## Contribute to DataJoint Python Documentation + +> Contributions to documentations are equivalently important to any code for the community, please help us to resolve any confusions in documentations. + +[Here](https://github.com/datajoint/datajoint-python/blob/master/docs/README.md) is the instructions for contributing documentations, or you can find the same instructions at `$PROJECT_DIR/docs/README.md` in the repository. + +[Back to top](#table-of-contents) + +## Setup Development Environment + +> We have [DevContainer](https://containers.dev/) ready for contributors to develop without setting up their environment. If you are familiar with DevContainer, Docker or Github Codespace, this is the recommended development environment for you. +> If you have never used Docker, it might be easier for you to use a virtual environment through `conda/mamba/venv`, it is also very straightforward to set up. + +### Prerequisites + +- Clone datajoint-python repository + +```bash +# If you have your SSH key set up with GitHub, you can clone using SSH +git clone git@github.com:datajoint/datajoint-python.git +# Otherwise, you can clone using HTTPS +git clone https://github.com/datajoint/datajoint-python.git +``` +- If you don't use DevContainer, then either install Anaconda/[Miniconda](https://www.anaconda.com/docs/getting-started/miniconda/install)/Mamba, or just use Python's built-in `venv` module without install anything else. + +### With Virtual Environment + +```bash +# Check if you have Python 3.9 or higher, if not please upgrade +python --version +# Create a virtual environment with venv +python -m venv .venv +source .venv/bin/activate +pip install -e .[dev] + +# Or create a virtual environment with conda +conda create -n dj python=3.13 # any 3.9+ is fine +conda activate dj +pip install -e .[dev] +``` + +[Back to top](#table-of-contents) + +### With DevContainer + +#### Launch Environment Here are some options that provide a great developer experience: @@ -26,37 +84,48 @@ Here are some options that provide a great developer experience: - Issue the following command in the terminal to build and run the Docker container: `HOST_UID=$(id -u) PY_VER=3.11 DJ_VERSION=$(grep -oP '\d+\.\d+\.\d+' datajoint/version.py) docker compose --profile test run --rm -it djtest -- sh -c 'pip install -qe ".[dev]" && bash'` - Issue the following command in the terminal to stop the Docker compose stack: `docker compose --profile test down` -## Features +[Back to top](#table-of-contents) -Once you've successfully launched the development environment, you'll be able to take advantage of our developer tooling to help improve productivity and quality. +## Extra Efficiency, Optional But Recommended -### Syntax Tests +### Pre-commit Hooks -The following will verify that there are no syntax errors. +We recommend using [pre-commit](https://pre-commit.com/) to automatically run linters and formatters on your code before committing. +To set up pre-commit, run the following command in your terminal: +```bash +pip install pre-commit +pre-commit install ``` -flake8 datajoint --count --select=E9,F63,F7,F82 --show-source --statistics + +You can manually run pre-commit on all files with the following command: + +```bash +pre-commit run --all-files ``` +This will run all the linters and formatters specified in the `.pre-commit-config.yaml` file. If all check passed, you can commit your code. Otherwise, you need to fix the failed checks and run the command again. -### Integration Tests +> Pre-commit will automatically run the linters and formatters on all staged files before committing. However, if your code doesn't follow the linters and formatters, the commit will fail. +> Some hooks will automatically fix your problem, and add the fixed files as git's `unstaged` files, you just need to add them(`git add .`) to git's `staged` files and commit again. +> Some hooks will not automatically fix your problem, so you need to check the pre-commit failed log to fix them manually and include the update to your `staged` files and commit again. -The following will verify there are no regression errors by running our test suite of unit and integration tests. +If you really don't want to use pre-commit, or if you don't like it, you can uninstall it with the following command: -- Entire test suite: - ``` - pytest -sv --cov-report term-missing --cov=datajoint tests - ``` +```bash +pre-commit uninstall +``` -- A single functional test: - ``` - pytest -sv tests/test_connection.py::test_dj_conn - ``` -- A single class test: - ``` - pytest -sv tests/test_aggr_regressions.py::TestIssue558 - ``` +But when you issue a pull request, the same linter and formatter check will run against your contribution, you are going to have the same failure as well. So without pre-commit, you need to **manually run these linters and formatters before committing your code**: + +- Syntax tests + +The following will verify that there are no syntax errors. -### Style Tests +``` +flake8 datajoint --count --select=E9,F63,F7,F82 --show-source --statistics +``` + +- Style tests The following will verify that there are no code styling errors. @@ -67,22 +136,44 @@ flake8 --ignore=E203,E722,W503 datajoint --count --max-complexity=62 --max-line- The following will ensure the codebase has been formatted with [black](https://black.readthedocs.io/en/stable/). ``` -black datajoint --check -v +black datajoint --check -v --diff ``` The following will ensure the test suite has been formatted with [black](https://black.readthedocs.io/en/stable/). ``` -black tests --check -v +black tests --check -v --diff ``` -### Jupyter +[Back to top](#table-of-contents) + +### Integration Tests + +The following will verify there are no regression errors by running our test suite of unit and integration tests. + +- Entire test suite: + ``` + pytest -sv --cov-report term-missing --cov=datajoint tests + ``` + +- A single functional test: + ``` + pytest -sv tests/test_connection.py::test_dj_conn + ``` +- A single class test: + ``` + pytest -sv tests/test_aggr_regressions.py::TestIssue558 + ``` + +[Back to top](#table-of-contents) -Jupyter notebooks are supported in this environment. This means that when you `import datajoint`, it will use the current state of the source. +### VSCode -Be sure to see the reference documentation if you are new to [running Jupyter notebooks w/ VSCode](https://code.visualstudio.com/docs/datascience/jupyter-notebooks#_create-or-open-a-jupyter-notebook). +#### Jupyter Extension -### Debugger +Be sure to go through this documentation if you are new to [Running Jupyter Notebooks with VSCode](https://code.visualstudio.com/docs/datascience/jupyter-notebooks#_create-or-open-a-jupyter-notebook). + +#### Debugger [VSCode Debugger](https://code.visualstudio.com/docs/editor/debugging) is a powerful tool that can really accelerate fixes. @@ -94,8 +185,12 @@ Try it as follows: - Select the `Run and Debug` tab - Start by clicking the button `Run and Debug` +[Back to top](#table-of-contents) + ### MySQL CLI +> Installation instruction is in [here](https://dev.mysql.com/doc/mysql-shell/8.0/en/mysql-shell-install.html) + It is often useful in development to connect to DataJoint's relational database backend directly using the MySQL CLI. Connect as follows to the database running within your developer environment: @@ -104,23 +199,4 @@ Connect as follows to the database running within your developer environment: mysql -hdb -uroot -ppassword ``` -### Documentation - -Our documentation is built using [MkDocs Material](https://squidfunk.github.io/mkdocs-material/). The easiest way to improve the documentation is by using the `docs/docker-compose.yaml` environment. The source can be modified in `docs/src` using markdown. - -The docs environment can be run using 3 modes: - -- **LIVE**: (*recommended*) This serves the docs locally. It supports live reloading on saves to `docs/src` files but does not support the docs version dropdown. Useful to see changes live. - ``` - MODE="LIVE" PACKAGE=datajoint UPSTREAM_REPO=https://github.com/datajoint/datajoint-python.git HOST_UID=$(id -u) docker compose -f docs/docker-compose.yaml up --build - ``` -- **QA**: This serves the docs locally. It supports the docs version dropdown but does not support live reloading. Useful as a final check. - ``` - MODE="QA" PACKAGE=datajoint UPSTREAM_REPO=https://github.com/datajoint/datajoint-python.git HOST_UID=$(id -u) docker compose -f docs/docker-compose.yaml up --build - ``` -- **BUILD**: This compiles the docs. Most useful for the docs deployment automation. Other modes are more useful to new contributors. - ``` - MODE="BUILD" PACKAGE=datajoint UPSTREAM_REPO=https://github.com/datajoint/datajoint-python.git HOST_UID=$(id -u) docker compose -f docs/docker-compose.yaml up --build - ``` - -When the docs are served locally, use the VSCode `PORTS` tab (next to `TERMINAL`) to manage access to the forwarded ports. Docs are served on port `8080`. +[Back to top](#table-of-contents) \ No newline at end of file