prepare repo for auto-formatters (#1546)

* prepare repo for aut-formatters

* fix CircleCI config

* reactivate lint jobs

* install libtinfo for linting c code

* disable git diff for required action

* fix config

* fix clang format

* fix lint failure behavior

* fix circleci consistency check

* try different failure messages

* change failure format

* fix config template

* use clang-format as pre-commit hook

* fix rev

* fix rev

* rename job from lint to format

* fix step names

* update template

* remove flake8-docstrings

* revert

* remove pydocstyle job

* remove pyupgrade as hook

* add usage explanation to contributing guide

* remove docstring job remnants

* add explanation of clang-format to contribution guide

* remove flake8 from unittest environments

* apply changes from auto formatters

* Change Black and usort version to match Meta's internal version

* Add formatting changes after changing black and usort version

* Add .clang-format to ensure clang changes align with Meta's internal workflow

* Add clang-format changes

* Merge main into branch

* Run pre-commit after merge

* Run clang-format after merge

* Fix regenerate.py merge conflict

* Fix merge issue

Co-authored-by: Abhinav Arora <[email protected]>

Author: pmeier

.circleci/config.yml

@@ -87,10 +87,62 @@ jobs:
     steps:
       - checkout
       - run:
+          name: Install check utilities
+          command: pip install --user --progress-bar=off jinja2 pyyaml
+      - run:
+          name: Check CircleCI config consistency
+          command: python .circleci/regenerate.py && git diff --quiet
+      - run:
+          when: on_fail
+          name: .circleci/config.yml not in sync with config.yml.in! Run '$ python .circleci/regenerate.py' to fix this.
+          command: exit 1
+
+  lint_python_and_config:
+    docker:
+      - image: circleci/python:3.7
+    steps:
+      - checkout
+      - run:
+          name: Install lint utilities
+          command: pip install --user --progress-bar=off pre-commit
+      - run:
+          name: Install pre-commit hooks
+          command: pre-commit install-hooks
+      - run:
+          name: Lint Python code and config files
+          command: pre-commit run --all-files
+      - run:
+          when: on_fail
+          name: Code format not compliant with the rules! Run '$ pre-commit run --all-files' to fix this.
+          command: exit 1
+
+  lint_c:
+    docker:
+      - image: circleci/python:3.7
+    steps:
+      - run:
+          name: Install additional system libraries
+          command: |
+            sudo apt update -qy
+            sudo apt install libtinfo5
+      - checkout
+      - run:
+          name: Install lint utilities
           command: |
-            pip install --user --progress-bar off jinja2 pyyaml
-            python .circleci/regenerate.py
-            git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1)
+            curl https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/clang-format-linux64 -o clang-format
+            chmod +x clang-format
+            ./clang-format --version
+      - run:
+          name: Lint C code
+          command: >
+            python run-clang-format.py
+            --recursive
+            --clang-format-executable=./clang-format
+            torchtext/csrc
+      - run:
+          when: on_fail
+          name: Code format not compliant with the rules! Run '$ python run-clang-format.py' to fix this.
+          command: exit 1
 
   binary_linux_wheel:
     <<: *binary_common
@@ -512,28 +564,14 @@ jobs:
             target=${tag:-main}
             ~/workspace/.circleci/build_docs/commit_docs.sh ~/workspace $target
 
-  docstring_parameters_sync:
-    <<: *binary_common
-    docker:
-      - image: continuumio/miniconda3
-    resource_class: medium
-    steps:
-      - attach_workspace:
-          at: ~/workspace
-      - designate_upload_channel
-      - checkout
-      - run:
-          name: Check parameters docstring sync
-          command: |
-            pip install --user pydocstyle
-            export PATH="$HOME/.local/bin:$PATH"
-            pydocstyle torchtext
-
-
 workflows:
-  build:
+  lint:
     jobs:
       - circleci_consistency
+      - lint_python_and_config
+      - lint_c
+  build:
+    jobs:
       {{ build_workflows() }}
   unittest:
     jobs:

.circleci/config.yml.in

@@ -14,48 +14,46 @@
 https://github.com/pytorch/vision/pull/1321#issuecomment-531033978
 """
 
+import os.path
+
 import jinja2
-from jinja2 import select_autoescape
 import yaml
-import os.path
+from jinja2 import select_autoescape
 
 
 PYTHON_VERSIONS = ["3.7", "3.8", "3.9"]
 
-DOC_VERSION = ('linux', '3.8')
+DOC_VERSION = ("linux", "3.8")
 
 
-def build_workflows(prefix='', upload=False, filter_branch=None, indentation=6):
+def build_workflows(prefix="", upload=False, filter_branch=None, indentation=6):
     w = []
     for btype in ["wheel", "conda"]:
         for os_type in ["linux", "macos", "windows"]:
             for python_version in PYTHON_VERSIONS:
                 fb = filter_branch
-                if not fb and (os_type == 'linux' and
-                               btype == 'wheel' and
-                               python_version == '3.8'):
+                if not fb and (os_type == "linux" and btype == "wheel" and python_version == "3.8"):
                     # the fields must match the build_docs "requires" dependency
-                    fb = '/.*/'
+                    fb = "/.*/"
                 w += build_workflow_pair(btype, os_type, python_version, fb, prefix, upload)
 
     if not filter_branch:
         # Build on every pull request, but upload only on nightly and tags
-        w += build_doc_job('/.*/')
-        w += upload_doc_job('nightly')
-        w += docstring_parameters_sync_job(None)
+        w += build_doc_job("/.*/")
+        w += upload_doc_job("nightly")
 
     return indent(indentation, w)
 
 
-def build_workflow_pair(btype, os_type, python_version, filter_branch, prefix='', upload=False):
+def build_workflow_pair(btype, os_type, python_version, filter_branch, prefix="", upload=False):
     w = []
     base_workflow_name = f"{prefix}binary_{os_type}_{btype}_py{python_version}"
     w.append(generate_base_workflow(base_workflow_name, python_version, filter_branch, os_type, btype))
 
     if upload:
         w.append(generate_upload_workflow(base_workflow_name, filter_branch, btype))
-        if filter_branch == 'nightly' and os_type in ['linux', 'windows']:
-            pydistro = 'pip' if btype == 'wheel' else 'conda'
+        if filter_branch == "nightly" and os_type in ["linux", "windows"]:
+            pydistro = "pip" if btype == "wheel" else "conda"
             w.append(generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version, os_type))
     return w
 
@@ -64,7 +62,9 @@ def build_doc_job(filter_branch):
     job = {
         "name": "build_docs",
         "python_version": "3.8",
-        "requires": ["binary_linux_wheel_py3.8", ],
+        "requires": [
+            "binary_linux_wheel_py3.8",
+        ],
     }
 
     if filter_branch:
@@ -77,26 +77,16 @@ def upload_doc_job(filter_branch):
         "name": "upload_docs",
         "context": "org-member",
         "python_version": "3.8",
-        "requires": ["build_docs", ],
+        "requires": [
+            "build_docs",
+        ],
     }
 
     if filter_branch:
         job["filters"] = gen_filter_branch_tree(filter_branch)
     return [{"upload_docs": job}]
 
 
-def docstring_parameters_sync_job(filter_branch):
-    job = {
-        "name": "docstring_parameters_sync",
-        "python_version": "3.8",
-        "requires": ["binary_linux_wheel_py3.8", ],
-    }
-
-    if filter_branch:
-        job["filters"] = gen_filter_branch_tree(filter_branch)
-    return [{"docstring_parameters_sync": job}]
-
-
 def generate_base_workflow(base_workflow_name, python_version, filter_branch, os_type, btype):
     d = {
         "name": base_workflow_name,
@@ -111,14 +101,12 @@ def generate_base_workflow(base_workflow_name, python_version, filter_branch, os
 
 def gen_filter_branch_tree(branch_name):
     return {
-        "branches": {
-            "only": branch_name
-        },
+        "branches": {"only": branch_name},
         "tags": {
             # Using a raw string here to avoid having to escape
             # anything
             "only": r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
-        }
+        },
     }
 
 
@@ -160,21 +148,16 @@ def indent(indentation, data_list):
 def unittest_workflows(indentation=6):
     w = []
     for os_type in ["linux", "windows"]:
-        for i, python_version in enumerate(PYTHON_VERSIONS):
-            w.append({
-                f"unittest_{os_type}": {
-                    "name": f"unittest_{os_type}_py{python_version}",
-                    "python_version": python_version,
-                }
-            })
-
-            if i == 0 and os_type == "linux":
-                w.append({
-                    "stylecheck": {
-                        "name": f"stylecheck_py{python_version}",
+        for python_version in PYTHON_VERSIONS:
+            w.append(
+                {
+                    f"unittest_{os_type}": {
+                        "name": f"unittest_{os_type}_py{python_version}",
                         "python_version": python_version,
                     }
-                })
+                }
+            )
+
     return indent(indentation, w)
 
 
@@ -183,12 +166,14 @@ def unittest_workflows(indentation=6):
     env = jinja2.Environment(
         loader=jinja2.FileSystemLoader(d),
         lstrip_blocks=True,
-        autoescape=select_autoescape(enabled_extensions=('html', 'xml')),
+        autoescape=select_autoescape(enabled_extensions=("html", "xml")),
     )
 
-    with open(os.path.join(d, 'config.yml'), 'w') as f:
-        f.write(env.get_template('config.yml.in').render(
-            build_workflows=build_workflows,
-            unittest_workflows=unittest_workflows,
-        ))
+    with open(os.path.join(d, "config.yml"), "w") as f:
+        f.write(
+            env.get_template("config.yml.in").render(
+                build_workflows=build_workflows,
+                unittest_workflows=unittest_workflows,
+            )
+        )
         f.write("\n")