Don't use bazel for third-party python dependencies.

Author: ChrisCummins

.github/workflows/ci.yaml

@@ -58,10 +58,7 @@ jobs:
 
             - name: Install python dependencies
               run: |
-                  python3 -m pip install --upgrade pip
-                  # Install python dependencies that fail with bazel pip import.
-                  grep -E '(numpy|torch)' programl/requirements.txt | xargs python3 -m pip install
-                  python3 -m pip install setuptools
+                  python3 -m pip install -r requirements.txt
                   echo /home/runner/.local/bin >> $GITHUB_PATH
 
             - name: whoami

README.md

@@ -185,6 +185,12 @@ flow analysis labels.
   download and use the correct bazel version).
 * Python ≥ 3.6
 
+Install the python dependencies using:
+
+```
+$ python -m pip install -r requirements.txt
+```
+
 Once you have the above requirements installed, test that everything is working
 by building and running full test suite:
 

WORKSPACE

@@ -29,31 +29,6 @@ load("@com_github_chriscummins_rules_bats//:bats.bzl", "bats_deps")
 
 bats_deps()
 
-# === Python requirements ===
-
-load(
-    "@rules_python//python:pip.bzl",
-    "pip3_import",
-    "pip_repositories",
-)
-
-pip_repositories()
-
-# //programl:requirements.txt
-
-pip3_import(
-    name = "programl_requirements",
-    timeout = 3600,
-    requirements = "@programl//:programl/requirements.txt",
-)
-
-load(
-    "@programl_requirements//:requirements.bzl",
-    programl_pip_install = "pip_install",
-)
-
-programl_pip_install()
-
 # === Protocol buffers ===
 
 load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
@@ -73,33 +48,3 @@ load("@com_github_grpc_grpc//bazel:grpc_extra_deps.bzl", "grpc_extra_deps")
 grpc_extra_deps()
 
 # ----------------- End ProGraML dependencies -----------------
-
-# //tasks:requirements.txt
-
-pip3_import(
-    name = "tasks_requirements",
-    timeout = 3600,
-    requirements = "@programl//:tasks/requirements.txt",
-)
-
-load(
-    "@tasks_requirements//:requirements.bzl",
-    tasks_pip_install = "pip_install",
-)
-
-tasks_pip_install()
-
-# //tests:requirements.txt
-
-pip3_import(
-    name = "tests_requirements",
-    timeout = 3600,
-    requirements = "@programl//:tests/requirements.txt",
-)
-
-load(
-    "@tests_requirements//:requirements.bzl",
-    tests_pip_install = "pip_install",
-)
-
-tests_pip_install()

bin/BUILD

@@ -116,7 +116,6 @@ py_binary(
         "//programl/proto:programl_py",
         "//programl/util/py:init_app",
         "//programl/util/py:stdin_fmt",
-        "//third_party/py/absl",
     ],
 )
 
@@ -143,7 +142,6 @@ py_binary(
         "//programl/util/py:init_app",
         "//programl/util/py:stdin_fmt",
         "//programl/util/py:stdout_fmt",
-        "//third_party/py/absl",
     ],
 )
 

programl/graph/format/py/BUILD

@@ -31,7 +31,6 @@ py_library(
     deps = [
         "//programl/proto:programl_py",
         "//programl/util/py:runfiles_path",
-        "//third_party/py/absl",
     ],
 )
 
@@ -61,7 +60,6 @@ py_library(
     srcs = ["graph_tuple.py"],
     visibility = ["//visibility:public"],
     deps = [
-        "//third_party/py/numpy",
     ],
 )
 
@@ -72,7 +70,6 @@ py_library(
     deps = [
         ":graph_tuple",
         "//programl/proto:programl_py",
-        "//third_party/py/numpy",
     ],
 )
 
@@ -104,6 +101,5 @@ py_library(
     visibility = ["//visibility:public"],
     deps = [
         "//programl/proto:programl_py",
-        "//third_party/py/networkx",
     ],
 )

programl/ir/llvm/BUILD

@@ -54,9 +54,7 @@ py_library(
         "//programl/util/py:pbutil",
         "//programl/util/py:progress",
         "//programl/util/py:runfiles_path",
-        "//third_party/py/absl",
-        "//third_party/py/ncc/inst2vec:inst2vec_preprocess",
-        "//third_party/py/numpy",
+        "//third_party/ncc/inst2vec:inst2vec_preprocess",
     ],
 )
 

programl/ir/llvm/inst2vec_encoder.py

@@ -27,7 +27,7 @@
 from programl.proto import node_pb2, program_graph_pb2
 from programl.util.py import decorators, pbutil, progress
 from programl.util.py.runfiles_path import runfiles_path
-from third_party.py.ncc.inst2vec import inst2vec_preprocess
+from third_party.ncc.inst2vec import inst2vec_preprocess
 
 DICTIONARY = runfiles_path(
     "programl/programl/ir/llvm/internal/inst2vec_augmented_dictionary.pickle"

programl/ir/llvm/py/BUILD

@@ -29,7 +29,6 @@ py_library(
     deps = [
         "//programl/proto:programl_py",
         "//programl/util/py:runfiles_path",
-        "//third_party/py/absl",
     ],
 )
 

programl/ir/xla/py/BUILD

@@ -28,7 +28,6 @@ py_library(
     visibility = ["//visibility:public"],
     deps = [
         "//programl/proto:programl_py",
-        "//third_party/py/networkx",
         "//third_party/tensorflow:xla_py",
     ],
 )

programl/models/BUILD

@@ -23,7 +23,6 @@ py_library(
     deps = [
         ":base_batch_builder",
         "//programl/util/py:humanize",
-        "//third_party/py/absl",
     ],
 )
 
@@ -53,11 +52,6 @@ py_library(
     name = "batch_results",
     srcs = ["batch_results.py"],
     visibility = ["//visibility:public"],
-    deps = [
-        "//third_party/py/absl",
-        "//third_party/py/numpy",
-        "//third_party/py/scikit_learn",
-    ],
 )
 
 py_library(
@@ -80,7 +74,6 @@ py_library(
         "//programl/models:rolling_results_builder",
         "//programl/proto:programl_py",
         "//programl/util/py:progress",
-        "//third_party/py/absl",
     ],
 )
 
@@ -103,6 +96,5 @@ py_library(
         ":batch_data",
         ":batch_results",
         ":rolling_results",
-        "//third_party/py/tqdm",
     ],
 )

programl/models/ggnn/BUILD

@@ -20,7 +20,6 @@ py_library(
     name = "aux_readout",
     srcs = ["aux_readout.py"],
     deps = [
-        "//third_party/py/torch",
     ],
 )
 
@@ -42,9 +41,6 @@ py_library(
         "//programl/models:model",
         "//programl/proto:programl_py",
         "//programl/util/py:progress",
-        "//third_party/py/absl",
-        "//third_party/py/numpy",
-        "//third_party/py/torch",
     ],
 )
 
@@ -55,15 +51,13 @@ py_library(
     deps = [
         "//programl/graph/format/py:graph_tuple",
         "//programl/proto:programl_py",
-        "//third_party/py/numpy",
     ],
 )
 
 py_library(
     name = "ggnn_layer",
     srcs = ["ggnn_layer.py"],
     deps = [
-        "//third_party/py/torch",
     ],
 )
 
@@ -76,8 +70,6 @@ py_library(
         ":loss",
         ":metrics",
         ":node_embeddings",
-        "//third_party/py/absl",
-        "//third_party/py/torch",
     ],
 )
 
@@ -88,23 +80,20 @@ py_library(
         ":ggnn_layer",
         ":messaging_layer",
         ":readout",
-        "//third_party/py/torch",
     ],
 )
 
 py_library(
     name = "linear_net",
     srcs = ["linear_net.py"],
     deps = [
-        "//third_party/py/torch",
     ],
 )
 
 py_library(
     name = "loss",
     srcs = ["loss.py"],
     deps = [
-        "//third_party/py/torch",
     ],
 )
 
@@ -114,32 +103,27 @@ py_library(
     deps = [
         ":linear_net",
         ":position_embeddings",
-        "//third_party/py/torch",
     ],
 )
 
 py_library(
     name = "metrics",
     srcs = ["metrics.py"],
     deps = [
-        "//third_party/py/torch",
     ],
 )
 
 py_library(
     name = "node_embeddings",
     srcs = ["node_embeddings.py"],
     deps = [
-        "//third_party/py/numpy",
-        "//third_party/py/torch",
     ],
 )
 
 py_library(
     name = "position_embeddings",
     srcs = ["position_embeddings.py"],
     deps = [
-        "//third_party/py/torch",
     ],
 )
 
@@ -148,6 +132,5 @@ py_library(
     srcs = ["readout.py"],
     deps = [
         ":linear_net",
-        "//third_party/py/torch",
     ],
 )

programl/models/lstm/BUILD

@@ -26,8 +26,6 @@ py_library(
         "//programl/models:model",
         "//programl/proto:programl_py",
         "//programl/util/py:progress",
-        "//third_party/py/absl",
-        "//third_party/py/numpy",
     ],
 )
 

programl/util/py/BUILD

@@ -34,7 +34,6 @@ py_library(
     visibility = ["//visibility:public"],
     deps = [
         "//programl",
-        "//third_party/py/absl",
     ],
 )
 
@@ -43,7 +42,6 @@ py_library(
     srcs = ["pbutil.py"],
     visibility = ["//visibility:public"],
     deps = [
-        "//third_party/py/protobuf",
     ],
 )
 
@@ -53,8 +51,6 @@ py_library(
     visibility = ["//visibility:public"],
     deps = [
         ":humanize",
-        "//third_party/py/absl",
-        "//third_party/py/tqdm",
     ],
 )
 
@@ -73,8 +69,6 @@ py_library(
     visibility = ["//visibility:public"],
     deps = [
         "//programl/util/py:pbutil",
-        "//third_party/py/absl",
-        "//third_party/py/protobuf",
     ],
 )
 
@@ -83,8 +77,6 @@ py_library(
     srcs = ["stdout_fmt.py"],
     visibility = ["//visibility:public"],
     deps = [
-        "//third_party/py/absl",
-        "//third_party/py/protobuf",
     ],
 )