mapper: utils: Passing hydra config to the child processes in serialized form with cloudpickle.

In Python 3.6, typing.List[str] cannot be pickled, therefore hydra
configuration cannot be passed with pickle. PEP 560 fixes the
issue in Python 3.7. As a workaround, we use more advanced serializtin
library cloudpickle.
See also: python/typing#511

Author: robkh

mocasin/mapper/utils.py

@@ -7,10 +7,12 @@
 import os
 from time import process_time
 
+import cloudpickle
 import csv
 import h5py
 import hydra
 import numpy as np
+import pickle
 
 from hydra.core.hydra_config import HydraConfig
 
@@ -162,9 +164,10 @@ def simulate(self, input_mappings):
         # As a workaround, suggested in
         # https://github.com/facebookresearch/hydra/issues/1005
         # we pass the hydra configuration to the child processes
-        config = None
+        cfg_pickled = None
         if HydraConfig.initialized():
             config = HydraConfig.get()
+            cfg_pickled = cloudpickle.dumps(config)
         for i, mapping in enumerate(mappings):
             # skip if this particular mapping is in the cache
             if lookups[i]:
@@ -174,7 +177,7 @@ def simulate(self, input_mappings):
                 self.platform, self.graph, mapping, self.trace
             )
 
-            simulations.append((simulation, config))
+            simulations.append((simulation, cfg_pickled))
         if self.parallel and len(simulations) > self.chunk_size:
             # since mappings are simulated in parallel, whole simulation time
             # is added later as offset
@@ -258,8 +261,9 @@ def run_simulation(arguments):
     # As a workaround, suggested in
     # https://github.com/facebookresearch/hydra/issues/1005
     # we pass the hydra configuration from the main process
-    simulation, config = arguments
-    if config:
+    simulation, cfg_pickled = arguments
+    if cfg_pickled:
+        config = pickle.loads(cfg_pickled)
         hydra.core.utils.configure_log(config.job_logging, config.verbose)
     with simulation:
         start_time = process_time()

setup.py

@@ -12,6 +12,7 @@
 
 install_requirements = [
     "arpeggio",
+    "cloudpickle",
     "cvxopt",
     "cvxpy!=1.1.8,<1.2" if sys.version_info < (3, 7) else "cvxpy!=1.1.8",
     "deap",