Update this branch because python 3.9 was released for real a while ago

Author: benclifford

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,16 @@
+# Description
+
+Please include a summary of the change and (optionally) which issue is fixed. Please also include
+relevant motivation and context.
+
+Fixes # (issue)
+
+## Type of change
+
+Choose which options apply, and delete the ones which do not apply.
+
+- Bug fix (non-breaking change that fixes an issue)
+- New feature (non-breaking change that adds functionality)
+- Breaking change (fix or feature that would cause existing functionality to not work as expected)
+- Documentation update
+- Code maintentance/cleanup

.travis.yml

@@ -1,13 +1,15 @@
 language: python
 python:
-  - "3.9-dev"
+  - "3.6"
+  - "3.7"
+  - "3.8"
+  - "3.9"
 
 env:
   global:
     - HYDRA_LAUNCHER=fork
     - OMPI_MCA_rmaps_base_oversubscribe=yes
     - MPI=mpich
-    - CCTOOLS_VERSION=7.0.11
 
 before_install:
     - sudo apt-get update -q
@@ -28,37 +30,50 @@ os:
 
 # command to run tests
 script:
-    - make flake8
-    - make lint
-
-    # This uses all of the configurations and tests as the base from which to
-    # run mypy checks - these are likely to capture most of the code used in
-    # parsl
-    - make mypy
-
       # do this before any testing, but not in-between tests
     - make clean_coverage
 
     # run the make targets for the two supported targets.
     - make test
 
+    # documentation checks:
+
+    - pip install .[docs]
+    - sudo apt-get install -y pandoc
+    - cd docs
+
+    # check we can build the docs without warnings
+    - make SPHINXOPTS=-W html
+
     # check that documentation stubs are up to date, as they are compiled from
     # python code but stored in version control rather than generated as part
-    # of doc build. prereqs for this will have been installed because of above
-    # pip install .[all] step
-    - cd docs
+    # of doc build.
     - rm -rfv stubs
     - sphinx-autogen reference.rst userguide/*rst devguide/*rst
     # this will both display any diffs in log output, and fail if there is any diff
     - git diff --exit-code
+
     - cd ..
 
     # assert that none of the runs in this test have put an ERROR message into a
-    # database manager log file or monitoring hub log file. It would be better if
+    # database manager log file or monitoring router log file. It would be better if
     # the tests themselves failed immediately when there was a monitoring error, but
     # in the absence of that, this is a dirty way to check.
     - bash -c '! grep ERROR runinfo*/*/database_manager.log'
-    - bash -c '! grep ERROR runinfo*/*/hub.log'
+    - bash -c '! grep ERROR runinfo*/*/monitoring_router.log'
+
+    # Check that parsl-visualize does not give errors.
+    # 'monitoring.db' needs to exist and have some interesting content in it
+    # which at time of writing comes from the htex_local_alternative test
+    # run.
+    - sudo apt-get install -y graphviz
+    - pip install .[monitoring]
+    - parsl-visualize &
+    # now wait for this to become responsive to connections
+    - wget http://127.0.0.1:8080/ --retry-connrefused --tries 30 --waitretry=1
+    # wget will return a failure code if any of the requested URLs don't return an HTTP 200 result
+    - wget http://127.0.0.1:8080/ --recursive --no-verbose --page-requisites --level=inf -e robots=off
+    - killall --wait parsl-visualize
 
     # check that 'all' install target works, even though we aren't doing any further
     # testing of what is installed

CoC.md renamed to CODE_OF_CONDUCT.md

@@ -3,7 +3,10 @@ Where to start
 
 We welcome contributions of any type (e.g., bug fixes, new features, reporting issues, documentation, etc).  If you're looking for a good place to get started you might like to peruse our current Git issues (those marked with `help wanted <https://github.com/Parsl/parsl/labels/help%20wanted>`_ are a good place to start).
 
-Please be aware of `Parsl's Code of Conduct<https://github.com/Parsl/parsl/blob/master/CoC.md>`_. 
+Please be aware of `Parsl's Code of Conduct <https://github.com/Parsl/parsl/blob/master/CODE_OF_CONDUCT.md>`_. 
+
+If you are not familiar with GitHub pull requests, the main mechanism to contribute changes to our code, there is `documentation available  <https://opensource.com/article/19/7/create-pull-request-github>`_.
+
 
 Coding conventions
 ------------------

CONTRIBUTING.rst

@@ -8,7 +8,7 @@ MPICH=mpich
 OPENMPI=openmpi
 EXECUTORS_PATH := $(shell ls -d parsl/executors/*/ | tr '\n' ':')
 export PATH := $(EXECUTORS_PATH):$(WORKQUEUE_INSTALL)/bin/:$(PATH)
-export CCTOOLS_VERSION=7.0.11
+export CCTOOLS_VERSION=7.1.11
 export HYDRA_LAUNCHER=fork
 export OMPI_MCA_rmaps_base_oversubscribe=yes
 MPI=$(MPICH)
@@ -24,9 +24,9 @@ virtualenv: ## create an activate a virtual env
 	echo "Run 'source $(VENV)/bin/activate' to activate the virtual environment"
 
 
-$(DEPS): virtualenv test-requirements.txt
+$(DEPS): test-requirements.txt requirements.txt
 	pip3 install --upgrade pip
-	pip3 install -r test-requirements.txt
+	pip3 install -r test-requirements.txt -r requirements.txt
 	touch $(DEPS)
 
 .PHONY: deps
@@ -52,42 +52,39 @@ mypy: ## run mypy checks
 	MYPYPATH=$(CWD)/mypy-stubs mypy parsl/app/ parsl/channels/ parsl/dataflow/ parsl/data_provider/ parsl/launchers parsl/providers/
 
 .PHONY: local_thread_test
-local_thread_test: $(DEPS) ## run all tests with local_thread config
-	pytest parsl -k "not cleannet" --config parsl/tests/configs/local_threads.py --cov=parsl --cov-append --cov-report= --random-order
+local_thread_test: ## run all tests with local_thread config
+	pytest parsl/tests/ -k "not cleannet" --config parsl/tests/configs/local_threads.py --cov=parsl --cov-append --cov-report= --random-order
 
 .PHONY: htex_local_test
-htex_local_test: $(DEPS) ## run all tests with htex_local config
-	PYTHONPATH=.  pytest parsl -k "not cleannet" --config parsl/tests/configs/htex_local.py --cov=parsl --cov-append --cov-report= --random-order
+htex_local_test: ## run all tests with htex_local config
+	PYTHONPATH=.  pytest parsl/tests/ -k "not cleannet" --config parsl/tests/configs/htex_local.py --cov=parsl --cov-append --cov-report= --random-order
 
 .PHONY: htex_local_alternate_test
-htex_local_alternate_test: $(DEPS) ## run all tests with htex_local config
-	echo "$(MPI)}"
-	parsl/executors/extreme_scale/install-mpi.sh $(MPI)
-	pip3 install ".[extreme_scale,monitoring]"
-	PYTHONPATH=.  pytest parsl -k "not cleannet" --config parsl/tests/configs/htex_local_alternate.py --cov=parsl --cov-append --cov-report= --random-order
+htex_local_alternate_test: ## run all tests with htex_local config
+	pip3 install ".[monitoring]"
+	PYTHONPATH=.  pytest parsl/tests/ -k "not cleannet" --config parsl/tests/configs/htex_local_alternate.py --cov=parsl --cov-append --cov-report= --random-order
 
 $(WORKQUEUE_INSTALL):
 	parsl/executors/workqueue/install-workqueue.sh
 
-
-work_queue_procs := $(shell ps aux | grep -E -e "[0-9]+:[0-9]+ work_queue_worker" | tr -s ' ' | cut -f 2 -d " ")
-work_queue_killcmd := $(if $(work_queue_procs), "kill" "-3" $(procs), "echo" "no work_queue_workers to running")
-
 .PHONY: workqueue_ex_test
-workqueue_ex_test: $(DEPS) $(WORKQUEUE_INSTALL)  ## run all tests with workqueue_ex config
-	pip3 install ".[extreme_scale]"
-	@$(work_queue_killcmd)
-	work_queue_worker localhost 9000  &> /dev/null &
-	PYTHONPATH=.:/tmp/cctools/lib/python3.5/site-packages  pytest parsl -k "not cleannet" --config parsl/tests/configs/workqueue_ex.py --cov=parsl --cov-append --cov-report= --random-order --bodge-dfk-per-test
-	@$(work_queue_killcmd)
+workqueue_ex_test: $(WORKQUEUE_INSTALL)  ## run all tests with workqueue_ex config
+	PYTHONPATH=.:/tmp/cctools/lib/python3.5/site-packages  pytest parsl/tests/ -k "not cleannet and not issue363" --config parsl/tests/configs/workqueue_ex.py --cov=parsl --cov-append --cov-report= --random-order
 
 .PHONY: config_local_test
-config_local_test: $(DEPS) ## run all tests with workqueue_ex config
+config_local_test: ## run all tests with workqueue_ex config
+	echo "$(MPI)"
+	parsl/executors/extreme_scale/install-mpi.sh $(MPI)
 	pip3 install ".[extreme_scale]"
-	PYTHONPATH=. pytest parsl -k "not cleannet" --config local --cov=parsl --cov-append --cov-report= --random-order
+	PYTHONPATH=. pytest parsl/tests/ -k "not cleannet" --config local --cov=parsl --cov-append --cov-report= --random-order
+
+.PHONY: site_test
+site_test:
+	pytest parsl/tests/ -k "not cleannet" ${SHARED_FS_OPTIONS} --config parsl/tests/site_tests/site_config_selector.py --cov=parsl --cov-append --cov-report= --random-order
+	pytest parsl/tests/site_tests/ ${SHARED_FS_OPTIONS} --config local
 
 .PHONY: test ## run all tests with all config types
-test: $(DEPS) clean_coverage lint flake8 local_thread_test htex_local_test htex_local_alternate_test workqueue_ex_test  config_local_test ## run all tests
+test: clean_coverage lint flake8 mypy local_thread_test htex_local_test htex_local_alternate_test workqueue_ex_test  config_local_test ## run all tests
 
 .PHONY: tag
 tag: ## create a tag in git. to run, do a 'make VERSION="version string" tag

Makefile

@@ -1,6 +1,6 @@
 Parsl - Parallel Scripting Library
 ==================================
-|licence| |build-status| |docs|
+|licence| |build-status| |docs| |NSF-1550588| |NSF-1550476| |NSF-1550562| |NSF-1550528|
 
 Parsl is a parallel programming library for Python. Parsl augments Python with simple, scalable, and flexible constructs for encoding parallelism. Developers annotate Python functions to specify opportunities for concurrent execution. These annotated functions, called apps, may represent pure Python functions or calls to external applications, whether sequential, multicore (e.g., CPU, GPU, accelerator), or multi-node MPI. Parsl further allows these calls to these apps, called tasks, to be connected by shared input/output data (e.g., Python objects or files) via which Parsl can construct a dynamic dependency graph of tasks.
 
@@ -15,7 +15,7 @@ Parsl can be used to implement various parallel computing paradigms:
 * Dynamic workflows in which the workflow is determined during execution
 * Interactive parallel programming through notebooks or another interactive mechanism
 
-The latest Parsl version available on PyPi is v0.9.0.
+The latest Parsl version available on PyPi is v1.0.0.
 
 .. |licence| image:: https://img.shields.io/badge/License-Apache%202.0-blue.svg
    :target: https://github.com/Parsl/parsl/blob/master/LICENSE
@@ -26,11 +26,23 @@ The latest Parsl version available on PyPi is v0.9.0.
 .. |docs| image:: https://readthedocs.org/projects/parsl/badge/?version=stable
    :target: http://parsl.readthedocs.io/en/stable/?badge=stable
    :alt: Documentation Status
-
+.. |NSF-1550588| image:: https://img.shields.io/badge/NSF-1550588-blue.svg
+   :target: https://nsf.gov/awardsearch/showAward?AWD_ID=1550588
+   :alt: NSF award info
+.. |NSF-1550476| image:: https://img.shields.io/badge/NSF-1550476-blue.svg
+   :target: https://nsf.gov/awardsearch/showAward?AWD_ID=1550476
+   :alt: NSF award info
+.. |NSF-1550562| image:: https://img.shields.io/badge/NSF-1550562-blue.svg
+   :target: https://nsf.gov/awardsearch/showAward?AWD_ID=1550562
+   :alt: NSF award info
+.. |NSF-1550528| image:: https://img.shields.io/badge/NSF-1550528-blue.svg
+   :target: https://nsf.gov/awardsearch/showAward?AWD_ID=1550528
+   :alt: NSF award info
+   
 QuickStart
 ==========
 
-Parsl is now available on PyPI, but first make sure you have Python3.5+ ::
+Parsl is now available on PyPI, but first make sure you have Python3.6+ ::
 
     $ python3 --version
 
@@ -42,7 +54,7 @@ To run the Parsl tutorial notebooks you will need to install Jupyter::
 
     $ pip3 install jupyter
 
-Detailed information about setting up Jupyter with Python3.5 is available `here <https://jupyter.readthedocs.io/en/latest/install.html>`_
+Detailed information about setting up Jupyter with Python is available `here <https://jupyter.readthedocs.io/en/latest/install.html>`_
 
 Note: Parsl uses an opt-in model to collect anonymous usage statistics for reporting and improvement purposes. To understand what stats are collected and enable collection please refer to the `usage tracking guide <http://parsl.readthedocs.io/en/stable/userguide/usage_tracking.html>`__
 
@@ -81,7 +93,7 @@ For Developers
 Requirements
 ============
 
-Parsl is supported in Python 3.5+. Requirements can be found `here <requirements.txt>`_. Requirements for running tests can be found `here <test-requirements.txt>`_.
+Parsl is supported in Python 3.6+. Requirements can be found `here <requirements.txt>`_. Requirements for running tests can be found `here <test-requirements.txt>`_.
 
 Code of Conduct
 ============

README.rst

@@ -65,7 +65,13 @@
             "givenName": "Ben",
             "familyName": "Glick",
             "email": "[email protected]"
-        }
+        },
+        {
+            "@id":"https://orcid.org/0000-0001-5147-0051",
+            "@type": "Person",
+            "givenName": "Kelly L.",
+            "familyName": "Rowland",
+            "email": "[email protected]"
     ],
     "softwareRequirements": [
         {
@@ -80,18 +86,6 @@
             },
             "runtimePlatform": "Python 3.6.4"
         },
-        {
-            "@type": "SoftwareApplication",
-            "identifier": "ipyparallel",
-            "name": "ipyparallel",
-            "provider": {
-                "@id": "https://pypi.org",
-                "@type": "Organization",
-                "name": "The Python Package Index",
-                "url": "https://pypi.org"
-            },
-            "runtimePlatform": "Python 3.6.4"
-        },
         {
             "@type": "SoftwareApplication",
             "identifier": "ipykernel",

codemeta.json

@@ -6,7 +6,7 @@ Documentation location
 
 Documentation is maintained in Python docstrings throughout the code. These are imported via the
 `autodoc <http://www.sphinx-doc.org/en/stable/ext/autodoc.html>`_ Sphinx extension in
-``docs/devguide/dev_docs.rst``. Individual stubs for user-facing classes (located in ``stubs``) are
+``docs/reference.rst``. Individual stubs for user-facing classes (located in ``stubs``) are
 generated automatically via sphinx-autogen.  Parsl modules, classes, and methods can be
 cross-referenced from a docstring by enclosing it in backticks (\`).
 
@@ -19,14 +19,14 @@ upon git commits.
 Local builds
 ------------
 
-To build the documentation locally, use
-::
+To build the documentation locally, use::
+
     $ make html
 
 Regenerate module stubs
 --------------------------
 
-If necessary, docstring stubs can be regenerated using
-::
+If necessary, docstring stubs can be regenerated using::
+
     $ sphinx-autogen reference.rst
 

docs/README.rst

@@ -40,9 +40,9 @@
     'sphinx.ext.napoleon'
 ]
 
-url = 'https://raw.githubusercontent.com/Parsl/parsl-tutorial/master/parsl-introduction.ipynb'
+url = 'https://raw.githubusercontent.com/Parsl/parsl-tutorial/master/1-parsl-introduction.ipynb'
 r = requests.get(url)
-with open(os.path.join(os.path.dirname(__file__), 'parsl-introduction.ipynb'), 'wb') as f:
+with open(os.path.join(os.path.dirname(__file__), '1-parsl-introduction.ipynb'), 'wb') as f:
     f.write(r.content)
 
 nbsphinx_execute = 'never'
@@ -76,8 +76,8 @@ def linkcode_resolve(domain, info):
 
 # General information about the project.
 project = 'Parsl'
-copyright = '2018, University of Chicago'
-author = 'The Parsl Project'
+copyright = '2018--2020, Parsl Project'
+author = 'Parsl Project'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -181,7 +181,7 @@ def linkcode_resolve(domain, info):
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = []
 
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied