Merge branch 'master' into posix_spawn-file_actions

Author: serhiy-storchaka

.gitattributes

@@ -37,3 +37,20 @@ Lib/test/test_importlib/data01/* -text
 *.proj text eol=crlf
 PCbuild/readme.txt text eol=crlf
 PC/readme.txt text eol=crlf
+
+# Generated files
+# https://github.com/github/linguist#generated-code
+Include/graminit.h          linguist-generated=true
+Python/graminit.h           linguist-generated=true
+Modules/clinic/*.h          linguist-generated=true
+Objects/clinic/*.h          linguist-generated=true
+PC/clinic/*.h               linguist-generated=true
+Python/clinic/*.h           linguist-generated=true
+Python/importlib.h          linguist-generated=true
+Python/importlib_external.h linguist-generated=true
+Include/Python-ast.h        linguist-generated=true
+Python/Python-ast.c         linguist-generated=true
+Include/opcode.h            linguist-generated=true
+Python/opcode_targets.h     linguist-generated=true
+Objects/typeslots.inc       linguist-generated=true
+Modules/unicodedata_db.h    linguist-generated=true

.github/CONTRIBUTING.rst

@@ -8,6 +8,10 @@ Build Status
 
   + `Stable buildbots <http://buildbot.python.org/3.x.stable/>`_
 
+- 3.7
+
+  + `Stable buildbots <http://buildbot.python.org/3.7.stable/>`_
+
 - 3.6
 
   + `Stable buildbots <http://buildbot.python.org/3.6.stable/>`_

.github/PULL_REQUEST_TEMPLATE.md

@@ -5,9 +5,11 @@ Please read this comment in its entirety. It's quite important.
 # Pull Request title
 
 It should be in the following format:
+
 ```
 bpo-NNNN: Summary of the changes made
 ```
+
 Where: bpo-NNNN refers to the issue number in the https://bugs.python.org.
 
 Most PRs will require an issue number. Trivial changes, like fixing a typo, do not need an issue.
@@ -16,9 +18,11 @@ Most PRs will require an issue number. Trivial changes, like fixing a typo, do n
 
 If this is a backport PR (PR made against branches other than `master`),
 please ensure that the PR title is in the following format:
+
 ```
 [X.Y] <title from the original PR> (GH-NNNN)
 ```
+
 Where: [X.Y] is the branch name, e.g. [3.6].
 
 GH-NNNN refers to the PR number from `master`.

.github/appveyor.yml

@@ -20,18 +20,19 @@ before_build:
         If (!$changes) {
           echo 'Only docs were updated, stopping build process.'
           Exit-AppveyorBuild
+        } else {
+          echo 'Doing full build due to non-doc changes in these files:'
+          echo $changes
         }
-        echo 'Doing full build due to non-doc changes in these files:'
-        echo $changes
       }
 
 
 build_script:
-  - cmd: PCbuild\build.bat -e
-  - cmd: PCbuild\win32\python.exe -m test.pythoninfo
+  - cmd: PCbuild\build.bat -e -p x64
+  - cmd: PCbuild\amd64\python.exe -m test.pythoninfo
 test_script:
-  - cmd: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 --fail-env-changed -j0
+  - cmd: PCbuild\rt.bat -x64 -q -uall -u-cpu -u-largefile -rwW --slowest --timeout=1200 --fail-env-changed -j0
 environment:
   HOST_PYTHON: C:\Python36\python.exe
 image:
-- Visual Studio 2017
+  - Visual Studio 2017

.travis.yml

@@ -12,7 +12,7 @@ cache:
 
 env:
   global:
-    - OPENSSL=1.1.0g
+    - OPENSSL=1.1.0h
     - OPENSSL_DIR="$HOME/multissl/openssl/${OPENSSL}"
     - PATH="${OPENSSL_DIR}/bin:$PATH"
     # Use -O3 because we don't use debugger on Travis-CI
@@ -40,6 +40,10 @@ matrix:
       # compiler here and the other to run the coverage build. Clang is preferred
       # in this instance for its better error messages.
       env: TESTING=cpython
+      addons:
+        apt:
+          packages:
+            - xvfb
     - os: linux
       language: python
       # Build the docs against a stable version of Python so code bugs don't hold up doc-related PRs.
@@ -60,25 +64,31 @@ matrix:
       env: OPTIONAL=true
       before_install:
         # Python 3 is needed for Argument Clinic and multissl
-        - brew install xz python3
+        - HOMEBREW_NO_AUTO_UPDATE=1 brew install xz python3
         - export PATH=$(brew --prefix)/bin:$(brew --prefix)/sbin:$PATH
     - os: linux
       language: c
       compiler: gcc
       env: OPTIONAL=true
+      addons:
+        apt:
+          packages:
+            - lcov
+            - xvfb
       before_script:
         - ./configure
-        - make -s -j4
+        - make coverage -s -j4
         # Need a venv that can parse covered code.
         - ./python -m venv venv
         - ./venv/bin/python -m pip install -U coverage
         - ./venv/bin/python -m test.pythoninfo
       script:
         # Skip tests that re-run the entire test suite.
-        - ./venv/bin/python -m coverage run --pylib -m test --fail-env-changed -uall,-cpu -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn -x test_concurrent_futures
+        - xvfb-run ./venv/bin/python -m coverage run --pylib -m test --fail-env-changed -uall,-cpu -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn -x test_concurrent_futures
       after_script:  # Probably should be after_success once test suite updated to run under coverage.py.
         # Make the `coverage` command available to Codecov w/ a version of Python that can parse all source files.
         - source ./venv/bin/activate
+        - make coverage-lcov
         - bash <(curl -s https://codecov.io/bash)
 
 
@@ -145,7 +155,7 @@ script:
   # Check that all symbols exported by libpython start with "Py" or "_Py"
   - make smelly
   # `-r -w` implicitly provided through `make buildbottest`.
-  - make buildbottest TESTOPTS="-j4 -uall,-cpu"
+  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then XVFB_RUN=xvfb-run; fi; $XVFB_RUN make buildbottest TESTOPTS="-j4 -uall,-cpu"
 
 notifications:
   email: false

.vsts/docs-release.yml

@@ -0,0 +1,43 @@
+# Current docs for the syntax of this file are at:
+#  https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md
+
+name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr)
+
+queue:
+  name: Hosted Linux Preview
+
+#variables:
+
+steps:
+- checkout: self
+  clean: true
+  fetchDepth: 5
+
+- script: sudo apt-get update && sudo apt-get install -qy --force-yes texlive-full 
+  displayName: 'Install LaTeX'
+
+- task: UsePythonVersion@0
+  displayName: 'Use Python 3.6 or later'
+  inputs:
+    versionSpec: '>=3.6'
+
+- script: python -m pip install sphinx blurb python-docs-theme
+  displayName: 'Install build dependencies'
+
+- script: make dist PYTHON=python SPHINXBUILD='python -m sphinx' BLURB='python -m blurb'
+  workingDirectory: '$(build.sourcesDirectory)/Doc'
+  displayName: 'Build documentation'
+
+- task: PublishBuildArtifacts@1
+  displayName: 'Publish build'
+  inputs:
+    PathToPublish: '$(build.sourcesDirectory)/Doc/build'
+    ArtifactName: build
+    publishLocation: Container
+
+- task: PublishBuildArtifacts@1
+  displayName: 'Publish dist'
+  inputs:
+    PathToPublish: '$(build.sourcesDirectory)/Doc/dist'
+    ArtifactName: dist
+    publishLocation: Container

.vsts/docs.yml

@@ -0,0 +1,44 @@
+# Current docs for the syntax of this file are at:
+#  https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md
+
+name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr)
+
+queue:
+  name: Hosted Linux Preview
+
+trigger:
+  branches:
+    include:
+    - master
+    - 3.7
+    - 3.6
+  paths:
+    include:
+    - Doc/*
+
+#variables:
+
+steps:
+- checkout: self
+  clean: true
+  fetchDepth: 5
+
+- task: UsePythonVersion@0
+  displayName: 'Use Python 3.6 or later'
+  inputs:
+    versionSpec: '>=3.6'
+
+- script: python -m pip install sphinx~=1.6.1 blurb python-docs-theme
+  displayName: 'Install build dependencies'
+
+- script: make check suspicious html PYTHON=python
+  workingDirectory: '$(build.sourcesDirectory)/Doc'
+  displayName: 'Build documentation'
+
+- task: PublishBuildArtifacts@1
+  displayName: 'Publish build'
+  condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
+  inputs:
+    PathToPublish: '$(build.sourcesDirectory)/Doc/build'
+    ArtifactName: build
+    publishLocation: Container

.vsts/linux-buildbot.yml

@@ -0,0 +1,72 @@
+# Current docs for the syntax of this file are at:
+#  https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md
+
+name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr)
+
+queue:
+  name: Hosted Linux Preview
+
+trigger:
+  branches:
+    include:
+    - master
+    - 3.7
+    - 3.6
+  paths:
+    exclude:
+    - Doc/*
+    - Tools/*
+
+variables:
+  # Copy-pasted from linux-deps.yml until template support arrives
+  OPENSSL: 1.1.0g
+  OPENSSL_DIR: "$(build.sourcesDirectory)/multissl/openssl/$(OPENSSL)"
+
+
+steps:
+- checkout: self
+  clean: true
+  fetchDepth: 5
+
+#- template: linux-deps.yml
+
+# See https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted-templates.md
+# For now, we copy/paste the steps
+- script: echo "deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update
+  displayName: 'Update apt-get lists'
+
+- script: echo ##vso[task.prependpath]$(OPENSSL_DIR)
+  displayName: 'Add $(OPENSSL_DIR) to PATH'
+- script: >
+    sudo apt-get -yq install
+    build-essential
+    zlib1g-dev
+    libbz2-dev
+    liblzma-dev
+    libncurses5-dev
+    libreadline6-dev
+    libsqlite3-dev
+    libssl-dev
+    libgdbm-dev
+    tk-dev
+    lzma
+    lzma-dev
+    liblzma-dev
+    libffi-dev
+    uuid-dev
+    xvfb
+  displayName: 'Install dependencies'
+- script: python3 Tools/ssl/multissltests.py --steps=library --base-directory $(build.sourcesDirectory)/multissl --openssl $(OPENSSL) --system Linux
+  displayName: 'python multissltests.py'
+
+- script: ./configure --with-pydebug
+  displayName: 'Configure CPython (debug)'
+
+- script: make -s -j4
+  displayName: 'Build CPython'
+
+- script: make pythoninfo
+  displayName: 'Display build info'
+
+- script: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu"
+  displayName: 'Tests'

.vsts/linux-coverage.yml

@@ -0,0 +1,78 @@
+# Current docs for the syntax of this file are at:
+#  https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md
+
+name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr)
+
+queue:
+  name: Hosted Linux Preview
+
+trigger:
+  branches:
+    include:
+    - master
+    - 3.7
+    - 3.6
+  paths:
+    exclude:
+    - Doc/*
+    - Tools/*
+
+variables:
+  # Copy-pasted from linux-deps.yml until template support arrives
+  OPENSSL: 1.1.0g
+  OPENSSL_DIR: "$(build.sourcesDirectory)/multissl/openssl/$(OPENSSL)"
+
+steps:
+- checkout: self
+  clean: true
+  fetchDepth: 5
+
+#- template: linux-deps.yml
+
+# See https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted-templates.md
+# For now, we copy/paste the steps
+- script: echo "deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update
+  displayName: 'Update apt-get lists'
+
+- script: echo ##vso[task.prependpath]$(OPENSSL_DIR)
+  displayName: 'Add $(OPENSSL_DIR) to PATH'
+- script: >
+    sudo apt-get -yq install
+    build-essential
+    zlib1g-dev
+    libbz2-dev
+    liblzma-dev
+    libncurses5-dev
+    libreadline6-dev
+    libsqlite3-dev
+    libssl-dev
+    libgdbm-dev
+    tk-dev
+    lzma
+    lzma-dev
+    liblzma-dev
+    libffi-dev
+    uuid-dev
+    xvfb
+  displayName: 'Install dependencies'
+- script: python3 Tools/ssl/multissltests.py --steps=library --base-directory $(build.sourcesDirectory)/multissl --openssl $(OPENSSL) --system Linux
+  displayName: 'python multissltests.py'
+
+
+- script: ./configure --with-pydebug
+  displayName: 'Configure CPython (debug)'
+
+- script: make -s -j4
+  displayName: 'Build CPython'
+
+- script: ./python -m venv venv && ./venv/bin/python -m pip install -U coverage
+  displayName: 'Set up virtual environment'
+
+- script: ./venv/bin/python -m test.pythoninfo
+  displayName: 'Display build info'
+
+- script: xvfb-run ./venv/bin/python -m coverage run --pylib -m test --fail-env-changed -uall,-cpu -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn -x test_concurrent_futures
+  displayName: 'Tests with coverage'
+
+- script: source ./venv/bin/activate && bash <(curl -s https://codecov.io/bash)
+  displayName: 'Publish code coverage results'