Skip to content

Commit 30179c8

Browse files
mpenkovmpenkov
committed
Merge branch 'release-4.3.2'
2 parents a60edb3 + 2e5f3d2 commit 30179c8

File tree

189 files changed

+3981
-5170
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

189 files changed

+3981
-5170
lines changed

‎.github/workflows/build-docs.yml

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
#
2+
# This workflow rebuilds documentation and stores the resulting patch as a
3+
# workflow artifact. We can then download the artifact, apply the patch, and
4+
# then push the changes.
5+
#
6+
# It's possible to do all this locally on a developer's machine, but it's not
7+
# trivial, because it requires many pre-requisites.
8+
#
9+
name: Rebuild documentation
10+
on: workflow_dispatch
11+
jobs:
12+
docs:
13+
name: Rebuild documentation
14+
timeout-minutes: 180
15+
runs-on: ubuntu-20.04
16+
defaults:
17+
run:
18+
shell: bash
19+
20+
steps:
21+
- uses: actions/checkout@v3
22+
- name: Setup up Python ${{ matrix.python }}
23+
uses: actions/setup-python@v4
24+
with:
25+
#
26+
# We use Py3.8 here for historical reasons.
27+
#
28+
python-version: "3.8"
29+
30+
- name: Update pip
31+
run: python -m pip install -U pip
32+
33+
- name: Install apt packages for LaTeX rendering
34+
run: |
35+
sudo apt-get -yq update
36+
sudo apt-get -yq remove texlive-binaries --purge
37+
sudo apt-get -yq --no-install-suggests --no-install-recommends --force-yes install dvipng texlive-latex-base texlive-latex-extra texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended latexmk
38+
sudo apt-get -yq install build-essential python3.8-dev
39+
- name: Install gensim and its dependencies
40+
run: pip install -e .[docs]
41+
42+
- name: Build documentation
43+
run: |
44+
python setup.py build_ext --inplace
45+
make -C docs/src clean html
46+
47+
- name: Check changes to prebuilt docs
48+
run: |
49+
git config user.email "[email protected]"
50+
git config user.name "Gensim Docs Build"
51+
if ! git diff --quiet @ ; then
52+
git add .
53+
branch="$GITHUB_HEAD_REF ($GITHUB_REF_NAME)"
54+
git commit -m "Import rebuilt documentation for branch $branch"
55+
git format-patch @^
56+
git bundle create prebuilt-docs-changes.bundle @^...@
57+
git reset --mixed @^
58+
git diff --exit-code --stat @
59+
fi
60+
61+
- name: Upload prebuilt docs changes
62+
if: always()
63+
uses: actions/upload-artifact@v3
64+
with:
65+
name: prebuilt-docs-changes
66+
path: |
67+
*.patch
68+
*.bundle

‎.github/workflows/build-wheels.yml

+103-13
Original file line numberDiff line numberDiff line change
@@ -36,32 +36,122 @@ jobs:
3636
platforms: all
3737

3838
- name: Build wheels
39-
uses: pypa/cibuildwheel@v2.12.0
39+
uses: pypa/cibuildwheel@v2.13.1
4040
env:
4141
CIBW_ARCHS_LINUX: x86_64 aarch64
4242
CIBW_ARCHS_MACOS: x86_64 arm64
43-
CIBW_ARCHS_WINDOWS: AMD64 x86 ARM64
44-
CIBW_BEFORE_BUILD: pip install numpy scipy
43+
CIBW_ARCHS_WINDOWS: AMD64 x86
4544
CIBW_SKIP: pp* cp36-* cp37-* *-win32 *_i686 *-musllinux_*
4645
CIBW_TEST_COMMAND: pytest -rfxEXs --durations=20 --disable-warnings --showlocals --pyargs gensim
4746
CIBW_TEST_REQUIRES: pytest testfixtures mock
48-
CIBW_TEST_SKIP: cp38* cp39* cp310* *_aarch64 *_arm64 *_universal2
47+
CIBW_TEST_SKIP: cp38* cp39* cp310* cp311* *_aarch64 *_arm64 *_universal2
48+
CIBW_BUILD_VERBOSITY: 3
49+
50+
- name: Upload wheels as artifacts
51+
if: always()
52+
uses: actions/upload-artifact@v3
53+
with:
54+
name: wheels-${{ matrix.os }}
55+
path: wheelhouse/*.whl
56+
57+
test:
58+
name: Test wheel for ${{ matrix.os }} Python ${{ matrix.python }}
59+
needs: build_wheels
60+
strategy:
61+
fail-fast: false
62+
matrix:
63+
include:
64+
- {python: '3.8', os: macos-11}
65+
- {python: '3.9', os: macos-11}
66+
- {python: '3.10', os: macos-11}
67+
- {python: '3.11', os: macos-11}
68+
69+
- {python: '3.8', os: ubuntu-20.04}
70+
- {python: '3.9', os: ubuntu-20.04}
71+
- {python: '3.10', os: ubuntu-20.04}
72+
- {python: '3.11', os: ubuntu-20.04}
73+
74+
- {python: '3.8', os: windows-2019}
75+
- {python: '3.9', os: windows-2019}
76+
77+
#
78+
# We know that the Windows Py3.10 wheels are broken. Don't test them,
79+
# because it will fail the entire workflow.
80+
#
81+
# https://github.com/RaRe-Technologies/gensim/issues/3489
82+
#
83+
# - {python: '3.10', os: windows-2019}
84+
- {python: '3.11', os: windows-2019}
85+
86+
runs-on: ${{ matrix.os }}
87+
steps:
88+
- name: Setup up Python ${{ matrix.python }}
89+
uses: actions/setup-python@v4
90+
with:
91+
python-version: ${{ matrix.python }}
92+
93+
- name: Downloads build artifacts
94+
uses: actions/download-artifact@v3
95+
with:
96+
path: artifacts/
97+
98+
#
99+
# We want to make sure our wheels run against older Numpy versions
100+
#
101+
- name: Install oldest-supported-numpy
102+
run: python -m pip install oldest-supported-numpy
103+
104+
#
105+
# Avoid checking out the entire gensim repo to get just one file
106+
#
107+
- name: Download installwheel.py
108+
run: curl "https://raw.githubusercontent.com/RaRe-Technologies/gensim/develop/.github/workflows/installwheel.py" --output installwheel.py --silent
109+
110+
- name: Install wheel
111+
run: python installwheel.py artifacts/wheels-${{ matrix.os }}
112+
113+
- name: Debug test environment
114+
run: |
115+
pip freeze
116+
python -c 'import numpy;print(numpy.__file__)'
117+
python -c 'import numpy;print(numpy.__version__)'
118+
119+
#
120+
# If the wheel was incorrectly built, then this will fail.
121+
# https://github.com/RaRe-Technologies/gensim/issues/3097
122+
#
123+
- name: Test wheel
124+
run: python -c 'import gensim'
125+
126+
upload:
127+
name: Upload to S3
128+
if: always()
129+
needs: build_wheels
130+
runs-on: ubuntu-latest
131+
steps:
132+
133+
- name: Install wheel uploading tool
134+
run: python -m pip install wheelhouse-uploader
135+
136+
- name: Downloads build artifacts
137+
uses: actions/download-artifact@v3
138+
with:
139+
path: artifacts/
140+
141+
- name: Check files
142+
run: tree artifacts/
143+
144+
- name: Move all wheels into one folder
145+
run: mkdir wheelhouse ; find artifacts/ -name '*.whl' -exec mv -v {} wheelhouse/ \;
49146

50147
- name: Upload wheels to s3://gensim-wheels
51148
#
52149
# Only do this if the credentials are set.
53150
# This means that PRs will still build wheels, but not upload them.
54151
# (PRs do not have access to secrets).
55152
#
56-
# The always() ensures this step runs even if a previous step fails.
57-
# We want to upload wheels whenever possible (even if e.g. tests failed)
58-
# because we don't want an innocuous test failure from blocking a release.
59-
#
60-
if: ${{ always() && env.WHEELHOUSE_UPLOADER_USERNAME && env.WHEELHOUSE_UPLOADER_SECRET }}
61-
run: |
62-
python -m pip install wheelhouse-uploader
63-
ls wheelhouse/*.whl
64-
python -m wheelhouse_uploader upload --local-folder wheelhouse/ --no-ssl-check gensim-wheels --provider S3 --no-enable-cdn
153+
if: ${{ env.WHEELHOUSE_UPLOADER_USERNAME && env.WHEELHOUSE_UPLOADER_SECRET }}
154+
run: python -m wheelhouse_uploader upload --local-folder wheelhouse/ --no-ssl-check gensim-wheels --provider S3 --no-enable-cdn
65155
env:
66156
WHEELHOUSE_UPLOADER_USERNAME: ${{ secrets.AWS_ACCESS_KEY_ID }}
67157
WHEELHOUSE_UPLOADER_SECRET: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

‎.github/workflows/installwheel.py

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
"""Install a wheel for the current platform."""
2+
import os
3+
import platform
4+
import subprocess
5+
import sys
6+
7+
8+
def main():
9+
subdir = sys.argv[1]
10+
vi = sys.version_info
11+
12+
if platform.system() in ('Linux', 'Darwin'):
13+
arch = 'x86_64'
14+
else:
15+
arch = 'amd64'
16+
17+
want = f'-cp{vi.major}{vi.minor}-'
18+
suffix = f'_{arch}.whl'
19+
20+
files = sorted(os.listdir(subdir))
21+
for f in files:
22+
if want in f and f.endswith(suffix):
23+
command = [sys.executable, '-m', 'pip', 'install', os.path.join(subdir, f)]
24+
subprocess.check_call(command)
25+
return 0
26+
27+
print(f'no matches for {want} / {suffix} in {subdir}:')
28+
print('\n'.join(files))
29+
30+
return 1
31+
32+
33+
34+
if __name__ == '__main__':
35+
sys.exit(main())

‎.github/workflows/test_wheel.py

+76
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
#!/usr/bin/env python
2+
"""Test a Gensim wheel stored on S3.
3+
4+
Downloads the wheel, installs it into a fresh working environment, and then runs gensim tests.
5+
6+
usage:
7+
8+
python test_wheel.py <url> $(which python3.10)
9+
10+
where the URL comes from http://gensim-wheels.s3-website-us-east-1.amazonaws.com/
11+
"""
12+
13+
import argparse
14+
import io
15+
import os
16+
import subprocess
17+
import tempfile
18+
import urllib.parse
19+
import urllib.request
20+
import shutil
21+
import sys
22+
23+
curr_dir = os.path.dirname(os.path.abspath(__file__))
24+
25+
26+
def run(*command, **kwargs):
27+
print("-" * 70, file=sys.stderr)
28+
print(" ".join(command), file=sys.stderr)
29+
print("-" * 70, file=sys.stderr)
30+
subprocess.check_call(command, **kwargs)
31+
32+
33+
def main():
34+
parser = argparse.ArgumentParser()
35+
parser.add_argument("wheel_path", help="The location of the wheel. May be a URL or local path")
36+
parser.add_argument("python", help="Which python binary to use to test the wheel")
37+
parser.add_argument("--gensim-path", default=os.path.expanduser("~/git/gensim"), help="Where the gensim repo lives")
38+
parser.add_argument("--keep", action="store_true", help="Do not delete the sandbox after testing")
39+
parser.add_argument("--test", default="test", help="Specify which tests to run")
40+
args = parser.parse_args()
41+
42+
_, python_version = subprocess.check_output([args.python, "--version"]).decode().strip().split(" ", 1)
43+
44+
try:
45+
tmpdir = tempfile.mkdtemp(prefix=f"test_wheel-py{python_version}-")
46+
47+
tmp_test_path = os.path.join(tmpdir, "test")
48+
shutil.copytree(os.path.join(args.gensim_path, "gensim/test"), tmp_test_path)
49+
50+
if args.wheel_path.startswith("http://") or args.wheel_path.startswith("https://"):
51+
parsed = urllib.parse.urlparse(args.wheel_path)
52+
filename = parsed.path.split('/')[-1]
53+
wheel_path = os.path.join(tmpdir, filename)
54+
urllib.request.urlretrieve(args.wheel_path, wheel_path)
55+
else:
56+
wheel_path = args.wheel_path
57+
58+
env_path = os.path.join(tmpdir, "env")
59+
run("virtualenv", "-p", args.python, env_path)
60+
61+
python_exe = os.path.join(tmpdir, "env/bin/python")
62+
run(python_exe, "-m", "pip", "install", wheel_path)
63+
run(python_exe, "-m", "pip", "install", "mock", "pytest", "testfixtures")
64+
65+
pytest_exe = os.path.join(tmpdir, "env/bin/pytest")
66+
run(pytest_exe, "-vvv", args.test, "--durations", "0", cwd=tmpdir)
67+
finally:
68+
if args.keep:
69+
print(f"keeping {tmpdir}, remove it yourself when done")
70+
else:
71+
shutil.rmtree(tmpdir)
72+
73+
74+
75+
if __name__ == "__main__":
76+
main()

0 commit comments

Comments
 (0)