From f90433b3fcb9619c64c8d3243c67907c06cb1182 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Mon, 7 Aug 2023 13:51:57 -0700 Subject: [PATCH 1/3] BLD: Add Cython Coverage option --- ci/deps/actions-310.yaml | 2 +- ci/deps/actions-311-downstream_compat.yaml | 2 +- ci/deps/actions-311-numpydev.yaml | 2 +- ci/deps/actions-311-pyarrownightly.yaml | 2 +- ci/deps/actions-311.yaml | 2 +- ci/deps/actions-39-minimum_versions.yaml | 2 +- ci/deps/actions-39.yaml | 2 +- ci/deps/actions-pypy-39.yaml | 2 +- ci/deps/circle-310-arm64.yaml | 2 +- environment.yml | 2 +- meson.build | 3 ++- meson.options | 1 + pandas/_libs/meson.build | 16 +++++++++++++--- pandas/_libs/tslibs/meson.build | 3 ++- pandas/_libs/window/meson.build | 4 ++-- pyproject.toml | 2 +- requirements-dev.txt | 2 +- 17 files changed, 32 insertions(+), 19 deletions(-) create mode 100644 meson.options diff --git a/ci/deps/actions-310.yaml b/ci/deps/actions-310.yaml index d8186d09bb9d4..3e2cc212052c2 100644 --- a/ci/deps/actions-310.yaml +++ b/ci/deps/actions-310.yaml @@ -7,7 +7,7 @@ dependencies: # build dependencies - versioneer[toml] - cython>=3.0.0 - - meson[ninja]=1.0.1 + - meson[ninja]=1.1.1 - meson-python=0.13.1 # test dependencies diff --git a/ci/deps/actions-311-downstream_compat.yaml b/ci/deps/actions-311-downstream_compat.yaml index 7cf7f805cee31..0c1f5e93201c0 100644 --- a/ci/deps/actions-311-downstream_compat.yaml +++ b/ci/deps/actions-311-downstream_compat.yaml @@ -8,7 +8,7 @@ dependencies: # build dependencies - versioneer[toml] - cython>=3.0.0 - - meson[ninja]=1.0.1 + - meson[ninja]=1.1.1 - meson-python=0.13.1 # test dependencies diff --git a/ci/deps/actions-311-numpydev.yaml b/ci/deps/actions-311-numpydev.yaml index 2cd4d5f3528f8..843418604fc73 100644 --- a/ci/deps/actions-311-numpydev.yaml +++ b/ci/deps/actions-311-numpydev.yaml @@ -6,7 +6,7 @@ dependencies: # build dependencies - versioneer[toml] - - meson[ninja]=1.0.1 + - meson[ninja]=1.1.1 - meson-python=0.13.1 # test dependencies diff --git a/ci/deps/actions-311-pyarrownightly.yaml b/ci/deps/actions-311-pyarrownightly.yaml index 3d5e2e99feb9c..a2e2b4a2ba79d 100644 --- a/ci/deps/actions-311-pyarrownightly.yaml +++ b/ci/deps/actions-311-pyarrownightly.yaml @@ -6,7 +6,7 @@ dependencies: # build dependencies - versioneer[toml] - - meson[ninja]=1.0.1 + - meson[ninja]=1.1.1 - cython>=3.0.0 - meson-python=0.13.1 diff --git a/ci/deps/actions-311.yaml b/ci/deps/actions-311.yaml index 67203201ea637..affb7a9636dcb 100644 --- a/ci/deps/actions-311.yaml +++ b/ci/deps/actions-311.yaml @@ -7,7 +7,7 @@ dependencies: # build dependencies - versioneer[toml] - cython>=3.0.0 - - meson[ninja]=1.0.1 + - meson[ninja]=1.1.1 - meson-python=0.13.1 # test dependencies diff --git a/ci/deps/actions-39-minimum_versions.yaml b/ci/deps/actions-39-minimum_versions.yaml index 52274180ccb48..02b430ba8c5fb 100644 --- a/ci/deps/actions-39-minimum_versions.yaml +++ b/ci/deps/actions-39-minimum_versions.yaml @@ -9,7 +9,7 @@ dependencies: # build dependencies - versioneer[toml] - cython>=3.0.0 - - meson[ninja]=1.0.1 + - meson[ninja]=1.1.1 - meson-python=0.13.1 # test dependencies diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index 658dfe032a42b..0fd7b6e65ec55 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -7,7 +7,7 @@ dependencies: # build dependencies - versioneer[toml] - cython>=3.0.0 - - meson[ninja]=1.0.1 + - meson[ninja]=1.1.1 - meson-python=0.13.1 # test dependencies diff --git a/ci/deps/actions-pypy-39.yaml b/ci/deps/actions-pypy-39.yaml index 292565e9640d9..2d8cd4620cfe8 100644 --- a/ci/deps/actions-pypy-39.yaml +++ b/ci/deps/actions-pypy-39.yaml @@ -10,7 +10,7 @@ dependencies: # build dependencies - versioneer[toml] - cython>=3.0.0 - - meson[ninja]=1.0.1 + - meson[ninja]=1.1.1 - meson-python=0.13.1 # test dependencies diff --git a/ci/deps/circle-310-arm64.yaml b/ci/deps/circle-310-arm64.yaml index 60bdc2b828dae..7a5fe88b2cf11 100644 --- a/ci/deps/circle-310-arm64.yaml +++ b/ci/deps/circle-310-arm64.yaml @@ -7,7 +7,7 @@ dependencies: # build dependencies - versioneer[toml] - cython>=3.0.0 - - meson[ninja]=1.0.1 + - meson[ninja]=1.1.1 - meson-python=0.13.1 # test dependencies diff --git a/environment.yml b/environment.yml index d0536ea81c8cc..0cc87f998754b 100644 --- a/environment.yml +++ b/environment.yml @@ -9,7 +9,7 @@ dependencies: # build dependencies - versioneer[toml] - cython=3.0.0 - - meson[ninja]=1.0.1 + - meson[ninja]=1.1.1 - meson-python=0.13.1 # test dependencies diff --git a/meson.build b/meson.build index a927b59abeaf9..a01453598facf 100644 --- a/meson.build +++ b/meson.build @@ -4,7 +4,7 @@ project( 'c', 'cpp', 'cython', version: run_command(['python', 'generate_version.py', '--print'], check: true).stdout().strip(), license: 'BSD-3', - meson_version: '>=1.0.1', + meson_version: '>=1.1', default_options: [ # TODO: investigate, does meson try to compile against debug Python # when buildtype = debug, this seems to be causing problems on CI @@ -33,6 +33,7 @@ add_project_arguments('-DNPY_NO_DEPRECATED_API=0', language : 'cpp') add_project_arguments('-DNPY_TARGET_VERSION=NPY_1_21_API_VERSION', language : 'c') add_project_arguments('-DNPY_TARGET_VERSION=NPY_1_21_API_VERSION', language : 'cpp') +cython_coverage = get_option('CYTHON_COVERAGE') if fs.exists('_version_meson.py') py.install_sources('_version_meson.py', pure: false, subdir: 'pandas') diff --git a/meson.options b/meson.options new file mode 100644 index 0000000000000..092b2e736c541 --- /dev/null +++ b/meson.options @@ -0,0 +1 @@ +option('CYTHON_COVERAGE', type: 'boolean', value: false) diff --git a/pandas/_libs/meson.build b/pandas/_libs/meson.build index f302c649bc7bd..6d4463b0cc97a 100644 --- a/pandas/_libs/meson.build +++ b/pandas/_libs/meson.build @@ -56,8 +56,6 @@ _intervaltree_helper = custom_target('intervaltree_helper_pxi', ) _khash_primitive_helper_dep = declare_dependency(sources: _khash_primitive_helper) -subdir('tslibs') - libs_sources = { # Dict of extension name -> dict of {sources, include_dirs, and deps} # numpy include dir is implicitly included @@ -100,12 +98,24 @@ libs_sources = { 'writers': {'sources': ['writers.pyx']} } +cython_args = ['--include-dir', meson.current_build_dir(), '-X always_allow_keywords=true'] +if cython_coverage + cython_args += ['-X linetrace=true'] + # Linetrace is not enough to ensure coverage is turned on + add_project_arguments('-DCYTHON_TRACE=1', language : 'c') + add_project_arguments('-DCYTHON_TRACE=1', language : 'cpp') + + add_project_arguments('-DCYTHON_TRACE_NOGIL=1', language : 'c') + add_project_arguments('-DCYTHON_TRACE_NOGIL=1', language : 'cpp') +endif + +subdir('tslibs') foreach ext_name, ext_dict : libs_sources py.extension_module( ext_name, ext_dict.get('sources'), - cython_args: ['--include-dir', meson.current_build_dir(), '-X always_allow_keywords=true'], + cython_args: cython_args, include_directories: [inc_np, inc_pd], dependencies: ext_dict.get('deps', ''), subdir: 'pandas/_libs', diff --git a/pandas/_libs/tslibs/meson.build b/pandas/_libs/tslibs/meson.build index 14d2eef46da20..5ae176880d1b8 100644 --- a/pandas/_libs/tslibs/meson.build +++ b/pandas/_libs/tslibs/meson.build @@ -23,7 +23,8 @@ foreach ext_name, ext_dict : tslibs_sources py.extension_module( ext_name, ext_dict.get('sources'), - cython_args: ['--include-dir', meson.current_build_dir(), '-X always_allow_keywords=true'], + #cython_args: ['--include-dir', meson.current_build_dir(), '-X always_allow_keywords=true'], + cython_args: cython_args, include_directories: [inc_np, inc_pd], dependencies: ext_dict.get('deps', ''), subdir: 'pandas/_libs/tslibs', diff --git a/pandas/_libs/window/meson.build b/pandas/_libs/window/meson.build index b83d8730f9447..11e5e50f73558 100644 --- a/pandas/_libs/window/meson.build +++ b/pandas/_libs/window/meson.build @@ -1,7 +1,7 @@ py.extension_module( 'aggregations', ['aggregations.pyx'], - cython_args: ['-X always_allow_keywords=true'], + cython_args: cython_args, #['-X always_allow_keywords=true'], include_directories: [inc_np, inc_pd], dependencies: [py_dep], subdir: 'pandas/_libs/window', @@ -12,7 +12,7 @@ py.extension_module( py.extension_module( 'indexers', ['indexers.pyx'], - cython_args: ['-X always_allow_keywords=true'], + cython_args: cython_args, #['-X always_allow_keywords=true'], include_directories: [inc_np, inc_pd], dependencies: [py_dep], subdir: 'pandas/_libs/window', diff --git a/pyproject.toml b/pyproject.toml index 62e4333d86ec3..d6b7277f561ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ # See https://github.com/scipy/scipy/pull/12940 for the AIX issue. requires = [ "meson-python==0.13.1", - "meson==1.0.1", + "meson==1.1.1", "wheel", "Cython>=3.0.0", # Note: sync with setup.py, environment.yml and asv.conf.json # Note: numpy 1.25 has a backwards compatible C API by default diff --git a/requirements-dev.txt b/requirements-dev.txt index 5bd34130cc525..0f514e8476521 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,7 +4,7 @@ pip versioneer[toml] cython==3.0.0 -meson[ninja]==1.0.1 +meson[ninja]==1.1.1 meson-python==0.13.1 pytest>=7.3.2 pytest-cov From 7f1b4c34e3c603a2155c5ed95fde596311c2e5c9 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Tue, 8 Aug 2023 11:21:20 +0200 Subject: [PATCH 2/3] bump more --- .github/workflows/unit-tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 1770d18d4eb41..f0d287ae04844 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -238,7 +238,7 @@ jobs: run: | /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev . ~/virtualenvs/pandas-dev/bin/activate - python -m pip install --no-cache-dir -U pip wheel setuptools meson[ninja]==1.0.1 meson-python==0.13.1 + python -m pip install --no-cache-dir -U pip wheel setuptools meson[ninja]==1.1.1 meson-python==0.13.1 python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1 python -m pip install --no-cache-dir --no-build-isolation -e . python -m pip list --no-cache-dir @@ -276,7 +276,7 @@ jobs: run: | /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev . ~/virtualenvs/pandas-dev/bin/activate - python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.0.1 + python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.1.1 python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1 python -m pip install --no-cache-dir --no-build-isolation -e . python -m pip list --no-cache-dir @@ -348,7 +348,7 @@ jobs: - name: Install dependencies run: | python --version - python -m pip install --upgrade pip setuptools wheel meson[ninja]==1.0.1 meson-python==0.13.1 + python -m pip install --upgrade pip setuptools wheel meson[ninja]==1.1.1 meson-python==0.13.1 python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy python -m pip install versioneer[toml] python -m pip install python-dateutil pytz tzdata cython hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-cov pytest-asyncio>=0.17 From ead496fc5ea8303e7244983c2e8f1f65957b193a Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 9 Aug 2023 07:15:06 +0200 Subject: [PATCH 3/3] fix segfault by declaring gil correctly --- pandas/_libs/parsers.pyx | 6 +++--- pandas/_libs/tslibs/meson.build | 1 - pandas/_libs/window/meson.build | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/pandas/_libs/parsers.pyx b/pandas/_libs/parsers.pyx index 6d66e21ce49f5..9baf644c4a921 100644 --- a/pandas/_libs/parsers.pyx +++ b/pandas/_libs/parsers.pyx @@ -254,8 +254,8 @@ cdef extern from "pandas/parser/pd_parser.h": int del_rd_source(void *src) - void* buffer_rd_bytes(void *source, size_t nbytes, - size_t *bytes_read, int *status, const char *encoding_errors) + void* buffer_rd_bytes(void *source, size_t nbytes, size_t *bytes_read, + int *status, const char *encoding_errors) nogil void uint_state_init(uint_state *self) int uint64_conflict(uint_state *self) @@ -325,7 +325,7 @@ cdef double round_trip_wrapper(const char *p, char **q, char decimal, cdef void* buffer_rd_bytes_wrapper(void *source, size_t nbytes, size_t *bytes_read, int *status, - const char *encoding_errors) noexcept: + const char *encoding_errors) noexcept nogil: return buffer_rd_bytes(source, nbytes, bytes_read, status, encoding_errors) cdef int del_rd_source_wrapper(void *src) noexcept: diff --git a/pandas/_libs/tslibs/meson.build b/pandas/_libs/tslibs/meson.build index 5ae176880d1b8..8bc15a4c0c842 100644 --- a/pandas/_libs/tslibs/meson.build +++ b/pandas/_libs/tslibs/meson.build @@ -23,7 +23,6 @@ foreach ext_name, ext_dict : tslibs_sources py.extension_module( ext_name, ext_dict.get('sources'), - #cython_args: ['--include-dir', meson.current_build_dir(), '-X always_allow_keywords=true'], cython_args: cython_args, include_directories: [inc_np, inc_pd], dependencies: ext_dict.get('deps', ''), diff --git a/pandas/_libs/window/meson.build b/pandas/_libs/window/meson.build index 11e5e50f73558..e5b8909f8e1cb 100644 --- a/pandas/_libs/window/meson.build +++ b/pandas/_libs/window/meson.build @@ -1,7 +1,7 @@ py.extension_module( 'aggregations', ['aggregations.pyx'], - cython_args: cython_args, #['-X always_allow_keywords=true'], + cython_args: cython_args, include_directories: [inc_np, inc_pd], dependencies: [py_dep], subdir: 'pandas/_libs/window',