From 0f5e9cfa80b9c7e53bcc035c4f4b072d5a696c36 Mon Sep 17 00:00:00 2001 From: Shantanu Date: Fri, 10 Jan 2025 18:01:05 -0800 Subject: [PATCH 1/2] Lock during experimental editable.rebuild --- pyproject.toml | 1 + .../resources/_editable_redirect.py | 82 +++++++++++-------- 2 files changed, 49 insertions(+), 34 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b4a42378..81213e2b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ dynamic = ["version"] dependencies = [ "exceptiongroup >=1.0; python_version<'3.11'", + "filelock >=3", "importlib-metadata >=4.13; python_version<'3.8'", "importlib-resources >=1.3; python_version<'3.9'", "packaging >=21.3", diff --git a/src/scikit_build_core/resources/_editable_redirect.py b/src/scikit_build_core/resources/_editable_redirect.py index e71812e7..169500c8 100644 --- a/src/scikit_build_core/resources/_editable_redirect.py +++ b/src/scikit_build_core/resources/_editable_redirect.py @@ -128,42 +128,56 @@ def rebuild(self) -> None: if verbose: print(f"Running cmake --build & --install in {self.path}") # noqa: T201 - result = subprocess.run( - ["cmake", "--build", ".", *self.build_options], - cwd=self.path, - stdout=sys.stderr if verbose else subprocess.PIPE, - env=env, - check=False, - text=True, - ) - if result.returncode and verbose: - print( # noqa: T201 - f"ERROR: {result.stdout}", - file=sys.stderr, + import filelock + + lock = filelock.FileLock(os.path.join(self.path, "editable_rebuild.lock")) + + try: + try: + lock.acquire(timeout=60) + except filelock.Timeout: + if verbose: + print(f"Still waiting to acquire rebuild lock in {self.path}...") # noqa: T201 + lock.acquire() + + result = subprocess.run( + ["cmake", "--build", ".", *self.build_options], + cwd=self.path, + stdout=sys.stderr if verbose else subprocess.PIPE, + env=env, + check=False, + text=True, ) - result.check_returncode() - - result = subprocess.run( - [ - "cmake", - "--install", - ".", - "--prefix", - self.install_dir, - *self.install_options, - ], - cwd=self.path, - stdout=sys.stderr if verbose else subprocess.PIPE, - env=env, - check=False, - text=True, - ) - if result.returncode and verbose: - print( # noqa: T201 - f"ERROR: {result.stdout}", - file=sys.stderr, + if result.returncode and verbose: + print( # noqa: T201 + f"ERROR: {result.stdout}", + file=sys.stderr, + ) + result.check_returncode() + + result = subprocess.run( + [ + "cmake", + "--install", + ".", + "--prefix", + self.install_dir, + *self.install_options, + ], + cwd=self.path, + stdout=sys.stderr if verbose else subprocess.PIPE, + env=env, + check=False, + text=True, ) - result.check_returncode() + if result.returncode and verbose: + print( # noqa: T201 + f"ERROR: {result.stdout}", + file=sys.stderr, + ) + result.check_returncode() + finally: + lock.release() def install( From 7369304f088cb693b1cc2d7bb6c3f344aa05c6a3 Mon Sep 17 00:00:00 2001 From: Shantanu Date: Fri, 10 Jan 2025 23:04:19 -0800 Subject: [PATCH 2/2] Use fcntl directly --- pyproject.toml | 1 - .../resources/_editable_redirect.py | 64 ++++++++++++++++--- 2 files changed, 55 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 81213e2b..b4a42378 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,6 @@ dynamic = ["version"] dependencies = [ "exceptiongroup >=1.0; python_version<'3.11'", - "filelock >=3", "importlib-metadata >=4.13; python_version<'3.8'", "importlib-resources >=1.3; python_version<'3.9'", "packaging >=21.3", diff --git a/src/scikit_build_core/resources/_editable_redirect.py b/src/scikit_build_core/resources/_editable_redirect.py index 169500c8..fdf08a9a 100644 --- a/src/scikit_build_core/resources/_editable_redirect.py +++ b/src/scikit_build_core/resources/_editable_redirect.py @@ -22,6 +22,59 @@ def __dir__() -> list[str]: return __all__ +class FileLockIfUnix: + def __init__(self, lock_file: str) -> None: + self.lock_file = lock_file + self.lock_file_fd: int | None = None + + def acquire(self) -> None: + # Based on filelock.BaseFileLock.acquire and filelock.UnixFileLock._acquire + try: + import fcntl + except ImportError: + return + import contextlib + import time + + poll_interval = 0.05 + log_interval = 60 + last_log = time.perf_counter() + + while True: + os.makedirs(os.path.dirname(self.lock_file), exist_ok=True) + open_flags = os.O_RDWR | os.O_TRUNC + if not os.path.exists(self.lock_file): + open_flags |= os.O_CREAT + + fd = os.open(self.lock_file, open_flags, 0o644) + with contextlib.suppress(PermissionError): # Lock is not owned by this UID + os.fchmod(fd, 0o644) + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + os.close(fd) + else: + self.lock_file_fd = fd + return + + now = time.perf_counter() + if now - last_log > log_interval: + last_log = now + print(f"Still waiting to acquire lock {self.lock_file}...") # noqa: T201 + + time.sleep(poll_interval) + + def release(self) -> None: + try: + import fcntl + except ImportError: + return + + assert isinstance(self.lock_file_fd, int) + fcntl.flock(self.lock_file_fd, fcntl.LOCK_UN) + os.close(self.lock_file_fd) + + class ScikitBuildRedirectingFinder(importlib.abc.MetaPathFinder): def __init__( self, @@ -128,17 +181,10 @@ def rebuild(self) -> None: if verbose: print(f"Running cmake --build & --install in {self.path}") # noqa: T201 - import filelock - - lock = filelock.FileLock(os.path.join(self.path, "editable_rebuild.lock")) + lock = FileLockIfUnix(os.path.join(self.path, "editable_rebuild.lock")) try: - try: - lock.acquire(timeout=60) - except filelock.Timeout: - if verbose: - print(f"Still waiting to acquire rebuild lock in {self.path}...") # noqa: T201 - lock.acquire() + lock.acquire() result = subprocess.run( ["cmake", "--build", ".", *self.build_options],