Update gitignore implementation to support more cases

Author: colinjc

.gitignore

@@ -25,3 +25,4 @@ build/
 .aider*
 uv.lock
 .local/
+.claude/settings.local.json

pyproject.toml

@@ -35,6 +35,7 @@ dependencies = [
     "gpuhunt==0.1.6",
     "argcomplete>=3.5.0",
     "humanize>=4.12.3",
+    "gitignore-parser>=0.1.12",
 ]
 
 [project.urls]

src/dstack/_internal/core/models/repos/local.py

@@ -2,13 +2,17 @@
 from pathlib import Path
 from typing import BinaryIO, Optional
 
+import humanize
 from typing_extensions import Literal
 
 from dstack._internal.core.models.repos.base import BaseRepoInfo, Repo
 from dstack._internal.utils.hash import get_sha256, slugify
 from dstack._internal.utils.ignore import GitIgnore
+from dstack._internal.utils.logging import get_logger
 from dstack._internal.utils.path import PathLike
 
+logger = get_logger(__name__)
+
 
 class LocalRepoInfo(BaseRepoInfo):
     repo_type: Literal["local"] = "local"
@@ -75,6 +79,8 @@ def write_code_file(self, fp: BinaryIO) -> str:
                 arcname="",
                 filter=TarIgnore(self.run_repo_data.repo_dir, globs=[".git"]),
             )
+
+        logger.debug(f"Code file size: {humanize.naturalsize(fp.tell())} bytes")
         return get_sha256(fp)
 
     def get_repo_info(self) -> LocalRepoInfo:

src/dstack/_internal/utils/ignore.py

@@ -1,7 +1,8 @@
-import fnmatch
-from itertools import zip_longest
+import os
 from pathlib import Path
-from typing import Dict, List, Optional
+from typing import List
+
+from gitignore_parser import parse_gitignore_str
 
 from dstack._internal.utils.path import PathLike
 
@@ -16,77 +17,83 @@ def __init__(
             if ignore_files is not None
             else [".gitignore", ".git/info/exclude", ".dstackignore"]
         )
-        self.ignore_globs: Dict[str, List[str]] = {".": globs or []}
-        self.load_recursive()
+        self.parser = None
+        self._create_combined_parser(globs or [])
 
-    def load_ignore_file(self, path: str, ignore_file: Path):
-        if path != "." and not path.startswith("./"):
-            path = "./" + path
-        if path not in self.ignore_globs:
-            self.ignore_globs[path] = []
-        with ignore_file.open("r") as f:
-            for line in f:
-                line = self.rstrip(line.rstrip("\n")).rstrip("/")
-                line = line.replace("\\ ", " ")
-                if line.startswith("#") or not line:
-                    continue
-                self.ignore_globs[path].append(line)
+    def _create_combined_parser(self, additional_globs: List[str]):
+        """Create a single parser from all ignore files and additional globs."""
+        all_patterns = []
 
-    def load_recursive(self, path: Optional[Path] = None):
-        path = path or self.root_dir
-        for ignore_file in self.ignore_files:
-            ignore_file = path / ignore_file
-            if ignore_file.exists():
-                self.load_ignore_file(str(path.relative_to(self.root_dir)), ignore_file)
+        # Collect patterns from all ignore files recursively
+        self._collect_patterns_recursive(self.root_dir, all_patterns)
 
-        for subdir in path.iterdir():
-            if not subdir.is_dir() or self.ignore(subdir.relative_to(self.root_dir)):
-                continue
-            self.load_recursive(subdir)
+        # Add additional glob patterns
+        all_patterns.extend(additional_globs)
 
-    @staticmethod
-    def rstrip(value: str) -> str:
-        end = len(value) - 1
-        while end >= 0:
-            if not value[end].isspace():
-                break
-            if end > 0 and value[end - 1] == "\\":
-                break  # escaped space
-            end -= 1
-        else:
-            return ""
-        return value[: end + 1]
+        self.parser = parse_gitignore_str("\n".join(all_patterns), self.root_dir)
 
-    @staticmethod
-    def fnmatch(name: str, pattern: str, sep="/") -> bool:
-        if pattern.startswith(sep):
-            name = sep + name
-        for n, p in zip_longest(
-            reversed(name.split(sep)), reversed(pattern.split(sep)), fillvalue=None
-        ):
-            if p == "**":
-                raise NotImplementedError()
-            if p is None:
-                return True
-            if n is None or not fnmatch.fnmatch(n, p):
-                return False
-        return True
+    def _collect_patterns_recursive(self, path: Path, patterns: List[str]):
+        """
+        Recursively collect patterns from all ignore files and combine them into a single gitignore,
+        with the root directory as the base path.
+        """
+        for ignore_file_name in self.ignore_files:
+            ignore_file = path / ignore_file_name
+            if ignore_file.exists():
+                try:
+                    # Get relative path from root to this directory
+                    if path == self.root_dir:
+                        prefix = ""
+                    else:
+                        prefix = path.relative_to(self.root_dir)
 
-    def ignore(self, path: PathLike, sep="/") -> bool:
-        if not path:
+                    # Read patterns and prefix them with directory path
+                    with ignore_file.open("r", encoding="utf-8", errors="ignore") as f:
+                        for line in f:
+                            line = line.strip()
+                            if line and not line.startswith("#"):
+                                if prefix:
+                                    # Prefix patterns with directory path for subdirectories
+                                    if line.startswith("/"):
+                                        # Absolute pattern within subdirectory
+                                        patterns.append(os.path.join(prefix, line[1:]))
+                                    else:
+                                        # Relative pattern within subdirectory
+                                        # Add pattern that matches files directly in the subdirectory
+                                        patterns.append(os.path.join(prefix, line))
+                                        # Add pattern that matches files in deeper subdirectories
+                                        patterns.append(os.path.join(prefix, "**", line))
+                                else:
+                                    # Root directory patterns
+                                    patterns.append(line)
+                except (OSError, UnicodeDecodeError):
+                    # Skip files we can't read
+                    continue
+
+        # Recursively process subdirectories
+        # Note: We need to check if directories should be ignored, but we can't
+        # use self.ignore() yet since we're still building the parser
+        # So we'll process all directories and let gitignore_parser handle the logic
+        try:
+            for subdir in path.iterdir():
+                if subdir.is_dir():
+                    self._collect_patterns_recursive(subdir, patterns)
+        except (OSError, PermissionError):
+            # Skip directories we can't read
+            pass
+
+    def ignore(self, path: PathLike) -> bool:
+        """Check if a path should be ignored."""
+        if not path or not self.parser:
             return False
+
         path = Path(path)
         if path.is_absolute():
-            path = path.relative_to(self.root_dir)
+            try:
+                path = path.relative_to(self.root_dir)
+            except ValueError:
+                return False
 
-        tokens = ("." + sep + str(path)).split(sep)
-        for i in range(1, len(tokens)):
-            parent = sep.join(tokens[:-i])
-            globs = self.ignore_globs.get(parent)
-            if not globs:
-                continue
-            name = sep.join(tokens[-i:])
-            for glob in globs:
-                if self.fnmatch(name, glob, sep=sep):
-                    return True
-        return False
+        # Convert to absolute path for gitignore_parser
+        abs_path = str(self.root_dir / path)
+        return self.parser(abs_path)