Initial commit

Author: jlowin

.gitignore

@@ -0,0 +1,10 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv

.python-version

@@ -0,0 +1 @@
+3.12

README-DEV.md

@@ -0,0 +1,71 @@
+# pasteurize development notes
+
+Internal development notes and goals for the pasteurize library. This document is for developer reference only.
+
+## Core functionality
+
+The library takes a source directory and produces a markdown representation of all relevant code files, either copying to clipboard or saving to a file.
+
+### Key features
+
+- Scan directory recursively for code files
+- Smart filtering using .gitignore if available
+- CLI interface with typer/rich for great UX
+- Configurable file extension inclusion/exclusion
+- Output to clipboard (default) or file
+- XML-style markdown formatting
+
+### Output format
+
+Each file will be formatted in markdown like:
+
+```xml
+<file path="path/to/file.py" language="python">
+def example():
+    pass
+</file>
+```
+
+## Technical decisions
+
+### Dependencies
+
+- typer - CLI interface
+- rich - Terminal formatting
+- pyperclip - Clipboard interaction
+- pathspec - .gitignore parsing
+- tomli - pyproject.toml parsing (if needed)
+
+### Project structure
+
+```
+pasteurize/
+├── pyproject.toml      # Project metadata and dependencies
+├── src/
+│   └── pasteurize/    
+│       ├── __init__.py
+│       ├── cli.py      # Typer CLI implementation
+│       ├── core.py     # Core scanning/processing logic
+│       └── format.py   # Markdown formatting utilities
+└── tests/
+    └── test_*.py      # Test files
+```
+
+### Development roadmap
+
+1. Basic file scanning with extension filtering
+2. Gitignore integration
+3. Markdown formatting
+4. Clipboard/file output handling
+5. CLI implementation
+6. Testing and documentation
+
+### Notes
+
+- Keep the core logic separate from CLI for better testing
+- Use pathlib for cross-platform path handling
+- Consider adding a config file option later for project-specific settings
+- May want to add file size limits and warnings
+- Consider adding syntax highlighting hints in markdown
+- Add support for detecting file language based on extension
+- Consider adding a way to exclude specific files/patterns beyond gitignore

README.md

@@ -0,0 +1,29 @@
+[project]
+name = "pasteurize"
+version = "0.1.0"
+description = "Convert source code directories into markdown for LLM context"
+authors = []
+dependencies = [
+    "typer>=0.9.0",
+    "rich>=13.7.0",
+    "pyperclip>=1.8.2",
+    "pathspec>=0.12.1",
+]
+requires-python = ">=3.9"
+readme = "README.md"
+license = { text = "MIT" }
+
+[project.scripts]
+pasteurize = "pasteurize.cli:app"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.rye]
+managed = true
+dev-dependencies = ["pytest>=8.0.0", "black>=24.2.0", "ruff>=0.2.1"]
+
+[tool.ruff]
+line-length = 88
+target-version = "py39"

pyproject.toml

@@ -0,0 +1,3 @@
+"""Convert source code directories into markdown for LLM context."""
+
+__version__ = "0.1.0"

src/pasteurize/__init__.py

@@ -0,0 +1,120 @@
+import typer
+from pathlib import Path
+from typing import Optional, List
+from rich.console import Console
+import pyperclip
+
+from .core import scan_directory
+from .format import format_files
+from .patterns import DEFAULT_EXTENSIONS
+
+app = typer.Typer(
+    no_args_is_help=True,  # Show help when no args provided
+    add_completion=False,  # Disable shell completion for simplicity
+)
+console = Console()
+
+
+@app.command()
+def main(
+    source: list[Path] = typer.Argument(
+        ...,
+        help="Source directories to process",
+        exists=True,
+        file_okay=False,
+        dir_okay=True,
+    ),
+    outfile: Optional[Path] = typer.Option(
+        None,
+        "--out",
+        "-o",
+        help="Output file path (defaults to clipboard)",
+    ),
+    include: Optional[str] = typer.Option(
+        None,
+        "--include",
+        "-i",
+        help="Extensions to include (comma-separated, e.g. 'py,js,ts'). "
+        "If not specified, uses default extensions.",
+    ),
+    exclude: Optional[List[str]] = typer.Option(
+        None,
+        "--exclude",
+        "-x",
+        help="Glob patterns to exclude (can be used multiple times). Examples:\n"
+        "--exclude '*.test.js' = exclude test files\n"
+        "--exclude '**/tests/**' = exclude tests directories\n"
+        "--exclude 'src/legacy/**' = exclude legacy directory\n"
+        "--exclude '*.min.js' = exclude minified JS",
+    ),
+    verbose: bool = typer.Option(
+        False,
+        "--verbose",
+        "-v",
+        help="Show verbose output",
+    ),
+) -> None:
+    """Convert source code files to markdown format for LLM context."""
+    try:
+        if verbose:
+            console.print(f"Scanning directories: {', '.join(str(s) for s in source)}")
+
+            if include:
+                console.print(f"Including extensions: {include}")
+            else:
+                console.print(
+                    f"Using default extensions: {','.join(sorted(DEFAULT_EXTENSIONS))}"
+                )
+
+            if exclude:
+                console.print(f"Exclude patterns: {', '.join(exclude)}")
+
+        # Process include extensions - if specified, use only those
+        include_exts = include.split(",") if include else list(DEFAULT_EXTENSIONS)
+
+        # Scan all directories and combine results
+        all_files = []
+        for directory in source:
+            files = scan_directory(
+                directory,
+                include=include_exts,
+                extra_patterns=exclude,
+                verbose=verbose,
+            )
+            all_files.extend(files)
+
+        # Remove any duplicates (in case of overlapping directories)
+        all_files = sorted(set(all_files))
+
+        if verbose:
+            console.print(f"Found {len(all_files)} unique files")
+            for f in all_files:
+                console.print(f"  {f}")
+
+        if not all_files:
+            console.print("[yellow]No files found matching criteria[/]")
+            raise typer.Exit(0)
+
+        try:
+            markdown = format_files(all_files, verbose=verbose)
+        except Exception as e:
+            console.print(f"[red]Error during formatting:[/] {str(e)}")
+            raise typer.Exit(1)
+
+        try:
+            if outfile:
+                outfile.write_text(markdown)
+                console.print(f"Output written to [green]{outfile}[/]")
+            else:
+                pyperclip.copy(markdown)
+                console.print(
+                    f"[green]Copied[/] {len(all_files)} files to clipboard "
+                    f"({len(markdown)} characters)"
+                )
+        except Exception as e:
+            console.print(f"[red]Error during output:[/] {str(e)}")
+            raise typer.Exit(1)
+
+    except Exception as e:
+        console.print(f"[red]Error:[/] {str(e)}")
+        raise typer.Exit(1)

src/pasteurize/cli.py

@@ -0,0 +1,143 @@
+from pathlib import Path
+from typing import Optional, Sequence
+import pathspec
+import sys
+
+from .patterns import DEFAULT_EXTENSIONS, EXCLUDED_DIRS, EXCLUDED_PATTERNS
+
+
+def find_gitignore(start_path: Path) -> Optional[Path]:
+    """Search for .gitignore file in current and parent directories."""
+    print(f"Searching for gitignore from: {start_path}", file=sys.stderr)
+    current = start_path.absolute()
+    while current != current.parent:
+        gitignore = current / ".gitignore"
+        if gitignore.is_file():
+            print(f"Found gitignore at: {gitignore}", file=sys.stderr)
+            return gitignore
+        current = current.parent
+    print("No gitignore found", file=sys.stderr)
+    return None
+
+
+def get_gitignore_spec(
+    path: Path, extra_patterns: Optional[list[str]] = None, verbose: bool = False
+) -> pathspec.PathSpec:
+    """Load .gitignore patterns and combine with our default exclusions."""
+    if verbose:
+        print(f"Getting gitignore spec for: {path}", file=sys.stderr)
+
+    patterns = list(EXCLUDED_PATTERNS)
+    if verbose:
+        print(f"Added {len(EXCLUDED_PATTERNS)} default patterns", file=sys.stderr)
+
+    # Add directory exclusions
+    dir_patterns = [f"{d}/" for d in EXCLUDED_DIRS]
+    patterns.extend(dir_patterns)
+    if verbose:
+        print(f"Added {len(dir_patterns)} directory exclusions", file=sys.stderr)
+
+    # Add any extra patterns provided
+    if extra_patterns:
+        patterns.extend(extra_patterns)
+        if verbose:
+            print(f"Added {len(extra_patterns)} extra patterns", file=sys.stderr)
+
+    # Add patterns from .gitignore if found
+    gitignore_path = find_gitignore(path) if verbose else None
+    if gitignore_path:
+        with open(gitignore_path) as f:
+            gitignore_patterns = [
+                line.strip() for line in f if line.strip() and not line.startswith("#")
+            ]
+            patterns.extend(gitignore_patterns)
+            if verbose:
+                print(
+                    f"Added {len(gitignore_patterns)} patterns from gitignore",
+                    file=sys.stderr,
+                )
+
+    if verbose:
+        print(f"Total patterns: {len(patterns)}", file=sys.stderr)
+    return pathspec.PathSpec.from_lines("gitwildmatch", patterns)
+
+
+def scan_directory(
+    path: Path,
+    include: Optional[Sequence[str]] = None,
+    extra_patterns: Optional[list[str]] = None,
+    verbose: bool = False,
+) -> list[Path]:
+    """
+    Scan directory for relevant files.
+
+    Args:
+        path: Directory to scan
+        include: File extensions to include (without dots)
+        extra_patterns: Additional gitignore-style patterns to exclude
+        verbose: Whether to print debug information
+
+    Returns:
+        List of paths to relevant files
+    """
+    if verbose:
+        print(f"\nScanning directory: {path}", file=sys.stderr)
+
+    if not path.is_dir():
+        raise ValueError(f"Path {path} is not a directory")
+
+    # Use provided extensions or defaults
+    include_set = {f".{ext.lstrip('.')}" for ext in (include or DEFAULT_EXTENSIONS)}
+
+    if verbose:
+        print(f"Include extensions: {include_set}", file=sys.stderr)
+
+    # Get combined gitignore and default exclusions
+    spec = get_gitignore_spec(path, extra_patterns, verbose)
+
+    result = []
+    processed = 0
+    skipped = 0
+
+    if verbose:
+        print("\nStarting file scan...", file=sys.stderr)
+
+    for file_path in path.rglob("*"):
+        processed += 1
+        if verbose and processed % 100 == 0:
+            print(
+                f"Processed {processed} files, found {len(result)}, skipped {skipped}...",
+                file=sys.stderr,
+            )
+
+        # Skip non-files
+        if not file_path.is_file():
+            skipped += 1
+            continue
+
+        # Get relative path for pattern matching
+        try:
+            rel_path = file_path.relative_to(path)
+        except ValueError:
+            skipped += 1
+            continue
+
+        # Skip excluded patterns
+        if spec.match_file(str(rel_path)):
+            skipped += 1
+            continue
+
+        # Apply extension filters
+        ext = file_path.suffix.lower()
+        if ext not in include_set:
+            skipped += 1
+            continue
+
+        result.append(file_path)
+
+    if verbose:
+        print(
+            f"\nScan complete: processed {processed} files, found {len(result)}, skipped {skipped}",
+            file=sys.stderr,
+        )
+    return sorted(result)