feat: distribution and testing improvements

- Improved self-test script: tests 10 real files with comprehensive checks
- Created setup.py for PyPI package distribution
- Created docstripper.rb Homebrew formula
- Created INSTALL.md with installation instructions

All tests passing: 10/10 files tested successfully

Author: kiku-jw

INSTALL.md

@@ -0,0 +1,64 @@
+# Installation Guide
+
+## PyPI Installation
+
+```bash
+pip install docstripper
+```
+
+After installation, use:
+```bash
+docstripper document.txt
+```
+
+## Homebrew Installation
+
+### Option 1: Install from Formula (when tap is created)
+
+```bash
+brew tap kiku-jw/docstripper
+brew install docstripper
+```
+
+### Option 2: Install from Local Formula
+
+```bash
+brew install --build-from-source docstripper.rb
+```
+
+After installation, use:
+```bash
+docstripper document.txt
+```
+
+## Manual Installation
+
+```bash
+# Clone repository
+git clone https://github.com/kiku-jw/DocStripper.git
+cd DocStripper
+
+# Make executable (optional, for direct usage)
+chmod +x tool.py
+
+# Use directly
+python tool.py document.txt
+
+# Or create symlink
+sudo ln -s $(pwd)/tool.py /usr/local/bin/docstripper
+```
+
+## Requirements
+
+- Python 3.9 or higher
+- For PDF support (optional): `pdftotext` from poppler-utils
+  - macOS: `brew install poppler`
+  - Ubuntu/Debian: `sudo apt-get install poppler-utils`
+  - Windows: Download from [poppler-windows releases](https://github.com/oschwartz10612/poppler-windows/releases/)
+
+## Verification
+
+Run the self-test script:
+```bash
+python scripts/self_test.py
+```

PLAN.md

@@ -86,10 +86,10 @@ RELEASE COMPLETE - v2.0.0 tagged and pushed
     14.2 Add ZIP download feedback notification ✅
     14.3 Add support snackbar after cleaning completion ✅
 
-15. Distribution and tooling
-    15.1 Prepare CLI for Homebrew formula
-    15.2 Prepare CLI for PyPI package
-    15.3 Improve self-test script for release validation
+15. Distribution and tooling ✅
+    15.1 Prepare CLI for Homebrew formula ✅
+    15.2 Prepare CLI for PyPI package ✅
+    15.3 Improve self-test script for release validation ✅
 
 16. Analytics (privacy-friendly) ✅
     16.1 Add Plausible or Umami analytics (1 line, no cookies) ✅

WORKLOG.md

@@ -39,4 +39,11 @@
 - Added privacy-friendly analytics (Plausible.io, no cookies)
 - Updated app.js version to 38
 
+2025-11-03T03:00:00Z — Distribution & Testing Improvements
+- Improved self-test script: now tests 10 real files from test_files/ directory with comprehensive checks
+- Created setup.py for PyPI package distribution
+- Created docstripper.rb Homebrew formula for macOS installation
+- Created INSTALL.md with installation instructions for all methods
+- All tests passing: 10/10 files tested successfully
+
 

docstripper.rb

@@ -0,0 +1,27 @@
+# Homebrew formula for DocStripper
+# To install: brew install --build-from-source docstripper.rb
+# Or add this tap first: brew tap kiku-jw/docstripper
+
+class Docstripper < Formula
+  desc "AI-powered batch document cleaner - Remove noise from text documents automatically"
+  homepage "https://github.com/kiku-jw/DocStripper"
+  url "https://github.com/kiku-jw/DocStripper/archive/refs/heads/main.zip"
+  version "2.1.0"
+  sha256 "" # Will be calculated on first release
+  license "MIT"
+
+  depends_on "[email protected]"
+
+  def install
+    # Install the tool as a Python script
+    bin.install "tool.py" => "docstripper"
+    # Make it executable
+    chmod 0755, bin/"docstripper"
+  end
+
+  test do
+    # Run self-test
+    system "#{bin}/docstripper", "--help"
+    system "python3", "#{Formula["[email protected]"].opt_bin}/python3", "-c", "import sys; sys.path.insert(0, '#{share}/docstripper'); from tool import DocStripper; print('OK')" if File.exist?("#{share}/docstripper/tool.py")
+  end
+end

scripts/self_test.py

@@ -1,5 +1,10 @@
 #!/usr/bin/env python3
+"""
+Self-test script for DocStripper
+Tests both unit tests and real files from test_files/
+"""
 import sys
+import os
 from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
@@ -11,7 +16,13 @@ def assert_contains(text, needle):
     assert needle in text, f"Expected to find '{needle}' in output"
 
 
-def run_tests():
+def assert_not_contains(text, needle):
+    assert needle not in text, f"Expected NOT to find '{needle}' in output"
+
+
+def run_unit_tests():
+    """Run basic unit tests"""
+    print("Running unit tests...")
     ds = DocStripper(dry_run=True)
 
     # Dehyphenation
@@ -59,10 +70,105 @@ def run_tests():
                                remove_headers=False)
     assert "Page 2" in cleaned
 
-    print("Self tests passed.")
+    print("✓ Unit tests passed")
+
+
+def run_file_tests():
+    """Test on real files from test_files/"""
+    print("\nRunning file tests...")
+    
+    repo_root = Path(__file__).resolve().parents[1]
+    test_files_dir = repo_root / "test_files"
+    
+    if not test_files_dir.exists():
+        print(f"⚠ test_files directory not found at {test_files_dir}")
+        return 0
+    
+    test_files = sorted(test_files_dir.glob("*.txt"))
+    
+    if not test_files:
+        print(f"⚠ No .txt files found in {test_files_dir}")
+        return 0
+    
+    print(f"Found {len(test_files)} test file(s)")
+    
+    ds = DocStripper(dry_run=True, 
+                     merge_lines=True,
+                     dehyphenate=True,
+                     normalize_ws=True,
+                     normalize_unicode=True,
+                     remove_headers=True)
+    
+    passed = 0
+    failed = 0
+    
+    for test_file in test_files[:10]:  # Limit to 10 files
+        try:
+            with open(test_file, 'r', encoding='utf-8', errors='ignore') as f:
+                content = f.read()
+            
+            if not content.strip():
+                print(f"  ⚠ {test_file.name}: empty file, skipping")
+                continue
+            
+            # Test cleaning
+            cleaned, stats = ds.clean_text(content,
+                                          merge_lines=True,
+                                          normalize_ws=True,
+                                          normalize_unicode=True,
+                                          dehyphenate=True,
+                                          remove_headers=True)
+            
+            # Basic sanity checks
+            assert len(cleaned) <= len(content) + 1000, f"Cleaned text too long for {test_file.name}"
+            assert isinstance(stats, dict), f"Stats should be dict for {test_file.name}"
+            
+            # Check that stats have expected keys
+            expected_keys = ['lines_removed', 'duplicates_collapsed', 'empty_lines_removed', 
+                           'header_footer_removed', 'dehyphenated_tokens', 'merged_lines']
+            for key in expected_keys:
+                assert key in stats, f"Missing stat '{key}' for {test_file.name}"
+            
+            print(f"  ✓ {test_file.name}: {len(content)} → {len(cleaned)} chars, "
+                  f"removed {stats.get('lines_removed', 0)} lines")
+            passed += 1
+            
+        except Exception as e:
+            print(f"  ✗ {test_file.name}: {e}")
+            failed += 1
+    
+    print(f"\nFile tests: {passed} passed, {failed} failed")
+    return failed
+
+
+def run_all_tests():
+    """Run all tests"""
+    print("=" * 60)
+    print("DocStripper Self-Test Suite")
+    print("=" * 60)
+    
+    try:
+        run_unit_tests()
+        file_failures = run_file_tests()
+        
+        print("\n" + "=" * 60)
+        if file_failures == 0:
+            print("✅ All tests passed!")
+            return 0
+        else:
+            print(f"❌ {file_failures} test(s) failed")
+            return 1
+    except AssertionError as e:
+        print(f"\n❌ Test failed: {e}")
+        return 1
+    except Exception as e:
+        print(f"\n❌ Unexpected error: {e}")
+        import traceback
+        traceback.print_exc()
+        return 1
 
 
 if __name__ == "__main__":
-    run_tests()
+    sys.exit(run_all_tests())
 
 

setup.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+"""
+Setup script for DocStripper PyPI package
+"""
+from setuptools import setup
+from pathlib import Path
+
+# Read README for long description
+readme_file = Path(__file__).parent / "README.md"
+long_description = readme_file.read_text(encoding='utf-8') if readme_file.exists() else ""
+
+# Read version from tool.py (extract __version__ if exists, otherwise use default)
+version = "2.1.0"
+try:
+    tool_file = Path(__file__).parent / "tool.py"
+    if tool_file.exists():
+        content = tool_file.read_text(encoding='utf-8')
+        # Try to find version in comments or use default
+        for line in content.split('\n')[:50]:
+            if 'version' in line.lower() and ('2.' in line or 'v2' in line.lower()):
+                import re
+                match = re.search(r'(\d+\.\d+\.\d+)', line)
+                if match:
+                    version = match.group(1)
+                    break
+except:
+    pass
+
+setup(
+    name="docstripper",
+    version=version,
+    author="Kiku",
+    author_email="",  # Add email if needed
+    description="AI-powered batch document cleaner - Remove noise from text documents automatically",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url="https://github.com/kiku-jw/DocStripper",
+    py_modules=["tool"],
+    scripts=[],
+    entry_points={
+        "console_scripts": [
+            "docstripper=tool:main",
+        ],
+    },
+    classifiers=[
+        "Development Status :: 4 - Beta",
+        "Intended Audience :: End Users/Desktop",
+        "License :: OSI Approved :: MIT License",
+        "Operating System :: OS Independent",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
+        "Topic :: Text Processing :: Filters",
+        "Topic :: Utilities",
+    ],
+    python_requires=">=3.9",
+    install_requires=[],  # No dependencies - uses only stdlib
+    extras_require={
+        "pdf": [],  # pdftotext is external dependency
+    },
+    keywords="document cleaner, text processing, pdf, docx, batch processing",
+    project_urls={
+        "Bug Reports": "https://github.com/kiku-jw/DocStripper/issues",
+        "Source": "https://github.com/kiku-jw/DocStripper",
+        "Documentation": "https://github.com/kiku-jw/DocStripper/wiki",
+    },
+)