feat(cli): add commit command for registering Parquet files

.github/workflows/release.yml

@@ -1,22 +1,62 @@
 name: Release
 
 on:
+  push:
+    tags: ["v*"]
   workflow_dispatch:
     inputs:
-      version:
-        description: 'Version to release (e.g., 0.1.0)'
+      tag:
+        description: "Release tag (e.g., v0.3.0)"
         required: true
         type: string
 
 env:
   RUST_BACKTRACE: 1
   CARGO_TERM_COLOR: always
+  RUSTFLAGS: -Dwarnings
 
 jobs:
+  # Determine the tag to use (from push or workflow_dispatch)
+  prepare:
+    name: Prepare Release
+    runs-on: ubuntu-latest
+    outputs:
+      tag: ${{ steps.tag.outputs.tag }}
+      version: ${{ steps.tag.outputs.version }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Determine tag
+        id: tag
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            TAG="${{ inputs.tag }}"
+          else
+            TAG="${{ github.ref_name }}"
+          fi
+          # Validate tag format
+          if [[ ! "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+.*$ ]]; then
+            echo "::error::Invalid tag format: $TAG (expected vX.Y.Z)"
+            exit 1
+          fi
+          VERSION="${TAG#v}"
+          echo "tag=$TAG" >> "$GITHUB_OUTPUT"
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+
+      - name: Verify Cargo.toml version matches tag
+        run: |
+          CARGO_VERSION=$(grep '^version = ' Cargo.toml | head -1 | sed 's/version = "\(.*\)"/\1/')
+          if [ "$CARGO_VERSION" != "${{ steps.tag.outputs.version }}" ]; then
+            echo "::error::Cargo.toml version ($CARGO_VERSION) does not match tag (${{ steps.tag.outputs.version }})"
+            exit 1
+          fi
+          echo "Version verified: $CARGO_VERSION"
+
   # Validate the release before publishing
   validate:
     name: Validate Release
     runs-on: ubuntu-latest
+    needs: prepare
     steps:
       - uses: actions/checkout@v5
 
@@ -25,14 +65,7 @@ jobs:
         with:
           components: rustfmt, clippy
 
-      - name: Verify version matches input
-        run: |
-          CARGO_VERSION=$(grep "^version" Cargo.toml | head -1 | cut -d'"' -f2)
-          if [ "$CARGO_VERSION" != "${{ github.event.inputs.version }}" ]; then
-            echo "Error: Cargo.toml version ($CARGO_VERSION) does not match input version (${{ github.event.inputs.version }})"
-            exit 1
-          fi
-          echo "Version verified: $CARGO_VERSION"
+      - uses: Swatinem/rust-cache@v2
 
       - name: Check formatting
         run: cargo fmt -- --check
@@ -54,40 +87,54 @@ jobs:
       - name: Check documentation
         run: cargo doc --no-deps --all-features
 
-  # Publish to crates.io
-  publish:
-    name: Publish to crates.io
+  # Cargo publish dry run
+  publish-check:
+    name: Cargo Publish (Dry Run)
     runs-on: ubuntu-latest
-    needs: validate
+    needs: [prepare, validate]
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v4
 
       - name: Install Rust
         uses: dtolnay/rust-toolchain@stable
 
-      - name: Publish to crates.io
-        run: cargo publish --token ${{ secrets.CARGO_REGISTRY_TOKEN }}
-        env:
-          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Cargo publish dry run
+        run: cargo publish --dry-run
 
   # Create GitHub release
   github-release:
     name: Create GitHub Release
     runs-on: ubuntu-latest
-    needs: publish
+    needs: [prepare, publish-check]
     permissions:
       contents: write
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v4
 
-      - name: Create Release
-        uses: actions/create-release@v1
+      - name: Create GitHub Release
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          tag_name: v${{ github.event.inputs.version }}
-          release_name: v${{ github.event.inputs.version }}
-          body: |
-            See [CHANGELOG.md](https://github.com/${{ github.repository }}/blob/main/CHANGELOG.md) for details.
-          draft: false
-          prerelease: false
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh release create ${{ needs.prepare.outputs.tag }} \
+            --title "${{ needs.prepare.outputs.tag }}" \
+            --generate-notes
+
+  # Publish to crates.io
+  publish:
+    name: Publish to crates.io
+    runs-on: ubuntu-latest
+    needs: [prepare, github-release]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@stable
+
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Publish to crates.io
+        env:
+          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+        run: cargo publish

.pre-commit-config.yaml

@@ -13,28 +13,18 @@ repos:
         entry: cargo fmt -- --check
         language: system
         pass_filenames: false
-        stages: [pre-commit, pre-push]
 
       - id: cargo-clippy
         name: cargo clippy
         entry: cargo clippy --locked --all-targets --all-features -- -D warnings
         language: system
         pass_filenames: false
-        stages: [pre-commit, pre-push]
-
-      - id: cargo-test
-        name: cargo test
-        entry: cargo test --locked --all-features
-        language: system
-        pass_filenames: false
-        stages: [pre-push]
 
       - id: rust-quality-thresholds
         name: Rust code quality thresholds
         entry: python3 scripts/enforce_quality.py --max-loc 550
         language: system
         pass_filenames: false
-        stages: [pre-commit, pre-push]
 
   - repo: https://github.com/compilerla/conventional-pre-commit
     rev: v3.3.0

AGENTS.md

@@ -88,6 +88,22 @@ icepick compact my_namespace.my_table --target-size 268435456
 icepick snapshot list my_namespace.my_table
 icepick snapshot cleanup my_namespace.my_table --dry-run
 icepick snapshot cleanup my_namespace.my_table --older-than-days 7 --retain-last 10
+
+# Commit Parquet files to a table
+icepick commit /data/**/*.parquet --namespace my_ns --table events --dry-run
+icepick commit /data/**/*.parquet --namespace my_ns --table events
+
+# Create new table from Parquet files
+icepick commit /data/**/*.parquet --namespace my_ns --table events \
+  --create --partition year:int,month:int
+
+# Specify explicit partition values (for non-Hive paths)
+icepick commit /flat/*.parquet --namespace my_ns --table events \
+  --partition-values year=2024,month=01
+
+# Use specific file as schema exemplar
+icepick commit /data/**/*.parquet --namespace my_ns --table events \
+  --exemplar /data/sample.parquet --create
 ```
 
 ## CORE CONCEPTS
@@ -363,6 +379,29 @@ if !plan.snapshots_to_remove.is_empty() {
 }
 ```
 
+### Pattern 9: Committing Parquet files
+
+```rust
+use icepick::catalog::register::{
+    introspect_parquet_file, parse_hive_partition_values, convert_partition_values,
+    register_data_files, RegisterOptions,
+};
+
+// Introspect a Parquet file (without partition extraction)
+let introspection = introspect_parquet_file(file_io, path, None).await?;
+
+// Extract partition values from Hive-style path
+let hive_values = parse_hive_partition_values(path);  // HashMap<String, String>
+
+// Convert to typed values using schema
+let typed_values = convert_partition_values(&hive_values, &schema)?;
+
+// Or provide explicit values
+let mut explicit = HashMap::new();
+explicit.insert("year".to_string(), "2024".to_string());
+let typed_values = convert_partition_values(&explicit, &schema)?;
+```
+
 ## INTEGRATION POINTS
 
 - **Async Runtime**: tokio (required for examples/tests, not enforced as dependency)

Cargo.toml

@@ -45,6 +45,7 @@ chrono = { version = "0.4.42", features = ["serde"] }
 [target.'cfg(not(target_family = "wasm"))'.dependencies]
 # Note: We use default-features to ensure proxy support works in all environments
 reqwest = { version = "0.12", features = ["json", "rustls-tls"] }
+opendal = { version = "0.54", default-features = false, features = ["services-fs"] }
 aws-sigv4 = { version = "1.3.6", default-features = false, features = ["sign-http"] }
 aws-credential-types = { version = "1.2", default-features = false }
 aws-config = { version = "1.8", default-features = false, features = ["rustls", "behavior-version-latest", "rt-tokio"] }
@@ -56,6 +57,7 @@ comfy-table = "7"
 bytesize = "1"
 humantime = "2"
 tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] }
+glob = "0.3"
 
 # WASM targets
 [target.'cfg(target_family = "wasm")'.dependencies]