diff --git a/examples/scratch/README.md b/examples/scratch/README.md index e2e50bb..454a808 100644 --- a/examples/scratch/README.md +++ b/examples/scratch/README.md @@ -50,6 +50,9 @@ ice create-table flowers.iris_no_copy --schema-from-parquet=file://iris.parquet local-mc cp iris.parquet local/bucket1/flowers/iris_no_copy/ ice insert flowers.iris_no_copy --no-copy s3://bucket1/flowers/iris_no_copy/iris.parquet +# delete partition(By default --dry-run=true is enabled to print the list of partitions that will be deleted) +ice delete nyc.taxis --dry-run=false + # inspect ice describe diff --git a/ice/src/main/java/com/altinity/ice/cli/Main.java b/ice/src/main/java/com/altinity/ice/cli/Main.java index 38a0d43..2444fe8 100644 --- a/ice/src/main/java/com/altinity/ice/cli/Main.java +++ b/ice/src/main/java/com/altinity/ice/cli/Main.java @@ -13,6 +13,7 @@ import com.altinity.ice.cli.internal.cmd.Check; import com.altinity.ice.cli.internal.cmd.CreateNamespace; import com.altinity.ice.cli.internal.cmd.CreateTable; +import com.altinity.ice.cli.internal.cmd.Delete; import com.altinity.ice.cli.internal.cmd.DeleteNamespace; import com.altinity.ice.cli.internal.cmd.DeleteTable; import com.altinity.ice.cli.internal.cmd.Describe; @@ -27,6 +28,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import java.io.IOException; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; @@ -426,6 +428,43 @@ void deleteNamespace( } } + @CommandLine.Command(name = "delete", description = "Delete data from catalog.") + void delete( + @CommandLine.Parameters( + arity = "1", + paramLabel = "", + description = "Table name (e.g. ns1.table1)") + String name, + @CommandLine.Option( + names = {"--partition"}, + description = + "JSON array of partition filters: [{\"name\": \"vendorId\", \"values\": [5, 6]}]. " + + "For timestamp columns, use ISO Datetime format YYYY-MM-ddTHH:mm:ss") + String partitionJson, + @CommandLine.Option( + names = "--dry-run", + description = "Log files that would be deleted without actually deleting them", + defaultValue = "true") + boolean dryRun) + throws IOException { + try (RESTCatalog catalog = loadCatalog(this.configFile())) { + List partitions = new ArrayList<>(); + if (partitionJson != null && !partitionJson.isEmpty()) { + ObjectMapper mapper = newObjectMapper(); + PartitionFilter[] parts = mapper.readValue(partitionJson, PartitionFilter[].class); + partitions = Arrays.asList(parts); + } + TableIdentifier tableId = TableIdentifier.parse(name); + + Delete.run(catalog, tableId, partitions, dryRun); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + public record PartitionFilter( + @JsonProperty("name") String name, @JsonProperty("values") List values) {} + private RESTCatalog loadCatalog() throws IOException { return loadCatalog(this.configFile()); } diff --git a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Delete.java b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Delete.java new file mode 100644 index 0000000..f9bde7d --- /dev/null +++ b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Delete.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2025 Altinity Inc and/or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +package com.altinity.ice.cli.internal.cmd; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.List; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.RewriteFiles; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableScan; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.rest.RESTCatalog; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public final class Delete { + + private static final Logger logger = LoggerFactory.getLogger(Delete.class); + + private Delete() {} + + public static void run( + RESTCatalog catalog, + TableIdentifier tableId, + List partitions, + boolean dryRun) + throws IOException, URISyntaxException { + + Table table = catalog.loadTable(tableId); + TableScan scan = table.newScan(); + if (partitions != null && !partitions.isEmpty()) { + org.apache.iceberg.expressions.Expression expr = null; + for (com.altinity.ice.cli.Main.PartitionFilter pf : partitions) { + org.apache.iceberg.expressions.Expression e = null; + for (Object value : pf.values()) { + org.apache.iceberg.expressions.Expression valueExpr = + org.apache.iceberg.expressions.Expressions.equal(pf.name(), value); + e = (e == null) ? valueExpr : org.apache.iceberg.expressions.Expressions.or(e, valueExpr); + } + expr = (expr == null) ? e : org.apache.iceberg.expressions.Expressions.and(expr, e); + } + scan = scan.filter(expr); + } + List filesToDelete = new ArrayList<>(); + + try (CloseableIterable tasks = scan.planFiles()) { + for (FileScanTask task : tasks) { + filesToDelete.add(task.file()); + } + } + + if (!filesToDelete.isEmpty()) { + if (dryRun) { + logger.info("Dry run: The following files would be deleted:"); + for (DataFile file : filesToDelete) { + logger.info(" {}", file.path()); + } + } else { + RewriteFiles rewrite = table.newRewrite(); + for (DataFile deleteFile : filesToDelete) { + rewrite.deleteFile(deleteFile); + } + rewrite.commit(); + logger.info("Partition(s) deleted."); + } + } else { + logger.info("No files found for the partition(s)."); + } + } +}