From e93e77502c4f08d3c4b60d8ae0e8920780f82f3b Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Mon, 12 May 2025 11:20:31 -0400 Subject: [PATCH 01/28] ice: Added support for positional delete file. --- .../main/java/com/altinity/ice/cli/Main.java | 21 +++-- .../ice/cli/internal/cmd/DeleteFile.java | 81 +++++++++++++++++++ 2 files changed, 97 insertions(+), 5 deletions(-) create mode 100644 ice/src/main/java/com/altinity/ice/cli/internal/cmd/DeleteFile.java diff --git a/ice/src/main/java/com/altinity/ice/cli/Main.java b/ice/src/main/java/com/altinity/ice/cli/Main.java index acda67f..bae9454 100644 --- a/ice/src/main/java/com/altinity/ice/cli/Main.java +++ b/ice/src/main/java/com/altinity/ice/cli/Main.java @@ -10,11 +10,7 @@ package com.altinity.ice.cli; import ch.qos.logback.classic.Level; -import com.altinity.ice.cli.internal.cmd.Check; -import com.altinity.ice.cli.internal.cmd.CreateTable; -import com.altinity.ice.cli.internal.cmd.DeleteTable; -import com.altinity.ice.cli.internal.cmd.Describe; -import com.altinity.ice.cli.internal.cmd.Insert; +import com.altinity.ice.cli.internal.cmd.*; import com.altinity.ice.cli.internal.config.Config; import com.altinity.ice.internal.picocli.VersionProvider; import com.altinity.ice.internal.strings.Strings; @@ -22,6 +18,7 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -356,6 +353,20 @@ void deleteTable( } } + @CommandLine.Command(name = "delete-file", description = "Delete file.") + void deleteFile( + @CommandLine.Option(names = "--file", description = "File to delete") String file, + @CommandLine.Option(names = "--pos", description = "Position to delete") int pos, + @CommandLine.Option(names = "--namespace", description = "Namespace name") String namespace, + @CommandLine.Option(names = "--table", description = "Table name") String tableName) + throws IOException { + try (RESTCatalog catalog = loadCatalog(this.configFile())) { + DeleteFile.run(catalog, file, pos, namespace, tableName); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + private RESTCatalog loadCatalog(String configFile) throws IOException { Config config = Config.load(configFile); RESTCatalog catalog = new RESTCatalog(); diff --git a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/DeleteFile.java b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/DeleteFile.java new file mode 100644 index 0000000..5636531 --- /dev/null +++ b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/DeleteFile.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2025 Altinity Inc and/or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +package com.altinity.ice.cli.internal.cmd; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import org.apache.iceberg.FileMetadata; +import org.apache.iceberg.Table; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.rest.RESTCatalog; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public final class DeleteFile { + + private static final Logger logger = LoggerFactory.getLogger(DeleteFile.class); + + private DeleteFile() {} + + public static void run( + RESTCatalog catalog, String file, int pos, String namespace, String tableName) + throws IOException, URISyntaxException { + + // if file is empty, + if (file == null || file.isEmpty()) { + throw new IllegalArgumentException("File is empty"); + } + + // Parse the S3 URL + URI fileUri = new URI(file); + String scheme = fileUri.getScheme(); + if (!"s3".equals(scheme)) { + throw new IllegalArgumentException("Only s3:// URLs are supported"); + } + + // Extract bucket and key + String bucket = fileUri.getHost(); + String key = fileUri.getPath().substring(1); // Remove leading slash + + // Create delete file in the same S3 location + String deleteKey = + key.substring(0, key.lastIndexOf('/') + 1) + + "delete-" + + System.currentTimeMillis() + + ".parquet"; + String deleteFileLocation = String.format("s3://%s/%s", bucket, deleteKey); + + // Load the table + TableIdentifier tableId = TableIdentifier.of(namespace, tableName); + Table table = catalog.loadTable(tableId); + + // Create the output file + OutputFile deleteOutput = table.io().newOutputFile(deleteFileLocation); + + // Get the data file info + var dataFile = table.newScan().planFiles().iterator().next().file(); + + org.apache.iceberg.DeleteFile deleteFile = + FileMetadata.deleteFileBuilder(table.spec()) + .ofPositionDeletes() + .withPath(deleteOutput.location()) + .withPartition(dataFile.partition()) + .withReferencedDataFile(dataFile.location()) + .withFileSizeInBytes(deleteOutput.toInputFile().getLength()) + .withRecordCount(1) + .build(); + + table.newRowDelta().addDeletes(deleteFile).commit(); + + logger.info("Position delete committed for file {} at position {}", file, pos); + } +} From 7b976f61ab436ddf969adc492961d7e04470e76d Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Wed, 14 May 2025 12:41:05 -0400 Subject: [PATCH 02/28] ice: Added support to delete file with partition name and value. --- .../main/java/com/altinity/ice/cli/Main.java | 21 ++++- .../ice/cli/internal/cmd/DeleteFile.java | 82 +++++++------------ 2 files changed, 48 insertions(+), 55 deletions(-) diff --git a/ice/src/main/java/com/altinity/ice/cli/Main.java b/ice/src/main/java/com/altinity/ice/cli/Main.java index bae9454..bf0b13e 100644 --- a/ice/src/main/java/com/altinity/ice/cli/Main.java +++ b/ice/src/main/java/com/altinity/ice/cli/Main.java @@ -355,18 +355,31 @@ void deleteTable( @CommandLine.Command(name = "delete-file", description = "Delete file.") void deleteFile( - @CommandLine.Option(names = "--file", description = "File to delete") String file, - @CommandLine.Option(names = "--pos", description = "Position to delete") int pos, @CommandLine.Option(names = "--namespace", description = "Namespace name") String namespace, - @CommandLine.Option(names = "--table", description = "Table name") String tableName) + @CommandLine.Option(names = "--table", description = "Table name") String tableName, + @CommandLine.Option( + names = {"--partition"}, + description = + "JSON array of partition filters: [{\"partition_name\": \"vendorId\", \"value\": 5}]") + String partitionJson) throws IOException { try (RESTCatalog catalog = loadCatalog(this.configFile())) { - DeleteFile.run(catalog, file, pos, namespace, tableName); + List partitions = new ArrayList<>(); + if (partitionJson != null && !partitionJson.isEmpty()) { + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true); + PartitionFilter[] parts = mapper.readValue(partitionJson, PartitionFilter[].class); + partitions = Arrays.asList(parts); + } + DeleteFile.run(catalog, namespace, tableName, partitions); } catch (URISyntaxException e) { throw new RuntimeException(e); } } + public record PartitionFilter( + @JsonProperty("partition_name") String partitionName, @JsonProperty("value") Object value) {} + private RESTCatalog loadCatalog(String configFile) throws IOException { Config config = Config.load(configFile); RESTCatalog catalog = new RESTCatalog(); diff --git a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/DeleteFile.java b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/DeleteFile.java index 5636531..e3e3852 100644 --- a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/DeleteFile.java +++ b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/DeleteFile.java @@ -10,12 +10,11 @@ package com.altinity.ice.cli.internal.cmd; import java.io.IOException; -import java.net.URI; import java.net.URISyntaxException; -import org.apache.iceberg.FileMetadata; -import org.apache.iceberg.Table; +import java.util.ArrayList; +import java.util.List; +import org.apache.iceberg.*; import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.io.OutputFile; import org.apache.iceberg.rest.RESTCatalog; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,55 +26,36 @@ public final class DeleteFile { private DeleteFile() {} public static void run( - RESTCatalog catalog, String file, int pos, String namespace, String tableName) + RESTCatalog catalog, + String namespace, + String tableName, + List partitions) throws IOException, URISyntaxException { - - // if file is empty, - if (file == null || file.isEmpty()) { - throw new IllegalArgumentException("File is empty"); + Table table = catalog.loadTable(TableIdentifier.of(namespace, tableName)); + TableScan scan = table.newScan(); + if (partitions != null && !partitions.isEmpty()) { + org.apache.iceberg.expressions.Expression expr = null; + for (com.altinity.ice.cli.Main.PartitionFilter pf : partitions) { + org.apache.iceberg.expressions.Expression e = + org.apache.iceberg.expressions.Expressions.equal(pf.partitionName(), pf.value()); + expr = (expr == null) ? e : org.apache.iceberg.expressions.Expressions.and(expr, e); + } + scan = scan.filter(expr); } - - // Parse the S3 URL - URI fileUri = new URI(file); - String scheme = fileUri.getScheme(); - if (!"s3".equals(scheme)) { - throw new IllegalArgumentException("Only s3:// URLs are supported"); + Iterable tasks = scan.planFiles(); + List filesToDelete = new ArrayList<>(); + for (FileScanTask task : tasks) { + filesToDelete.add(task.file()); + } + if (!filesToDelete.isEmpty()) { + RewriteFiles rewrite = table.newRewrite(); + for (DataFile deleteFile : filesToDelete) { + rewrite.deleteFile(deleteFile); + } + rewrite.commit(); + logger.info("Partition(s) deleted."); + } else { + logger.info("No files found for the partition(s)."); } - - // Extract bucket and key - String bucket = fileUri.getHost(); - String key = fileUri.getPath().substring(1); // Remove leading slash - - // Create delete file in the same S3 location - String deleteKey = - key.substring(0, key.lastIndexOf('/') + 1) - + "delete-" - + System.currentTimeMillis() - + ".parquet"; - String deleteFileLocation = String.format("s3://%s/%s", bucket, deleteKey); - - // Load the table - TableIdentifier tableId = TableIdentifier.of(namespace, tableName); - Table table = catalog.loadTable(tableId); - - // Create the output file - OutputFile deleteOutput = table.io().newOutputFile(deleteFileLocation); - - // Get the data file info - var dataFile = table.newScan().planFiles().iterator().next().file(); - - org.apache.iceberg.DeleteFile deleteFile = - FileMetadata.deleteFileBuilder(table.spec()) - .ofPositionDeletes() - .withPath(deleteOutput.location()) - .withPartition(dataFile.partition()) - .withReferencedDataFile(dataFile.location()) - .withFileSizeInBytes(deleteOutput.toInputFile().getLength()) - .withRecordCount(1) - .build(); - - table.newRowDelta().addDeletes(deleteFile).commit(); - - logger.info("Position delete committed for file {} at position {}", file, pos); } } From 615b4308ad6229f001ccb5140b9893f76a9ec0c5 Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Thu, 15 May 2025 14:44:14 -0400 Subject: [PATCH 03/28] ice: Added support to delete file with partition name and value. --- examples/scratch/README.md | 3 +++ ice/src/main/java/com/altinity/ice/cli/Main.java | 9 ++++++--- .../com/altinity/ice/cli/internal/cmd/DeleteFile.java | 1 + 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/examples/scratch/README.md b/examples/scratch/README.md index e2e50bb..597b2ef 100644 --- a/examples/scratch/README.md +++ b/examples/scratch/README.md @@ -50,6 +50,9 @@ ice create-table flowers.iris_no_copy --schema-from-parquet=file://iris.parquet local-mc cp iris.parquet local/bucket1/flowers/iris_no_copy/ ice insert flowers.iris_no_copy --no-copy s3://bucket1/flowers/iris_no_copy/iris.parquet +# delete partition +ice delete-file --namespace=nyc --table=taxismay15 --partition='[{"partition_name": "tpep_pickup_datetime", "value": "2025-01-01T00:00:00"}]' + # inspect ice describe diff --git a/ice/src/main/java/com/altinity/ice/cli/Main.java b/ice/src/main/java/com/altinity/ice/cli/Main.java index 8ec0b3d..6ca9d07 100644 --- a/ice/src/main/java/com/altinity/ice/cli/Main.java +++ b/ice/src/main/java/com/altinity/ice/cli/Main.java @@ -366,12 +366,15 @@ void deleteTable( @CommandLine.Command(name = "delete-file", description = "Delete file.") void deleteFile( - @CommandLine.Option(names = "--namespace", description = "Namespace name") String namespace, - @CommandLine.Option(names = "--table", description = "Table name") String tableName, + @CommandLine.Option(names = "--namespace", description = "Namespace name", required = true) + String namespace, + @CommandLine.Option(names = "--table", description = "Table name", required = true) + String tableName, @CommandLine.Option( names = {"--partition"}, description = - "JSON array of partition filters: [{\"partition_name\": \"vendorId\", \"value\": 5}]") + "JSON array of partition filters: [{\"partition_name\": \"vendorId\", \"value\": 5}]. " + + "For timestmap columns, use ISO Datetime format YYYY-MM-ddTHH:mm:ss") String partitionJson) throws IOException { try (RESTCatalog catalog = loadCatalog(this.configFile())) { diff --git a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/DeleteFile.java b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/DeleteFile.java index e3e3852..57038d7 100644 --- a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/DeleteFile.java +++ b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/DeleteFile.java @@ -31,6 +31,7 @@ public static void run( String tableName, List partitions) throws IOException, URISyntaxException { + Table table = catalog.loadTable(TableIdentifier.of(namespace, tableName)); TableScan scan = table.newScan(); if (partitions != null && !partitions.isEmpty()) { From 40838b441a9e2f0a49f5ef4f52152f50f323a0ae Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Thu, 15 May 2025 14:52:05 -0400 Subject: [PATCH 04/28] ice: Fix formatting errors. --- ice/src/main/java/com/altinity/ice/cli/Main.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ice/src/main/java/com/altinity/ice/cli/Main.java b/ice/src/main/java/com/altinity/ice/cli/Main.java index 6ca9d07..b23efd2 100644 --- a/ice/src/main/java/com/altinity/ice/cli/Main.java +++ b/ice/src/main/java/com/altinity/ice/cli/Main.java @@ -373,8 +373,8 @@ void deleteFile( @CommandLine.Option( names = {"--partition"}, description = - "JSON array of partition filters: [{\"partition_name\": \"vendorId\", \"value\": 5}]. " + - "For timestmap columns, use ISO Datetime format YYYY-MM-ddTHH:mm:ss") + "JSON array of partition filters: [{\"partition_name\": \"vendorId\", \"value\": 5}]. " + + "For timestmap columns, use ISO Datetime format YYYY-MM-ddTHH:mm:ss") String partitionJson) throws IOException { try (RESTCatalog catalog = loadCatalog(this.configFile())) { From 539d8898ee9a72070cbc55854ce0e4755da08e6f Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Thu, 15 May 2025 14:54:06 -0400 Subject: [PATCH 05/28] ice: Remove wildcard imports --- ice/src/main/java/com/altinity/ice/cli/Main.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ice/src/main/java/com/altinity/ice/cli/Main.java b/ice/src/main/java/com/altinity/ice/cli/Main.java index b23efd2..169b209 100644 --- a/ice/src/main/java/com/altinity/ice/cli/Main.java +++ b/ice/src/main/java/com/altinity/ice/cli/Main.java @@ -10,7 +10,12 @@ package com.altinity.ice.cli; import ch.qos.logback.classic.Level; -import com.altinity.ice.cli.internal.cmd.*; +import com.altinity.ice.cli.internal.cmd.Check; +import com.altinity.ice.cli.internal.cmd.CreateTable; +import com.altinity.ice.cli.internal.cmd.DeleteTable; +import com.altinity.ice.cli.internal.cmd.Describe; +import com.altinity.ice.cli.internal.cmd.Insert; +import com.altinity.ice.cli.internal.cmd.DeleteFile; import com.altinity.ice.cli.internal.config.Config; import com.altinity.ice.internal.picocli.VersionProvider; import com.altinity.ice.internal.strings.Strings; From 40a313a24868f91830b9f01fba8227ef89d32d0a Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Thu, 15 May 2025 15:03:09 -0400 Subject: [PATCH 06/28] ice: Fix formatting. --- ice/src/main/java/com/altinity/ice/cli/Main.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ice/src/main/java/com/altinity/ice/cli/Main.java b/ice/src/main/java/com/altinity/ice/cli/Main.java index 169b209..dbfd7ca 100644 --- a/ice/src/main/java/com/altinity/ice/cli/Main.java +++ b/ice/src/main/java/com/altinity/ice/cli/Main.java @@ -12,10 +12,10 @@ import ch.qos.logback.classic.Level; import com.altinity.ice.cli.internal.cmd.Check; import com.altinity.ice.cli.internal.cmd.CreateTable; +import com.altinity.ice.cli.internal.cmd.DeleteFile; import com.altinity.ice.cli.internal.cmd.DeleteTable; import com.altinity.ice.cli.internal.cmd.Describe; import com.altinity.ice.cli.internal.cmd.Insert; -import com.altinity.ice.cli.internal.cmd.DeleteFile; import com.altinity.ice.cli.internal.config.Config; import com.altinity.ice.internal.picocli.VersionProvider; import com.altinity.ice.internal.strings.Strings; From d8ca2ee085556270b6890c62c12f0c2ecb829ee0 Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Wed, 21 May 2025 17:47:27 -0400 Subject: [PATCH 07/28] ice: Renamed DeleteFile to DeletePartition, Added --dry-run option. --- examples/scratch/README.md | 4 +-- .../main/java/com/altinity/ice/cli/Main.java | 20 +++++++------ .../{DeleteFile.java => DeletePartition.java} | 28 ++++++++++++------- 3 files changed, 32 insertions(+), 20 deletions(-) rename ice/src/main/java/com/altinity/ice/cli/internal/cmd/{DeleteFile.java => DeletePartition.java} (76%) diff --git a/examples/scratch/README.md b/examples/scratch/README.md index 597b2ef..5410b6f 100644 --- a/examples/scratch/README.md +++ b/examples/scratch/README.md @@ -50,8 +50,8 @@ ice create-table flowers.iris_no_copy --schema-from-parquet=file://iris.parquet local-mc cp iris.parquet local/bucket1/flowers/iris_no_copy/ ice insert flowers.iris_no_copy --no-copy s3://bucket1/flowers/iris_no_copy/iris.parquet -# delete partition -ice delete-file --namespace=nyc --table=taxismay15 --partition='[{"partition_name": "tpep_pickup_datetime", "value": "2025-01-01T00:00:00"}]' +# delete partition(Add --dry-run to print the list of partitions that will be deleted) +ice delete --namespace=nyc --table=taxismay15 --partition='[{"name": "tpep_pickup_datetime", "value": "2025-01-01T00:00:00"}]' # inspect ice describe diff --git a/ice/src/main/java/com/altinity/ice/cli/Main.java b/ice/src/main/java/com/altinity/ice/cli/Main.java index dbfd7ca..c766a78 100644 --- a/ice/src/main/java/com/altinity/ice/cli/Main.java +++ b/ice/src/main/java/com/altinity/ice/cli/Main.java @@ -12,7 +12,7 @@ import ch.qos.logback.classic.Level; import com.altinity.ice.cli.internal.cmd.Check; import com.altinity.ice.cli.internal.cmd.CreateTable; -import com.altinity.ice.cli.internal.cmd.DeleteFile; +import com.altinity.ice.cli.internal.cmd.DeletePartition; import com.altinity.ice.cli.internal.cmd.DeleteTable; import com.altinity.ice.cli.internal.cmd.Describe; import com.altinity.ice.cli.internal.cmd.Insert; @@ -369,8 +369,8 @@ void deleteTable( } } - @CommandLine.Command(name = "delete-file", description = "Delete file.") - void deleteFile( + @CommandLine.Command(name = "delete", description = "Delete Partition(s).") + void deletePartition( @CommandLine.Option(names = "--namespace", description = "Namespace name", required = true) String namespace, @CommandLine.Option(names = "--table", description = "Table name", required = true) @@ -378,9 +378,13 @@ void deleteFile( @CommandLine.Option( names = {"--partition"}, description = - "JSON array of partition filters: [{\"partition_name\": \"vendorId\", \"value\": 5}]. " - + "For timestmap columns, use ISO Datetime format YYYY-MM-ddTHH:mm:ss") - String partitionJson) + "JSON array of partition filters: [{\"name\": \"vendorId\", \"value\": 5}]. " + + "For timestamp columns, use ISO Datetime format YYYY-MM-ddTHH:mm:ss") + String partitionJson, + @CommandLine.Option( + names = "--dry-run", + description = "Log files that would be deleted without actually deleting them") + boolean dryRun) throws IOException { try (RESTCatalog catalog = loadCatalog(this.configFile())) { List partitions = new ArrayList<>(); @@ -390,14 +394,14 @@ void deleteFile( PartitionFilter[] parts = mapper.readValue(partitionJson, PartitionFilter[].class); partitions = Arrays.asList(parts); } - DeleteFile.run(catalog, namespace, tableName, partitions); + DeletePartition.run(catalog, namespace, tableName, partitions, dryRun); } catch (URISyntaxException e) { throw new RuntimeException(e); } } public record PartitionFilter( - @JsonProperty("partition_name") String partitionName, @JsonProperty("value") Object value) {} + @JsonProperty("name") String name, @JsonProperty("value") Object value) {} private RESTCatalog loadCatalog(String configFile) throws IOException { Config config = Config.load(configFile); diff --git a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/DeleteFile.java b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/DeletePartition.java similarity index 76% rename from ice/src/main/java/com/altinity/ice/cli/internal/cmd/DeleteFile.java rename to ice/src/main/java/com/altinity/ice/cli/internal/cmd/DeletePartition.java index 57038d7..bb62eaa 100644 --- a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/DeleteFile.java +++ b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/DeletePartition.java @@ -19,17 +19,18 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public final class DeleteFile { +public final class DeletePartition { - private static final Logger logger = LoggerFactory.getLogger(DeleteFile.class); + private static final Logger logger = LoggerFactory.getLogger(DeletePartition.class); - private DeleteFile() {} + private DeletePartition() {} public static void run( RESTCatalog catalog, String namespace, String tableName, - List partitions) + List partitions, + boolean dryRun) throws IOException, URISyntaxException { Table table = catalog.loadTable(TableIdentifier.of(namespace, tableName)); @@ -38,7 +39,7 @@ public static void run( org.apache.iceberg.expressions.Expression expr = null; for (com.altinity.ice.cli.Main.PartitionFilter pf : partitions) { org.apache.iceberg.expressions.Expression e = - org.apache.iceberg.expressions.Expressions.equal(pf.partitionName(), pf.value()); + org.apache.iceberg.expressions.Expressions.equal(pf.name(), pf.value()); expr = (expr == null) ? e : org.apache.iceberg.expressions.Expressions.and(expr, e); } scan = scan.filter(expr); @@ -49,12 +50,19 @@ public static void run( filesToDelete.add(task.file()); } if (!filesToDelete.isEmpty()) { - RewriteFiles rewrite = table.newRewrite(); - for (DataFile deleteFile : filesToDelete) { - rewrite.deleteFile(deleteFile); + if (dryRun) { + logger.info("Dry run: The following files would be deleted:"); + for (DataFile file : filesToDelete) { + logger.info(" {}", file.path()); + } + } else { + RewriteFiles rewrite = table.newRewrite(); + for (DataFile deleteFile : filesToDelete) { + rewrite.deleteFile(deleteFile); + } + rewrite.commit(); + logger.info("Partition(s) deleted."); } - rewrite.commit(); - logger.info("Partition(s) deleted."); } else { logger.info("No files found for the partition(s)."); } From 7aa2075fda213a07b99164c5ae4b79033e642fcd Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Sat, 24 May 2025 12:43:24 -0400 Subject: [PATCH 08/28] ice: DeletePartition , add support to delete a list of values. --- ice/src/main/java/com/altinity/ice/cli/Main.java | 4 ++-- .../altinity/ice/cli/internal/cmd/DeletePartition.java | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/ice/src/main/java/com/altinity/ice/cli/Main.java b/ice/src/main/java/com/altinity/ice/cli/Main.java index c766a78..a9c61cf 100644 --- a/ice/src/main/java/com/altinity/ice/cli/Main.java +++ b/ice/src/main/java/com/altinity/ice/cli/Main.java @@ -378,7 +378,7 @@ void deletePartition( @CommandLine.Option( names = {"--partition"}, description = - "JSON array of partition filters: [{\"name\": \"vendorId\", \"value\": 5}]. " + "JSON array of partition filters: [{\"name\": \"vendorId\", \"values\": [5, 6]}]. " + "For timestamp columns, use ISO Datetime format YYYY-MM-ddTHH:mm:ss") String partitionJson, @CommandLine.Option( @@ -401,7 +401,7 @@ void deletePartition( } public record PartitionFilter( - @JsonProperty("name") String name, @JsonProperty("value") Object value) {} + @JsonProperty("name") String name, @JsonProperty("values") List values) {} private RESTCatalog loadCatalog(String configFile) throws IOException { Config config = Config.load(configFile); diff --git a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/DeletePartition.java b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/DeletePartition.java index bb62eaa..7ec4037 100644 --- a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/DeletePartition.java +++ b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/DeletePartition.java @@ -38,8 +38,12 @@ public static void run( if (partitions != null && !partitions.isEmpty()) { org.apache.iceberg.expressions.Expression expr = null; for (com.altinity.ice.cli.Main.PartitionFilter pf : partitions) { - org.apache.iceberg.expressions.Expression e = - org.apache.iceberg.expressions.Expressions.equal(pf.name(), pf.value()); + org.apache.iceberg.expressions.Expression e = null; + for (Object value : pf.values()) { + org.apache.iceberg.expressions.Expression valueExpr = + org.apache.iceberg.expressions.Expressions.equal(pf.name(), value); + e = (e == null) ? valueExpr : org.apache.iceberg.expressions.Expressions.or(e, valueExpr); + } expr = (expr == null) ? e : org.apache.iceberg.expressions.Expressions.and(expr, e); } scan = scan.filter(expr); From 84247ec01625630b93db0aee67b4f20d84b206a8 Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Thu, 5 Jun 2025 08:41:43 -0400 Subject: [PATCH 09/28] Added Integration test to start rest catalog server with shutdown --- .../com/altinity/ice/rest/catalog/Main.java | 40 ++++++--- .../ice/rest/catalog/RESTCatalogIT.java | 81 +++++++++++++++++++ .../src/test/resources/ice-rest-catalog.yaml | 17 ++++ 3 files changed, 129 insertions(+), 9 deletions(-) create mode 100644 ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogIT.java create mode 100644 ice-rest-catalog/src/test/resources/ice-rest-catalog.yaml diff --git a/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/Main.java b/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/Main.java index 9c8797c..b32de15 100644 --- a/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/Main.java +++ b/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/Main.java @@ -34,6 +34,7 @@ import java.io.IOException; import java.io.PrintWriter; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -47,6 +48,7 @@ import org.eclipse.jetty.http.MimeTypes; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.handler.ShutdownHandler; import org.eclipse.jetty.server.handler.gzip.GzipHandler; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; @@ -83,7 +85,7 @@ public String configFile() { return configFile; } - private Main() {} + public Main() {} private static Server createServer( String host, int port, Catalog catalog, Config config, Map icebergConfig) { @@ -115,7 +117,8 @@ private static Server createBaseServer( RESTCatalogHandler restCatalogAdapter; if (requireAuth) { - mux.insertHandler(createAuthorizationHandler(config.bearerTokens(), config)); + var authHandler = createAuthorizationHandler(config.bearerTokens(), config); + mux.insertHandler(authHandler); restCatalogAdapter = new RESTCatalogAdapter(catalog); var loadTableConfig = config.toIcebergLoadTableConfig(); @@ -133,6 +136,25 @@ private static Server createBaseServer( new RESTCatalogMiddlewareTableAWCredentials( restCatalogAdapter, awsCredentialsProviders::get); } + // pick first available bearer token for shutdown + String shutdownToken = + Arrays.stream(config.bearerTokens()) + .map(Config.Token::value) + .filter(Objects::nonNull) + .findFirst() + .orElseThrow( + () -> new IllegalStateException("No bearer tokens configured for shutdown")); + + var shutdownHandler = new ShutdownHandler(shutdownToken, false, false); + shutdownHandler.setHandler(mux); + + var server = new Server(); + overrideJettyDefaults(server); + server.setHandler(shutdownHandler); + var servletHolder = new ServletHolder(new RESTCatalogServlet(restCatalogAdapter)); + mux.addServlet(servletHolder, "/*"); + return server; + } else { restCatalogAdapter = new RESTCatalogAdapter(catalog); var loadTableConfig = config.toIcebergLoadTableConfig(); @@ -149,15 +171,15 @@ private static Server createBaseServer( new RESTCatalogMiddlewareTableAWCredentials( restCatalogAdapter, uid -> awsCredentialsProvider); } - } - var h = new ServletHolder(new RESTCatalogServlet(restCatalogAdapter)); - mux.addServlet(h, "/*"); + var h = new ServletHolder(new RESTCatalogServlet(restCatalogAdapter)); + mux.addServlet(h, "/*"); - var s = new Server(); - overrideJettyDefaults(s); - s.setHandler(mux); - return s; + var s = new Server(); + overrideJettyDefaults(s); + s.setHandler(mux); + return s; + } } private static Map createAwsCredentialsProviders( diff --git a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogIT.java b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogIT.java new file mode 100644 index 0000000..82dd52c --- /dev/null +++ b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogIT.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2025 Altinity Inc and/or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +package com.altinity.ice.rest.catalog; + +import org.testng.annotations.AfterClass; +import org.testng.annotations.Test; +import picocli.CommandLine; + +public class RESTCatalogIT { + + private Thread serverThread; + private Main mainCommand; + + @Test + public void testMainCommandWithConfig() throws InterruptedException { + // Provide your CLI arguments here — replace with an actual test config file path + String[] args = {"--config", "src/test/resources/ice-rest-catalog.yaml"}; + + // Create a new instance of the command class + mainCommand = new Main(); + + // Start the server in a separate thread + serverThread = + new Thread( + () -> { + try { + new CommandLine(mainCommand).execute(args); + } catch (Exception e) { + System.err.println("Error running server: " + e.getMessage()); + } + }); + serverThread.start(); + + // Give the server time to start up + Thread.sleep(5000); + } + + @AfterClass + public void tearDown() { + try { + // Get the admin address from the config + String adminAddr = System.getProperty("ice.rest.catalog.admin.addr", "localhost:5000"); + + // Get the shutdown token from the config + String shutdownToken = "foo"; + + // Create HTTP client and send shutdown request + java.net.http.HttpClient client = java.net.http.HttpClient.newHttpClient(); + java.net.http.HttpRequest request = + java.net.http.HttpRequest.newBuilder() + .uri(java.net.URI.create("http://" + adminAddr + "/shutdown?token=" + shutdownToken)) + // .header("Authorization", "bearer " + shutdownToken) + .POST(java.net.http.HttpRequest.BodyPublishers.noBody()) + .build(); + + java.net.http.HttpResponse response = + client.send(request, java.net.http.HttpResponse.BodyHandlers.ofString()); + + if (response.statusCode() != 200) { + System.err.println("Failed to shutdown server. Status code: " + response.statusCode()); + } + + // Wait for the server thread to finish + if (serverThread != null && serverThread.isAlive()) { + serverThread.join(5000); // Wait up to 5 seconds for graceful shutdown + if (serverThread.isAlive()) { + serverThread.interrupt(); // Force interrupt if still running + } + } + } catch (Exception e) { + System.err.println("Error during server shutdown: " + e.getMessage()); + } + } +} diff --git a/ice-rest-catalog/src/test/resources/ice-rest-catalog.yaml b/ice-rest-catalog/src/test/resources/ice-rest-catalog.yaml new file mode 100644 index 0000000..1601e2c --- /dev/null +++ b/ice-rest-catalog/src/test/resources/ice-rest-catalog.yaml @@ -0,0 +1,17 @@ +uri: jdbc:sqlite:file:data/ice-rest-catalog/db.sqlite?journal_mode=WAL&synchronous=OFF&journal_size_limit=500 + +# To use etcd instead of sqlite, start etcd with `etcd --data-dir=data/etcd`, then uncomment the line below +#uri: etcd:http://localhost:2379 + +warehouse: s3://bucket1 + +s3: + endpoint: http://localhost:9000 + pathStyleAccess: true + accessKeyID: miniouser + secretAccessKey: miniopassword + region: minio + +bearerTokens: + - value: foo + From dd5ee8549cb6a9e668deeb38ad0f0b0d89d077d8 Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Thu, 5 Jun 2025 19:39:17 -0400 Subject: [PATCH 10/28] Merged conflicts from main. --- ice/src/main/java/com/altinity/ice/cli/Main.java | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/ice/src/main/java/com/altinity/ice/cli/Main.java b/ice/src/main/java/com/altinity/ice/cli/Main.java index 2cd681d..54e79c5 100644 --- a/ice/src/main/java/com/altinity/ice/cli/Main.java +++ b/ice/src/main/java/com/altinity/ice/cli/Main.java @@ -170,15 +170,13 @@ void createTable( List partitions = new ArrayList<>(); if (sortOrderJson != null && !sortOrderJson.isEmpty()) { - ObjectMapper mapper = new ObjectMapper(); - mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true); + ObjectMapper mapper = newObjectMapper(); IceSortOrder[] orders = mapper.readValue(sortOrderJson, IceSortOrder[].class); sortOrders = Arrays.asList(orders); } if (partitionJson != null && !partitionJson.isEmpty()) { - ObjectMapper mapper = new ObjectMapper(); - mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true); + ObjectMapper mapper = newObjectMapper(); IcePartition[] parts = mapper.readValue(partitionJson, IcePartition[].class); partitions = Arrays.asList(parts); } @@ -284,16 +282,14 @@ void insert( List partitions = null; if (partitionJson != null && !partitionJson.isEmpty()) { - ObjectMapper mapper = new ObjectMapper(); - mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true); + ObjectMapper mapper = newObjectMapper(); IcePartition[] parts = mapper.readValue(partitionJson, IcePartition[].class); partitions = Arrays.asList(parts); } List sortOrders = null; if (sortOrderJson != null && !sortOrderJson.isEmpty()) { - ObjectMapper mapper = new ObjectMapper(); - mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true); + ObjectMapper mapper = newObjectMapper(); IceSortOrder[] orders = mapper.readValue(sortOrderJson, IceSortOrder[].class); sortOrders = Arrays.asList(orders); } @@ -442,8 +438,7 @@ void deletePartition( try (RESTCatalog catalog = loadCatalog(this.configFile())) { List partitions = new ArrayList<>(); if (partitionJson != null && !partitionJson.isEmpty()) { - ObjectMapper mapper = new ObjectMapper(); - mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true); + ObjectMapper mapper = newObjectMapper(); PartitionFilter[] parts = mapper.readValue(partitionJson, PartitionFilter[].class); partitions = Arrays.asList(parts); } From 00dae4648c6a1f3b5f27950900116bfc0f45cc25 Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Thu, 5 Jun 2025 19:41:11 -0400 Subject: [PATCH 11/28] Merged conflicts from main. --- ice/src/main/java/com/altinity/ice/cli/Main.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ice/src/main/java/com/altinity/ice/cli/Main.java b/ice/src/main/java/com/altinity/ice/cli/Main.java index 54e79c5..945b04a 100644 --- a/ice/src/main/java/com/altinity/ice/cli/Main.java +++ b/ice/src/main/java/com/altinity/ice/cli/Main.java @@ -10,7 +10,14 @@ package com.altinity.ice.cli; import ch.qos.logback.classic.Level; -import com.altinity.ice.cli.internal.cmd.*; +import com.altinity.ice.cli.internal.cmd.Check; +import com.altinity.ice.cli.internal.cmd.CreateNamespace; +import com.altinity.ice.cli.internal.cmd.CreateTable; +import com.altinity.ice.cli.internal.cmd.DeleteNamespace; +import com.altinity.ice.cli.internal.cmd.DeleteTable; +import com.altinity.ice.cli.internal.cmd.Describe; +import com.altinity.ice.cli.internal.cmd.Insert; +import com.altinity.ice.cli.internal.cmd.Scan; import com.altinity.ice.cli.internal.config.Config; import com.altinity.ice.internal.picocli.VersionProvider; import com.altinity.ice.internal.strings.Strings; From c14da5444498ce1cd67ad68c2b2e58327fcddbde Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Thu, 5 Jun 2025 19:41:48 -0400 Subject: [PATCH 12/28] Merged conflicts from main. --- ice/src/main/java/com/altinity/ice/cli/Main.java | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/ice/src/main/java/com/altinity/ice/cli/Main.java b/ice/src/main/java/com/altinity/ice/cli/Main.java index 945b04a..54e79c5 100644 --- a/ice/src/main/java/com/altinity/ice/cli/Main.java +++ b/ice/src/main/java/com/altinity/ice/cli/Main.java @@ -10,14 +10,7 @@ package com.altinity.ice.cli; import ch.qos.logback.classic.Level; -import com.altinity.ice.cli.internal.cmd.Check; -import com.altinity.ice.cli.internal.cmd.CreateNamespace; -import com.altinity.ice.cli.internal.cmd.CreateTable; -import com.altinity.ice.cli.internal.cmd.DeleteNamespace; -import com.altinity.ice.cli.internal.cmd.DeleteTable; -import com.altinity.ice.cli.internal.cmd.Describe; -import com.altinity.ice.cli.internal.cmd.Insert; -import com.altinity.ice.cli.internal.cmd.Scan; +import com.altinity.ice.cli.internal.cmd.*; import com.altinity.ice.cli.internal.config.Config; import com.altinity.ice.internal.picocli.VersionProvider; import com.altinity.ice.internal.strings.Strings; From 13e0751e7bd1efec13eb2d4ed5e7b384003e769f Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Thu, 5 Jun 2025 19:43:16 -0400 Subject: [PATCH 13/28] Fixed imports --- ice/src/main/java/com/altinity/ice/cli/Main.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ice/src/main/java/com/altinity/ice/cli/Main.java b/ice/src/main/java/com/altinity/ice/cli/Main.java index 54e79c5..6521605 100644 --- a/ice/src/main/java/com/altinity/ice/cli/Main.java +++ b/ice/src/main/java/com/altinity/ice/cli/Main.java @@ -10,7 +10,15 @@ package com.altinity.ice.cli; import ch.qos.logback.classic.Level; -import com.altinity.ice.cli.internal.cmd.*; +import com.altinity.ice.cli.internal.cmd.Check; +import com.altinity.ice.cli.internal.cmd.CreateNamespace; +import com.altinity.ice.cli.internal.cmd.CreateTable; +import com.altinity.ice.cli.internal.cmd.DeleteNamespace; +import com.altinity.ice.cli.internal.cmd.DeletePartition; +import com.altinity.ice.cli.internal.cmd.DeleteTable; +import com.altinity.ice.cli.internal.cmd.Describe; +import com.altinity.ice.cli.internal.cmd.Insert; +import com.altinity.ice.cli.internal.cmd.Scan; import com.altinity.ice.cli.internal.config.Config; import com.altinity.ice.internal.picocli.VersionProvider; import com.altinity.ice.internal.strings.Strings; From 2e5795a40e0526c1c36b4a7b44aa2c3d78092e24 Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Thu, 5 Jun 2025 19:50:50 -0400 Subject: [PATCH 14/28] revert devbox.json changes. --- examples/docker-compose/devbox.json | 1 + examples/eks/devbox.json | 3 ++- examples/localfileio/devbox.json | 3 ++- examples/localfileio/devbox.lock | 38 ++++++++++++++++++++++++++++- examples/scratch/devbox.json | 3 ++- 5 files changed, 44 insertions(+), 4 deletions(-) diff --git a/examples/docker-compose/devbox.json b/examples/docker-compose/devbox.json index 4998fc5..c077607 100644 --- a/examples/docker-compose/devbox.json +++ b/examples/docker-compose/devbox.json @@ -1,6 +1,7 @@ { "$schema": "https://raw.githubusercontent.com/jetify-com/devbox/0.10.7/.schema/devbox.schema.json", "packages": [ + "jdk-headless@21.0.7+6" ], "env": { "AT": "ice:examples/docker-compose" diff --git a/examples/eks/devbox.json b/examples/eks/devbox.json index 848f8fe..be3a4e9 100644 --- a/examples/eks/devbox.json +++ b/examples/eks/devbox.json @@ -5,7 +5,8 @@ "kubectl@latest", "envsubst@latest", "awscli2@latest", - "etcd_3_5@latest" + "etcd_3_5@latest", + "jdk-headless@21.0.7+6" ], "env": { "AT": "ice:examples/eks" diff --git a/examples/localfileio/devbox.json b/examples/localfileio/devbox.json index 4672af0..f8be8f9 100644 --- a/examples/localfileio/devbox.json +++ b/examples/localfileio/devbox.json @@ -4,6 +4,7 @@ "AT": "ice:examples/localfileio" }, "packages": { - "sqlite": "latest" + "sqlite": "latest", + "jdk-headless": "21.0.7+6" } } diff --git a/examples/localfileio/devbox.lock b/examples/localfileio/devbox.lock index 91e4d1b..a665e0c 100644 --- a/examples/localfileio/devbox.lock +++ b/examples/localfileio/devbox.lock @@ -2,7 +2,43 @@ "lockfile_version": "1", "packages": { "github:NixOS/nixpkgs/nixpkgs-unstable": { - "resolved": "github:NixOS/nixpkgs/29335f23bea5e34228349ea739f31ee79e267b88?lastModified=1745804731&narHash=sha256-v%2FsK3AS0QKu%2FTu5sHIfddiEHCvrbNYPv8X10Fpux68g%3D" + "resolved": "github:NixOS/nixpkgs/e4b09e47ace7d87de083786b404bf232eb6c89d8?lastModified=1748856973&narHash=sha256-RlTsJUvvr8ErjPBsiwrGbbHYW8XbB%2Foek0Gi78XdWKg%3D" + }, + "jdk-headless@21.0.7+6": { + "last_modified": "2025-05-16T20:19:48Z", + "resolved": "github:NixOS/nixpkgs/12a55407652e04dcf2309436eb06fef0d3713ef3#jdk21_headless", + "source": "devbox-search", + "version": "21.0.7+6", + "systems": { + "aarch64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/r4isdsjpz1i2zi93y6cjl53m0ngvbww3-openjdk-headless-21.0.7+6", + "default": true + }, + { + "name": "debug", + "path": "/nix/store/s7h859cnwfbr0ss7dlssphbhhivdc63p-openjdk-headless-21.0.7+6-debug" + } + ], + "store_path": "/nix/store/r4isdsjpz1i2zi93y6cjl53m0ngvbww3-openjdk-headless-21.0.7+6" + }, + "x86_64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/jfxsvqhkxza9vrmpjw2nfxyz32883q1b-openjdk-headless-21.0.7+6", + "default": true + }, + { + "name": "debug", + "path": "/nix/store/ffknhap8n6nh733c33z714a9whmqz392-openjdk-headless-21.0.7+6-debug" + } + ], + "store_path": "/nix/store/jfxsvqhkxza9vrmpjw2nfxyz32883q1b-openjdk-headless-21.0.7+6" + } + } }, "sqlite@latest": { "last_modified": "2025-05-29T08:12:58Z", diff --git a/examples/scratch/devbox.json b/examples/scratch/devbox.json index 6583158..c5c08f2 100644 --- a/examples/scratch/devbox.json +++ b/examples/scratch/devbox.json @@ -4,7 +4,8 @@ "minio@latest", "minio-client@latest", "sqlite@latest", - "etcd_3_5@latest" + "etcd_3_5@latest", + "jdk-headless@21.0.7+6" ], "env": { "AT": "ice:examples/scratch" From 4f93ceb5bddd8371cfa9b1b2f202a52b77fc0001 Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Wed, 18 Jun 2025 11:57:08 -0400 Subject: [PATCH 15/28] Added RESTCatalogIT integration test to start ice rest server and validate ice CLI commands. --- .../ice/rest/catalog/RESTCatalogIT.java | 32 +++++++++++++------ .../src/test/resources/ice-rest-cli.yaml | 14 ++++++++ .../main/java/com/altinity/ice/cli/Main.java | 2 +- 3 files changed, 37 insertions(+), 11 deletions(-) create mode 100644 ice-rest-catalog/src/test/resources/ice-rest-cli.yaml diff --git a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogIT.java b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogIT.java index fb0bd3f..9b5aa01 100644 --- a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogIT.java +++ b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogIT.java @@ -45,19 +45,31 @@ public void testMainCommandWithConfig() throws InterruptedException { mainCommand = new Main(); // Start the server in a separate thread - serverThread = - new Thread( - () -> { - try { - new CommandLine(mainCommand).execute(args); - } catch (Exception e) { - System.err.println("Error running server: " + e.getMessage()); - } - }); + serverThread = new Thread(() -> { + try { + new CommandLine(mainCommand).execute(args); + } catch (Exception e) { + System.err.println("Error running server: " + e.getMessage()); + } + }); serverThread.start(); // Give the server time to start up - Thread.sleep(5000); + Thread.sleep(2000); + + // Create namespace and table using Ice CLI + String namespace = "test_namespace"; + String tableName = "test_table"; + + // Delete namespace before creating it + new CommandLine(new com.altinity.ice.cli.Main()) + .execute("--config", "src/test/resources/ice-rest-cli.yaml", "delete-namespace", namespace); + + // Create namespace + new CommandLine(new com.altinity.ice.cli.Main()) + .execute("--config", "src/test/resources/ice-rest-cli.yaml", "create-namespace", namespace); + + } @AfterClass diff --git a/ice-rest-catalog/src/test/resources/ice-rest-cli.yaml b/ice-rest-catalog/src/test/resources/ice-rest-cli.yaml new file mode 100644 index 0000000..b9993e2 --- /dev/null +++ b/ice-rest-catalog/src/test/resources/ice-rest-cli.yaml @@ -0,0 +1,14 @@ + +# To use etcd instead of sqlite, start etcd with `etcd --data-dir=data/etcd`, then uncomment the line below +#uri: etcd:http://localhost:2379 +s3: + endpoint: http://localhost:9000 + pathStyleAccess: true + accessKeyID: miniouser + secretAccessKey: miniopassword + region: minio + +bearerToken: foo +#bearerToken: +# - value: foo + diff --git a/ice/src/main/java/com/altinity/ice/cli/Main.java b/ice/src/main/java/com/altinity/ice/cli/Main.java index b902674..e5379d4 100644 --- a/ice/src/main/java/com/altinity/ice/cli/Main.java +++ b/ice/src/main/java/com/altinity/ice/cli/Main.java @@ -75,7 +75,7 @@ public String configFile() { scope = CommandLine.ScopeType.INHERIT) private String logLevel; - private Main() {} + public Main() {} @CommandLine.Command(name = "check", description = "Check configuration.") void check() throws IOException { From 86c0e7fb2130a3fb36ae4fcfa6c2ef23ca84e53d Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Thu, 19 Jun 2025 07:42:22 -0400 Subject: [PATCH 16/28] Reverted back change to shutDown token. --- .../com/altinity/ice/rest/catalog/Main.java | 238 ++++++++---------- 1 file changed, 108 insertions(+), 130 deletions(-) diff --git a/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/Main.java b/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/Main.java index b32de15..d2cfe6b 100644 --- a/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/Main.java +++ b/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/Main.java @@ -34,7 +34,6 @@ import java.io.IOException; import java.io.PrintWriter; import java.nio.charset.StandardCharsets; -import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -48,7 +47,6 @@ import org.eclipse.jetty.http.MimeTypes; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; -import org.eclipse.jetty.server.handler.ShutdownHandler; import org.eclipse.jetty.server.handler.gzip.GzipHandler; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; @@ -63,19 +61,19 @@ import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; @CommandLine.Command( - name = "ice-rest-catalog", - description = "Iceberg REST Catalog.", - mixinStandardHelpOptions = true, - scope = CommandLine.ScopeType.INHERIT, - versionProvider = VersionProvider.class, - subcommands = com.altinity.ice.cli.Main.class) + name = "ice-rest-catalog", + description = "Iceberg REST Catalog.", + mixinStandardHelpOptions = true, + scope = CommandLine.ScopeType.INHERIT, + versionProvider = VersionProvider.class, + subcommands = com.altinity.ice.cli.Main.class) public final class Main implements Callable { private static final Logger logger = LoggerFactory.getLogger(Main.class); @CommandLine.Option( - names = {"-c", "--config"}, - description = "/path/to/config.yaml ($CWD/.ice-rest-catalog.yaml by default)") + names = {"-c", "--config"}, + description = "/path/to/config.yaml ($CWD/.ice-rest-catalog.yaml by default)") String configFile; public String configFile() { @@ -85,10 +83,10 @@ public String configFile() { return configFile; } - public Main() {} + private Main() {} private static Server createServer( - String host, int port, Catalog catalog, Config config, Map icebergConfig) { + String host, int port, Catalog catalog, Config config, Map icebergConfig) { var s = createBaseServer(catalog, config, icebergConfig, true); ServerConnector connector = new ServerConnector(s); connector.setHost(host); @@ -98,7 +96,7 @@ private static Server createServer( } private static Server createAdminServer( - String host, int port, Catalog catalog, Config config, Map icebergConfig) { + String host, int port, Catalog catalog, Config config, Map icebergConfig) { var s = createBaseServer(catalog, config, icebergConfig, false); ServerConnector connector = new ServerConnector(s); connector.setHost(host); @@ -108,7 +106,7 @@ private static Server createAdminServer( } private static Server createBaseServer( - Catalog catalog, Config config, Map icebergConfig, boolean requireAuth) { + Catalog catalog, Config config, Map icebergConfig, boolean requireAuth) { var mux = new ServletContextHandler(ServletContextHandler.NO_SESSIONS); mux.insertHandler(new GzipHandler()); // TODO: RequestLogHandler @@ -116,89 +114,69 @@ private static Server createBaseServer( // TODO: ShutdownHandler RESTCatalogHandler restCatalogAdapter; + String warehouse = icebergConfig.getOrDefault(CatalogProperties.WAREHOUSE_LOCATION, ""); + boolean awsAuth = + warehouse.startsWith("s3://") + || warehouse.startsWith("arn:aws:s3tables:"); // FIXME: arn:aws-cn:s3tables if (requireAuth) { - var authHandler = createAuthorizationHandler(config.bearerTokens(), config); - mux.insertHandler(authHandler); + mux.insertHandler(createAuthorizationHandler(config.bearerTokens(), config)); restCatalogAdapter = new RESTCatalogAdapter(catalog); var loadTableConfig = config.toIcebergLoadTableConfig(); if (!loadTableConfig.isEmpty()) { restCatalogAdapter = - new RESTCatalogMiddlewareTableConfig(restCatalogAdapter, loadTableConfig); + new RESTCatalogMiddlewareTableConfig(restCatalogAdapter, loadTableConfig); } - if (icebergConfig - .getOrDefault(CatalogProperties.WAREHOUSE_LOCATION, "") - .startsWith("s3://")) { + if (awsAuth) { Map awsCredentialsProviders = - createAwsCredentialsProviders(config.bearerTokens(), config, icebergConfig); + createAwsCredentialsProviders(config.bearerTokens(), config, icebergConfig); restCatalogAdapter = - new RESTCatalogMiddlewareTableAWCredentials( - restCatalogAdapter, awsCredentialsProviders::get); + new RESTCatalogMiddlewareTableAWCredentials( + restCatalogAdapter, awsCredentialsProviders::get); } - // pick first available bearer token for shutdown - String shutdownToken = - Arrays.stream(config.bearerTokens()) - .map(Config.Token::value) - .filter(Objects::nonNull) - .findFirst() - .orElseThrow( - () -> new IllegalStateException("No bearer tokens configured for shutdown")); - - var shutdownHandler = new ShutdownHandler(shutdownToken, false, false); - shutdownHandler.setHandler(mux); - - var server = new Server(); - overrideJettyDefaults(server); - server.setHandler(shutdownHandler); - var servletHolder = new ServletHolder(new RESTCatalogServlet(restCatalogAdapter)); - mux.addServlet(servletHolder, "/*"); - return server; - } else { restCatalogAdapter = new RESTCatalogAdapter(catalog); var loadTableConfig = config.toIcebergLoadTableConfig(); if (!loadTableConfig.isEmpty()) { restCatalogAdapter = - new RESTCatalogMiddlewareTableConfig(restCatalogAdapter, loadTableConfig); + new RESTCatalogMiddlewareTableConfig(restCatalogAdapter, loadTableConfig); } - if (icebergConfig - .getOrDefault(CatalogProperties.WAREHOUSE_LOCATION, "") - .startsWith("s3://")) { + if (awsAuth) { DefaultCredentialsProvider awsCredentialsProvider = DefaultCredentialsProvider.create(); restCatalogAdapter = - new RESTCatalogMiddlewareTableAWCredentials( - restCatalogAdapter, uid -> awsCredentialsProvider); + new RESTCatalogMiddlewareTableAWCredentials( + restCatalogAdapter, uid -> awsCredentialsProvider); } + } - var h = new ServletHolder(new RESTCatalogServlet(restCatalogAdapter)); - mux.addServlet(h, "/*"); + var h = new ServletHolder(new RESTCatalogServlet(restCatalogAdapter)); + mux.addServlet(h, "/*"); - var s = new Server(); - overrideJettyDefaults(s); - s.setHandler(mux); - return s; - } + var s = new Server(); + overrideJettyDefaults(s); + s.setHandler(mux); + return s; } private static Map createAwsCredentialsProviders( - Config.Token[] tokens, Config config, Map icebergConfig) { + Config.Token[] tokens, Config config, Map icebergConfig) { AwsCredentialsProvider awsCredentialsProvider; if (icebergConfig.containsKey(S3FileIOProperties.ACCESS_KEY_ID)) { if (icebergConfig.containsKey(S3FileIOProperties.SESSION_TOKEN)) { awsCredentialsProvider = - StaticCredentialsProvider.create( - AwsSessionCredentials.create( - icebergConfig.get(S3FileIOProperties.ACCESS_KEY_ID), - icebergConfig.get(S3FileIOProperties.SECRET_ACCESS_KEY), - icebergConfig.get(S3FileIOProperties.SESSION_TOKEN))); + StaticCredentialsProvider.create( + AwsSessionCredentials.create( + icebergConfig.get(S3FileIOProperties.ACCESS_KEY_ID), + icebergConfig.get(S3FileIOProperties.SECRET_ACCESS_KEY), + icebergConfig.get(S3FileIOProperties.SESSION_TOKEN))); } else { awsCredentialsProvider = - StaticCredentialsProvider.create( - AwsBasicCredentials.create( - icebergConfig.get(S3FileIOProperties.ACCESS_KEY_ID), - icebergConfig.get(S3FileIOProperties.SECRET_ACCESS_KEY))); + StaticCredentialsProvider.create( + AwsBasicCredentials.create( + icebergConfig.get(S3FileIOProperties.ACCESS_KEY_ID), + icebergConfig.get(S3FileIOProperties.SECRET_ACCESS_KEY))); } } else { awsCredentialsProvider = DefaultCredentialsProvider.create(); @@ -206,36 +184,36 @@ private static Map createAwsCredentialsProviders Map awsCredentialsProviders = new HashMap<>(); for (Config.Token token : tokens) { awsCredentialsProviders.put( - token.resourceName(), - !token.accessConfig().hasAWSAssumeRoleARN() - ? awsCredentialsProvider - : CredentialsProvider.assumeRule( - awsCredentialsProvider, - token.accessConfig().awsAssumeRoleARN(), - getUserAgentForToken(token))); + token.resourceName(), + !token.accessConfig().hasAWSAssumeRoleARN() + ? awsCredentialsProvider + : CredentialsProvider.assumeRule( + awsCredentialsProvider, + token.accessConfig().awsAssumeRoleARN(), + getUserAgentForToken(token))); } if (config.anonymousAccess().enabled()) { var token = new Config.Token("anonymous", "", config.anonymousAccess().accessConfig()); awsCredentialsProviders.put( - token.resourceName(), - !token.accessConfig().hasAWSAssumeRoleARN() - ? awsCredentialsProvider - : CredentialsProvider.assumeRule( - awsCredentialsProvider, - token.accessConfig().awsAssumeRoleARN(), - getUserAgentForToken(token))); + token.resourceName(), + !token.accessConfig().hasAWSAssumeRoleARN() + ? awsCredentialsProvider + : CredentialsProvider.assumeRule( + awsCredentialsProvider, + token.accessConfig().awsAssumeRoleARN(), + getUserAgentForToken(token))); } return awsCredentialsProviders; } private static String getUserAgentForToken(Config.Token token) { return !Strings.isNullOrEmpty(token.name()) - ? "ice-rest-catalog." + token.name() - : "ice-rest-catalog"; + ? "ice-rest-catalog." + token.name() + : "ice-rest-catalog"; } private static RESTCatalogAuthorizationHandler createAuthorizationHandler( - Config.Token[] tokens, Config config) { + Config.Token[] tokens, Config config) { Session anonymousSession = null; if (config.anonymousAccess().enabled()) { var t = new Config.Token("anonymous", "", config.anonymousAccess().accessConfig()); @@ -244,7 +222,7 @@ private static RESTCatalogAuthorizationHandler createAuthorizationHandler( } if (tokens.length == 0 && anonymousSession == null) { throw new IllegalArgumentException( - "invalid config: either set anonymousAccess.enabled to true or provide tokens via bearerTokens"); + "invalid config: either set anonymousAccess.enabled to true or provide tokens via bearerTokens"); } return new RESTCatalogAuthorizationHandler(tokens, anonymousSession); } @@ -255,19 +233,19 @@ private static Server createDebugServer(String host, int port) { mux.addServlet(new ServletHolder(new PrometheusMetricsServlet()), "/metrics"); var h = - new ServletHolder( - new HttpServlet() { - @Override - protected void doGet(HttpServletRequest req, HttpServletResponse resp) - throws IOException { - resp.setStatus(HttpServletResponse.SC_OK); - resp.setContentType(MimeTypes.Type.TEXT_PLAIN.asString()); - resp.setCharacterEncoding(StandardCharsets.UTF_8.name()); - try (PrintWriter w = resp.getWriter()) { - w.write("OK"); - } - } - }); + new ServletHolder( + new HttpServlet() { + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) + throws IOException { + resp.setStatus(HttpServletResponse.SC_OK); + resp.setContentType(MimeTypes.Type.TEXT_PLAIN.asString()); + resp.setCharacterEncoding(StandardCharsets.UTF_8.name()); + try (PrintWriter w = resp.getWriter()) { + w.write("OK"); + } + } + }); mux.addServlet(h, "/healthz"); // TODO: provide proper impl @@ -297,28 +275,28 @@ public Integer call() throws Exception { var icebergConfig = config.toIcebergConfig(); logger.debug( - "Iceberg configuration: {}", - icebergConfig.entrySet().stream() - .map(e -> !e.getKey().contains("key") ? e.getKey() + "=" + e.getValue() : e.getKey()) - .sorted() - .collect(Collectors.joining(", "))); + "Iceberg configuration: {}", + icebergConfig.entrySet().stream() + .map(e -> !e.getKey().contains("key") ? e.getKey() + "=" + e.getValue() : e.getKey()) + .sorted() + .collect(Collectors.joining(", "))); ObjectMapper om = new ObjectMapper(); for (Config.Token t : config.bearerTokens()) { if (Strings.isNullOrEmpty(t.name())) { logger.info( - "Catalog accessible via bearer token named \"{}\" (config: {})", - Objects.requireNonNullElse(t.name(), ""), - om.writeValueAsString(t.accessConfig())); + "Catalog accessible via bearer token named \"{}\" (config: {})", + Objects.requireNonNullElse(t.name(), ""), + om.writeValueAsString(t.accessConfig())); } else { logger.info( - "Catalog accessible via bearer token (config: {})", - om.writeValueAsString(t.accessConfig())); + "Catalog accessible via bearer token (config: {})", + om.writeValueAsString(t.accessConfig())); } } if (config.anonymousAccess().enabled()) { logger.warn( - "Anonymous access enabled (config: {})", - om.writeValueAsString(config.anonymousAccess().accessConfig())); + "Anonymous access enabled (config: {})", + om.writeValueAsString(config.anonymousAccess().accessConfig())); } // FIXME: remove @@ -348,19 +326,19 @@ public Integer call() throws Exception { if (!Strings.isNullOrEmpty(config.adminAddr())) { HostAndPort adminHostAndPort = HostAndPort.fromString(config.adminAddr()); Server adminServer = - createAdminServer( - adminHostAndPort.getHost(), - adminHostAndPort.getPort(), - catalog, - config, - icebergConfig); + createAdminServer( + adminHostAndPort.getHost(), + adminHostAndPort.getPort(), + catalog, + config, + icebergConfig); adminServer.start(); logger.warn("Serving admin endpoint at http://{}/v1/{config,*}", adminHostAndPort); } HostAndPort hostAndPort = HostAndPort.fromString(config.addr()); Server httpServer = - createServer(hostAndPort.getHost(), hostAndPort.getPort(), catalog, config, icebergConfig); + createServer(hostAndPort.getHost(), hostAndPort.getPort(), catalog, config, icebergConfig); httpServer.start(); logger.info("Serving http://{}/v1/{config,*}", hostAndPort); @@ -380,11 +358,11 @@ private void initializeMaintenanceScheduler(Catalog catalog, Config config) { } try { MaintenanceScheduler scheduler = - new MaintenanceScheduler( - catalog, config.maintenanceSchedule(), config.snapshotTTLInDays()); + new MaintenanceScheduler( + catalog, config.maintenanceSchedule(), config.snapshotTTLInDays()); scheduler.startScheduledMaintenance(); logger.info( - "Maintenance scheduler initialized with schedule: {}", config.maintenanceSchedule()); + "Maintenance scheduler initialized with schedule: {}", config.maintenanceSchedule()); } catch (Exception e) { logger.error("Failed to initialize maintenance scheduler", e); throw new RuntimeException(e); @@ -395,33 +373,33 @@ private static Catalog newEctdCatalog(Map config) { // TODO: remove; params all verified by config String uri = config.getOrDefault(CatalogProperties.URI, "etcd:http://localhost:2379"); Preconditions.checkArgument( - !Strings.isNullOrEmpty(uri), "etcd catalog: \"%s\" required", CatalogProperties.URI); + !Strings.isNullOrEmpty(uri), "etcd catalog: \"%s\" required", CatalogProperties.URI); String inputWarehouseLocation = config.get(CatalogProperties.WAREHOUSE_LOCATION); Preconditions.checkArgument( - !Strings.isNullOrEmpty(inputWarehouseLocation), - "etcd catalog: \"%s\" required", - CatalogProperties.WAREHOUSE_LOCATION); + !Strings.isNullOrEmpty(inputWarehouseLocation), + "etcd catalog: \"%s\" required", + CatalogProperties.WAREHOUSE_LOCATION); String ioImpl = config.get(CatalogProperties.FILE_IO_IMPL); Preconditions.checkArgument( - !Strings.isNullOrEmpty(ioImpl), - "etcd catalog: \"%s\" required", - CatalogProperties.FILE_IO_IMPL); + !Strings.isNullOrEmpty(ioImpl), + "etcd catalog: \"%s\" required", + CatalogProperties.FILE_IO_IMPL); var io = CatalogUtil.loadFileIO(ioImpl, config, null); return new EtcdCatalog( - "default", Strings.removePrefix(uri, "etcd:"), inputWarehouseLocation, io); + "default", Strings.removePrefix(uri, "etcd:"), inputWarehouseLocation, io); } public static void main(String[] args) throws Exception { SLF4JBridgeHandler.install(); CommandLine cmd = new CommandLine(new Main()); cmd.setExecutionExceptionHandler( - (Exception ex, CommandLine self, CommandLine.ParseResult res) -> { - logger.error("Fatal", ex); - return 1; - }); + (Exception ex, CommandLine self, CommandLine.ParseResult res) -> { + logger.error("Fatal", ex); + return 1; + }); int exitCode = cmd.execute(args); System.exit(exitCode); } From 2396bc1b3f1b40c1f23913f3104194806214d83b Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Thu, 19 Jun 2025 07:49:07 -0400 Subject: [PATCH 17/28] formatting changes. --- .../com/altinity/ice/rest/catalog/Main.java | 190 +++++++++--------- 1 file changed, 95 insertions(+), 95 deletions(-) diff --git a/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/Main.java b/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/Main.java index d2cfe6b..feaab56 100644 --- a/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/Main.java +++ b/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/Main.java @@ -61,19 +61,19 @@ import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; @CommandLine.Command( - name = "ice-rest-catalog", - description = "Iceberg REST Catalog.", - mixinStandardHelpOptions = true, - scope = CommandLine.ScopeType.INHERIT, - versionProvider = VersionProvider.class, - subcommands = com.altinity.ice.cli.Main.class) + name = "ice-rest-catalog", + description = "Iceberg REST Catalog.", + mixinStandardHelpOptions = true, + scope = CommandLine.ScopeType.INHERIT, + versionProvider = VersionProvider.class, + subcommands = com.altinity.ice.cli.Main.class) public final class Main implements Callable { private static final Logger logger = LoggerFactory.getLogger(Main.class); @CommandLine.Option( - names = {"-c", "--config"}, - description = "/path/to/config.yaml ($CWD/.ice-rest-catalog.yaml by default)") + names = {"-c", "--config"}, + description = "/path/to/config.yaml ($CWD/.ice-rest-catalog.yaml by default)") String configFile; public String configFile() { @@ -86,7 +86,7 @@ public String configFile() { private Main() {} private static Server createServer( - String host, int port, Catalog catalog, Config config, Map icebergConfig) { + String host, int port, Catalog catalog, Config config, Map icebergConfig) { var s = createBaseServer(catalog, config, icebergConfig, true); ServerConnector connector = new ServerConnector(s); connector.setHost(host); @@ -96,7 +96,7 @@ private static Server createServer( } private static Server createAdminServer( - String host, int port, Catalog catalog, Config config, Map icebergConfig) { + String host, int port, Catalog catalog, Config config, Map icebergConfig) { var s = createBaseServer(catalog, config, icebergConfig, false); ServerConnector connector = new ServerConnector(s); connector.setHost(host); @@ -106,7 +106,7 @@ private static Server createAdminServer( } private static Server createBaseServer( - Catalog catalog, Config config, Map icebergConfig, boolean requireAuth) { + Catalog catalog, Config config, Map icebergConfig, boolean requireAuth) { var mux = new ServletContextHandler(ServletContextHandler.NO_SESSIONS); mux.insertHandler(new GzipHandler()); // TODO: RequestLogHandler @@ -116,8 +116,8 @@ private static Server createBaseServer( RESTCatalogHandler restCatalogAdapter; String warehouse = icebergConfig.getOrDefault(CatalogProperties.WAREHOUSE_LOCATION, ""); boolean awsAuth = - warehouse.startsWith("s3://") - || warehouse.startsWith("arn:aws:s3tables:"); // FIXME: arn:aws-cn:s3tables + warehouse.startsWith("s3://") + || warehouse.startsWith("arn:aws:s3tables:"); // FIXME: arn:aws-cn:s3tables if (requireAuth) { mux.insertHandler(createAuthorizationHandler(config.bearerTokens(), config)); @@ -125,29 +125,29 @@ private static Server createBaseServer( var loadTableConfig = config.toIcebergLoadTableConfig(); if (!loadTableConfig.isEmpty()) { restCatalogAdapter = - new RESTCatalogMiddlewareTableConfig(restCatalogAdapter, loadTableConfig); + new RESTCatalogMiddlewareTableConfig(restCatalogAdapter, loadTableConfig); } if (awsAuth) { Map awsCredentialsProviders = - createAwsCredentialsProviders(config.bearerTokens(), config, icebergConfig); + createAwsCredentialsProviders(config.bearerTokens(), config, icebergConfig); restCatalogAdapter = - new RESTCatalogMiddlewareTableAWCredentials( - restCatalogAdapter, awsCredentialsProviders::get); + new RESTCatalogMiddlewareTableAWCredentials( + restCatalogAdapter, awsCredentialsProviders::get); } } else { restCatalogAdapter = new RESTCatalogAdapter(catalog); var loadTableConfig = config.toIcebergLoadTableConfig(); if (!loadTableConfig.isEmpty()) { restCatalogAdapter = - new RESTCatalogMiddlewareTableConfig(restCatalogAdapter, loadTableConfig); + new RESTCatalogMiddlewareTableConfig(restCatalogAdapter, loadTableConfig); } if (awsAuth) { DefaultCredentialsProvider awsCredentialsProvider = DefaultCredentialsProvider.create(); restCatalogAdapter = - new RESTCatalogMiddlewareTableAWCredentials( - restCatalogAdapter, uid -> awsCredentialsProvider); + new RESTCatalogMiddlewareTableAWCredentials( + restCatalogAdapter, uid -> awsCredentialsProvider); } } @@ -161,22 +161,22 @@ private static Server createBaseServer( } private static Map createAwsCredentialsProviders( - Config.Token[] tokens, Config config, Map icebergConfig) { + Config.Token[] tokens, Config config, Map icebergConfig) { AwsCredentialsProvider awsCredentialsProvider; if (icebergConfig.containsKey(S3FileIOProperties.ACCESS_KEY_ID)) { if (icebergConfig.containsKey(S3FileIOProperties.SESSION_TOKEN)) { awsCredentialsProvider = - StaticCredentialsProvider.create( - AwsSessionCredentials.create( - icebergConfig.get(S3FileIOProperties.ACCESS_KEY_ID), - icebergConfig.get(S3FileIOProperties.SECRET_ACCESS_KEY), - icebergConfig.get(S3FileIOProperties.SESSION_TOKEN))); + StaticCredentialsProvider.create( + AwsSessionCredentials.create( + icebergConfig.get(S3FileIOProperties.ACCESS_KEY_ID), + icebergConfig.get(S3FileIOProperties.SECRET_ACCESS_KEY), + icebergConfig.get(S3FileIOProperties.SESSION_TOKEN))); } else { awsCredentialsProvider = - StaticCredentialsProvider.create( - AwsBasicCredentials.create( - icebergConfig.get(S3FileIOProperties.ACCESS_KEY_ID), - icebergConfig.get(S3FileIOProperties.SECRET_ACCESS_KEY))); + StaticCredentialsProvider.create( + AwsBasicCredentials.create( + icebergConfig.get(S3FileIOProperties.ACCESS_KEY_ID), + icebergConfig.get(S3FileIOProperties.SECRET_ACCESS_KEY))); } } else { awsCredentialsProvider = DefaultCredentialsProvider.create(); @@ -184,36 +184,36 @@ private static Map createAwsCredentialsProviders Map awsCredentialsProviders = new HashMap<>(); for (Config.Token token : tokens) { awsCredentialsProviders.put( - token.resourceName(), - !token.accessConfig().hasAWSAssumeRoleARN() - ? awsCredentialsProvider - : CredentialsProvider.assumeRule( - awsCredentialsProvider, - token.accessConfig().awsAssumeRoleARN(), - getUserAgentForToken(token))); + token.resourceName(), + !token.accessConfig().hasAWSAssumeRoleARN() + ? awsCredentialsProvider + : CredentialsProvider.assumeRule( + awsCredentialsProvider, + token.accessConfig().awsAssumeRoleARN(), + getUserAgentForToken(token))); } if (config.anonymousAccess().enabled()) { var token = new Config.Token("anonymous", "", config.anonymousAccess().accessConfig()); awsCredentialsProviders.put( - token.resourceName(), - !token.accessConfig().hasAWSAssumeRoleARN() - ? awsCredentialsProvider - : CredentialsProvider.assumeRule( - awsCredentialsProvider, - token.accessConfig().awsAssumeRoleARN(), - getUserAgentForToken(token))); + token.resourceName(), + !token.accessConfig().hasAWSAssumeRoleARN() + ? awsCredentialsProvider + : CredentialsProvider.assumeRule( + awsCredentialsProvider, + token.accessConfig().awsAssumeRoleARN(), + getUserAgentForToken(token))); } return awsCredentialsProviders; } private static String getUserAgentForToken(Config.Token token) { return !Strings.isNullOrEmpty(token.name()) - ? "ice-rest-catalog." + token.name() - : "ice-rest-catalog"; + ? "ice-rest-catalog." + token.name() + : "ice-rest-catalog"; } private static RESTCatalogAuthorizationHandler createAuthorizationHandler( - Config.Token[] tokens, Config config) { + Config.Token[] tokens, Config config) { Session anonymousSession = null; if (config.anonymousAccess().enabled()) { var t = new Config.Token("anonymous", "", config.anonymousAccess().accessConfig()); @@ -222,7 +222,7 @@ private static RESTCatalogAuthorizationHandler createAuthorizationHandler( } if (tokens.length == 0 && anonymousSession == null) { throw new IllegalArgumentException( - "invalid config: either set anonymousAccess.enabled to true or provide tokens via bearerTokens"); + "invalid config: either set anonymousAccess.enabled to true or provide tokens via bearerTokens"); } return new RESTCatalogAuthorizationHandler(tokens, anonymousSession); } @@ -233,19 +233,19 @@ private static Server createDebugServer(String host, int port) { mux.addServlet(new ServletHolder(new PrometheusMetricsServlet()), "/metrics"); var h = - new ServletHolder( - new HttpServlet() { - @Override - protected void doGet(HttpServletRequest req, HttpServletResponse resp) - throws IOException { - resp.setStatus(HttpServletResponse.SC_OK); - resp.setContentType(MimeTypes.Type.TEXT_PLAIN.asString()); - resp.setCharacterEncoding(StandardCharsets.UTF_8.name()); - try (PrintWriter w = resp.getWriter()) { - w.write("OK"); - } - } - }); + new ServletHolder( + new HttpServlet() { + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) + throws IOException { + resp.setStatus(HttpServletResponse.SC_OK); + resp.setContentType(MimeTypes.Type.TEXT_PLAIN.asString()); + resp.setCharacterEncoding(StandardCharsets.UTF_8.name()); + try (PrintWriter w = resp.getWriter()) { + w.write("OK"); + } + } + }); mux.addServlet(h, "/healthz"); // TODO: provide proper impl @@ -275,28 +275,28 @@ public Integer call() throws Exception { var icebergConfig = config.toIcebergConfig(); logger.debug( - "Iceberg configuration: {}", - icebergConfig.entrySet().stream() - .map(e -> !e.getKey().contains("key") ? e.getKey() + "=" + e.getValue() : e.getKey()) - .sorted() - .collect(Collectors.joining(", "))); + "Iceberg configuration: {}", + icebergConfig.entrySet().stream() + .map(e -> !e.getKey().contains("key") ? e.getKey() + "=" + e.getValue() : e.getKey()) + .sorted() + .collect(Collectors.joining(", "))); ObjectMapper om = new ObjectMapper(); for (Config.Token t : config.bearerTokens()) { if (Strings.isNullOrEmpty(t.name())) { logger.info( - "Catalog accessible via bearer token named \"{}\" (config: {})", - Objects.requireNonNullElse(t.name(), ""), - om.writeValueAsString(t.accessConfig())); + "Catalog accessible via bearer token named \"{}\" (config: {})", + Objects.requireNonNullElse(t.name(), ""), + om.writeValueAsString(t.accessConfig())); } else { logger.info( - "Catalog accessible via bearer token (config: {})", - om.writeValueAsString(t.accessConfig())); + "Catalog accessible via bearer token (config: {})", + om.writeValueAsString(t.accessConfig())); } } if (config.anonymousAccess().enabled()) { logger.warn( - "Anonymous access enabled (config: {})", - om.writeValueAsString(config.anonymousAccess().accessConfig())); + "Anonymous access enabled (config: {})", + om.writeValueAsString(config.anonymousAccess().accessConfig())); } // FIXME: remove @@ -326,19 +326,19 @@ public Integer call() throws Exception { if (!Strings.isNullOrEmpty(config.adminAddr())) { HostAndPort adminHostAndPort = HostAndPort.fromString(config.adminAddr()); Server adminServer = - createAdminServer( - adminHostAndPort.getHost(), - adminHostAndPort.getPort(), - catalog, - config, - icebergConfig); + createAdminServer( + adminHostAndPort.getHost(), + adminHostAndPort.getPort(), + catalog, + config, + icebergConfig); adminServer.start(); logger.warn("Serving admin endpoint at http://{}/v1/{config,*}", adminHostAndPort); } HostAndPort hostAndPort = HostAndPort.fromString(config.addr()); Server httpServer = - createServer(hostAndPort.getHost(), hostAndPort.getPort(), catalog, config, icebergConfig); + createServer(hostAndPort.getHost(), hostAndPort.getPort(), catalog, config, icebergConfig); httpServer.start(); logger.info("Serving http://{}/v1/{config,*}", hostAndPort); @@ -358,11 +358,11 @@ private void initializeMaintenanceScheduler(Catalog catalog, Config config) { } try { MaintenanceScheduler scheduler = - new MaintenanceScheduler( - catalog, config.maintenanceSchedule(), config.snapshotTTLInDays()); + new MaintenanceScheduler( + catalog, config.maintenanceSchedule(), config.snapshotTTLInDays()); scheduler.startScheduledMaintenance(); logger.info( - "Maintenance scheduler initialized with schedule: {}", config.maintenanceSchedule()); + "Maintenance scheduler initialized with schedule: {}", config.maintenanceSchedule()); } catch (Exception e) { logger.error("Failed to initialize maintenance scheduler", e); throw new RuntimeException(e); @@ -373,33 +373,33 @@ private static Catalog newEctdCatalog(Map config) { // TODO: remove; params all verified by config String uri = config.getOrDefault(CatalogProperties.URI, "etcd:http://localhost:2379"); Preconditions.checkArgument( - !Strings.isNullOrEmpty(uri), "etcd catalog: \"%s\" required", CatalogProperties.URI); + !Strings.isNullOrEmpty(uri), "etcd catalog: \"%s\" required", CatalogProperties.URI); String inputWarehouseLocation = config.get(CatalogProperties.WAREHOUSE_LOCATION); Preconditions.checkArgument( - !Strings.isNullOrEmpty(inputWarehouseLocation), - "etcd catalog: \"%s\" required", - CatalogProperties.WAREHOUSE_LOCATION); + !Strings.isNullOrEmpty(inputWarehouseLocation), + "etcd catalog: \"%s\" required", + CatalogProperties.WAREHOUSE_LOCATION); String ioImpl = config.get(CatalogProperties.FILE_IO_IMPL); Preconditions.checkArgument( - !Strings.isNullOrEmpty(ioImpl), - "etcd catalog: \"%s\" required", - CatalogProperties.FILE_IO_IMPL); + !Strings.isNullOrEmpty(ioImpl), + "etcd catalog: \"%s\" required", + CatalogProperties.FILE_IO_IMPL); var io = CatalogUtil.loadFileIO(ioImpl, config, null); return new EtcdCatalog( - "default", Strings.removePrefix(uri, "etcd:"), inputWarehouseLocation, io); + "default", Strings.removePrefix(uri, "etcd:"), inputWarehouseLocation, io); } public static void main(String[] args) throws Exception { SLF4JBridgeHandler.install(); CommandLine cmd = new CommandLine(new Main()); cmd.setExecutionExceptionHandler( - (Exception ex, CommandLine self, CommandLine.ParseResult res) -> { - logger.error("Fatal", ex); - return 1; - }); + (Exception ex, CommandLine self, CommandLine.ParseResult res) -> { + logger.error("Fatal", ex); + return 1; + }); int exitCode = cmd.execute(args); System.exit(exitCode); } From 7f7f1a8d237508919fd805930a837ae9bab48404 Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Fri, 20 Jun 2025 00:24:19 -0400 Subject: [PATCH 18/28] Modified deletePartition to use namespace.tablename as argument. Modified Integration test to start admin server and use etcdcatalog --- .../com/altinity/ice/rest/catalog/Main.java | 12 +- .../ice/rest/catalog/RESTCatalogIT.java | 171 +++++++++++++----- .../main/java/com/altinity/ice/cli/Main.java | 13 +- .../ice/cli/internal/cmd/DeletePartition.java | 5 +- 4 files changed, 143 insertions(+), 58 deletions(-) diff --git a/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/Main.java b/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/Main.java index feaab56..891a813 100644 --- a/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/Main.java +++ b/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/Main.java @@ -85,8 +85,8 @@ public String configFile() { private Main() {} - private static Server createServer( - String host, int port, Catalog catalog, Config config, Map icebergConfig) { + static Server createServer( + String host, int port, Catalog catalog, Config config, Map icebergConfig) { var s = createBaseServer(catalog, config, icebergConfig, true); ServerConnector connector = new ServerConnector(s); connector.setHost(host); @@ -95,8 +95,8 @@ private static Server createServer( return s; } - private static Server createAdminServer( - String host, int port, Catalog catalog, Config config, Map icebergConfig) { + static Server createAdminServer( + String host, int port, Catalog catalog, Config config, Map icebergConfig) { var s = createBaseServer(catalog, config, icebergConfig, false); ServerConnector connector = new ServerConnector(s); connector.setHost(host); @@ -227,7 +227,7 @@ private static RESTCatalogAuthorizationHandler createAuthorizationHandler( return new RESTCatalogAuthorizationHandler(tokens, anonymousSession); } - private static Server createDebugServer(String host, int port) { + static Server createDebugServer(String host, int port) { var mux = new ServletContextHandler(ServletContextHandler.NO_SESSIONS); mux.insertHandler(new GzipHandler()); @@ -369,7 +369,7 @@ private void initializeMaintenanceScheduler(Catalog catalog, Config config) { } } - private static Catalog newEctdCatalog(Map config) { + static Catalog newEctdCatalog(Map config) { // TODO: remove; params all verified by config String uri = config.getOrDefault(CatalogProperties.URI, "etcd:http://localhost:2379"); Preconditions.checkArgument( diff --git a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogIT.java b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogIT.java index 9b5aa01..84c386d 100644 --- a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogIT.java +++ b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogIT.java @@ -10,64 +10,134 @@ package com.altinity.ice.rest.catalog; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Consumer; + +import com.altinity.ice.internal.strings.Strings; +import com.altinity.ice.rest.catalog.internal.config.Config; +import com.altinity.ice.rest.catalog.internal.etcd.EtcdCatalog; +import com.google.common.net.HostAndPort; +import io.etcd.jetcd.KV; +import io.etcd.jetcd.Txn; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.inmemory.InMemoryFileIO; +import org.eclipse.jetty.server.Server; +import org.testcontainers.containers.GenericContainer; import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; import picocli.CommandLine; +import static com.altinity.ice.rest.catalog.Main.*; + public class RESTCatalogIT { private Thread serverThread; - private Main mainCommand; - - private void shutdownServer(String adminAddr, String shutdownToken) - throws IOException, InterruptedException { - java.net.http.HttpClient client = java.net.http.HttpClient.newHttpClient(); - java.net.http.HttpRequest request = - java.net.http.HttpRequest.newBuilder() - .uri(java.net.URI.create("http://" + adminAddr + "/shutdown?token=" + shutdownToken)) - .POST(java.net.http.HttpRequest.BodyPublishers.noBody()) - .build(); - - java.net.http.HttpResponse response = - client.send(request, java.net.http.HttpResponse.BodyHandlers.ofString()); - - if (response.statusCode() != 200) { - System.err.println("Failed to shutdown server. Status code: " + response.statusCode()); - } + + private EtcdCatalog catalog; + private Consumer preKvtx; + private Server httpServer; + private Server adminServer; + + @SuppressWarnings("rawtypes") + private final GenericContainer etcd = + new GenericContainer("bitnami/etcd:3.5.21") + .withExposedPorts(2379, 2380) + .withEnv("ALLOW_NONE_AUTHENTICATION", "yes"); + + @BeforeClass + public void setUp() { + etcd.start(); + String uri = "http://" + etcd.getHost() + ":" + etcd.getMappedPort(2379); + catalog = + new EtcdCatalog("default", uri, "/foo", new InMemoryFileIO()) { + + @Override + protected Txn kvtx() { + if (preKvtx != null) { + var x = preKvtx; + preKvtx = null; + x.accept(this.kv); + } + return super.kvtx(); + } + }; } - @Test - public void testMainCommandWithConfig() throws InterruptedException { + @Test() + public void testMainCommandWithConfig() throws Exception { // Provide your CLI arguments here — replace with an actual test config file path - String[] args = {"--config", "src/test/resources/ice-rest-catalog.yaml"}; - // Create a new instance of the command class - mainCommand = new Main(); + // Start server in a separate thread + serverThread = + new Thread( + () -> { + try { + var config = Config.load("src/test/resources/ice-rest-catalog.yaml"); + + // Use RESTCatalog directly + Map icebergConfig = new HashMap<>(); + icebergConfig.put("uri", "http://localhost:5000"); + icebergConfig.put("warehouse", "s3://my-bucket/warehouse"); + // Revert back to rest catalog + icebergConfig.put("catalog-impl", "org.apache.iceberg.rest.RESTCatalog"); + icebergConfig.put("io-impl", "org.apache.iceberg.aws.s3.S3FileIO"); + // Catalog catalog = newEctdCatalog(icebergConfig); + // need to implement custom org.apache.iceberg.rest.RESTClient) + if (!Strings.isNullOrEmpty(config.adminAddr())) { + HostAndPort adminHostAndPort = HostAndPort.fromString(config.adminAddr()); + adminServer = + createAdminServer( + adminHostAndPort.getHost(), + adminHostAndPort.getPort(), + catalog, + config, + icebergConfig); + adminServer.start(); + } + + HostAndPort hostAndPort = HostAndPort.fromString(config.addr()); + httpServer = + createServer( + hostAndPort.getHost(), + hostAndPort.getPort(), + catalog, + config, + icebergConfig); + httpServer.start(); + httpServer.join(); + } catch (Exception e) { + System.err.println("Error in server thread: " + e.getMessage()); + e.printStackTrace(); + } + }); - // Start the server in a separate thread - serverThread = new Thread(() -> { - try { - new CommandLine(mainCommand).execute(args); - } catch (Exception e) { - System.err.println("Error running server: " + e.getMessage()); - } - }); serverThread.start(); // Give the server time to start up - Thread.sleep(2000); + Thread.sleep(5000); // Create namespace and table using Ice CLI - String namespace = "test_namespace"; - String tableName = "test_table"; + String table = "nyc.taxis"; + + // insert data + new CommandLine(new com.altinity.ice.cli.Main()) + .execute("--config", "src/test/resources/ice-rest-cli.yaml", "insert", table, + "https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2025-01.parquet", "--partition='[{\"column\":\"tpep_pickup_datetime\",\"transform\":\"day\"}]'"); + + // Scan command + new CommandLine(new com.altinity.ice.cli.Main()) + .execute("--config", "src/test/resources/ice-rest-cli.yaml", "scan", table); - // Delete namespace before creating it + // Delete table new CommandLine(new com.altinity.ice.cli.Main()) - .execute("--config", "src/test/resources/ice-rest-cli.yaml", "delete-namespace", namespace); + .execute("--config", "src/test/resources/ice-rest-cli.yaml", "delete-table", table); - // Create namespace + // Delete namespace new CommandLine(new com.altinity.ice.cli.Main()) - .execute("--config", "src/test/resources/ice-rest-cli.yaml", "create-namespace", namespace); + .execute("--config", "src/test/resources/ice-rest-cli.yaml", "delete-namespace", "nyc"); } @@ -75,14 +145,22 @@ public void testMainCommandWithConfig() throws InterruptedException { @AfterClass public void tearDown() { try { - // Get the admin address from the config - String adminAddr = System.getProperty("ice.rest.catalog.admin.addr", "localhost:5000"); - - // Get the shutdown token from the config - String shutdownToken = "foo"; + // Stop the servers first + if (httpServer != null && httpServer.isRunning()) { + try { + httpServer.stop(); + } catch (Exception e) { + System.err.println("Error stopping http server: " + e.getMessage()); + } + } - // Shutdown the server - shutdownServer(adminAddr, shutdownToken); + if (adminServer != null && adminServer.isRunning()) { + try { + adminServer.stop(); + } catch (Exception e) { + System.err.println("Error stopping admin server: " + e.getMessage()); + } + } // Wait for the server thread to finish if (serverThread != null && serverThread.isAlive()) { @@ -91,6 +169,11 @@ public void tearDown() { serverThread.interrupt(); // Force interrupt if still running } } + + // Stop the etcd container + if (etcd != null && etcd.isRunning()) { + etcd.stop(); + } } catch (Exception e) { System.err.println("Error during server shutdown: " + e.getMessage()); } diff --git a/ice/src/main/java/com/altinity/ice/cli/Main.java b/ice/src/main/java/com/altinity/ice/cli/Main.java index e5379d4..80f5c47 100644 --- a/ice/src/main/java/com/altinity/ice/cli/Main.java +++ b/ice/src/main/java/com/altinity/ice/cli/Main.java @@ -430,10 +430,11 @@ void deleteNamespace( @CommandLine.Command(name = "delete", description = "Delete Partition(s).") void deletePartition( - @CommandLine.Option(names = "--namespace", description = "Namespace name", required = true) - String namespace, - @CommandLine.Option(names = "--table", description = "Table name", required = true) - String tableName, + @CommandLine.Parameters( + arity = "1", + paramLabel = "", + description = "Table name (e.g. ns1.table1)") + String name, @CommandLine.Option( names = {"--partition"}, description = @@ -452,7 +453,9 @@ void deletePartition( PartitionFilter[] parts = mapper.readValue(partitionJson, PartitionFilter[].class); partitions = Arrays.asList(parts); } - DeletePartition.run(catalog, namespace, tableName, partitions, dryRun); + TableIdentifier tableId = TableIdentifier.parse(name); + + DeletePartition.run(catalog, tableId, partitions, dryRun); } catch (URISyntaxException e) { throw new RuntimeException(e); } diff --git a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/DeletePartition.java b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/DeletePartition.java index 7ec4037..80636bd 100644 --- a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/DeletePartition.java +++ b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/DeletePartition.java @@ -27,13 +27,12 @@ private DeletePartition() {} public static void run( RESTCatalog catalog, - String namespace, - String tableName, + TableIdentifier tableId, List partitions, boolean dryRun) throws IOException, URISyntaxException { - Table table = catalog.loadTable(TableIdentifier.of(namespace, tableName)); + Table table = catalog.loadTable(tableId); TableScan scan = table.newScan(); if (partitions != null && !partitions.isEmpty()) { org.apache.iceberg.expressions.Expression expr = null; From b75d4af1d52463cf4da6baa9a428db477c0797ef Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Sat, 21 Jun 2025 18:41:06 -0400 Subject: [PATCH 19/28] Added insert command to RESTCatalogIT --- .../main/java/com/altinity/ice/rest/catalog/Main.java | 4 ++-- .../com/altinity/ice/rest/catalog/RESTCatalogIT.java | 10 +++------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/Main.java b/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/Main.java index 891a813..9f9df8c 100644 --- a/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/Main.java +++ b/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/Main.java @@ -86,7 +86,7 @@ public String configFile() { private Main() {} static Server createServer( - String host, int port, Catalog catalog, Config config, Map icebergConfig) { + String host, int port, Catalog catalog, Config config, Map icebergConfig) { var s = createBaseServer(catalog, config, icebergConfig, true); ServerConnector connector = new ServerConnector(s); connector.setHost(host); @@ -96,7 +96,7 @@ static Server createServer( } static Server createAdminServer( - String host, int port, Catalog catalog, Config config, Map icebergConfig) { + String host, int port, Catalog catalog, Config config, Map icebergConfig) { var s = createBaseServer(catalog, config, icebergConfig, false); ServerConnector connector = new ServerConnector(s); connector.setHost(host); diff --git a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogIT.java b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogIT.java index 84c386d..1c9d0e9 100644 --- a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogIT.java +++ b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogIT.java @@ -9,7 +9,6 @@ */ package com.altinity.ice.rest.catalog; -import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.function.Consumer; @@ -20,8 +19,6 @@ import com.google.common.net.HostAndPort; import io.etcd.jetcd.KV; import io.etcd.jetcd.Txn; -import org.apache.iceberg.CatalogUtil; -import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.inmemory.InMemoryFileIO; import org.eclipse.jetty.server.Server; import org.testcontainers.containers.GenericContainer; @@ -30,7 +27,8 @@ import org.testng.annotations.Test; import picocli.CommandLine; -import static com.altinity.ice.rest.catalog.Main.*; +import static com.altinity.ice.rest.catalog.Main.createAdminServer; +import static com.altinity.ice.rest.catalog.Main.createServer; public class RESTCatalogIT { @@ -124,7 +122,7 @@ public void testMainCommandWithConfig() throws Exception { // insert data new CommandLine(new com.altinity.ice.cli.Main()) - .execute("--config", "src/test/resources/ice-rest-cli.yaml", "insert", table, + .execute("--config", "src/test/resources/ice-rest-cli.yaml", "insert", table, "https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2025-01.parquet", "--partition='[{\"column\":\"tpep_pickup_datetime\",\"transform\":\"day\"}]'"); // Scan command @@ -138,8 +136,6 @@ public void testMainCommandWithConfig() throws Exception { // Delete namespace new CommandLine(new com.altinity.ice.cli.Main()) .execute("--config", "src/test/resources/ice-rest-cli.yaml", "delete-namespace", "nyc"); - - } @AfterClass From 3b4ae26e6b53ce243ee7d821e8ec6fadfafe7897 Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Sat, 28 Jun 2025 09:52:54 -0400 Subject: [PATCH 20/28] Addressed PR review comments. --- .../ice/rest/catalog/RESTCatalogIT.java | 177 ------------------ .../main/java/com/altinity/ice/cli/Main.java | 11 +- .../cmd/{DeletePartition.java => Delete.java} | 6 +- 3 files changed, 9 insertions(+), 185 deletions(-) delete mode 100644 ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogIT.java rename ice/src/main/java/com/altinity/ice/cli/internal/cmd/{DeletePartition.java => Delete.java} (96%) diff --git a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogIT.java b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogIT.java deleted file mode 100644 index 1c9d0e9..0000000 --- a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogIT.java +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Copyright (c) 2025 Altinity Inc and/or its affiliates. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - */ -package com.altinity.ice.rest.catalog; - -import java.util.HashMap; -import java.util.Map; -import java.util.function.Consumer; - -import com.altinity.ice.internal.strings.Strings; -import com.altinity.ice.rest.catalog.internal.config.Config; -import com.altinity.ice.rest.catalog.internal.etcd.EtcdCatalog; -import com.google.common.net.HostAndPort; -import io.etcd.jetcd.KV; -import io.etcd.jetcd.Txn; -import org.apache.iceberg.inmemory.InMemoryFileIO; -import org.eclipse.jetty.server.Server; -import org.testcontainers.containers.GenericContainer; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Test; -import picocli.CommandLine; - -import static com.altinity.ice.rest.catalog.Main.createAdminServer; -import static com.altinity.ice.rest.catalog.Main.createServer; - -public class RESTCatalogIT { - - private Thread serverThread; - - private EtcdCatalog catalog; - private Consumer preKvtx; - private Server httpServer; - private Server adminServer; - - @SuppressWarnings("rawtypes") - private final GenericContainer etcd = - new GenericContainer("bitnami/etcd:3.5.21") - .withExposedPorts(2379, 2380) - .withEnv("ALLOW_NONE_AUTHENTICATION", "yes"); - - @BeforeClass - public void setUp() { - etcd.start(); - String uri = "http://" + etcd.getHost() + ":" + etcd.getMappedPort(2379); - catalog = - new EtcdCatalog("default", uri, "/foo", new InMemoryFileIO()) { - - @Override - protected Txn kvtx() { - if (preKvtx != null) { - var x = preKvtx; - preKvtx = null; - x.accept(this.kv); - } - return super.kvtx(); - } - }; - } - - @Test() - public void testMainCommandWithConfig() throws Exception { - // Provide your CLI arguments here — replace with an actual test config file path - - // Start server in a separate thread - serverThread = - new Thread( - () -> { - try { - var config = Config.load("src/test/resources/ice-rest-catalog.yaml"); - - // Use RESTCatalog directly - Map icebergConfig = new HashMap<>(); - icebergConfig.put("uri", "http://localhost:5000"); - icebergConfig.put("warehouse", "s3://my-bucket/warehouse"); - // Revert back to rest catalog - icebergConfig.put("catalog-impl", "org.apache.iceberg.rest.RESTCatalog"); - icebergConfig.put("io-impl", "org.apache.iceberg.aws.s3.S3FileIO"); - // Catalog catalog = newEctdCatalog(icebergConfig); - // need to implement custom org.apache.iceberg.rest.RESTClient) - if (!Strings.isNullOrEmpty(config.adminAddr())) { - HostAndPort adminHostAndPort = HostAndPort.fromString(config.adminAddr()); - adminServer = - createAdminServer( - adminHostAndPort.getHost(), - adminHostAndPort.getPort(), - catalog, - config, - icebergConfig); - adminServer.start(); - } - - HostAndPort hostAndPort = HostAndPort.fromString(config.addr()); - httpServer = - createServer( - hostAndPort.getHost(), - hostAndPort.getPort(), - catalog, - config, - icebergConfig); - httpServer.start(); - httpServer.join(); - } catch (Exception e) { - System.err.println("Error in server thread: " + e.getMessage()); - e.printStackTrace(); - } - }); - - serverThread.start(); - - // Give the server time to start up - Thread.sleep(5000); - - // Create namespace and table using Ice CLI - String table = "nyc.taxis"; - - // insert data - new CommandLine(new com.altinity.ice.cli.Main()) - .execute("--config", "src/test/resources/ice-rest-cli.yaml", "insert", table, - "https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2025-01.parquet", "--partition='[{\"column\":\"tpep_pickup_datetime\",\"transform\":\"day\"}]'"); - - // Scan command - new CommandLine(new com.altinity.ice.cli.Main()) - .execute("--config", "src/test/resources/ice-rest-cli.yaml", "scan", table); - - // Delete table - new CommandLine(new com.altinity.ice.cli.Main()) - .execute("--config", "src/test/resources/ice-rest-cli.yaml", "delete-table", table); - - // Delete namespace - new CommandLine(new com.altinity.ice.cli.Main()) - .execute("--config", "src/test/resources/ice-rest-cli.yaml", "delete-namespace", "nyc"); - } - - @AfterClass - public void tearDown() { - try { - // Stop the servers first - if (httpServer != null && httpServer.isRunning()) { - try { - httpServer.stop(); - } catch (Exception e) { - System.err.println("Error stopping http server: " + e.getMessage()); - } - } - - if (adminServer != null && adminServer.isRunning()) { - try { - adminServer.stop(); - } catch (Exception e) { - System.err.println("Error stopping admin server: " + e.getMessage()); - } - } - - // Wait for the server thread to finish - if (serverThread != null && serverThread.isAlive()) { - serverThread.join(5000); // Wait up to 5 seconds for graceful shutdown - if (serverThread.isAlive()) { - serverThread.interrupt(); // Force interrupt if still running - } - } - - // Stop the etcd container - if (etcd != null && etcd.isRunning()) { - etcd.stop(); - } - } catch (Exception e) { - System.err.println("Error during server shutdown: " + e.getMessage()); - } - } -} diff --git a/ice/src/main/java/com/altinity/ice/cli/Main.java b/ice/src/main/java/com/altinity/ice/cli/Main.java index 80f5c47..60487b5 100644 --- a/ice/src/main/java/com/altinity/ice/cli/Main.java +++ b/ice/src/main/java/com/altinity/ice/cli/Main.java @@ -13,8 +13,8 @@ import com.altinity.ice.cli.internal.cmd.Check; import com.altinity.ice.cli.internal.cmd.CreateNamespace; import com.altinity.ice.cli.internal.cmd.CreateTable; +import com.altinity.ice.cli.internal.cmd.Delete; import com.altinity.ice.cli.internal.cmd.DeleteNamespace; -import com.altinity.ice.cli.internal.cmd.DeletePartition; import com.altinity.ice.cli.internal.cmd.DeleteTable; import com.altinity.ice.cli.internal.cmd.Describe; import com.altinity.ice.cli.internal.cmd.Insert; @@ -428,8 +428,8 @@ void deleteNamespace( } } - @CommandLine.Command(name = "delete", description = "Delete Partition(s).") - void deletePartition( + @CommandLine.Command(name = "delete", description = "Delete data from catalog.") + void delete( @CommandLine.Parameters( arity = "1", paramLabel = "", @@ -443,7 +443,8 @@ void deletePartition( String partitionJson, @CommandLine.Option( names = "--dry-run", - description = "Log files that would be deleted without actually deleting them") + description = "Log files that would be deleted without actually deleting them", + defaultValue = "true") boolean dryRun) throws IOException { try (RESTCatalog catalog = loadCatalog(this.configFile())) { @@ -455,7 +456,7 @@ void deletePartition( } TableIdentifier tableId = TableIdentifier.parse(name); - DeletePartition.run(catalog, tableId, partitions, dryRun); + Delete.run(catalog, tableId, partitions, dryRun); } catch (URISyntaxException e) { throw new RuntimeException(e); } diff --git a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/DeletePartition.java b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Delete.java similarity index 96% rename from ice/src/main/java/com/altinity/ice/cli/internal/cmd/DeletePartition.java rename to ice/src/main/java/com/altinity/ice/cli/internal/cmd/Delete.java index 80636bd..0f6361d 100644 --- a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/DeletePartition.java +++ b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Delete.java @@ -19,11 +19,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public final class DeletePartition { +public final class Delete { - private static final Logger logger = LoggerFactory.getLogger(DeletePartition.class); + private static final Logger logger = LoggerFactory.getLogger(Delete.class); - private DeletePartition() {} + private Delete() {} public static void run( RESTCatalog catalog, From 12ab3060a7a9470f22f89111087dd11c8a327e40 Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Sat, 28 Jun 2025 10:09:31 -0400 Subject: [PATCH 21/28] Reverted back removal of private --- .idea/.gitignore | 8 ++++++++ .../src/main/java/com/altinity/ice/rest/catalog/Main.java | 8 ++++---- 2 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 .idea/.gitignore diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..13566b8 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/Main.java b/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/Main.java index 9f9df8c..feaab56 100644 --- a/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/Main.java +++ b/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/Main.java @@ -85,7 +85,7 @@ public String configFile() { private Main() {} - static Server createServer( + private static Server createServer( String host, int port, Catalog catalog, Config config, Map icebergConfig) { var s = createBaseServer(catalog, config, icebergConfig, true); ServerConnector connector = new ServerConnector(s); @@ -95,7 +95,7 @@ static Server createServer( return s; } - static Server createAdminServer( + private static Server createAdminServer( String host, int port, Catalog catalog, Config config, Map icebergConfig) { var s = createBaseServer(catalog, config, icebergConfig, false); ServerConnector connector = new ServerConnector(s); @@ -227,7 +227,7 @@ private static RESTCatalogAuthorizationHandler createAuthorizationHandler( return new RESTCatalogAuthorizationHandler(tokens, anonymousSession); } - static Server createDebugServer(String host, int port) { + private static Server createDebugServer(String host, int port) { var mux = new ServletContextHandler(ServletContextHandler.NO_SESSIONS); mux.insertHandler(new GzipHandler()); @@ -369,7 +369,7 @@ private void initializeMaintenanceScheduler(Catalog catalog, Config config) { } } - static Catalog newEctdCatalog(Map config) { + private static Catalog newEctdCatalog(Map config) { // TODO: remove; params all verified by config String uri = config.getOrDefault(CatalogProperties.URI, "etcd:http://localhost:2379"); Preconditions.checkArgument( From bce53f22d433d1fc1dbd960906d8f701db4332a6 Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Sat, 28 Jun 2025 10:11:49 -0400 Subject: [PATCH 22/28] Removed yaml files added for Integration test. --- .idea/.gitignore | 8 -------- .../src/test/resources/ice-rest-catalog.yaml | 17 ----------------- .../src/test/resources/ice-rest-cli.yaml | 14 -------------- 3 files changed, 39 deletions(-) delete mode 100644 .idea/.gitignore delete mode 100644 ice-rest-catalog/src/test/resources/ice-rest-catalog.yaml delete mode 100644 ice-rest-catalog/src/test/resources/ice-rest-cli.yaml diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 13566b8..0000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Editor-based HTTP Client requests -/httpRequests/ -# Datasource local storage ignored files -/dataSources/ -/dataSources.local.xml diff --git a/ice-rest-catalog/src/test/resources/ice-rest-catalog.yaml b/ice-rest-catalog/src/test/resources/ice-rest-catalog.yaml deleted file mode 100644 index 1601e2c..0000000 --- a/ice-rest-catalog/src/test/resources/ice-rest-catalog.yaml +++ /dev/null @@ -1,17 +0,0 @@ -uri: jdbc:sqlite:file:data/ice-rest-catalog/db.sqlite?journal_mode=WAL&synchronous=OFF&journal_size_limit=500 - -# To use etcd instead of sqlite, start etcd with `etcd --data-dir=data/etcd`, then uncomment the line below -#uri: etcd:http://localhost:2379 - -warehouse: s3://bucket1 - -s3: - endpoint: http://localhost:9000 - pathStyleAccess: true - accessKeyID: miniouser - secretAccessKey: miniopassword - region: minio - -bearerTokens: - - value: foo - diff --git a/ice-rest-catalog/src/test/resources/ice-rest-cli.yaml b/ice-rest-catalog/src/test/resources/ice-rest-cli.yaml deleted file mode 100644 index b9993e2..0000000 --- a/ice-rest-catalog/src/test/resources/ice-rest-cli.yaml +++ /dev/null @@ -1,14 +0,0 @@ - -# To use etcd instead of sqlite, start etcd with `etcd --data-dir=data/etcd`, then uncomment the line below -#uri: etcd:http://localhost:2379 -s3: - endpoint: http://localhost:9000 - pathStyleAccess: true - accessKeyID: miniouser - secretAccessKey: miniopassword - region: minio - -bearerToken: foo -#bearerToken: -# - value: foo - From 271c86464784dc30eaea0c1823104c6cfaad42f4 Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Sat, 28 Jun 2025 10:13:02 -0400 Subject: [PATCH 23/28] reverted back private change to Main --- ice/src/main/java/com/altinity/ice/cli/Main.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ice/src/main/java/com/altinity/ice/cli/Main.java b/ice/src/main/java/com/altinity/ice/cli/Main.java index 60487b5..2444fe8 100644 --- a/ice/src/main/java/com/altinity/ice/cli/Main.java +++ b/ice/src/main/java/com/altinity/ice/cli/Main.java @@ -75,7 +75,7 @@ public String configFile() { scope = CommandLine.ScopeType.INHERIT) private String logLevel; - public Main() {} + private Main() {} @CommandLine.Command(name = "check", description = "Check configuration.") void check() throws IOException { From a4748555d66a8583b8cb4c9eb0bd89fa7055d421 Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Sun, 29 Jun 2025 17:23:43 -0400 Subject: [PATCH 24/28] Updated README.md with example --- examples/scratch/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/scratch/README.md b/examples/scratch/README.md index 5410b6f..454a808 100644 --- a/examples/scratch/README.md +++ b/examples/scratch/README.md @@ -50,8 +50,8 @@ ice create-table flowers.iris_no_copy --schema-from-parquet=file://iris.parquet local-mc cp iris.parquet local/bucket1/flowers/iris_no_copy/ ice insert flowers.iris_no_copy --no-copy s3://bucket1/flowers/iris_no_copy/iris.parquet -# delete partition(Add --dry-run to print the list of partitions that will be deleted) -ice delete --namespace=nyc --table=taxismay15 --partition='[{"name": "tpep_pickup_datetime", "value": "2025-01-01T00:00:00"}]' +# delete partition(By default --dry-run=true is enabled to print the list of partitions that will be deleted) +ice delete nyc.taxis --dry-run=false # inspect ice describe From 9f5a8bebedeed564a4eae3862886878567f96683 Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Mon, 30 Jun 2025 10:52:44 -0400 Subject: [PATCH 25/28] Removed wildcard imports, replaced Iterable with CloseableIterable and added close() --- .../com/altinity/ice/cli/internal/cmd/Delete.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Delete.java b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Delete.java index 0f6361d..457ba84 100644 --- a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Delete.java +++ b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Delete.java @@ -13,8 +13,13 @@ import java.net.URISyntaxException; import java.util.ArrayList; import java.util.List; -import org.apache.iceberg.*; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.RewriteFiles; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableScan; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.rest.RESTCatalog; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,11 +52,13 @@ public static void run( } scan = scan.filter(expr); } - Iterable tasks = scan.planFiles(); + CloseableIterable tasks = scan.planFiles(); List filesToDelete = new ArrayList<>(); for (FileScanTask task : tasks) { filesToDelete.add(task.file()); } + tasks.close(); + if (!filesToDelete.isEmpty()) { if (dryRun) { logger.info("Dry run: The following files would be deleted:"); From 344b0b38cbd543f8e77deea0ba817d5025a1610b Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Tue, 1 Jul 2025 11:51:44 -0500 Subject: [PATCH 26/28] Add try/catch to close CloseableIterable. --- .../com/altinity/ice/cli/internal/cmd/Delete.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Delete.java b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Delete.java index 457ba84..34d06fe 100644 --- a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Delete.java +++ b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Delete.java @@ -52,12 +52,16 @@ public static void run( } scan = scan.filter(expr); } - CloseableIterable tasks = scan.planFiles(); List filesToDelete = new ArrayList<>(); - for (FileScanTask task : tasks) { - filesToDelete.add(task.file()); + + try (CloseableIterable tasks = scan.planFiles()) { + for (FileScanTask task : tasks) { + filesToDelete.add(task.file()); + } + } catch (Exception e) { + logger.error("Error getting files to delete: {}", e.getMessage()); + throw e; } - tasks.close(); if (!filesToDelete.isEmpty()) { if (dryRun) { From 23c389a03707965d8239932deee5f7b46a47a945 Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Wed, 2 Jul 2025 11:32:47 -0500 Subject: [PATCH 27/28] Remove logging line before throwing exception. --- ice/src/main/java/com/altinity/ice/cli/internal/cmd/Delete.java | 1 - 1 file changed, 1 deletion(-) diff --git a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Delete.java b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Delete.java index 34d06fe..47c4003 100644 --- a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Delete.java +++ b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Delete.java @@ -59,7 +59,6 @@ public static void run( filesToDelete.add(task.file()); } } catch (Exception e) { - logger.error("Error getting files to delete: {}", e.getMessage()); throw e; } From ee0c9155e1481d8a934618c6c0e50b2ab50a00be Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Mon, 7 Jul 2025 16:19:40 -0500 Subject: [PATCH 28/28] Removed catch block. --- ice/src/main/java/com/altinity/ice/cli/internal/cmd/Delete.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Delete.java b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Delete.java index 47c4003..f9bde7d 100644 --- a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Delete.java +++ b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Delete.java @@ -58,8 +58,6 @@ public static void run( for (FileScanTask task : tasks) { filesToDelete.add(task.file()); } - } catch (Exception e) { - throw e; } if (!filesToDelete.isEmpty()) {