diff --git a/examples/scratch/README.md b/examples/scratch/README.md index 454a808..9d45f2c 100644 --- a/examples/scratch/README.md +++ b/examples/scratch/README.md @@ -56,6 +56,12 @@ ice delete nyc.taxis --dry-run=false # inspect ice describe +# Alter table (add column) +ice alter-table flowers.iris --operations '[{"operation": "add column", "column_name": "age", "type": "int", "comment": "user age"}]' + +# Alter table (drop column) +ice alter-table flowers.iris --operations '[{"operation": "drop column", "column_name": "age"}]' + # open ClickHouse shell, then try SQL below clickhouse local ``` diff --git a/ice/src/main/java/com/altinity/ice/cli/Main.java b/ice/src/main/java/com/altinity/ice/cli/Main.java index 2444fe8..fae5dfc 100644 --- a/ice/src/main/java/com/altinity/ice/cli/Main.java +++ b/ice/src/main/java/com/altinity/ice/cli/Main.java @@ -10,6 +10,7 @@ package com.altinity.ice.cli; import ch.qos.logback.classic.Level; +import com.altinity.ice.cli.internal.cmd.AlterTable; import com.altinity.ice.cli.internal.cmd.Check; import com.altinity.ice.cli.internal.cmd.CreateNamespace; import com.altinity.ice.cli.internal.cmd.CreateTable; @@ -34,6 +35,7 @@ import java.util.Base64; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Scanner; import java.util.stream.Collectors; import org.apache.iceberg.catalog.Namespace; @@ -428,6 +430,36 @@ void deleteNamespace( } } + @CommandLine.Command( + name = "alter-table", + description = + "Alter table schema by adding or dropping columns. Supported operations: add column, drop column. Example: ice alter-table ns1.table1 --operations '[{\"operation\": \"add column\", \"column_name\": \"age\", \"type\": \"int\", \"comment\": \"user age\"}]'") + void alterTable( + @CommandLine.Parameters( + arity = "1", + paramLabel = "", + description = "Table name (e.g. ns1.table1)") + String name, + @CommandLine.Option( + names = {"--operations"}, + required = true, + description = + "JSON array of operations: [{'operation': 'add column', 'column_name': 'age', 'type': 'int', 'comment': 'user age'}, {'operation': 'drop column', 'column_name': 'col_name'}]") + String operationsJson) + throws IOException { + try (RESTCatalog catalog = loadCatalog(this.configFile())) { + TableIdentifier tableId = TableIdentifier.parse(name); + + ObjectMapper mapper = newObjectMapper(); + List> operations = + mapper.readValue( + operationsJson, + mapper.getTypeFactory().constructCollectionType(List.class, Map.class)); + + AlterTable.run(catalog, tableId, operations); + } + } + @CommandLine.Command(name = "delete", description = "Delete data from catalog.") void delete( @CommandLine.Parameters( diff --git a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/AlterTable.java b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/AlterTable.java new file mode 100644 index 0000000..a0f96ff --- /dev/null +++ b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/AlterTable.java @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2025 Altinity Inc and/or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +package com.altinity.ice.cli.internal.cmd; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import org.apache.iceberg.Table; +import org.apache.iceberg.Transaction; +import org.apache.iceberg.UpdateSchema; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.types.Types; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class AlterTable { + private static final Logger logger = LoggerFactory.getLogger(AlterTable.class); + + // Static constants for column definition keys + private static final String OPERATION_KEY = "operation"; + private static final String COLUMN_NAME_KEY = "column_name"; + private static final String TYPE_KEY = "type"; + private static final String COMMENT_KEY = "comment"; + + private AlterTable() {} + + public enum OperationType { + ADD("add column"), + DROP("drop column"); + + private final String key; + + OperationType(String key) { + this.key = key; + } + + public String getKey() { + return key; + } + + public static OperationType fromKey(String key) { + for (OperationType type : values()) { + if (type.key.equals(key)) { + return type; + } + } + throw new IllegalArgumentException("Unsupported operation type: " + key); + } + } + + public record ColumnDefinition( + @JsonProperty("operation") String operation, + @JsonProperty("column_name") String columnName, + @JsonProperty("type") String type, + @JsonProperty("comment") String comment) {} + + public static void run( + Catalog catalog, TableIdentifier tableId, List> operations) + throws IOException { + + Table table = catalog.loadTable(tableId); + + // Apply schema changes + Transaction transaction = table.newTransaction(); + UpdateSchema updateSchema = transaction.updateSchema(); + + for (Map operation : operations) { + + // get the operation type + ColumnDefinition columnDef = parseColumnDefinitionMap(operation); + OperationType operationType = OperationType.fromKey(columnDef.operation()); + + switch (operationType) { + case ADD: + Types.NestedField field = + parseColumnType(columnDef.columnName(), columnDef.type(), columnDef.comment()); + updateSchema.addColumn(columnDef.columnName(), field.type(), columnDef.comment()); + logger.info("Adding column '{}' to table: {}", columnDef.columnName(), tableId); + break; + case DROP: + // Validate that the column exists + if (table.schema().findField(columnDef.columnName()) == null) { + throw new IllegalArgumentException( + "Column '" + columnDef.columnName() + "' does not exist in table: " + tableId); + } + updateSchema.deleteColumn(columnDef.columnName()); + logger.info("Dropping column '{}' from table: {}", columnDef.columnName(), tableId); + break; + default: + throw new IllegalArgumentException("Unsupported operation type: " + operationType); + } + } + + updateSchema.commit(); + transaction.commitTransaction(); + + logger.info("Successfully applied {} operations to table: {}", operations.size(), tableId); + } + + static ColumnDefinition parseColumnDefinitionMap(Map operationMap) { + try { + String operation = operationMap.get(OPERATION_KEY); + String columnName = operationMap.get(COLUMN_NAME_KEY); + String type = operationMap.get(TYPE_KEY); + String comment = operationMap.get(COMMENT_KEY); + + if (operation == null || operation.trim().isEmpty()) { + throw new IllegalArgumentException(OPERATION_KEY + " is required and cannot be empty"); + } + + if (columnName == null || columnName.trim().isEmpty()) { + throw new IllegalArgumentException(COLUMN_NAME_KEY + " is required and cannot be empty"); + } + + // For drop column operations, type is not required + OperationType operationType = OperationType.fromKey(operation); + if (operationType == OperationType.ADD) { + if (type == null || type.trim().isEmpty()) { + throw new IllegalArgumentException( + TYPE_KEY + " is required and cannot be empty for add column operations"); + } + if (comment == null || comment.trim().isEmpty()) { + throw new IllegalArgumentException( + COMMENT_KEY + " is required and cannot be empty for add column operations"); + } + } + + return new ColumnDefinition(operation, columnName, type, comment); + } catch (Exception e) { + throw new IllegalArgumentException("Invalid column definition: " + e.getMessage()); + } + } + + private static Types.NestedField parseColumnType(String name, String type, String comment) { + Types.NestedField field; + + switch (type.toLowerCase()) { + case "string": + case "varchar": + field = Types.NestedField.optional(-1, name, Types.StringType.get(), comment); + break; + case "int": + case "integer": + field = Types.NestedField.optional(-1, name, Types.IntegerType.get(), comment); + break; + case "long": + case "bigint": + field = Types.NestedField.optional(-1, name, Types.LongType.get(), comment); + break; + case "double": + field = Types.NestedField.optional(-1, name, Types.DoubleType.get(), comment); + break; + case "float": + field = Types.NestedField.optional(-1, name, Types.FloatType.get(), comment); + break; + case "boolean": + field = Types.NestedField.optional(-1, name, Types.BooleanType.get(), comment); + break; + case "date": + field = Types.NestedField.optional(-1, name, Types.DateType.get(), comment); + break; + case "timestamp": + field = Types.NestedField.optional(-1, name, Types.TimestampType.withoutZone(), comment); + break; + case "timestamptz": + field = Types.NestedField.optional(-1, name, Types.TimestampType.withZone(), comment); + break; + case "binary": + field = Types.NestedField.optional(-1, name, Types.BinaryType.get(), comment); + break; + default: + throw new IllegalArgumentException( + "Unsupported column type: " + + type + + ". Supported types: string, int, long, double, float, boolean, date, timestamp, timestamptz, binary"); + } + + return field; + } +} diff --git a/ice/src/test/java/com/altinity/ice/cli/internal/cmd/AlterTableTest.java b/ice/src/test/java/com/altinity/ice/cli/internal/cmd/AlterTableTest.java new file mode 100644 index 0000000..cc83d64 --- /dev/null +++ b/ice/src/test/java/com/altinity/ice/cli/internal/cmd/AlterTableTest.java @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2025 Altinity Inc and/or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +package com.altinity.ice.cli.internal.cmd; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; + +import java.util.HashMap; +import java.util.Map; +import org.testng.annotations.Test; + +public class AlterTableTest { + + @Test + public void testParseColumnDefinitionJson() { + + Map operation = new HashMap<>(); + operation.put("operation", "add column"); + operation.put("column_name", "age"); + operation.put("type", "int"); + operation.put("comment", "User age"); + AlterTable.ColumnDefinition columnDef = AlterTable.parseColumnDefinitionMap(operation); + + assertEquals(columnDef.columnName(), "age"); + assertEquals(columnDef.type(), "int"); + assertEquals(columnDef.comment(), "User age"); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testParseColumnDefinitionJsonWithoutComment() { + + Map operation = new HashMap<>(); + operation.put("operation", "add column"); + operation.put("column_name", "name"); + operation.put("type", "string"); + AlterTable.ColumnDefinition columnDef = AlterTable.parseColumnDefinitionMap(operation); + + assertEquals(columnDef.columnName(), "name"); + assertEquals(columnDef.type(), "string"); + assertNull(columnDef.comment()); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testParseColumnDefinitionJsonMissingColumnName() { + + Map operation = new HashMap<>(); + operation.put("operation", "add column"); + operation.put("type", "int"); + operation.put("comment", "User age"); + AlterTable.parseColumnDefinitionMap(operation); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testParseColumnDefinitionJsonMissingType() { + + Map operation = new HashMap<>(); + operation.put("operation", "add column"); + operation.put("column_name", "age"); + operation.put("comment", "User age"); + AlterTable.parseColumnDefinitionMap(operation); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testParseColumnDefinitionJsonEmptyColumnName() { + + Map operation = new HashMap<>(); + operation.put("operation", "add column"); + operation.put("column_name", ""); + operation.put("type", "int"); + AlterTable.parseColumnDefinitionMap(operation); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testParseColumnDefinitionJsonEmptyType() { + + Map operation = new HashMap<>(); + operation.put("operation", "add column"); + operation.put("column_name", "age"); + operation.put("type", ""); + AlterTable.parseColumnDefinitionMap(operation); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testParseColumnDefinitionJsonInvalidJson() { + + Map operation = new HashMap<>(); + operation.put("operation2", "add column"); + operation.put("column_name", "age"); + operation.put("type", "int"); + AlterTable.parseColumnDefinitionMap(operation); + } + + @Test + public void testOperationTypeFromKey() { + assertEquals(AlterTable.OperationType.fromKey("add column"), AlterTable.OperationType.ADD); + assertEquals(AlterTable.OperationType.fromKey("drop column"), AlterTable.OperationType.DROP); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testOperationTypeFromKeyInvalid() { + AlterTable.OperationType.fromKey("invalid"); + } + + @Test + public void testParseColumnDefinitionMapForDropColumn() { + + // Test drop column operation - only needs column_name + Map operation = new HashMap<>(); + operation.put("operation", "drop column"); + operation.put("column_name", "age"); + AlterTable.ColumnDefinition columnDef = AlterTable.parseColumnDefinitionMap(operation); + + assertEquals(columnDef.operation(), "drop column"); + assertEquals(columnDef.columnName(), "age"); + assertNull(columnDef.type()); + assertNull(columnDef.comment()); + } +}