diff --git a/cassandra-bigtable-migration-tools/cassandra-bigtable-proxy/translators/common/utils.go b/cassandra-bigtable-migration-tools/cassandra-bigtable-proxy/translators/common/utils.go index b78549ff..abf9632e 100644 --- a/cassandra-bigtable-migration-tools/cassandra-bigtable-proxy/translators/common/utils.go +++ b/cassandra-bigtable-migration-tools/cassandra-bigtable-proxy/translators/common/utils.go @@ -970,14 +970,25 @@ func ParseAs(a cql.IAsSpecContext) (string, error) { return "", nil } - alias := a.OBJECT_NAME().GetText() - if utilities.IsReservedCqlKeyword(alias) { + alias := NormalizeCqlIdentifier(a.OBJECT_NAME().GetText()) + if !IsQuotedIdentifier(a.OBJECT_NAME().GetText()) && utilities.IsReservedCqlKeyword(alias) { return "", fmt.Errorf("cannot use reserved word as alias: '%s'", alias) } return alias, nil } +func NormalizeCqlIdentifier(identifier string) string { + if IsQuotedIdentifier(identifier) { + return TrimDoubleQuotes(identifier) + } + return strings.ToLower(identifier) +} + +func IsQuotedIdentifier(identifier string) bool { + return strings.HasPrefix(identifier, "\"") && strings.HasSuffix(identifier, "\"") +} + func ConvertStrictConditionsToRowKeyValues(table *schemaMapping.TableSchema, conditions []types.Condition) ([]types.DynamicValue, error) { if len(conditions) > len(table.PrimaryKeys) { return nil, fmt.Errorf("only primary keys supported in where clause") diff --git a/cassandra-bigtable-migration-tools/cassandra-bigtable-proxy/translators/select_translator/select_query_builder.go b/cassandra-bigtable-migration-tools/cassandra-bigtable-proxy/translators/select_translator/select_query_builder.go index 638b429e..1ccc1b0a 100644 --- a/cassandra-bigtable-migration-tools/cassandra-bigtable-proxy/translators/select_translator/select_query_builder.go +++ b/cassandra-bigtable-migration-tools/cassandra-bigtable-proxy/translators/select_translator/select_query_builder.go @@ -35,10 +35,14 @@ func createBtqlSelectClause(tableConfig *sm.TableSchema, s *types.SelectClause, return strings.Join(columns, ", "), nil } +func formatBtqlAlias(alias string) string { + return "`" + strings.NewReplacer("\\", "\\\\", "`", "\\`").Replace(alias) + "`" +} + func createBtqlFunc(col types.SelectedColumn, tableConfig *sm.TableSchema) (string, error) { if col.Func == types.FuncCodeCount && col.ColumnName == "*" { if col.Alias != "" { - return "count(*) as " + col.Alias, nil + return "count(*) as " + formatBtqlAlias(col.Alias), nil } return "count(*)", nil } @@ -56,7 +60,7 @@ func createBtqlFunc(col types.SelectedColumn, tableConfig *sm.TableSchema) (stri if col.Func == types.FuncCodeWriteTime { // todo what about collections? if col.Alias != "" { - return fmt.Sprintf("UNIX_MICROS(WRITE_TIMESTAMP(%s, '%s')) AS %s", colMeta.ColumnFamily, colMeta.Name, col.Alias), nil + return fmt.Sprintf("UNIX_MICROS(WRITE_TIMESTAMP(%s, '%s')) AS %s", colMeta.ColumnFamily, colMeta.Name, formatBtqlAlias(col.Alias)), nil } return fmt.Sprintf("UNIX_MICROS(WRITE_TIMESTAMP(%s, '%s'))", colMeta.ColumnFamily, colMeta.Name), nil } @@ -67,14 +71,14 @@ func createBtqlFunc(col types.SelectedColumn, tableConfig *sm.TableSchema) (stri return "", err } if col.Alias != "" { - return fmt.Sprintf("%s AS %s", castValue, col.Alias), nil + return fmt.Sprintf("%s AS %s", castValue, formatBtqlAlias(col.Alias)), nil } return castValue, nil } if col.Func == types.FuncCodeNow { if col.Alias != "" { - return "CURRENT_TIMESTAMP() AS " + col.Alias, nil // CURRENT_TIMESTAMP returns a timestamp, we'll convert it to UUID in result processor if needed, but it's better to return raw bits if we can. + return "CURRENT_TIMESTAMP() AS " + formatBtqlAlias(col.Alias), nil // CURRENT_TIMESTAMP returns a timestamp, we'll convert it to UUID in result processor if needed, but it's better to return raw bits if we can. } return "CURRENT_TIMESTAMP()", nil } @@ -86,7 +90,7 @@ func createBtqlFunc(col types.SelectedColumn, tableConfig *sm.TableSchema) (stri column := fmt.Sprintf("%s(%s)", col.Func.String(), castValue) if col.Alias != "" { - column = column + " as " + col.Alias + column = column + " as " + formatBtqlAlias(col.Alias) } return column, nil @@ -115,7 +119,7 @@ func createBtqlSelectCol(tableConfig *sm.TableSchema, selectedColumn types.Selec } if selectedColumn.Alias != "" { - sql = fmt.Sprintf("%s as %s", sql, selectedColumn.Alias) + sql = fmt.Sprintf("%s as %s", sql, formatBtqlAlias(selectedColumn.Alias)) } return sql, nil } @@ -195,7 +199,7 @@ func createBigtableSql(t *SelectTranslator, st *types.PreparedSelectQuery) (stri for _, col := range st.GroupByColumns { lookupCol := col if _, ok := aliasToColumn[col]; ok { - groupByKeys = append(groupByKeys, col) + groupByKeys = append(groupByKeys, formatBtqlAlias(col)) } else { if colMeta, ok := tableConfig.Columns[types.ColumnName(lookupCol)]; ok { if !colMeta.CQLType.IsCollection() { @@ -218,7 +222,7 @@ func createBigtableSql(t *SelectTranslator, st *types.PreparedSelectQuery) (stri for _, orderByCol := range st.OrderBy.Columns { lookupCol := orderByCol.Column if _, ok := aliasToColumn[orderByCol.Column]; ok { - orderByClauses = append(orderByClauses, orderByCol.Column+" "+string(orderByCol.Operation)) + orderByClauses = append(orderByClauses, formatBtqlAlias(orderByCol.Column)+" "+string(orderByCol.Operation)) } else { if colMeta, ok := tableConfig.Columns[types.ColumnName(lookupCol)]; ok { if colMeta.IsPrimaryKey { diff --git a/cassandra-bigtable-migration-tools/cassandra-bigtable-proxy/translators/select_translator/translator_select_test.go b/cassandra-bigtable-migration-tools/cassandra-bigtable-proxy/translators/select_translator/translator_select_test.go index 4fad7465..161ffa25 100644 --- a/cassandra-bigtable-migration-tools/cassandra-bigtable-proxy/translators/select_translator/translator_select_test.go +++ b/cassandra-bigtable-migration-tools/cassandra-bigtable-proxy/translators/select_translator/translator_select_test.go @@ -28,16 +28,17 @@ import ( ) type Want struct { - Keyspace types.Keyspace - Table types.TableName - TranslatedQuery string - SelectClause *types.SelectClause - Conditions []types.Condition - CachedBTPrepare *bigtable.PreparedStatement - OrderBy types.OrderBy - GroupByColumns []string - LimitValue types.DynamicValue - AllParams []*types.ParameterMetadata + Keyspace types.Keyspace + Table types.TableName + TranslatedQuery string + SelectClause *types.SelectClause + Conditions []types.Condition + CachedBTPrepare *bigtable.PreparedStatement + OrderBy types.OrderBy + GroupByColumns []string + LimitValue types.DynamicValue + AllParams []*types.ParameterMetadata + ResultColumnNames []string } func TestTranslator_TranslateSelectQuerytoBigtable(t *testing.T) { @@ -105,7 +106,7 @@ func TestTranslator_TranslateSelectQuerytoBigtable(t *testing.T) { name: "Select query with list contains clause", query: `select col_int as name from test_keyspace.test_table where list_text CONTAINS 'test';`, want: &Want{ - TranslatedQuery: "SELECT TO_INT64(`cf1`['col_int']) as name FROM test_table WHERE ARRAY_INCLUDES(MAP_VALUES(`list_text`), 'test');", + TranslatedQuery: "SELECT TO_INT64(`cf1`['col_int']) as `name` FROM test_table WHERE ARRAY_INCLUDES(MAP_VALUES(`list_text`), 'test');", Table: "test_table", Keyspace: "test_keyspace", SelectClause: &types.SelectClause{ @@ -131,7 +132,7 @@ func TestTranslator_TranslateSelectQuerytoBigtable(t *testing.T) { name: "Select query with map contains key clause", query: `select col_int as name from test_keyspace.test_table where map_text_text CONTAINS KEY 'test';`, want: &Want{ - TranslatedQuery: "SELECT TO_INT64(`cf1`['col_int']) as name FROM test_table WHERE MAP_CONTAINS_KEY(`map_text_text`, 'test');", + TranslatedQuery: "SELECT TO_INT64(`cf1`['col_int']) as `name` FROM test_table WHERE MAP_CONTAINS_KEY(`map_text_text`, 'test');", Table: "test_table", Keyspace: "test_keyspace", SelectClause: &types.SelectClause{ @@ -156,7 +157,7 @@ func TestTranslator_TranslateSelectQuerytoBigtable(t *testing.T) { name: "Select query with set contains clause", query: `select col_int as name from test_keyspace.test_table where set_text CONTAINS 'test';`, want: &Want{ - TranslatedQuery: "SELECT TO_INT64(`cf1`['col_int']) as name FROM test_table WHERE MAP_CONTAINS_KEY(`set_text`, 'test');", + TranslatedQuery: "SELECT TO_INT64(`cf1`['col_int']) as `name` FROM test_table WHERE MAP_CONTAINS_KEY(`set_text`, 'test');", Table: "test_table", Keyspace: "test_keyspace", SelectClause: &types.SelectClause{ @@ -581,7 +582,7 @@ func TestTranslator_TranslateSelectQuerytoBigtable(t *testing.T) { name: "Writetime CqlQuery with as keyword", query: `select pk1, WRITETIME(col_int) as name from test_keyspace.test_table where pk1 = 'test';`, want: &Want{ - TranslatedQuery: "SELECT pk1, UNIX_MICROS(WRITE_TIMESTAMP(cf1, 'col_int')) AS name FROM test_table WHERE pk1 = 'test';", + TranslatedQuery: "SELECT pk1, UNIX_MICROS(WRITE_TIMESTAMP(cf1, 'col_int')) AS `name` FROM test_table WHERE pk1 = 'test';", Table: "test_table", Keyspace: "test_keyspace", SelectClause: &types.SelectClause{ @@ -607,7 +608,7 @@ func TestTranslator_TranslateSelectQuerytoBigtable(t *testing.T) { name: "As CqlQuery", query: `select col_int as name from test_keyspace.test_table where pk1 = 'test';`, want: &Want{ - TranslatedQuery: "SELECT TO_INT64(`cf1`['col_int']) as name FROM test_table WHERE pk1 = 'test';", + TranslatedQuery: "SELECT TO_INT64(`cf1`['col_int']) as `name` FROM test_table WHERE pk1 = 'test';", Table: "test_table", Keyspace: "test_keyspace", SelectClause: &types.SelectClause{ @@ -628,6 +629,246 @@ func TestTranslator_TranslateSelectQuerytoBigtable(t *testing.T) { AllParams: []*types.ParameterMetadata{}}, sessionKeyspace: "test_keyspace", }, + { + name: "As CqlQuery with quoted alias", + query: `select col_int as "TableBindName" from test_keyspace.test_table where pk1 = 'test';`, + want: &Want{ + TranslatedQuery: "SELECT TO_INT64(`cf1`['col_int']) as `TableBindName` FROM test_table WHERE pk1 = 'test';", + Table: "test_table", + Keyspace: "test_keyspace", + SelectClause: &types.SelectClause{ + Columns: []types.SelectedColumn{ + *types.NewSelectedColumn("col_int", "col_int", "TableBindName", types.TypeInt), + }, + }, + Conditions: []types.Condition{ + { + Column: mockdata.GetColumnOrDie("test_keyspace", "test_table", "pk1"), + Operator: "=", + Value: types.NewLiteralValue("test"), + }, + }, + OrderBy: types.OrderBy{IsOrderBy: false}, + AllParams: []*types.ParameterMetadata{}, + ResultColumnNames: []string{"TableBindName"}, + }, + sessionKeyspace: "test_keyspace", + }, + { + name: "As CqlQuery with unquoted mixed-case alias", + query: `select col_int as TableBindName from test_keyspace.test_table where pk1 = 'test';`, + want: &Want{ + TranslatedQuery: "SELECT TO_INT64(`cf1`['col_int']) as `tablebindname` FROM test_table WHERE pk1 = 'test';", + Table: "test_table", + Keyspace: "test_keyspace", + SelectClause: &types.SelectClause{ + Columns: []types.SelectedColumn{ + *types.NewSelectedColumn("col_int", "col_int", "tablebindname", types.TypeInt), + }, + }, + Conditions: []types.Condition{ + { + Column: mockdata.GetColumnOrDie("test_keyspace", "test_table", "pk1"), + Operator: "=", + Value: types.NewLiteralValue("test"), + }, + }, + OrderBy: types.OrderBy{IsOrderBy: false}, + AllParams: []*types.ParameterMetadata{}, + ResultColumnNames: []string{"tablebindname"}, + }, + sessionKeyspace: "test_keyspace", + }, + { + name: "As CqlQuery with quoted alias ending in backslash", + query: `select col_int as "TrailingBackslash\" from test_keyspace.test_table where pk1 = 'test';`, + want: &Want{ + TranslatedQuery: "SELECT TO_INT64(`cf1`['col_int']) as `TrailingBackslash\\\\` FROM test_table WHERE pk1 = 'test';", + Table: "test_table", + Keyspace: "test_keyspace", + SelectClause: &types.SelectClause{ + Columns: []types.SelectedColumn{ + *types.NewSelectedColumn("col_int", "col_int", "TrailingBackslash\\", types.TypeInt), + }, + }, + Conditions: []types.Condition{ + { + Column: mockdata.GetColumnOrDie("test_keyspace", "test_table", "pk1"), + Operator: "=", + Value: types.NewLiteralValue("test"), + }, + }, + OrderBy: types.OrderBy{IsOrderBy: false}, + AllParams: []*types.ParameterMetadata{}, + ResultColumnNames: []string{"TrailingBackslash\\"}, + }, + sessionKeyspace: "test_keyspace", + }, + { + name: "As CqlQuery with quoted reserved alias", + query: `select col_int as "select" from test_keyspace.test_table where pk1 = 'test';`, + want: &Want{ + TranslatedQuery: "SELECT TO_INT64(`cf1`['col_int']) as `select` FROM test_table WHERE pk1 = 'test';", + Table: "test_table", + Keyspace: "test_keyspace", + SelectClause: &types.SelectClause{ + Columns: []types.SelectedColumn{ + *types.NewSelectedColumn("col_int", "col_int", "select", types.TypeInt), + }, + }, + Conditions: []types.Condition{ + { + Column: mockdata.GetColumnOrDie("test_keyspace", "test_table", "pk1"), + Operator: "=", + Value: types.NewLiteralValue("test"), + }, + }, + OrderBy: types.OrderBy{IsOrderBy: false}, + AllParams: []*types.ParameterMetadata{}, + ResultColumnNames: []string{"select"}, + }, + sessionKeyspace: "test_keyspace", + }, + { + name: "Writetime CqlQuery with quoted alias", + query: `select WRITETIME(col_int) as "WriteTimeAlias" from test_keyspace.test_table where pk1 = 'test';`, + want: &Want{ + TranslatedQuery: "SELECT UNIX_MICROS(WRITE_TIMESTAMP(cf1, 'col_int')) AS `WriteTimeAlias` FROM test_table WHERE pk1 = 'test';", + Table: "test_table", + Keyspace: "test_keyspace", + SelectClause: &types.SelectClause{ + Columns: []types.SelectedColumn{ + *types.NewSelectedColumnFunction("WRITETIME(col_int)", "col_int", "WriteTimeAlias", types.TypeBigInt, types.FuncCodeWriteTime), + }, + }, + Conditions: []types.Condition{ + { + Column: mockdata.GetColumnOrDie("test_keyspace", "test_table", "pk1"), + Operator: "=", + Value: types.NewLiteralValue("test"), + }, + }, + OrderBy: types.OrderBy{IsOrderBy: false}, + AllParams: []*types.ParameterMetadata{}, + ResultColumnNames: []string{"WriteTimeAlias"}, + }, + sessionKeyspace: "test_keyspace", + }, + { + name: "Count CqlQuery with quoted alias", + query: `select count(*) as "TotalRows" from test_keyspace.test_table where pk1 = 'test';`, + want: &Want{ + TranslatedQuery: "SELECT count(*) as `TotalRows` FROM test_table WHERE pk1 = 'test';", + Table: "test_table", + Keyspace: "test_keyspace", + SelectClause: &types.SelectClause{ + Columns: []types.SelectedColumn{ + *types.NewSelectedColumnFunction("system.count(*)", "*", "TotalRows", types.TypeBigInt, types.FuncCodeCount), + }, + }, + Conditions: []types.Condition{ + { + Column: mockdata.GetColumnOrDie("test_keyspace", "test_table", "pk1"), + Operator: "=", + Value: types.NewLiteralValue("test"), + }, + }, + OrderBy: types.OrderBy{IsOrderBy: false}, + AllParams: []*types.ParameterMetadata{}, + ResultColumnNames: []string{"TotalRows"}, + }, + sessionKeyspace: "test_keyspace", + }, + { + name: "ORDER BY quoted alias", + query: `select pk1, count(col_int) as "TotalRows" from test_keyspace.test_table where pk1 = 'test' GROUP BY pk1 ORDER BY "TotalRows" DESC;`, + want: &Want{ + TranslatedQuery: "SELECT pk1, count(TO_INT64(`cf1`['col_int'])) as `TotalRows` FROM test_table WHERE pk1 = 'test' GROUP BY pk1 ORDER BY `TotalRows` desc;", + Table: "test_table", + Keyspace: "test_keyspace", + SelectClause: &types.SelectClause{ + Columns: []types.SelectedColumn{ + *types.NewSelectedColumn("pk1", "pk1", "", types.TypeVarchar), + *types.NewSelectedColumnFunction("system.count(col_int)", "col_int", "TotalRows", types.TypeBigInt, types.FuncCodeCount), + }, + }, + Conditions: []types.Condition{ + { + Column: mockdata.GetColumnOrDie("test_keyspace", "test_table", "pk1"), + Operator: "=", + Value: types.NewLiteralValue("test"), + }, + }, + GroupByColumns: []string{"pk1"}, + OrderBy: types.OrderBy{ + IsOrderBy: true, + Columns: []types.OrderByColumn{ + {Column: "TotalRows", Operation: types.Desc}, + }, + }, + AllParams: []*types.ParameterMetadata{}, + ResultColumnNames: []string{"pk1", "TotalRows"}, + }, + sessionKeyspace: "test_keyspace", + }, + { + name: "ORDER BY unquoted mixed-case alias", + query: `select pk1, count(col_int) as TotalRows from test_keyspace.test_table where pk1 = 'test' GROUP BY pk1 ORDER BY TotalRows DESC;`, + want: &Want{ + TranslatedQuery: "SELECT pk1, count(TO_INT64(`cf1`['col_int'])) as `totalrows` FROM test_table WHERE pk1 = 'test' GROUP BY pk1 ORDER BY `totalrows` desc;", + Table: "test_table", + Keyspace: "test_keyspace", + SelectClause: &types.SelectClause{ + Columns: []types.SelectedColumn{ + *types.NewSelectedColumn("pk1", "pk1", "", types.TypeVarchar), + *types.NewSelectedColumnFunction("system.count(col_int)", "col_int", "totalrows", types.TypeBigInt, types.FuncCodeCount), + }, + }, + Conditions: []types.Condition{ + { + Column: mockdata.GetColumnOrDie("test_keyspace", "test_table", "pk1"), + Operator: "=", + Value: types.NewLiteralValue("test"), + }, + }, + GroupByColumns: []string{"pk1"}, + OrderBy: types.OrderBy{ + IsOrderBy: true, + Columns: []types.OrderByColumn{ + {Column: "totalrows", Operation: types.Desc}, + }, + }, + AllParams: []*types.ParameterMetadata{}, + ResultColumnNames: []string{"pk1", "totalrows"}, + }, + sessionKeyspace: "test_keyspace", + }, + { + name: "GROUP BY quoted alias", + query: `select pk1 as "PkAlias" from test_keyspace.test_table where pk1 = 'test' GROUP BY "PkAlias";`, + want: &Want{ + TranslatedQuery: "SELECT pk1 as `PkAlias` FROM test_table WHERE pk1 = 'test' GROUP BY `PkAlias`;", + Table: "test_table", + Keyspace: "test_keyspace", + SelectClause: &types.SelectClause{ + Columns: []types.SelectedColumn{ + *types.NewSelectedColumn("pk1", "pk1", "PkAlias", types.TypeVarchar), + }, + }, + Conditions: []types.Condition{ + { + Column: mockdata.GetColumnOrDie("test_keyspace", "test_table", "pk1"), + Operator: "=", + Value: types.NewLiteralValue("test"), + }, + }, + GroupByColumns: []string{"PkAlias"}, + OrderBy: types.OrderBy{IsOrderBy: false}, + AllParams: []*types.ParameterMetadata{}, + ResultColumnNames: []string{"PkAlias"}, + }, + sessionKeyspace: "test_keyspace", + }, { name: "CqlQuery without Columns", query: `select from test_keyspace.test_table where pk1 = 'test'`, @@ -1476,6 +1717,13 @@ func TestTranslator_TranslateSelectQuerytoBigtable(t *testing.T) { assert.Equal(t, tt.want.OrderBy, gotSelect.OrderBy) assert.Equal(t, tt.want.GroupByColumns, gotSelect.GroupByColumns) assert.ElementsMatch(t, tt.want.AllParams, gotSelect.Params.Ordered()) + if tt.want.ResultColumnNames != nil { + var gotColumnNames []string + for _, col := range gotSelect.ResultColumnMetadata { + gotColumnNames = append(gotColumnNames, col.Name) + } + assert.Equal(t, tt.want.ResultColumnNames, gotColumnNames) + } }) } } diff --git a/cassandra-bigtable-migration-tools/cassandra-bigtable-proxy/translators/select_translator/utils.go b/cassandra-bigtable-migration-tools/cassandra-bigtable-proxy/translators/select_translator/utils.go index 355c5492..a42940a7 100644 --- a/cassandra-bigtable-migration-tools/cassandra-bigtable-proxy/translators/select_translator/utils.go +++ b/cassandra-bigtable-migration-tools/cassandra-bigtable-proxy/translators/select_translator/utils.go @@ -88,7 +88,7 @@ func parseOrderByFromSelect(input cql.IOrderSpecContext) (types.OrderBy, error) return types.OrderBy{}, fmt.Errorf("order_by section not have proper values") } - colName := strings.TrimSpace(object.GetText()) + colName := common.NormalizeCqlIdentifier(strings.TrimSpace(object.GetText())) if colName == "" { return types.OrderBy{}, fmt.Errorf("order_by section has empty column name") } @@ -136,7 +136,7 @@ func parseGroupByColumn(input cql.IGroupSpecContext) []string { return nil } - colName := object.GetText() + colName := common.NormalizeCqlIdentifier(object.GetText()) columns = append(columns, colName) }