diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/tds.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/tds.pure index 62b9f9a0cbd..891bc5b5e7c 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/tds.pure +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/tds.pure @@ -325,7 +325,7 @@ function columns = $simpleCols->map(cs|let offset = $columnSpecifications->indexOf($cs); ^TDSColumn(offset = $offset, name = $cs.name, - type = $cs.func->functionReturnType().rawType->toOne()->cast(@DataType), + type = $cs.func->functionReturnType().rawType->toOne(), documentation = $cs.documentation); )); @@ -555,7 +555,7 @@ function let newTds = ^TabularDataSet( columns=$tds.columns->concatenate($newColumnFunctions->size()->range()->map(index | let col = $newColumnFunctions->at($index); - ^TDSColumn(offset=$tds.columns->size() + $index, name=$col.name, type=$col.func->functionReturnType().rawType->toOne()->cast(@DataType)); + ^TDSColumn(offset=$tds.columns->size() + $index, name=$col.name, type=$col.func->functionReturnType().rawType->toOne()); ))); //todo: remove this by making parent an association @@ -581,7 +581,7 @@ function let newTds = ^TabularDataSet( columns=$newColumnFunctions->size()->range()->map(index | let col = $newColumnFunctions->at($index); - ^TDSColumn(offset=$index, name=$col.name, type=$col.func->functionReturnType().rawType->toOne()->cast(@DataType)); + ^TDSColumn(offset=$index, name=$col.name, type=$col.func->functionReturnType().rawType->toOne()); )); //todo: remove this by making parent an association @@ -641,7 +641,7 @@ function ); let aggOutputCols = range($aggValues->size())->map(index| let name = $aggValues->at($index).name; - let colType = $aggValues->at($index).aggregateFn->functionReturnType().rawType->toOne()->cast(@DataType); + let colType = $aggValues->at($index).aggregateFn->functionReturnType().rawType->toOne(); ^TDSColumn(offset = $index + $columns->size(), type = $colType, @@ -850,7 +850,7 @@ function meta::pure::tds::groupBy(set:K[*], functions:meta::pure::metamod let columns = $columnInfo->map(info | ^TDSColumn( name = $info.first, offset = $columnInfo->indexOf($info), - type = $info.second->functionReturnType().rawType->toOne()->cast(@DataType))); + type = $info.second->functionReturnType().rawType->toOne())); let tds = ^TabularDataSet(columns=$columns); diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/tdsSchema.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/tdsSchema.pure index f06932611a6..42eed4dd9e5 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/tdsSchema.pure +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/tdsSchema.pure @@ -236,7 +236,7 @@ function <> meta::pure::tds::schema::resolveSchemaImpl(fe : Func let colNames = $fe.parametersValues->at(2)->reactivate($openVars)->cast(@String); let cols = $colFuncs->zip($colNames)->map(p| - ^TDSColumn(name = $p.second, type = $p.first->functionReturnType().rawType->toOne()->cast(@DataType)) + ^TDSColumn(name = $p.second, type = $p.first->functionReturnType().rawType->toOne()) ); createSchemaState($cols); }), @@ -259,7 +259,7 @@ function <> meta::pure::tds::schema::resolveSchemaImpl(fe : Func let colNames = $fe.parametersValues->at(3)->reactivate($openVars)->cast(@String); let cols = $groupByColLambdas->concatenate($aggColLambdas.aggregateFn)->zip($colNames)->map(p| - ^TDSColumn(name=$p.second, type=$p.first->functionReturnType().rawType->toOne()->cast(@DataType)); + ^TDSColumn(name=$p.second, type=$p.first->functionReturnType().rawType->toOne()); ); createSchemaState($cols); @@ -271,7 +271,7 @@ function <> meta::pure::tds::schema::resolveSchemaImpl(fe : Func let aggSpecifications = $fe.parametersValues->at(2)->reactivate($openVars)->cast(@meta::pure::tds::AggregateValue)->toOneMany(); let aggColumns = $aggSpecifications->map(aggSpecification | - ^TDSColumn(name=$aggSpecification.name, type=$aggSpecification.aggregateFn->functionReturnType().rawType->toOne()->cast(@DataType)); + ^TDSColumn(name=$aggSpecification.name, type=$aggSpecification.aggregateFn->functionReturnType().rawType->toOne()); ); $tdsSchema.groupBy($colNamesToGroupBy, $aggColumns); @@ -308,7 +308,7 @@ function <> meta::pure::tds::schema::resolveSchemaImpl(fe : Func lf:LambdaFunction[1]|$lf->functionReturnType().typeArguments->at(1) ]); - let colType = $fType.rawType->toOne()->cast(@DataType); + let colType = $fType.rawType->toOne(); let colName = $fe.parametersValues->at(4)->reactivate($openVars)->cast(@String)->toOne(); $tdsSchema.olap(^TDSColumn(name=$colName, type=$colType)); @@ -354,7 +354,7 @@ function meta::pure::tds::schema::resolveProject(colSpecs : ColumnSpecification< { let cols = $colSpecs->map(colSpec| $colSpec->match([ - bcs : BasicColumnSpecification[1]|^TDSColumn(name = $bcs.name, type = $bcs.func->functionReturnType().rawType->toOne()->cast(@DataType)) + bcs : BasicColumnSpecification[1]|^TDSColumn(name = $bcs.name, type = $bcs.func->functionReturnType().rawType->toOne()) ]) ); createSchemaState($cols); diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/testTdsSchema.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/testTdsSchema.pure index 67472bfd305..9965c8db240 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/testTdsSchema.pure +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/testTdsSchema.pure @@ -218,7 +218,8 @@ function <> meta::pure::tds::schema::tests::resolveSchemaTest() : Bo col(p|%2018-12-12,'constDate'), col(p|true,'constBoolean'), col(p|1,'constInteger'), - col(p|1.5,'constFloat') + col(p|1.5,'constFloat'), + col(p|^meta::pure::metamodel::variant::Variant(), 'constVariant') ]) }); diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-postgresSql/legend-engine-xt-relationalStore-postgresSqlModel-pure/src/main/resources/core_external_store_relational_postgres_sql_model/schema_metamodel.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-postgresSql/legend-engine-xt-relationalStore-postgresSqlModel-pure/src/main/resources/core_external_store_relational_postgres_sql_model/schema_metamodel.pure index a4c4656f5ff..bccc9c8e6fa 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-postgresSql/legend-engine-xt-relationalStore-postgresSqlModel-pure/src/main/resources/core_external_store_relational_postgres_sql_model/schema_metamodel.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-postgresSql/legend-engine-xt-relationalStore-postgresSqlModel-pure/src/main/resources/core_external_store_relational_postgres_sql_model/schema_metamodel.pure @@ -13,6 +13,11 @@ Class meta::external::query::sql::schema::metamodel::EnumSchemaColumn extends me <> type: String[1]; } +Class meta::external::query::sql::schema::metamodel::VariantSchemaColumn extends meta::external::query::sql::schema::metamodel::SchemaColumn +{ + <> type: String[1]; +} + Class meta::external::query::sql::schema::metamodel::Schema { diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-postgres-server/src/main/java/org/finos/legend/engine/postgres/protocol/wire/serialization/types/JsonType.java b/legend-engine-xts-sql/legend-engine-xt-sql-postgres-server/src/main/java/org/finos/legend/engine/postgres/protocol/wire/serialization/types/JsonType.java index d8376821819..9f83bdc972c 100644 --- a/legend-engine-xts-sql/legend-engine-xt-sql-postgres-server/src/main/java/org/finos/legend/engine/postgres/protocol/wire/serialization/types/JsonType.java +++ b/legend-engine-xts-sql/legend-engine-xt-sql-postgres-server/src/main/java/org/finos/legend/engine/postgres/protocol/wire/serialization/types/JsonType.java @@ -21,11 +21,15 @@ package org.finos.legend.engine.postgres.protocol.wire.serialization.types; +import com.fasterxml.jackson.databind.ObjectMapper; import io.netty.buffer.ByteBuf; +import java.io.IOException; + public class JsonType extends PGType { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); public static final JsonType INSTANCE = new JsonType(); static final int OID = 114; @@ -67,26 +71,35 @@ public int writeAsBinary(ByteBuf buffer, Object value) @Override protected byte[] encodeAsUTF8Text(Object value) { - /* if (value instanceof String str) { - return str.getBytes(StandardCharsets.UTF_8); - } - try { - XContentBuilder builder = JsonXContent.contentBuilder(); - if (value instanceof List values) { - builder.startArray(); - for (Object o : values) { - builder.value(o); + try + { + if (value instanceof String) + { + String str = (String) value; + // If the string is already valid JSON (object or array), send as-is + if (isJsonObjectOrArray(str)) + { + // Re-serialize to normalize (compact) the JSON + Object parsed = OBJECT_MAPPER.readValue(str, Object.class); + return OBJECT_MAPPER.writeValueAsBytes(parsed); } - builder.endArray(); - } else { - builder.map((Map) value); } - builder.close(); - return BytesReference.toBytes(BytesReference.bytes(builder)); - } catch (IOException e) { - throw new RuntimeException(e); - }*/ - throw new UnsupportedOperationException("Not implemented"); + return OBJECT_MAPPER.writeValueAsBytes(value); + } + catch (IOException e) + { + throw new RuntimeException("Failed to encode value as JSON", e); + } + } + + private static boolean isJsonObjectOrArray(String str) + { + if (str.isEmpty()) + { + return false; + } + char first = str.charAt(0); + return first == '{' || first == '['; } @Override @@ -100,17 +113,13 @@ public Object readBinaryValue(ByteBuf buffer, int valueLength) @Override Object decodeUTF8Text(byte[] bytes) { - /*try { - XContentParser parser = JsonXContent.JSON_XCONTENT.createParser( - NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, bytes); - if (bytes.length > 1 && bytes[0] == '[') { - parser.nextToken(); - return parser.list(); - } - return parser.map(); - } catch (IOException e) { - throw new RuntimeException(e); - }*/ - throw new UnsupportedOperationException("Not implemented"); + try + { + return OBJECT_MAPPER.readValue(bytes, Object.class); + } + catch (IOException e) + { + throw new RuntimeException("Failed to decode JSON value", e); + } } } diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-postgres-server/src/test/java/org/finos/legend/engine/postgres/PostgresServerTypeMappingTest.java b/legend-engine-xts-sql/legend-engine-xt-sql-postgres-server/src/test/java/org/finos/legend/engine/postgres/PostgresServerTypeMappingTest.java index 074dc342515..5f27f80918a 100644 --- a/legend-engine-xts-sql/legend-engine-xt-sql-postgres-server/src/test/java/org/finos/legend/engine/postgres/PostgresServerTypeMappingTest.java +++ b/legend-engine-xts-sql/legend-engine-xt-sql-postgres-server/src/test/java/org/finos/legend/engine/postgres/PostgresServerTypeMappingTest.java @@ -34,6 +34,7 @@ import java.util.TimeZone; +import org.postgresql.util.PGobject; import org.eclipse.collections.api.factory.Lists; import org.finos.legend.engine.postgres.protocol.sql.handler.legend.statement.result.LegendDataType; import org.finos.legend.engine.postgres.protocol.wire.auth.identity.AnonymousIdentityProvider; @@ -288,6 +289,24 @@ public void testNumberAsDouble() throws Exception validate(LegendDataType.NUMBER, "5.5", "float8", 5.5D); } + @Test + public void testJSON() throws Exception + { + validate(LegendDataType.VARIANT, "\"hello\"", "json", pgJson("\"hello\"")); + validate(LegendDataType.VARIANT, "\"{\\\"a\\\": 1}\"", "json", pgJson("{\"a\":1}")); + validate(LegendDataType.VARIANT, "\"[1,2,3]\"", "json", pgJson("[1,2,3]")); + validate(LegendDataType.VARIANT, "123", "json", pgJson("123")); + validate(LegendDataType.VARIANT, "null", "json", null); + } + + private static PGobject pgJson(String value) throws Exception + { + PGobject pgObject = new PGobject(); + pgObject.setType("json"); + pgObject.setValue(value); + return pgObject; + } + public void validate(String legendDataType, String legendValue, String expectedColumnType, Object expectedValue) throws Exception { diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-transformation/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/tests/testTranspile.pure b/legend-engine-xts-sql/legend-engine-xt-sql-transformation/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/tests/testTranspile.pure index 6e75515c818..204e42ed266 100644 --- a/legend-engine-xts-sql/legend-engine-xt-sql-transformation/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/tests/testTranspile.pure +++ b/legend-engine-xts-sql/legend-engine-xt-sql-transformation/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/tests/testTranspile.pure @@ -4269,7 +4269,7 @@ function <> meta::external::query::sql::transformation::queryToPure:: function meta::external::query::sql::transformation::queryToPure::tests::testSchemaService1(relation:Boolean[1]):Boolean[1] { - let sqlString = 'SELECT * FROM service."/service/service1"'; + let sqlString = 'SELECT *, cast(String as json) as "JSON" FROM service."/service/service1"'; let actualSchema = $sqlString->processQuery(false, $relation).columns->meta::external::query::sql::tdsColsToSchema(); @@ -4280,7 +4280,9 @@ function meta::external::query::sql::transformation::queryToPure::tests::testSch let expectedCol5 = ^PrimitiveSchemaColumn(name='StrictDate', type=meta::external::query::sql::schema::metamodel::PrimitiveType.StrictDate); let expectedCol6 = ^PrimitiveSchemaColumn(name='DateTime', type=meta::external::query::sql::schema::metamodel::PrimitiveType.DateTime); let expectedCol7 = ^PrimitiveSchemaColumn(name='String', type=meta::external::query::sql::schema::metamodel::PrimitiveType.String); - let expectedSchema = ^Schema(columns=[$expectedCol1, $expectedCol2, $expectedCol3, $expectedCol4, $expectedCol5, $expectedCol6, $expectedCol7]); + let expectedCol8 = ^VariantSchemaColumn(name='JSON', type=meta::pure::metamodel::variant::Variant->elementToPath()); + + let expectedSchema = ^Schema(columns=[$expectedCol1, $expectedCol2, $expectedCol3, $expectedCol4, $expectedCol5, $expectedCol6, $expectedCol7, $expectedCol8]); assertEquals($expectedSchema, $actualSchema); } diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-transformation/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/schema.pure b/legend-engine-xts-sql/legend-engine-xt-sql-transformation/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/schema.pure index 8fc8f1da922..579f5498196 100644 --- a/legend-engine-xts-sql/legend-engine-xt-sql-transformation/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/schema.pure +++ b/legend-engine-xts-sql/legend-engine-xt-sql-transformation/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/schema.pure @@ -41,6 +41,11 @@ function meta::external::query::sql::schema::tdsColToSchemaCol(col: TDSColumn[1] let columnPrimitiveType = $col.type->match([ s:PrimitiveType[1] | ^meta::external::query::sql::schema::metamodel::PrimitiveSchemaColumn(type=$tdsType->meta::external::query::sql::schema::typeToPrimitiveType(),name=$columnName), e:Enumeration[1] | ^meta::external::query::sql::schema::metamodel::EnumSchemaColumn(type=$tdsType->meta::pure::functions::meta::elementToPath(),name=$columnName), + c:Class[1] | if ([ + pair(| $c == meta::pure::metamodel::variant::Variant, | ^meta::external::query::sql::schema::metamodel::VariantSchemaColumn(type = meta::pure::metamodel::variant::Variant->elementToPath(),name=$columnName)) + ], + | fail('Unsupported type on column: ' + $columnName + ' (' + $tdsType->meta::pure::functions::meta::elementToPath() + '), only primitive types and enums are supported') + );, a:Any[*] | fail('Unsupported type on column: ' + $columnName + ' (' + $tdsType->meta::pure::functions::meta::elementToPath() + '), only primitive types and enums are supported'); ])->cast(@meta::external::query::sql::schema::metamodel::SchemaColumn); }