Legend SQL - support variant column types in resultsets

legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/tds.pure

@@ -325,7 +325,7 @@ function
                columns = $simpleCols->map(cs|let offset = $columnSpecifications->indexOf($cs);
                                                        ^TDSColumn(offset = $offset,
                                                                   name = $cs.name,
-                                                                  type = $cs.func->functionReturnType().rawType->toOne()->cast(@DataType),
+                                                                  type = $cs.func->functionReturnType().rawType->toOne(),
                                                                   documentation = $cs.documentation);
                                                 ));
 
@@ -555,7 +555,7 @@ function
    let newTds = ^TabularDataSet(
       columns=$tds.columns->concatenate($newColumnFunctions->size()->range()->map(index |
                                                 let col = $newColumnFunctions->at($index);
-                                                ^TDSColumn(offset=$tds.columns->size() + $index, name=$col.name, type=$col.func->functionReturnType().rawType->toOne()->cast(@DataType));
+                                                ^TDSColumn(offset=$tds.columns->size() + $index, name=$col.name, type=$col.func->functionReturnType().rawType->toOne());
                                                 )));
 
    //todo: remove this by making parent an association
@@ -581,7 +581,7 @@ function
    let newTds = ^TabularDataSet(
       columns=$newColumnFunctions->size()->range()->map(index |
                                                 let col = $newColumnFunctions->at($index);
-                                                ^TDSColumn(offset=$index, name=$col.name, type=$col.func->functionReturnType().rawType->toOne()->cast(@DataType));
+                                                ^TDSColumn(offset=$index, name=$col.name, type=$col.func->functionReturnType().rawType->toOne());
                                                 ));
 
    //todo: remove this by making parent an association
@@ -641,7 +641,7 @@ function
                                     );
    let aggOutputCols = range($aggValues->size())->map(index|
          let name = $aggValues->at($index).name;
-         let colType = $aggValues->at($index).aggregateFn->functionReturnType().rawType->toOne()->cast(@DataType);
+         let colType = $aggValues->at($index).aggregateFn->functionReturnType().rawType->toOne();
 
          ^TDSColumn(offset = $index + $columns->size(),
                      type = $colType,
@@ -850,7 +850,7 @@ function meta::pure::tds::groupBy<K,V,U>(set:K[*], functions:meta::pure::metamod
    let columns = $columnInfo->map(info | ^TDSColumn(
       name = $info.first,
       offset = $columnInfo->indexOf($info),
-      type = $info.second->functionReturnType().rawType->toOne()->cast(@DataType)));
+      type = $info.second->functionReturnType().rawType->toOne()));
 
    let tds = ^TabularDataSet(columns=$columns);
 

legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/tdsSchema.pure

@@ -236,7 +236,7 @@ function <<access.private>> meta::pure::tds::schema::resolveSchemaImpl(fe : Func
             let colNames = $fe.parametersValues->at(2)->reactivate($openVars)->cast(@String);
 
             let cols = $colFuncs->zip($colNames)->map(p|
-               ^TDSColumn(name = $p.second, type = $p.first->functionReturnType().rawType->toOne()->cast(@DataType))
+               ^TDSColumn(name = $p.second, type = $p.first->functionReturnType().rawType->toOne())
                );
             createSchemaState($cols);
          }),
@@ -259,7 +259,7 @@ function <<access.private>> meta::pure::tds::schema::resolveSchemaImpl(fe : Func
             let colNames = $fe.parametersValues->at(3)->reactivate($openVars)->cast(@String);
 
             let cols = $groupByColLambdas->concatenate($aggColLambdas.aggregateFn)->zip($colNames)->map(p|
-                  ^TDSColumn(name=$p.second, type=$p.first->functionReturnType().rawType->toOne()->cast(@DataType));
+                  ^TDSColumn(name=$p.second, type=$p.first->functionReturnType().rawType->toOne());
                );
 
             createSchemaState($cols);
@@ -271,7 +271,7 @@ function <<access.private>> meta::pure::tds::schema::resolveSchemaImpl(fe : Func
             let aggSpecifications = $fe.parametersValues->at(2)->reactivate($openVars)->cast(@meta::pure::tds::AggregateValue<Any,Any>)->toOneMany();
 
             let aggColumns = $aggSpecifications->map(aggSpecification |
-               ^TDSColumn(name=$aggSpecification.name, type=$aggSpecification.aggregateFn->functionReturnType().rawType->toOne()->cast(@DataType));
+               ^TDSColumn(name=$aggSpecification.name, type=$aggSpecification.aggregateFn->functionReturnType().rawType->toOne());
                );
 
             $tdsSchema.groupBy($colNamesToGroupBy, $aggColumns);
@@ -308,7 +308,7 @@ function <<access.private>> meta::pure::tds::schema::resolveSchemaImpl(fe : Func
                lf:LambdaFunction<Any>[1]|$lf->functionReturnType().typeArguments->at(1)
             ]);
 
-            let colType = $fType.rawType->toOne()->cast(@DataType);
+            let colType = $fType.rawType->toOne();
             let colName = $fe.parametersValues->at(4)->reactivate($openVars)->cast(@String)->toOne();
 
             $tdsSchema.olap(^TDSColumn(name=$colName, type=$colType));
@@ -354,7 +354,7 @@ function meta::pure::tds::schema::resolveProject(colSpecs : ColumnSpecification<
 {
    let cols = $colSpecs->map(colSpec|
                $colSpec->match([
-                  bcs : BasicColumnSpecification<Any>[1]|^TDSColumn(name = $bcs.name, type = $bcs.func->functionReturnType().rawType->toOne()->cast(@DataType))
+                  bcs : BasicColumnSpecification<Any>[1]|^TDSColumn(name = $bcs.name, type = $bcs.func->functionReturnType().rawType->toOne())
                   ])
                );
    createSchemaState($cols);

legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/testTdsSchema.pure

@@ -218,7 +218,8 @@ function  <<test.Test>> meta::pure::tds::schema::tests::resolveSchemaTest() : Bo
                col(p|%2018-12-12,'constDate'),
                col(p|true,'constBoolean'),
                col(p|1,'constInteger'),
-               col(p|1.5,'constFloat')
+               col(p|1.5,'constFloat'),
+               col(p|^meta::pure::metamodel::variant::Variant(), 'constVariant')
             ])
    });
 

legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-postgresSql/legend-engine-xt-relationalStore-postgresSqlModel-pure/src/main/resources/core_external_store_relational_postgres_sql_model/schema_metamodel.pure

@@ -13,6 +13,11 @@ Class meta::external::query::sql::schema::metamodel::EnumSchemaColumn extends me
   <<equality.Key>> type: String[1];
 }
 
+Class meta::external::query::sql::schema::metamodel::VariantSchemaColumn extends meta::external::query::sql::schema::metamodel::SchemaColumn
+{
+  <<equality.Key>> type: String[1];
+}
+
 
 Class meta::external::query::sql::schema::metamodel::Schema
 {

legend-engine-xts-sql/legend-engine-xt-sql-postgres-server/src/main/java/org/finos/legend/engine/postgres/protocol/wire/serialization/types/JsonType.java

@@ -21,11 +21,15 @@
 
 package org.finos.legend.engine.postgres.protocol.wire.serialization.types;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import io.netty.buffer.ByteBuf;
 
+import java.io.IOException;
+
 public class JsonType extends PGType<Object>
 {
 
+    private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
     public static final JsonType INSTANCE = new JsonType();
     static final int OID = 114;
 
@@ -67,26 +71,35 @@ public int writeAsBinary(ByteBuf buffer, Object value)
     @Override
     protected byte[] encodeAsUTF8Text(Object value)
     {
-       /* if (value instanceof String str) {
-            return str.getBytes(StandardCharsets.UTF_8);
-        }
-        try {
-            XContentBuilder builder = JsonXContent.contentBuilder();
-            if (value instanceof List<?> values) {
-                builder.startArray();
-                for (Object o : values) {
-                    builder.value(o);
+        try
+        {
+            if (value instanceof String)
+            {
+                String str = (String) value;
+                // If the string is already valid JSON (object or array), send as-is
+                if (isJsonObjectOrArray(str))
+                {
+                    // Re-serialize to normalize (compact) the JSON
+                    Object parsed = OBJECT_MAPPER.readValue(str, Object.class);
+                    return OBJECT_MAPPER.writeValueAsBytes(parsed);
                 }
-                builder.endArray();
-            } else {
-                builder.map((Map) value);
             }
-            builder.close();
-            return BytesReference.toBytes(BytesReference.bytes(builder));
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }*/
-        throw new UnsupportedOperationException("Not implemented");
+            return OBJECT_MAPPER.writeValueAsBytes(value);
+        }
+        catch (IOException e)
+        {
+            throw new RuntimeException("Failed to encode value as JSON", e);
+        }
+    }
+
+    private static boolean isJsonObjectOrArray(String str)
+    {
+        if (str.isEmpty())
+        {
+            return false;
+        }
+        char first = str.charAt(0);
+        return first == '{' || first == '[';
     }
 
     @Override
@@ -100,17 +113,13 @@ public Object readBinaryValue(ByteBuf buffer, int valueLength)
     @Override
     Object decodeUTF8Text(byte[] bytes)
     {
-        /*try {
-            XContentParser parser = JsonXContent.JSON_XCONTENT.createParser(
-                NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, bytes);
-            if (bytes.length > 1 && bytes[0] == '[') {
-                parser.nextToken();
-                return parser.list();
-            }
-            return parser.map();
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }*/
-        throw new UnsupportedOperationException("Not implemented");
+        try
+        {
+            return OBJECT_MAPPER.readValue(bytes, Object.class);
+        }
+        catch (IOException e)
+        {
+            throw new RuntimeException("Failed to decode JSON value", e);
+        }
     }
 }

legend-engine-xts-sql/legend-engine-xt-sql-postgres-server/src/test/java/org/finos/legend/engine/postgres/PostgresServerTypeMappingTest.java

@@ -34,6 +34,7 @@
 
 import java.util.TimeZone;
 
+import org.postgresql.util.PGobject;
 import org.eclipse.collections.api.factory.Lists;
 import org.finos.legend.engine.postgres.protocol.sql.handler.legend.statement.result.LegendDataType;
 import org.finos.legend.engine.postgres.protocol.wire.auth.identity.AnonymousIdentityProvider;
@@ -288,6 +289,24 @@ public void testNumberAsDouble() throws Exception
         validate(LegendDataType.NUMBER, "5.5", "float8", 5.5D);
     }
 
+    @Test
+    public void testJSON() throws Exception
+    {
+        validate(LegendDataType.VARIANT, "\"hello\"", "json", pgJson("\"hello\""));
+        validate(LegendDataType.VARIANT, "\"{\\\"a\\\": 1}\"", "json", pgJson("{\"a\":1}"));
+        validate(LegendDataType.VARIANT, "\"[1,2,3]\"", "json", pgJson("[1,2,3]"));
+        validate(LegendDataType.VARIANT, "123", "json", pgJson("123"));
+        validate(LegendDataType.VARIANT, "null", "json", null);
+    }
+
+    private static PGobject pgJson(String value) throws Exception
+    {
+        PGobject pgObject = new PGobject();
+        pgObject.setType("json");
+        pgObject.setValue(value);
+        return pgObject;
+    }
+
 
     public void validate(String legendDataType, String legendValue, String expectedColumnType, Object expectedValue) throws Exception
     {

legend-engine-xts-sql/legend-engine-xt-sql-transformation/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/tests/testTranspile.pure

@@ -4269,7 +4269,7 @@ function <<test.Test>> meta::external::query::sql::transformation::queryToPure::
 
 function meta::external::query::sql::transformation::queryToPure::tests::testSchemaService1(relation:Boolean[1]):Boolean[1]
 {
-  let sqlString = 'SELECT * FROM service."/service/service1"';
+  let sqlString = 'SELECT *, cast(String as json) as "JSON" FROM service."/service/service1"';
 
   let actualSchema = $sqlString->processQuery(false, $relation).columns->meta::external::query::sql::tdsColsToSchema();
 
@@ -4280,7 +4280,9 @@ function meta::external::query::sql::transformation::queryToPure::tests::testSch
   let expectedCol5 = ^PrimitiveSchemaColumn(name='StrictDate',   type=meta::external::query::sql::schema::metamodel::PrimitiveType.StrictDate);
   let expectedCol6 = ^PrimitiveSchemaColumn(name='DateTime',     type=meta::external::query::sql::schema::metamodel::PrimitiveType.DateTime);
   let expectedCol7 = ^PrimitiveSchemaColumn(name='String',       type=meta::external::query::sql::schema::metamodel::PrimitiveType.String);
-  let expectedSchema = ^Schema(columns=[$expectedCol1, $expectedCol2, $expectedCol3, $expectedCol4, $expectedCol5, $expectedCol6, $expectedCol7]);
+  let expectedCol8 = ^VariantSchemaColumn(name='JSON',           type=meta::pure::metamodel::variant::Variant->elementToPath());
+
+  let expectedSchema = ^Schema(columns=[$expectedCol1, $expectedCol2, $expectedCol3, $expectedCol4, $expectedCol5, $expectedCol6, $expectedCol7, $expectedCol8]);
 
   assertEquals($expectedSchema, $actualSchema);
 }

legend-engine-xts-sql/legend-engine-xt-sql-transformation/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/schema.pure

@@ -41,6 +41,11 @@ function meta::external::query::sql::schema::tdsColToSchemaCol(col: TDSColumn[1]
   let columnPrimitiveType = $col.type->match([
     s:PrimitiveType[1]      | ^meta::external::query::sql::schema::metamodel::PrimitiveSchemaColumn(type=$tdsType->meta::external::query::sql::schema::typeToPrimitiveType(),name=$columnName),
     e:Enumeration<Any>[1]   | ^meta::external::query::sql::schema::metamodel::EnumSchemaColumn(type=$tdsType->meta::pure::functions::meta::elementToPath(),name=$columnName),
+    c:Class<Any>[1]         | if ([
+                                pair(| $c == meta::pure::metamodel::variant::Variant, | ^meta::external::query::sql::schema::metamodel::VariantSchemaColumn(type = meta::pure::metamodel::variant::Variant->elementToPath(),name=$columnName))
+                              ],
+                                | fail('Unsupported type on column: ' + $columnName + ' (' + $tdsType->meta::pure::functions::meta::elementToPath() + '), only primitive types and enums are supported')
+                              );,
     a:Any[*]                | fail('Unsupported type on column: ' + $columnName + ' (' + $tdsType->meta::pure::functions::meta::elementToPath() + '), only primitive types and enums are supported');
   ])->cast(@meta::external::query::sql::schema::metamodel::SchemaColumn);
 }