Optimize clinical data derived queries

src/integration/java/org/cbioportal/infrastructure/repository/clickhouse/clinical_data/ClickHouseClinicalDataRepositoryIntegrationTest.java

@@ -7,6 +7,7 @@
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import org.cbioportal.AbstractClickhouseIntegrationTest;
 import org.cbioportal.domain.clinical_data.ClinicalData;
@@ -26,6 +27,10 @@ class ClickHouseClinicalDataRepositoryIntegrationTest extends AbstractClickhouse
   private ClickhouseClinicalDataMapper mapper;
 
   // Test data based on actual cBioPortal public dataset
+  private static final List<String> TEST_STUDY_IDS = Arrays.asList(
+      "acc_tcga_pan_can_atlas_2018"
+  );
+
   private static final List<String> TEST_SAMPLE_UNIQUE_IDS = Arrays.asList(
       "acc_tcga_pan_can_atlas_2018_TCGA-OR-A5J1-01",
       "acc_tcga_pan_can_atlas_2018_TCGA-OR-A5J2-01",
@@ -59,6 +64,7 @@ void testFetchClinicalDataId_WithSampleData() {
     List<ClinicalData> result = repository.fetchClinicalDataId(
         TEST_SAMPLE_UNIQUE_IDS,
         COMMON_SAMPLE_ATTRIBUTES,
+        TEST_STUDY_IDS,
         ClinicalDataType.SAMPLE
     );
 
@@ -79,6 +85,7 @@ void testFetchClinicalDataSummary_WithPatientData() {
     List<ClinicalData> result = repository.fetchClinicalDataSummary(
         TEST_PATIENT_UNIQUE_IDS,
         COMMON_PATIENT_ATTRIBUTES,
+        TEST_STUDY_IDS,
         ClinicalDataType.PATIENT
     );
 
@@ -103,6 +110,7 @@ void testFetchClinicalDataDetailed_WithSpecificValues() {
             "acc_tcga_pan_can_atlas_2018_TCGA-OR-A5J2-01"
         ),
         Arrays.asList("ANEUPLOIDY_SCORE", "SAMPLE_TYPE"),
+        TEST_STUDY_IDS,
         ClinicalDataType.SAMPLE
     );
 
@@ -143,6 +151,7 @@ void testFetchClinicalDataMeta_WithSampleData() {
     Integer count = repository.fetchClinicalDataMeta(
         TEST_SAMPLE_UNIQUE_IDS,
         COMMON_SAMPLE_ATTRIBUTES,
+        TEST_STUDY_IDS,
         ClinicalDataType.SAMPLE
     );
 
@@ -152,6 +161,7 @@ void testFetchClinicalDataMeta_WithSampleData() {
     List<ClinicalData> actualData = repository.fetchClinicalDataSummary(
         TEST_SAMPLE_UNIQUE_IDS,
         COMMON_SAMPLE_ATTRIBUTES,
+        TEST_STUDY_IDS,
         ClinicalDataType.SAMPLE
     );
     assertEquals(actualData.size(), count.intValue());
@@ -163,6 +173,7 @@ void testFetchClinicalDataId_WithEmptyInput() {
     List<ClinicalData> result = repository.fetchClinicalDataId(
         List.of(),
         COMMON_SAMPLE_ATTRIBUTES,
+        Collections.emptyList(),
         ClinicalDataType.SAMPLE
     );
 
@@ -176,6 +187,7 @@ void testFetchClinicalDataMeta_WithEmptyInput() {
     Integer count = repository.fetchClinicalDataMeta(
         List.of(),
         COMMON_SAMPLE_ATTRIBUTES,
+        Collections.emptyList(),
         ClinicalDataType.SAMPLE
     );
 
@@ -193,11 +205,11 @@ void testProjectionConsistency_SameEntitiesReturned() {
     List<String> testAttrs = List.of("SAMPLE_TYPE");
 
     List<ClinicalData> idResults = repository.fetchClinicalDataId(
-        testIds, testAttrs, ClinicalDataType.SAMPLE
+        testIds, testAttrs, TEST_STUDY_IDS, ClinicalDataType.SAMPLE
     );
 
     List<ClinicalData> summaryResults = repository.fetchClinicalDataSummary(
-        testIds, testAttrs, ClinicalDataType.SAMPLE
+        testIds, testAttrs, TEST_STUDY_IDS, ClinicalDataType.SAMPLE
     );
 
     // Then - should return same entities with different levels of detail
@@ -229,12 +241,14 @@ void testClinicalDataType_PatientVsSample() {
     List<ClinicalData> sampleData = repository.fetchClinicalDataSummary(
         TEST_SAMPLE_UNIQUE_IDS,
         COMMON_SAMPLE_ATTRIBUTES,
+        TEST_STUDY_IDS,
         ClinicalDataType.SAMPLE
     );
 
     List<ClinicalData> patientData = repository.fetchClinicalDataSummary(
         TEST_PATIENT_UNIQUE_IDS,
         COMMON_PATIENT_ATTRIBUTES,
+        TEST_STUDY_IDS,
         ClinicalDataType.PATIENT
     );
 

src/main/java/org/cbioportal/domain/clinical_data/repository/ClinicalDataRepository.java

@@ -60,7 +60,10 @@ List<ClinicalDataCountItem> getClinicalDataCounts(
    * @see ClinicalDataType
    */
   List<org.cbioportal.domain.clinical_data.ClinicalData> fetchClinicalDataId(
-      List<String> uniqueIds, List<String> attributeIds, ClinicalDataType clinicalDataType);
+      List<String> uniqueIds,
+      List<String> attributeIds,
+      List<String> studyIds,
+      ClinicalDataType clinicalDataType);
 
   /**
    * Retrieves clinical data with SUMMARY projection (basic data with values).
@@ -76,7 +79,10 @@ List<org.cbioportal.domain.clinical_data.ClinicalData> fetchClinicalDataId(
    * @see ClinicalDataType
    */
   List<org.cbioportal.domain.clinical_data.ClinicalData> fetchClinicalDataSummary(
-      List<String> uniqueIds, List<String> attributeIds, ClinicalDataType clinicalDataType);
+      List<String> uniqueIds,
+      List<String> attributeIds,
+      List<String> studyIds,
+      ClinicalDataType clinicalDataType);
 
   /**
    * Retrieves clinical data with DETAILED projection (complete data set).
@@ -93,7 +99,10 @@ List<org.cbioportal.domain.clinical_data.ClinicalData> fetchClinicalDataSummary(
    * @see ClinicalDataType
    */
   List<org.cbioportal.domain.clinical_data.ClinicalData> fetchClinicalDataDetailed(
-      List<String> uniqueIds, List<String> attributeIds, ClinicalDataType clinicalDataType);
+      List<String> uniqueIds,
+      List<String> attributeIds,
+      List<String> studyIds,
+      ClinicalDataType clinicalDataType);
 
   /**
    * Retrieves the count of clinical data records matching the specified criteria.
@@ -109,5 +118,8 @@ List<org.cbioportal.domain.clinical_data.ClinicalData> fetchClinicalDataDetailed
    * @see ClinicalDataType
    */
   Integer fetchClinicalDataMeta(
-      List<String> uniqueIds, List<String> attributeIds, ClinicalDataType clinicalDataType);
+      List<String> uniqueIds,
+      List<String> attributeIds,
+      List<String> studyIds,
+      ClinicalDataType clinicalDataType);
 }

src/main/java/org/cbioportal/domain/clinical_data/usecase/FetchClinicalDataMetaUseCase.java

@@ -62,7 +62,13 @@ public Integer execute(
       uniqueIds.add(identifier.getStudyId() + '_' + identifier.getEntityId());
     }
     List<String> attributeIds = clinicalDataMultiStudyFilter.getAttributeIds();
+    List<String> studyIds =
+        clinicalDataMultiStudyFilter.getIdentifiers().stream()
+            .map(ClinicalDataIdentifier::getStudyId)
+            .distinct()
+            .toList();
 
-    return clinicalDataRepository.fetchClinicalDataMeta(uniqueIds, attributeIds, clinicalDataType);
+    return clinicalDataRepository.fetchClinicalDataMeta(
+        uniqueIds, attributeIds, studyIds, clinicalDataType);
   }
 }

src/main/java/org/cbioportal/domain/clinical_data/usecase/FetchClinicalDataUseCase.java

@@ -74,6 +74,12 @@ public List<ClinicalData> execute(
       ClinicalDataMultiStudyFilter clinicalDataMultiStudyFilter,
       ClinicalDataType clinicalDataType,
       ProjectionType projectionType) {
+    List<String> studyIds =
+        clinicalDataMultiStudyFilter.getIdentifiers().stream()
+            .map(ClinicalDataIdentifier::getStudyId)
+            .distinct()
+            .toList();
+
     // Transform filter identifiers into unique IDs for repository layer
     List<String> uniqueIds = new ArrayList<>();
     for (ClinicalDataIdentifier identifier : clinicalDataMultiStudyFilter.getIdentifiers()) {
@@ -84,13 +90,14 @@ public List<ClinicalData> execute(
     // Route to appropriate repository method based on projection type
     return switch (projectionType) {
       case ID ->
-          clinicalDataRepository.fetchClinicalDataId(uniqueIds, attributeIds, clinicalDataType);
+          clinicalDataRepository.fetchClinicalDataId(
+              uniqueIds, attributeIds, studyIds, clinicalDataType);
       case SUMMARY ->
           clinicalDataRepository.fetchClinicalDataSummary(
-              uniqueIds, attributeIds, clinicalDataType);
+              uniqueIds, attributeIds, studyIds, clinicalDataType);
       case DETAILED ->
           clinicalDataRepository.fetchClinicalDataDetailed(
-              uniqueIds, attributeIds, clinicalDataType);
+              uniqueIds, attributeIds, studyIds, clinicalDataType);
       default -> Collections.emptyList();
     };
   }

src/main/java/org/cbioportal/infrastructure/repository/clickhouse/clinical_data/ClickhouseClinicalDataMapper.java

@@ -59,7 +59,10 @@ List<ClinicalData> getPatientClinicalDataFromStudyViewFilter(
    * @return list of clinical data records with minimal field set
    */
   List<ClinicalData> fetchClinicalDataId(
-      List<String> uniqueIds, List<String> attributeIds, String clinicalDataType);
+      List<String> uniqueIds,
+      List<String> attributeIds,
+      List<String> studyIds,
+      String clinicalDataType);
 
   /**
    * Retrieves clinical data with SUMMARY projection (basic data with values).
@@ -73,7 +76,10 @@ List<ClinicalData> fetchClinicalDataId(
    * @return list of clinical data records with basic field set including values
    */
   List<ClinicalData> fetchClinicalDataSummary(
-      List<String> uniqueIds, List<String> attributeIds, String clinicalDataType);
+      List<String> uniqueIds,
+      List<String> attributeIds,
+      List<String> studyIds,
+      String clinicalDataType);
 
   /**
    * Retrieves clinical data with DETAILED projection (complete data set).
@@ -88,7 +94,10 @@ List<ClinicalData> fetchClinicalDataSummary(
    * @return list of clinical data records with complete field set including attribute metadata
    */
   List<ClinicalData> fetchClinicalDataDetailed(
-      List<String> uniqueIds, List<String> attributeIds, String clinicalDataType);
+      List<String> uniqueIds,
+      List<String> attributeIds,
+      List<String> studyIds,
+      String clinicalDataType);
 
   /**
    * Retrieves the count of clinical data records matching the specified criteria.
@@ -103,5 +112,8 @@ List<ClinicalData> fetchClinicalDataDetailed(
    * @return total number of clinical data records matching the criteria
    */
   Integer fetchClinicalDataMeta(
-      List<String> uniqueIds, List<String> attributeIds, String clinicalDataType);
+      List<String> uniqueIds,
+      List<String> attributeIds,
+      List<String> studyIds,
+      String clinicalDataType);
 }

src/main/java/org/cbioportal/infrastructure/repository/clickhouse/clinical_data/ClickhouseClinicalDataRepository.java

@@ -47,39 +47,55 @@ public List<ClinicalDataCountItem> getClinicalDataCounts(
 
   @Override
   public List<ClinicalData> fetchClinicalDataId(
-      List<String> uniqueIds, List<String> attributeIds, ClinicalDataType clinicalDataType) {
+      List<String> uniqueIds,
+      List<String> attributeIds,
+      List<String> studyIds,
+      ClinicalDataType clinicalDataType) {
     if (CollectionUtils.isEmpty(uniqueIds)) {
       return Collections.emptyList();
     }
-    return mapper.fetchClinicalDataId(uniqueIds, attributeIds, clinicalDataType.toString());
+    return mapper.fetchClinicalDataId(
+        uniqueIds, attributeIds, studyIds, clinicalDataType.toString());
   }
 
   @Override
   public List<ClinicalData> fetchClinicalDataSummary(
-      List<String> uniqueIds, List<String> attributeIds, ClinicalDataType clinicalDataType) {
+      List<String> uniqueIds,
+      List<String> attributeIds,
+      List<String> studyIds,
+      ClinicalDataType clinicalDataType) {
     if (CollectionUtils.isEmpty(uniqueIds)) {
       return Collections.emptyList();
     }
-    return mapper.fetchClinicalDataSummary(uniqueIds, attributeIds, clinicalDataType.toString());
+    return mapper.fetchClinicalDataSummary(
+        uniqueIds, attributeIds, studyIds, clinicalDataType.toString());
   }
 
   @Override
   public List<ClinicalData> fetchClinicalDataDetailed(
-      List<String> uniqueIds, List<String> attributeIds, ClinicalDataType clinicalDataType) {
+      List<String> uniqueIds,
+      List<String> attributeIds,
+      List<String> studyIds,
+      ClinicalDataType clinicalDataType) {
     if (CollectionUtils.isEmpty(uniqueIds)) {
       return Collections.emptyList();
     }
-    return mapper.fetchClinicalDataDetailed(uniqueIds, attributeIds, clinicalDataType.toString());
+    return mapper.fetchClinicalDataDetailed(
+        uniqueIds, attributeIds, studyIds, clinicalDataType.toString());
   }
 
   @Override
   public Integer fetchClinicalDataMeta(
-      List<String> uniqueIds, List<String> attributeIds, ClinicalDataType clinicalDataType) {
+      List<String> uniqueIds,
+      List<String> attributeIds,
+      List<String> studyIds,
+      ClinicalDataType clinicalDataType) {
     if (CollectionUtils.isEmpty(uniqueIds)) {
       return 0;
     }
     Integer cnt =
-        mapper.fetchClinicalDataMeta(uniqueIds, attributeIds, clinicalDataType.toString());
+        mapper.fetchClinicalDataMeta(
+            uniqueIds, attributeIds, studyIds, clinicalDataType.toString());
     return cnt == null ? 0 : cnt;
   }
 }

src/main/resources/db-scripts/clickhouse/clickhouse.sql

@@ -316,7 +316,7 @@ CREATE TABLE IF NOT EXISTS clinical_data_derived
     type LowCardinality(String)
 )
     ENGINE=MergeTree
-        ORDER BY (type, attribute_name, sample_unique_id);
+        ORDER BY (cancer_study_identifier, type, attribute_name, sample_unique_id);
 
 -- Insert sample attribute data
 INSERT INTO TABLE clinical_data_derived