Merge pull request #5049 from broadinstitute/dev

Dev

Author: hanars

.github/workflows/unit-tests.yml

@@ -41,7 +41,7 @@ jobs:
           # Maps tcp port 5432 on service container to the host
           - 5432:5432
       clickhouse:
-        image: bitnami/clickhouse:latest
+        image: bitnamilegacy/clickhouse:latest
         ports:
           - 9000:9000   # Native client interface
         volumes:

CHANGELOG.md

@@ -3,7 +3,10 @@
 ## dev
 
 ## 9/18/25
-# Deprecate hail backend support
+* Store gene ids on SavedVariant model (REQUIRES DB MIGRATION)
+
+## 9/29/25
+* Deprecate hail backend support
 
 ## 8/7/25
 * Deprecate support for lifting existing projects from GRCH37 to GRCh38

clickhouse_search/backend/functions.py

@@ -7,6 +7,12 @@
 class Array(Func):
     function = 'array'
 
+    def _resolve_output_field(self):
+        output_field = super()._resolve_output_field()
+        if not isinstance(output_field, ArrayField):
+            output_field = ArrayField(base_field=output_field)
+        return output_field
+
 
 class ArrayConcat(Func):
     function = 'arrayConcat'

clickhouse_search/fixtures/clickhouse_saved_variants.json

@@ -213,7 +213,7 @@
     "pk": 101,
     "fields": {
       "key": 101,
-      "variant_id": "suffix_19107_DEL"
+      "variant_id": "suffix_19107_DEL_013746"
     }
 }, {
     "model": "clickhouse_search.annotationssv",
@@ -225,7 +225,7 @@
       "end_chrom": null,
       "pos": 249045487,
       "end": 249045898,
-      "variant_id": "suffix_19107_DEL",
+      "variant_id": "suffix_19107_DEL_013746",
       "lifted_over_chrom": null,
       "lifted_over_pos": null,
       "rg37_locus_end": [null, null],
@@ -250,7 +250,7 @@
     "pk": 111,
     "fields": {
       "key": 111,
-      "variant_id": "prefix_70191_DEL"
+      "variant_id": "prefix_19107_DEL"
     }
 }, {
     "model": "clickhouse_search.annotationsgcnv",
@@ -261,7 +261,7 @@
       "chrom": "1",
       "pos": 249045487,
       "end": 249045898,
-      "variant_id": "prefix_70191_DEL",
+      "variant_id": "prefix_19107_DEL",
       "lifted_over_chrom": "2",
       "lifted_over_pos": null,
       "rg37_locus_end": [null, null],

clickhouse_search/fixtures/clickhouse_search.json

@@ -48,7 +48,7 @@
       "populations": [
         [35805, 0.29499999, 121372, 0.41530353, 0, 24061, 5872],
         [72672, 0.28899795, 251462, 0.4116475, 0, 11567],
-        [0, 0, 0, 0, 0, 0],
+        [927, 0.03444932, 26912, 0.04027665, 0, 0],
         [65154, 0.246152, 264690, 47604, 8775]
       ],
       "sorted_transcript_consequences": [
@@ -124,7 +124,7 @@
       ],
       "sorted_transcript_consequences": [
         [null, null, ["splice_donor_variant"], true, null, "ENSG00000097046"],
-        [null, 1, ["missense_variant"], false, null, "ENSG00000097046"],
+        [0.1, 1, ["missense_variant"], false, null, "ENSG00000097046"],
         [null, null, ["missense_variant"], false, null, "ENSG00000097046"]
       ],
       "sorted_motif_feature_consequences": [
@@ -681,7 +681,7 @@
       ],
       "sign": 1,
       "calls": [
-        ["HG00733", 1, 99, 0.6530612111091614, 49],
+        ["HG00733", null, 0, null, null],
         ["HG00731", 2, 48, 1, 16]
       ]
     }

clickhouse_search/fixtures/clickhouse_transcripts.json

@@ -113,7 +113,7 @@
           ], null]
         ],
         [
-          [null], null, "protein_coding", 1, null, ["missense_variant"], [null, null], "ENSG00000097046",
+          [0.1], null, "protein_coding", 1, null, ["missense_variant"], [null, null], "ENSG00000097046",
           "ENST00000350997.12:c.375+139G>A", null, [1, 11], [null, []], "missense_variant", null, "NM_013402.7",
           "NM_013402.7", [false], "ENST00000350997", 1, [null, null, null, [
             null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null

clickhouse_search/management/commands/set_saved_variant_key.py

@@ -26,6 +26,7 @@ def setUpTestData(cls):
     @mock.patch('seqr.utils.file_utils.subprocess.Popen')
     def test_command(self, mock_subprocess):
         mock_subprocess.return_value.stdout = self.MOCK_GCNV_DATA
+        mock_subprocess.return_value.wait.return_value = 0
 
         call_command('set_saved_variant_key')
         self.assert_json_logs(user=None, expected=[
@@ -39,21 +40,14 @@ def test_command(self, mock_subprocess):
             ('Updated batch of 2', None),
             ('Updated keys for 2 SNV_INDEL (GRCh38) variants', None),
             ('Finding keys for 2 SV_WGS (GRCh38) variant ids', None),
-            ('Found 0 keys', None),
-            ('Finding keys for 2 SV_WES (GRCh38) variant ids', None),
-            ('Found 0 keys', None),
-            ('2 SV variants have no key, 0 of which have no search data, 0 of which are known to have dropped out of the callset.', None),
-            ('==> gsutil cat gs://seqr-datasets-gcnv/GRCh38/RDG_WES_Broad_Internal/v4/CMG_gCNV_2022_annotated.ensembl.round2_3.strvctvre.tsv.gz | gunzip -c -q - ', None),
-            ('Mapping reloaded SV_WES IDs to latest version', None),
-            ('Finding keys for 1 SV_WES (GRCh38) variant ids', None),
             ('Found 1 keys', None),
             ('Updated batch of 1', None),
-            ('Updated keys for 1 SV_WES (GRCh38) variants', None),
-            ('Mapping reloaded SV_WGS IDs to latest version', None),
-            ('Finding keys for 1 SV_WGS (GRCh38) variant ids', None),
+            ('Updated keys for 1 SV_WGS (GRCh38) variants', None),
+            ('No key found for 1 variants', None),
+            ('Finding keys for 1 SV_WES (GRCh38) variant ids', None),
             ('Found 1 keys', None),
             ('Updated batch of 1', None),
-            ('Updated keys for 1 SV_WGS (GRCh38) variants', None),
+            ('Updated keys for 1 SV_WES (GRCh38) variants', None),
             ('Finding keys for 7 SNV_INDEL (GRCh37) variant ids', None),
             ('Found 1 keys', None),
             ('Updated batch of 1', None),
@@ -70,9 +64,9 @@ def test_command(self, mock_subprocess):
             {'guid': 'SV0000002_1248367227_r0390_100', 'key': 100, 'variant_id': '1-248367227-TC-T', 'dataset_type': 'SNV_INDEL', 'genotypes': {'I000004_hg00731': {'numAlt': 2}, 'I000005_hg00732': {'numAlt': 1}}, 'saved_variant_json': {}},
             {'guid': 'SV0000006_1248367227_r0003_tes', 'key': 100, 'variant_id': '1-248367227-TC-T', 'dataset_type': 'SNV_INDEL', 'genotypes': {'I000002_na19675': mock.ANY, 'I000017_na20889': mock.ANY}, 'saved_variant_json': {}},
             {'guid': 'SV0000006_1248367227_r0004_non', 'key': 100, 'variant_id': '1-248367227-TC-T', 'dataset_type': 'SNV_INDEL', 'genotypes': {'I000018_na21234': {'sampleId': 'NA20885', 'ab': 0.0, 'gq': 99.0, 'numAlt': 1}}, 'saved_variant_json': {}},
-            {'guid': 'SV0000007_prefix_19107_DEL_r00', 'key': 111, 'variant_id': 'prefix_70191_DEL', 'dataset_type': 'SV_WES', 'genotypes': {'I000017_na20889': { 'cn': 1, 'sampleId': 'NA20885', 'numAlt': -1,  'defragged': False, 'qs': 33, 'numExon': 2}}, 'saved_variant_json': {}},
+            {'guid': 'SV0000007_prefix_19107_DEL_r00', 'key': 111, 'variant_id': 'prefix_19107_DEL', 'dataset_type': 'SV_WES', 'genotypes': {'I000017_na20889': { 'cn': 1, 'sampleId': 'NA20885', 'numAlt': -1,  'defragged': False, 'qs': 33, 'numExon': 2}}, 'saved_variant_json': {}},
             {'guid': 'SV0000009_25000014783_r0004_no', 'key': 100, 'variant_id': 'M-14783-T-C', 'dataset_type': 'MITO', 'genotypes': {'I000018_na21234': mock.ANY}, 'saved_variant_json': {}},
-            {'guid': 'SV0000013_prefix_19107_DEL_r00', 'key': 101, 'variant_id': 'suffix_19107_DEL', 'dataset_type': 'SV_WGS', 'genotypes': {'I000018_na21234': mock.ANY}, 'saved_variant_json': {}},
+            {'guid': 'SV0000013_prefix_19107_DEL_r00', 'key': 101, 'variant_id': 'suffix_19107_DEL_013746', 'dataset_type': 'SV_WGS', 'genotypes': {'I000018_na21234': mock.ANY}, 'saved_variant_json': {}},
             {'guid': 'SV0027166_191912634_r0384_rare', 'key': None, 'variant_id': '19-1912634-C-T', 'dataset_type': None, 'genotypes': mock.ANY, 'saved_variant_json': mock.ANY},
             {'guid': 'SV0027167_191912633_r0384_rare', 'key': None, 'variant_id': '19-1912633-G-T', 'dataset_type': None, 'genotypes': mock.ANY, 'saved_variant_json': mock.ANY},
             {'guid': 'SV0027168_191912632_r0384_rare', 'key': None, 'variant_id': '19-1912632-G-C', 'dataset_type': None, 'genotypes': mock.ANY, 'saved_variant_json': mock.ANY},
@@ -106,6 +100,7 @@ class SetSavedVariantKeyFailedMappingTest(SetSavedVariantKeyTest):
     @mock.patch('seqr.utils.file_utils.subprocess.Popen')
     def test_command(self, mock_subprocess):
         mock_subprocess.return_value.stdout = self.MOCK_GCNV_DATA
+        mock_subprocess.return_value.wait.return_value = 0
 
         call_command('set_saved_variant_key')
         self.assert_json_logs(user=None, expected=[
@@ -121,12 +116,8 @@ def test_command(self, mock_subprocess):
             ('Finding keys for 2 SV_WES (GRCh38) variant ids', None),
             ('Found 0 keys', None),
             ('2 SV variants have no key, 0 of which have no search data, 0 of which are known to have dropped out of the callset.', None),
-            ('==> gsutil cat gs://seqr-datasets-gcnv/GRCh38/RDG_WES_Broad_Internal/v4/CMG_gCNV_2022_annotated.ensembl.round2_3.strvctvre.tsv.gz | gunzip -c -q - ', None),
-            ('Mapping reloaded SV_WGS IDs to latest version', None),
-            ('Finding keys for 1 SV_WGS (GRCh38) variant ids', None),
-            ('Found 0 keys', None),
-            ("1 variants failed ID mapping: ['suffix_19107_DEL']", None),
             ('1 remaining SV WES variants prefix_19107_DEL - 12', None),
+            ('1 remaining SV WGS variants suffix_19107_DEL_013746 - fam14', None),
             ('Finding keys for 7 SNV_INDEL (GRCh37) variant ids', None),
             ('Found 0 keys', None),
             ('7 variants have no key, 0 of which have no search data, 7 of which are absent from the hail backend.', None),