Merge pull request #2309 from broadinstitute/development

Release 1.103

Author: eweitz

Gemfile.lock

@@ -372,7 +372,7 @@ GEM
     puma (5.6.9)
       nio4r (~> 2.0)
     racc (1.8.1)
-    rack (2.2.14)
+    rack (2.2.19)
     rack-brotli (1.1.0)
       brotli (>= 0.1.7)
       rack (>= 1.4)
@@ -546,6 +546,7 @@ GEM
 PLATFORMS
   arm64-darwin-21
   arm64-darwin-23
+  arm64-darwin-24
   x86_64-darwin-19
   x86_64-darwin-20
   x86_64-darwin-21

app/controllers/api/v1/search_controller.rb

@@ -282,9 +282,14 @@ def index
           sort_type = params[:order].to_sym
         end
 
+        result_types = params[:data_types]&.split(',')&.map(&:to_sym) || []
+        @studies = StudySearchService.filter_results_by_data_type(@studies, result_types)
+
         # convert to array to allow appending external search results (Azul, TDR, etc.)
         @studies = @studies.to_a
 
+        # filter results by file type, if requested
+
         # perform Azul search if there are facets/terms provided by user, and they requested HCA results
         # run this before inferred search so that they are weighted and sorted correctly
         # skip if user is searching inside a collection or they are performing global gene search
@@ -391,6 +396,7 @@ def index
               # only show results where we found a hit in gene search
               @inferred_studies = Study.where(:id.in => new_genes[:study_ids])
             end
+            @inferred_studies = StudySearchService.filter_results_by_data_type(@inferred_studies, result_types)
             @inferred_accessions = @inferred_studies.pluck(:accession)
             logger.info "Found #{@inferred_accessions.count} inferred matches: #{@inferred_accessions}"
             @matching_accessions += @inferred_accessions
@@ -399,7 +405,6 @@ def index
         end
 
         @matching_accessions = @studies.map { |study| self.class.get_study_attribute(study, :accession) }
-        logger.info "studies_by_facet: #{@studies_by_facet}"
         logger.info "Final list of matching studies: #{@matching_accessions}"
         @results = @studies.paginate(page: params[:page], per_page: Study.per_page)
         if params[:export].present?

app/controllers/api/v1/study_files_controller.rb

@@ -331,8 +331,19 @@ def perform_update!(study_file)
         end
         if safe_file_params[:custom_color_updates]
           parsed_update = JSON.parse(safe_file_params[:custom_color_updates])
-          safe_file_params['cluster_file_info'] = {custom_colors: ClusterFileInfo.merge_color_updates(study_file, parsed_update)}
+          safe_file_params['cluster_file_info'] = {
+            custom_colors: ClusterFileInfo.merge_color_updates(study_file, parsed_update)
+          }
           safe_file_params.delete(:custom_color_updates)
+          if safe_file_params[:global_color_update]
+            @study.clustering_files.reject { |f| f.id.to_s == params[:id] }.each do |file|
+              update_params = {
+                'cluster_file_info' => { custom_colors: ClusterFileInfo.merge_color_updates(file, parsed_update) }
+              }
+              file.delay.update(update_params)
+            end
+          end
+          safe_file_params.delete(:global_color_update)
         end
 
         # manually check first if species/assembly was supplied by name
@@ -741,7 +752,7 @@ def study_file_params
           :human_fastq_url, :human_data, :use_metadata_convention, :cluster_type, :generation, :x_axis_label,
           :y_axis_label, :z_axis_label, :x_axis_min, :x_axis_max, :y_axis_min, :y_axis_max, :z_axis_min, :z_axis_max,
           :species, :assembly, :external_link_url, :external_link_title, :external_link_description, :parse_on_upload,
-          :custom_color_updates, :reference_anndata_file,
+          :custom_color_updates, :global_color_update, :reference_anndata_file,
           spatial_cluster_associations: [],
           options: [
             :cluster_group_id, :font_family, :font_size, :font_color, :matrix_id, :submission_id, :bam_id, :bed_id,

app/javascript/components/search/controls/OptionsButton.jsx

@@ -10,19 +10,24 @@ export default function OptionsButton() {
   const searchContext = useContext(StudySearchContext)
   const [showOptions, setShowOptions] = useState(false)
   const configuredOptions = [
-    { searchProp: 'external', value: 'hca', label: 'Include HCA results' }
+    { searchProp: 'external', value: 'hca', label: 'Include HCA results' },
+    { searchProp: 'data_types', value: 'raw_counts', label: 'Has raw counts', multiple: true },
+    { searchProp: 'data_types', value: 'diff_exp', label: 'Has differential expression', multiple: true },
+    { searchProp: 'data_types', value: 'spatial', label: 'Has spatial data', multiple: true }
   ]
 
   const optionsPopover = <Popover data-analytics-name='search-options-menu' id='search-options-menu'>
     <ul className="facet-filter-list">
       {
-        configuredOptions.map((option) => {
+        configuredOptions.map((option, index) => {
         return <OptionsControl
-          key={option.searchProp}
+          key={`${option.searchProp}-${index}`}
           searchContext={searchContext}
           searchProp={option.searchProp}
           value={option.value}
-          label={option.label}/>
+          label={option.label}
+          multiple={option.multiple}
+        />
         })
       }
     </ul>

app/javascript/components/search/controls/OptionsControl.jsx

@@ -1,20 +1,44 @@
 import React, { useState } from 'react'
 
-export default function OptionsControl({searchContext, searchProp, value, label}) {
-  const defaultChecked = searchContext.params[searchProp] === value
+/** checkbox control for adding optional parameters to search query */
+export default function OptionsControl({ searchContext, searchProp, value, label, multiple = false }) {
+  const defaultChecked = isDefaultChecked()
   const [isChecked, setIsChecked] = useState(defaultChecked)
 
+  /** return existing url query params for this option */
+  function getExistingOpts() {
+    return searchContext.params[searchProp]?.split(',').filter(o => o !== '') || []
+  }
+
+  /** set the default state for this option checkbox */
+  function isDefaultChecked() {
+    if (multiple) {
+      return getExistingOpts().includes(value)
+    } else {
+      return searchContext.params[searchProp] === value
+    }
+  }
+
   /** toggle state of checkbox */
   function toggleCheckbox(checked) {
     setIsChecked(checked)
-    searchContext.updateSearch({ [searchProp] : checked ? value : null })
+    if (multiple) {
+      const existingOpts = getExistingOpts()
+      const newOpts = checked ? existingOpts.concat(value) : existingOpts.filter(v => v !== value)
+      searchContext.updateSearch({ [searchProp] : newOpts.join(',') })
+    } else {
+      searchContext.updateSearch({ [searchProp] : checked ? value : null })
+    }
   }
 
   return (
     <li id={`options-control-${searchProp}`} key={`options-control-${searchProp}`}>
       <label>
-        <input type="checkbox" checked={isChecked} onChange={() => {toggleCheckbox(!isChecked)}}/>
-          <span onClick={() => {toggleCheckbox(!isChecked)}} >{ label }</span>
+        <input data-testid={`options-checkbox-${searchProp}-${value}`}
+               type="checkbox"
+               checked={isChecked}
+               onChange={() => {toggleCheckbox(!isChecked)}}/>
+        <span onClick={() => {toggleCheckbox(!isChecked)}} >{ label }</span>
       </label>
     </li>
   )

app/javascript/components/search/results/SearchQueryDisplay.jsx

@@ -78,6 +78,8 @@ export const ClearAllButton = () => {
     const emptySearchParams = {
       terms: '',
       genes: '',
+      external: '',
+      data_types: '',
       facets: emptyFilters
     }
     selectionContext.updateSelection(emptySearchParams, true)

app/javascript/components/visualization/ScatterPlot.jsx

@@ -143,13 +143,14 @@ function RawScatterPlot({
   }, [editedCustomColors])
 
   /** Save any changes to the legend colors */
-  async function saveCustomColors(newColors) {
+  async function saveCustomColors(newColors, globalColorUpdate = false) {
     const colorObj = {}
     // read the annotation name off of scatterData to ensure it's the real name, and not '' or '_default'
     colorObj[scatterData?.annotParams?.name] = newColors
     const newFileObj = {
       _id: scatterData?.clusterFileId,
-      custom_color_updates: colorObj
+      custom_color_updates: colorObj,
+      global_color_update: globalColorUpdate
     }
     setIsLoading(true)
     try {

app/javascript/components/visualization/controls/ScatterPlotLegend.jsx

@@ -1,6 +1,13 @@
 import React, { useEffect, useState } from 'react'
 import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'
-import { faPalette, faExternalLinkAlt, faTimes, faSearch } from '@fortawesome/free-solid-svg-icons'
+import {
+  faPalette,
+  faExternalLinkAlt,
+  faTimes,
+  faSearch,
+  faFileUpload,
+  faGlobe, faFileDownload
+} from '@fortawesome/free-solid-svg-icons'
 import Modal from 'react-bootstrap/lib/Modal'
 import { HexColorPicker, HexColorInput } from 'react-colorful'
 import Button from 'react-bootstrap/lib/Button'
@@ -215,6 +222,9 @@ export default function ScatterPlotLegend({
 }) {
   // is the user currently in color-editing mode
   const [showColorControls, setShowColorControls] = useState(false)
+  const [globalColorUpdate, setGlobalColorUpdate] = useState(false)
+  const [toggleClassName, setToggleClassName] = useState('fa-toggle-off')
+
   // whether a request to the server to save colors is pending
   const labels = getLegendSortedLabels(countsByLabel)
   const numLabels = labels.length
@@ -246,18 +256,40 @@ export default function ScatterPlotLegend({
   /** resets any unsaved changes to user colors and clears custom colors */
   async function resetColors() {
     setEditedCustomColors({})
-    await saveCustomColors({})
+    await saveCustomColors({}, globalColorUpdate)
     setShowColorControls(false)
   }
 
   /** save the colors to the server */
   async function saveColors() {
     // merge the user picked colors with existing custom colors so previously saved values are preserved
     const colorsToSave = Object.assign(customColors, editedCustomColors)
-    await saveCustomColors(colorsToSave)
+    await saveCustomColors(colorsToSave, globalColorUpdate)
     setShowColorControls(false)
   }
 
+  function exportColors() {
+    const colorMap = Object.keys(customColors).length > 0 ? customColors : refColorMap
+    const lines = Object.entries(colorMap).map(([label, color]) => {
+      return `${label}\t${color}\n`
+    })
+
+    // Create an element with an anchor link and connect this to the blob
+    const element = document.createElement('a')
+    const colorExport = new Blob(lines, { type: 'text/plain' })
+    element.href = URL.createObjectURL(colorExport)
+
+    // name the file and indicate it should download
+    element.download = `${name}_color_map.tsv`
+
+    // Simulate clicking the link resulting in downloading the file
+    document.body.appendChild(element)
+    element.click()
+
+    // Cleanup
+    document.body.removeChild(element)
+  }
+
   /** collect general information when a user's mouse enters the legend  */
   function logMouseEnter() {
     log('hover:scatterlegend', { numLabels })
@@ -323,6 +355,40 @@ export default function ScatterPlotLegend({
     setLabelsToShow(labels)
   }
 
+  /** handle clicking global color update toggle */
+  function handleToggleGlobalColor() {
+    const toggleClass = toggleClassName === 'fa-toggle-on' ? 'fa-toggle-off' : 'fa-toggle-on'
+    setGlobalColorUpdate(!globalColorUpdate)
+    setToggleClassName(toggleClass)
+  }
+
+  /** read uploaded manifest and apply colors to current scatter plot */
+  function readColorManifest(file) {
+    const colorUpdate = {}
+    const fileReader = new FileReader()
+    fileReader.onloadend = () => {
+      const lines = fileReader.result.trim().split(/\n/)
+      lines.map((line, _) => {
+        const entry = line.split(/[\t,]/).map((l, _) => {return l.trim()})
+        const label = entry[0]
+        const color = entry[1]
+        colorUpdate[label] = color
+      })
+      saveCustomColors(colorUpdate, globalColorUpdate)
+    }
+    fileReader.readAsText(file)
+  }
+
+  const globalSwitch =
+    <label htmlFor="global-color-update"
+           data-toggle="tooltip"
+           className="color-update-toggle"
+           title="Apply color changes globally for this annotation">
+      <span className={globalColorUpdate ? 'text-info' : 'text-muted'} onClick={handleToggleGlobalColor}>Global&nbsp;
+        <i className={`fa fa-fw ${toggleClassName}`}></i>
+      </span>
+    </label>
+
   return (
     <div
       className={`scatter-legend ${filteredClass}`}
@@ -379,18 +445,41 @@ export default function ScatterPlotLegend({
                   <a role="button" className="pull-right" data-analytics-name="legend-color-picker-cancel" onClick={cancelColors}>
                     Cancel
                   </a><br/>
-                  &nbsp;
+                  {globalSwitch}
                   <a role="button" className="pull-right" data-analytics-name="legend-color-picker-reset" onClick={resetColors}>
                     Reset to defaults
                   </a>
                 </div>
               </>
             }
             { !showColorControls &&
-              <a role="button" data-analytics-name="legend-color-picker-show" onClick={() => setShowColorControls(true)}>
-                Customize colors <FontAwesomeIcon icon={faPalette}/>
-              </a>
-            }
+              <>
+                <a role="button"
+                   className='customize-color-palette'
+                   data-analytics-name="legend-color-picker-show"
+                   onClick={() => setShowColorControls(true)}
+                >
+                  Customize <FontAwesomeIcon icon={faPalette}/>
+                </a>
+                {globalSwitch}
+                <label htmlFor="color-manifest-upload"
+                       data-toggle="tooltip"
+                       className="icon-button"
+                       title="Upload a manifest of annotation labels to color hex codes">
+                  <input id="color-manifest-upload"
+                         type="file"
+                         onChange={e => readColorManifest(e.target.files[0])}/>
+                  <FontAwesomeIcon className="action fa-lg" icon={faFileUpload} />
+                </label>
+                <label htmlFor="color-manifest-export"
+                       data-toggle="tooltip"
+                       title="Export current color manifest"
+                       onClick={exportColors}
+                >
+                  <FontAwesomeIcon className="action fa-lg" icon={faFileDownload} />
+                </label>
+              </>
+      }
           </div>
         }
         <div>

app/javascript/lib/scp-api.jsx

@@ -926,7 +926,7 @@ export async function fetchSearch(type, searchParams, mock=false) {
 export function buildSearchQueryString(type, searchParams) {
   const facetsParam = buildFacetQueryString(searchParams.facets)
 
-  const params = ['page', 'order', 'terms', 'external', 'export', 'preset', 'genes', 'genePage']
+  const params = ['page', 'order', 'terms', 'external', 'export', 'data_types', 'preset', 'genes', 'genePage']
   let otherParamString = params.map(param => {
     return searchParams[param] ? `&${param}=${searchParams[param]}` : ''
   }).join('')

app/javascript/providers/GeneSearchProvider.jsx

@@ -128,6 +128,7 @@ export function buildParamsFromQuery(query, preset) {
     genes: cleanGeneParams,
     terms: queryParams.terms ? queryParams.terms : '',
     external: queryParams.external ? queryParams.external : '',
+    data_types: queryParams.data_types ? queryParams.data_types : '',
     facets: buildFacetsFromQueryString(queryParams.facets),
     preset: preset ? preset : queryString.preset_search
   }