From d997ba0ee7a54e2d15dd72829021dff51af410e5 Mon Sep 17 00:00:00 2001 From: bistline Date: Tue, 1 Jul 2025 14:33:06 -0400 Subject: [PATCH 01/15] Adding text export option for search results --- app/controllers/api/v1/search_controller.rb | 21 ++++- .../api/v1/study_search_results_objects.rb | 79 +++++++++++++++++++ test/api/search_controller_test.rb | 12 +++ 3 files changed, 109 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/v1/search_controller.rb b/app/controllers/api/v1/search_controller.rb index a818e5450..24ca83f64 100644 --- a/app/controllers/api/v1/search_controller.rb +++ b/app/controllers/api/v1/search_controller.rb @@ -29,10 +29,10 @@ class SearchController < ApiBaseController parameter do key :name, :type key :in, :query - key :description, 'Type of query to perform (study- or cell-based)' + key :description, 'Type of query to perform (study- or gene-based)' key :required, true key :type, :string - key :enum, ['study', 'cell'] + key :enum, %w[study gene] end parameter do key :name, :facets @@ -94,6 +94,14 @@ class SearchController < ApiBaseController key :type, :string key :enum, [:recent, :popular] end + parameter do + key :name, :export + key :in, :query + key :description, 'Export results as a file' + key :required, false + key :type, :string + key :enum, %w[tsv] + end response 200 do key :description, 'Search parameters, Studies and StudyFiles' schema do @@ -394,7 +402,14 @@ def index logger.info "Final list of matching studies: #{@matching_accessions}" @results = @studies.paginate(page: params[:page], per_page: Study.per_page) - render json: search_results_obj, status: 200 + if params[:export].present? + send_data results_text_export, + filename: "scp_search_results_#{Time.now.strftime('%Y%m%d_%H%M%S')}.#{params[:export]}", + status: :ok, + x_sendfile: true + else + render json: search_results_obj, status: 200 + end end swagger_path '/search/facets' do diff --git a/app/controllers/api/v1/study_search_results_objects.rb b/app/controllers/api/v1/study_search_results_objects.rb index f8839f956..411d2b5cb 100644 --- a/app/controllers/api/v1/study_search_results_objects.rb +++ b/app/controllers/api/v1/study_search_results_objects.rb @@ -8,6 +8,13 @@ module StudySearchResultsObjects # list of metadata names to include in cohort responses COHORT_METADATA = %w[disease__ontology_label organ__ontology_label species__ontology_label sex library_preparation_protocol__ontology_label].freeze + # headers for TSV text export of search results + TEXT_HEADERS = [ + 'Study source', 'Accession', 'Name', 'Description', 'Public', 'Detached', 'Cell count', 'Gene count', + 'Study URL', 'Disease', 'Organ', 'Species', 'Sex', 'Library preparation protocol', 'Facet matches', + 'Term matches' + ].freeze + COMMA_SPACE = ', '.freeze def search_results_obj response_obj = { @@ -112,6 +119,78 @@ def cohort_metadata(study) cohort_entries end + def results_text_export + lines = [TEXT_HEADERS.join("\t")] + @studies.each { |study| lines << study_text_export(study).join("\t") } + lines.join("\n") + end + + def study_text_export(study) + if study.is_a?(Study) + metadata = cohort_metadata(study).with_indifferent_access + term_matches = study.search_weight(@term_list || []) + facet_data = @studies_by_facet&.[](study.accession) || {} + text_to_facet = @metadata_matches&.[](study.accession) || {} + facet_matches = Api::V1::StudySearchResultsObjects.merge_facet_matches(facet_data, text_to_facet) + [ + 'SCP', + study.accession, + study.name, + study.description, + study.public, + study.detached, + study.cell_count, + study.gene_count, + result_url_for(study), + metadata[:disease].join(COMMA_SPACE), + metadata[:organ].join(COMMA_SPACE), + metadata[:species].join(COMMA_SPACE), + metadata[:sex].join(COMMA_SPACE), + metadata[:library_preparation_protocol].join(COMMA_SPACE), + Api::V1::StudySearchResultsObjects.facet_results_as_text(facet_matches), + term_matches[:terms].keys.join(COMMA_SPACE) + ] + else + [ + study[:hca_result] ? 'HCA' : 'TDR', + study[:accession], + study[:name], + study[:description].gsub(/\n/, ''), + true, + false, + 0, + 0, + result_url_for(study), + study.dig(:metadata, :disease).join(COMMA_SPACE), + study.dig(:metadata, :organ).join(COMMA_SPACE), + study.dig(:metadata, :species).join(COMMA_SPACE), + study.dig(:metadata, :sex).join(COMMA_SPACE), + study.dig(:metadata, :library_preparation_protocol).join(COMMA_SPACE), + Api::V1::StudySearchResultsObjects.facet_results_as_text(@studies_by_facet[study[:accession]]), + nil + ] + end + end + + # returns a URL for the study, either SCP or HCA + def result_url_for(study) + if study.is_a?(Study) + view_study_url(accession: study.accession, study_name: study.url_safe_name) + else + "https://data.humancellatlas.org/explore/projects/#{study[:hca_project_id]}" + end + end + + # flatten facet matches into a text string for export + def self.facet_results_as_text(facets) + entries = [] + facets.delete(:facet_search_weight) + facets.each do |facet_name, filters| + entries << "#{facet_name}:#{filters.map { |f| f[:name] }.join('|')}" + end + entries.join(COMMA_SPACE) + end + # merge in multiple facet match data objects into a single merged entity for a given study def self.merge_facet_matches(existing_data, new_data) study_data = existing_data || {} diff --git a/test/api/search_controller_test.rb b/test/api/search_controller_test.rb index 6482fa201..ab1fac713 100644 --- a/test/api/search_controller_test.rb +++ b/test/api/search_controller_test.rb @@ -547,4 +547,16 @@ class SearchControllerTest < ActionDispatch::IntegrationTest positive_mock.verify end end + + test 'should return text results when requested' do + other_matches = Study.viewable(@user).any_of({ description: /#{HOMO_SAPIENS_FILTER[:name]}/ }, + { description: /#{NO_DISEASE_FILTER[:name]}/ }).pluck(:accession) + facet_query = "species:#{HOMO_SAPIENS_FILTER[:id]}#{FACET_DELIM}disease:#{NO_DISEASE_FILTER[:id]}" + execute_http_request(:get, api_v1_search_path(type: 'study', facets: facet_query, export: 'tsv')) + assert_response :success + lines = json.split(/\n/)[1..] + all_accessions = (@convention_accessions + other_matches.map(&:accession)).flatten.uniq.sort + found_accessions = lines.map { |l| l.split(/\t/)[1] }.flatten.uniq.sort + assert_equal all_accessions, found_accessions + end end From b268668993d31bac463d5ffb552d08aeae0a9175 Mon Sep 17 00:00:00 2001 From: bistline Date: Wed, 2 Jul 2025 11:25:04 -0400 Subject: [PATCH 02/15] Adding button for exporting results, new js API method --- app/controllers/api/v1/search_controller.rb | 6 ++-- .../api/v1/study_search_results_objects.rb | 3 +- .../search/results/ResultsExport.jsx | 29 ++++++++++++++++++ .../search/results/ResultsPanel.jsx | 6 +++- .../search/results/SearchQueryDisplay.jsx | 4 ++- app/javascript/lib/scp-api.jsx | 30 ++++++++++++++++++- 6 files changed, 72 insertions(+), 6 deletions(-) create mode 100644 app/javascript/components/search/results/ResultsExport.jsx diff --git a/app/controllers/api/v1/search_controller.rb b/app/controllers/api/v1/search_controller.rb index 24ca83f64..a2040a0b0 100644 --- a/app/controllers/api/v1/search_controller.rb +++ b/app/controllers/api/v1/search_controller.rb @@ -399,7 +399,7 @@ def index end @matching_accessions = @studies.map { |study| self.class.get_study_attribute(study, :accession) } - + logger.info "studies_by_facet: #{@studies_by_facet}" logger.info "Final list of matching studies: #{@matching_accessions}" @results = @studies.paginate(page: params[:page], per_page: Study.per_page) if params[:export].present? @@ -672,7 +672,9 @@ def self.match_results_by_filter(search_result:, result_key:, facets:) else matching_facet[:filters].detect do |filter| filters = search_result[result_key].is_a?(Array) ? search_result[result_key] : [search_result[result_key]] - filters.include?(filter[:id]) || filters.include?(filter[:name]) + filters.include?(filter[:id]) || + filters.include?(filter[:name]) || + filters.include?(filter[:id].gsub(/_/, ':')) # handle malformed IDs end end end diff --git a/app/controllers/api/v1/study_search_results_objects.rb b/app/controllers/api/v1/study_search_results_objects.rb index 411d2b5cb..5c96de1c6 100644 --- a/app/controllers/api/v1/study_search_results_objects.rb +++ b/app/controllers/api/v1/study_search_results_objects.rb @@ -132,6 +132,7 @@ def study_text_export(study) facet_data = @studies_by_facet&.[](study.accession) || {} text_to_facet = @metadata_matches&.[](study.accession) || {} facet_matches = Api::V1::StudySearchResultsObjects.merge_facet_matches(facet_data, text_to_facet) + inferred = @inferred_accessions&.include?(study.accession) [ 'SCP', study.accession, @@ -148,7 +149,7 @@ def study_text_export(study) metadata[:sex].join(COMMA_SPACE), metadata[:library_preparation_protocol].join(COMMA_SPACE), Api::V1::StudySearchResultsObjects.facet_results_as_text(facet_matches), - term_matches[:terms].keys.join(COMMA_SPACE) + inferred ? "inferred text match on #{@inferred_terms.join(COMMA_SPACE)}" : term_matches[:terms].keys.join(COMMA_SPACE) ] else [ diff --git a/app/javascript/components/search/results/ResultsExport.jsx b/app/javascript/components/search/results/ResultsExport.jsx new file mode 100644 index 000000000..aee0229b0 --- /dev/null +++ b/app/javascript/components/search/results/ResultsExport.jsx @@ -0,0 +1,29 @@ +import React, { useState } from 'react' +import Button from 'react-bootstrap/lib/Button' +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome' +import { faFileExport } from '@fortawesome/free-solid-svg-icons' +import { exportSearchResultsText } from '~/lib/scp-api' + +export default function ResultsExport({ studySearchState }) { + const [exporting, setExporting] = useState(false) + + /** export results to a file */ + async function exportResults() { + setExporting(true) + await exportSearchResultsText(studySearchState.params).then(() => { + setExporting(false) + }) + } + + return ( + + ) +} diff --git a/app/javascript/components/search/results/ResultsPanel.jsx b/app/javascript/components/search/results/ResultsPanel.jsx index 509a758b0..acce5cdab 100644 --- a/app/javascript/components/search/results/ResultsPanel.jsx +++ b/app/javascript/components/search/results/ResultsPanel.jsx @@ -46,7 +46,11 @@ const ResultsPanel = ({ studySearchState, studyComponent, noResultsDisplay, book } else if (results.studies && results.studies.length > 0) { panelContent = ( <> - { } + { } + { } { /** displays a summary of an executed search. * e.g. (Text contains (stomach)) AND (Metadata contains (organ: brain)) */ -export default function SearchQueryDisplay({ terms, facets, bookmarks }) { +export default function SearchQueryDisplay({ terms, facets, bookmarks, studySearchState }) { const hasFacets = facets && facets.length > 0 const hasTerms = terms && terms.length > 0 if (!hasFacets && !hasTerms) { @@ -133,6 +134,7 @@ export default function SearchQueryDisplay({ terms, facets, bookmarks }) { : {termsDisplay}{facetsDisplay} + ) diff --git a/app/javascript/lib/scp-api.jsx b/app/javascript/lib/scp-api.jsx index bf8928d90..4940076d2 100644 --- a/app/javascript/lib/scp-api.jsx +++ b/app/javascript/lib/scp-api.jsx @@ -510,6 +510,34 @@ export async function downloadBucketFile(bucketId, filePath) { document.body.removeChild(element) } +export async function exportSearchResultsText(searchParams, mock=false) { + searchParams.export = 'tsv' + const path = `/search?${buildSearchQueryString('study', searchParams)}` + const init = { + method: 'GET', + headers: { + Authorization: `Bearer ${getOAuthToken()}` + } + } + const [searchResults, perfTimes] = await scpApi(path, init, mock, false, false) + const fileName = searchResults.headers.get('Content-Disposition').split('"')[1] + const dataBlob = await searchResults.blob() + + // Create an element with an anchor link and connect this to the blob + const element = document.createElement('a') + element.href = URL.createObjectURL(dataBlob) + + // name the file and indicate it should download + element.download = fileName + + // Simulate clicking the link resulting in downloading the file + document.body.appendChild(element) + element.click() + + // Cleanup + document.body.removeChild(element) +} + /** * Returns initial content for the "Explore" tab in Study Overview * @@ -898,7 +926,7 @@ export async function fetchSearch(type, searchParams, mock=false) { export function buildSearchQueryString(type, searchParams) { const facetsParam = buildFacetQueryString(searchParams.facets) - const params = ['page', 'order', 'terms', 'external', 'preset', 'genes', 'genePage'] + const params = ['page', 'order', 'terms', 'external', 'export', 'preset', 'genes', 'genePage'] let otherParamString = params.map(param => { return searchParams[param] ? `&${param}=${searchParams[param]}` : '' }).join('') From bcadd9a4e4d16def929283dd170e0ef08c88cca1 Mon Sep 17 00:00:00 2001 From: bistline Date: Mon, 21 Jul 2025 10:57:37 -0400 Subject: [PATCH 03/15] adding component test --- .../search/results/ResultsExport.jsx | 4 +- test/js/search/results-export.test.js | 39 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 test/js/search/results-export.test.js diff --git a/app/javascript/components/search/results/ResultsExport.jsx b/app/javascript/components/search/results/ResultsExport.jsx index aee0229b0..bdc89cbfe 100644 --- a/app/javascript/components/search/results/ResultsExport.jsx +++ b/app/javascript/components/search/results/ResultsExport.jsx @@ -5,6 +5,7 @@ import { faFileExport } from '@fortawesome/free-solid-svg-icons' import { exportSearchResultsText } from '~/lib/scp-api' export default function ResultsExport({ studySearchState }) { + const hasResults = studySearchState?.results?.studies && studySearchState.results.studies.length > 0 const [exporting, setExporting] = useState(false) /** export results to a file */ @@ -18,7 +19,8 @@ export default function ResultsExport({ studySearchState }) { return (