Skip to content

Commit 61724a6

Browse files
authored
Merge pull request #2209 from broadinstitute/development
Release 1.91.0
2 parents 6bf9e2a + 09b1cd0 commit 61724a6

22 files changed

+547
-155
lines changed

Gemfile.lock

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -329,11 +329,11 @@ GEM
329329
net-protocol
330330
netrc (0.11.0)
331331
nio4r (2.7.4)
332-
nokogiri (1.18.2-arm64-darwin)
332+
nokogiri (1.18.3-arm64-darwin)
333333
racc (~> 1.4)
334-
nokogiri (1.18.2-x86_64-darwin)
334+
nokogiri (1.18.3-x86_64-darwin)
335335
racc (~> 1.4)
336-
nokogiri (1.18.2-x86_64-linux-gnu)
336+
nokogiri (1.18.3-x86_64-linux-gnu)
337337
racc (~> 1.4)
338338
oauth2 (1.4.7)
339339
faraday (>= 0.8, < 2.0)
@@ -368,7 +368,7 @@ GEM
368368
puma (5.6.9)
369369
nio4r (~> 2.0)
370370
racc (1.8.1)
371-
rack (2.2.10)
371+
rack (2.2.11)
372372
rack-brotli (1.1.0)
373373
brotli (>= 0.1.7)
374374
rack (>= 1.4)

app/javascript/components/upload/UploadWizard.jsx

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,8 +55,8 @@ const ALL_POSSIBLE_STEPS = [
5555
AnnDataUploadStep,
5656
DifferentialExpressionStep,
5757
SpatialStep,
58-
CoordinateLabelStep,
5958
SequenceFileStep,
59+
CoordinateLabelStep,
6060
GeneListStep,
6161
MiscellaneousStep,
6262
SeuratStep,
@@ -101,7 +101,7 @@ export function RawUploadWizard({ studyAccession, name }) {
101101
// set the additional steps to display, based on classic or AnnData experience
102102
if (isAnnDataExperience) {
103103
MAIN_STEPS = MAIN_STEPS_ANNDATA
104-
SUPPLEMENTAL_STEPS = ALL_POSSIBLE_STEPS.slice(6, 7)
104+
SUPPLEMENTAL_STEPS = ALL_POSSIBLE_STEPS.slice(6, 8)
105105
// SUPPLEMENTAL_STEPS.splice(1, 0, DifferentialExpressionStep)
106106
// TODO enable after raw counts are sorted for AnnData (SCP-5110)
107107
NON_VISUALIZABLE_STEPS = ALL_POSSIBLE_STEPS.slice(10, 12)

app/javascript/components/upload/upload-utils.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -332,6 +332,7 @@ const sequenceExtensions = [
332332
]
333333
const baiExtensions = ['.bai']
334334
const tbiExtensions = ['.tbi']
335+
const csiExtensions = ['.csi'] // Like TBI, but supports larger chromosomes
335336
const annDataExtensions = ['.h5', '.h5ad', '.hdf5']
336337
const seuratExtensions = ['.Rds', '.rds', '.RDS', '.seuratdata', '.h5seurat', '.h5Seurat', '.seuratdisk', '.Rda', '.rda']
337338
const miscExtensions = baseMiscExtensions.concat(mtxExtensions, annDataExtensions, seuratExtensions)
@@ -342,7 +343,7 @@ export const FileTypeExtensions = {
342343
misc: miscExtensions.concat(miscExtensions.map(ext => `${ext}.gz`)),
343344
sequence: sequenceExtensions,
344345
bai: baiExtensions,
345-
tbi: tbiExtensions,
346+
tbi: tbiExtensions.concat(csiExtensions),
346347
annData: annDataExtensions,
347348
seurat: seuratExtensions
348349
}

app/javascript/lib/validation/log-validation.js

Lines changed: 23 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,25 @@ function getTrimmedIssueMessages(issues) {
1818
}).slice(0, 20) // Show <= 20 messages
1919
}
2020

21+
/** Get shared issue props for file-validation and study-validation */
22+
export function getWarningAndErrorProps(errors, warnings) {
23+
const errorMessages = getTrimmedIssueMessages(errors)
24+
const warningMessages = getTrimmedIssueMessages(warnings)
25+
26+
const errorTypes = Array.from(new Set(errors.map(columns => columns[1])))
27+
const warningTypes = Array.from(new Set(warnings.map(columns => columns[1])))
28+
29+
return {
30+
errors: errorMessages,
31+
warnings: warningMessages,
32+
errorTypes, warningTypes,
33+
numErrors: errors.length,
34+
numWarnings: warnings.length,
35+
numErrorTypes: errorTypes.length,
36+
numWarningTypes: warningTypes.length
37+
}
38+
}
39+
2140
/** Get properties about this validation run to log to Mixpanel */
2241
export function getLogProps(fileInfo, issueObj, perfTimes) {
2342
const { errors, warnings, summary } = issueObj
@@ -45,23 +64,13 @@ export function getLogProps(fileInfo, issueObj, perfTimes) {
4564
}
4665
return Object.assign({ status: 'success' }, defaultProps)
4766
} else {
48-
const errorMessages = getTrimmedIssueMessages(errors)
49-
const warningMessages = getTrimmedIssueMessages(warnings)
67+
const issueProps = getWarningAndErrorProps(errors, warnings)
5068

51-
const errorTypes = Array.from(new Set(errors.map(columns => columns[1])))
52-
const warningTypes = Array.from(new Set(warnings.map(columns => columns[1])))
69+
const withIssueProps = Object.assign(defaultProps, issueProps)
5370

54-
return Object.assign(defaultProps, {
71+
return Object.assign(withIssueProps, {
5572
status: 'failure',
56-
summary,
57-
numErrors: errors.length,
58-
numWarnings: warnings.length,
59-
errors: errorMessages,
60-
warnings: warningMessages,
61-
numErrorTypes: errorTypes.length,
62-
numWarningTypes: warningTypes.length,
63-
errorTypes,
64-
warningTypes
73+
summary
6574
})
6675
}
6776
}

app/javascript/lib/validation/validate-file-content.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ import { parseAnnDataFile } from './validate-anndata'
3535
const MAX_GZIP_FILESIZE = 50 * oneMiB
3636

3737
/** File extensions / suffixes that indicate content must be gzipped */
38-
const EXTENSIONS_MUST_GZIP = ['gz', 'bam', 'tbi']
38+
const EXTENSIONS_MUST_GZIP = ['gz', 'bam', 'tbi', 'csi']
3939

4040

4141
/**
Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
import { getWarningAndErrorProps } from '~/lib/validation/log-validation'
2+
import { log } from '~/lib/metrics-api'
3+
4+
/** Ensure a name is provided for the study */
5+
function validateName(input) {
6+
const issues = []
7+
8+
const name = input.value
9+
if (name === '') {
10+
const msg = 'Enter a name for your study.'
11+
issues.push(['error', 'missing-name', msg])
12+
}
13+
14+
return issues
15+
}
16+
17+
/** Convert Date string to string format in "Data release date" UI */
18+
function dateToMMDDYYYY(dateString) {
19+
const date = new Date(dateString)
20+
const rawMonth = date.getMonth() + 1; // Months are zero-indexed, so add 1
21+
const rawDay = date.getDate();
22+
const year = date.getFullYear();
23+
24+
const month = rawMonth.toString().padStart(2, '0');
25+
const day = rawDay.toString().padStart(2, '0');
26+
27+
const MMDDYYYY = `${month}/${day}/${year}`;
28+
29+
return MMDDYYYY
30+
}
31+
32+
/** Ensure any embargo date is between tomorrow and max embargo date */
33+
export function validateEmbargo(embargoInput) {
34+
const issues = []
35+
36+
const rawEmbargoDate = embargoInput.value
37+
const embargoDate = new Date(embargoInput.value)
38+
const maxDate = new Date(embargoInput.max)
39+
40+
const tomorrow = new Date();
41+
tomorrow.setDate(tomorrow.getDate() + 1);
42+
43+
if (
44+
rawEmbargoDate !== '' &&
45+
(embargoDate > maxDate || embargoDate < tomorrow)
46+
) {
47+
const tomorrowFormatted = dateToMMDDYYYY(tomorrow)
48+
const maxDateFormatted = dateToMMDDYYYY(maxDate)
49+
50+
const msg =
51+
`If embargoed, date must be between ` +
52+
`tomorrow (${tomorrowFormatted}) and ${maxDateFormatted}.`
53+
54+
issues.push(['error', 'invalid-embargo', msg])
55+
}
56+
57+
return issues
58+
}
59+
60+
/** Ensure a billing project is selected */
61+
function validateBillingProject(input) {
62+
const issues = []
63+
64+
const billingProject = input.value
65+
if (billingProject === '') {
66+
const msg = 'Pick a billing project from above menu.'
67+
issues.push(['error', 'missing-billing-project', msg])
68+
}
69+
70+
return issues
71+
}
72+
73+
/** Ensure a workspace is selected, if using existing workspace */
74+
function validateWorkspace(input, studyForm) {
75+
const issues = []
76+
77+
const workspace = input.value
78+
const useExistingWorkspace =
79+
studyForm.querySelector('#study_use_existing_workspace').value === '1'
80+
81+
if (useExistingWorkspace && workspace === '') {
82+
const msg = 'Enter a workspace name, or set "Use existing workspace?" to "No".'
83+
issues.push(['error', 'missing-workspace', msg])
84+
}
85+
86+
return issues
87+
}
88+
89+
/** Add or remove error classes from field elements around given element */
90+
function updateErrorState(element, addOrRemove) {
91+
const fieldDiv = element.closest('[class^="col-md"]')
92+
93+
if (addOrRemove === 'add') {
94+
fieldDiv.querySelector('label').classList.add('text-danger')
95+
fieldDiv.classList.add('has-error', 'has-feedback')
96+
} else {
97+
fieldDiv.querySelector('label').classList.remove('text-danger')
98+
fieldDiv.classList.remove('has-error', 'has-feedback')
99+
}
100+
}
101+
102+
/** Render messages for any validation issue for given input element */
103+
function writeValidationMessage(input, issues) {
104+
if (issues.length === 0) {return}
105+
106+
// Consider below if we need to deal with multiple errors per field
107+
// See ValidationMessage.jsx for model to follow
108+
// const messageList = '<ul>' + issues.map(issue => {
109+
// const msg = issue[2]
110+
// return `<li className="validation-error">${msg}</li>`
111+
// }).join('') + '</ul>
112+
113+
const message = issues[0][2]
114+
115+
const messageHtml = `<div class="validation-error">${message}</div>`
116+
117+
updateErrorState(input, 'add')
118+
input.insertAdjacentHTML('afterend', messageHtml)
119+
}
120+
121+
/** Validate given field, get issues, write any messages */
122+
function checkField(studyForm, field, validateFns, issues) {
123+
const input = studyForm.querySelector(`#study_${field}`)
124+
let fieldIssues
125+
if (field === 'firecloud_workspace') {
126+
fieldIssues = validateFns[field](input, studyForm)
127+
} else {
128+
fieldIssues = validateFns[field](input)
129+
}
130+
issues = issues.concat(fieldIssues)
131+
writeValidationMessage(input, fieldIssues)
132+
133+
return issues
134+
}
135+
136+
/** Get event data to log to Bard / Mixpanel */
137+
function getLogProps(issues) {
138+
const warnings = issues.filter(issue => issue[0] === 'warn')
139+
const errors = issues.filter(issue => issue[0] === 'error')
140+
141+
const issueProps = getWarningAndErrorProps(errors, warnings)
142+
const status = errors.length === 0 ? 'success' : 'failure'
143+
144+
const logProps = Object.assign(issueProps, {
145+
status
146+
})
147+
148+
return logProps
149+
}
150+
151+
/** Validation form in "Create study" page */
152+
export function validateStudy(studyForm) {
153+
let issues = []
154+
155+
// Clear any prior error messages
156+
document.querySelectorAll('.validation-error').forEach(error => {
157+
updateErrorState(error, 'remove')
158+
error.remove()
159+
})
160+
161+
const validateFns = {
162+
'name': validateName,
163+
'firecloud_project': validateBillingProject, // "Terra billing project"
164+
'embargo': validateEmbargo, // "Data release date"
165+
'firecloud_workspace': validateWorkspace // "Existing Terra workspace"
166+
}
167+
168+
const fields = Object.keys(validateFns)
169+
fields.forEach(field => {
170+
issues = checkField(studyForm, field, validateFns, issues)
171+
})
172+
173+
const logProps = getLogProps(issues)
174+
175+
log('study-validation', logProps)
176+
177+
return issues
178+
}

app/javascript/vite/application.js

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import ClusterAssociationSelect from '~/components/upload/ClusterAssociationSele
1515
import RawAssociationSelect from '~/components/upload/RawAssociationSelect'
1616
import { getFeatureFlagsWithDefaults } from '~/providers/UserProvider'
1717
import ValidateFile from '~/lib/validation/validate-file'
18+
import { validateStudy } from '~/lib/validation/validate-study'
1819
import { setupSentry } from '~/lib/sentry-logging'
1920
import { adjustGlobalHeader, mitigateStudyOverviewTitleTruncation } from '~/lib/layout-utils'
2021
import { clearOldServiceWorkerCaches } from '~/lib/service-worker-cache'
@@ -132,6 +133,7 @@ window.SCP.eventsToLog.forEach(eventToLog => {
132133

133134
window.SCP.getFeatureFlagsWithDefaults = getFeatureFlagsWithDefaults
134135
window.SCP.validateRemoteFile = validateRemoteFile
136+
window.SCP.validateStudy = validateStudy
135137
window.SCP.API = ScpApi
136138

137139
window.Spinner = Spinner

app/lib/differential_expression_service.rb

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,19 @@ def self.run_differential_expression_job(cluster_group, study, user, annotation_
163163
params_object.machine_type = machine_type if machine_type.present? # override :machine_type if specified
164164
return true if dry_run # exit before submission if specified as annotation was already validated
165165

166-
if params_object.valid?
166+
# check if there's already a job running using these parameters and exit if so
167+
job_params = ApplicationController.batch_api_client.format_command_line(
168+
study_file:, action: :differential_expression, user_metrics_uuid: user.metrics_uuid, params_object:
169+
)
170+
running = ApplicationController.batch_api_client.find_matching_jobs(
171+
params: job_params, job_states: BatchApiClient::RUNNING_STATES
172+
)
173+
if running.any?
174+
log_message "Found #{running.count} running DE jobs: #{running.map(&:name).join(', ')}"
175+
log_message "Matching these parameters: #{job_params.join(' ')}"
176+
log_message "Exiting without queuing new job"
177+
false
178+
elsif params_object.valid?
167179
# launch DE job
168180
job = IngestJob.new(study:, study_file:, user:, action: :differential_expression, params_object:)
169181
job.delay.push_remote_and_launch_ingest

app/mailers/single_cell_mailer.rb

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -282,16 +282,6 @@ def user_notification(user, subject, message)
282282
end
283283
end
284284

285-
# nightly sanity check email looking for missing files
286-
def sanity_check(missing_files)
287-
@missing_files = missing_files
288-
@admins = User.where(admin: true).map(&:email)
289-
290-
mail(to: @admins, subject: "[Single Cell Portal Admin Notification #{Rails.env != 'production' ? " (#{Rails.env})" : nil}]: Sanity check results: #{@missing_files.size} files missing") do |format|
291-
format.html
292-
end
293-
end
294-
295285
# collect usage statistics for the given day and email admins
296286
def nightly_admin_report
297287
@admins = User.where(admin: true).map(&:email)
@@ -324,9 +314,6 @@ def nightly_admin_report
324314
# disk usage
325315
@disk_stats = SummaryStatsUtils.disk_usage
326316

327-
# storage sanity check
328-
@missing_files = SummaryStatsUtils.storage_sanity_check
329-
330317
mail(to: @admins, subject: "[Single Cell Portal Admin Notification#{Rails.env != 'production' ? " (#{Rails.env})" : nil}]: Nightly Server Report for #{@today}") do |format|
331318
format.html { render layout: 'nightly_admin_report' }
332319
end

app/models/batch_api_client.rb

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,22 @@ def list_jobs(page_token: nil)
9595
service.list_project_location_jobs(project_location, page_token:)
9696
end
9797

98+
# find any existing jobs using command line parameters & job states
99+
#
100+
# * *params*
101+
# - +params+ (Array<String>) => array of params sent to job to match on
102+
# - +job_state+ (Array<String>) => optional states to filter on (defaults to completed jobs)
103+
#
104+
# * *returns*
105+
# - (Array<Google::Apis::BatchV1::Job>)
106+
def find_matching_jobs(params: [], job_states: COMPLETED_STATES)
107+
jobs = list_jobs.jobs
108+
jobs.select do |job|
109+
commands = get_job_command_line(job:)
110+
(params & commands).sort == params.sort && job_states.include?(job.status.state)
111+
end
112+
end
113+
98114
# main handler to create and run a Batch API job
99115
#
100116
# * *params*

0 commit comments

Comments
 (0)