Add new task to flag invalid ClinVar ids

server/Gemfile

@@ -110,9 +110,12 @@ end
 
 group :test do
   # Adds support for Capybara system testing and selenium driver
-  gem "capybara", ">= 3.26"
-  gem "simplecov"
-  gem "simplecov-lcov"
+
+  gem 'capybara', '>= 3.26'
+  gem 'simplecov'
+  gem 'simplecov-lcov'
+  gem 'mocha', require: false
+
 end
 
 gem "csv", "~> 3.3"

server/Gemfile.lock

@@ -263,9 +263,11 @@ GEM
     method_source (1.1.0)
     mini_magick (4.13.2)
     mini_mime (1.1.5)
-    mini_portile2 (2.8.8)
     minitest (5.25.5)
+    mocha (2.7.1)
+      ruby2_keywords (>= 0.0.5)
     msgpack (1.8.0)
+    mini_portile2 (2.8.8)
     multi_json (1.15.0)
     multi_xml (0.7.1)
       bigdecimal (~> 3.1)
@@ -619,6 +621,7 @@ DEPENDENCIES
   kramdown (~> 2.4.0)
   kramdown-parser-gfm (~> 1.1.0)
   listen (~> 3.3)
+  mocha
   net-ftp (~> 0.3.3)
   obo (~> 0.1.5)
   oj (~> 3.16.3)

server/app/jobs/find_invalid_clinvar_records.rb

@@ -0,0 +1,72 @@
+class FindInvalidClinvarRecords < ApplicationJob
+  KNOWN_GOOD_ID = '16609'
+
+  def perform
+    #Bail early if this fails. Maybe the site is down or something.
+    check_clinvar_id(KNOWN_GOOD_ID) do |succeeded|
+      if !succeeded
+        raise StandardError.new("ClinVar ID #{KNOWN_GOOD_ID} did not return a 200. Aborting checks.")
+      end
+    end
+
+    Variant.joins(:clinvar_entries).distinct.find_each do |variant|
+      variant.clinvar_entries.each do |clinvar_entry|
+        clinvar_id = clinvar_entry.clinvar_id
+
+        if clinvar_id == 'N/A' || clinvar_id == 'NONE FOUND'
+          next
+        end
+
+        if has_existing_flag?(variant, clinvar_id)
+          next
+        end
+
+        check_clinvar_id(clinvar_id) do |succeeded|
+          if !succeeded
+            flag_variant(variant, clinvar_id)
+          end
+        end
+
+        sleep 0.25
+      end
+    end
+  end
+
+  private
+  def url_for_clinvar_id(id)
+    "https://www.ncbi.nlm.nih.gov/clinvar/variation/#{id}/"
+  end
+
+  def check_clinvar_id(id)
+    url = URI.parse(url_for_clinvar_id(id))
+    response = Net::HTTP.get_response(url)
+    case response.code
+    when "200"
+      yield true
+    else
+      yield false
+    end
+  end
+
+  def has_existing_flag?(variant, clinvar_id)
+    verbiage = flag_verbiage(variant, clinvar_id)
+    variant.flags.where(state: 'open', flagging_user_id: Constants::CIVICBOT_USER_ID)
+      .any? { |f| f.open_activity.note == verbiage }
+  end
+
+  def flag_variant(variant, clinvar_id)
+    verbiage = flag_verbiage(variant, clinvar_id)
+    civicbot_user = User.find(Constants::CIVICBOT_USER_ID)
+
+    Activities::FlagEntity.new(
+      flagging_user: civicbot_user,
+      flaggable: variant,
+      organization_id: nil,
+      note: verbiage
+    ).perform
+  end
+
+  def flag_verbiage(variant, clinvar_id)
+    "This Variant links to a ClinVar ID that is no longer valid: #{clinvar_id}"
+  end
+end

server/config/scheduled_tasks.yml

@@ -134,6 +134,12 @@ FindRetractedSources:
   queue: default
   class: FindRetractedSources
 
+FindInvalidClinvarRecords:
+  cron: '0 22 1 * *'
+  description: Find and flag Variants that link to invalid ClinVar Records
+  queue: default
+  class: FindInvalidClinvarRecords
+
 GenerateNightlyUniprotFile:
   cron: '0 3 * * *'
   description: Update the nightly Uniprot id to gene mapping file

server/test/fixtures/clinvar_entries.yml

@@ -0,0 +1,2 @@
+one:
+  clinvar_id: '12345'

server/test/fixtures/users.yml

@@ -12,3 +12,8 @@ admin:
   username: admin
   name: admin
   role: admin
+
+civicbot:
+  id: 385
+  name: civicbot
+  role: editor

server/test/jobs/find_invalid_clinvar_records_test.rb

@@ -0,0 +1,57 @@
+require 'test_helper'
+
+class FindInvalidClinvarRecordsTest < ActiveJob::TestCase
+  setup do
+    @job = FindInvalidClinvarRecords.new
+    @job.stubs(:check_clinvar_id).with(FindInvalidClinvarRecords::KNOWN_GOOD_ID).yields(true)
+    @job.stubs(:check_clinvar_id).with { |id| id != FindInvalidClinvarRecords::KNOWN_GOOD_ID }.yields(false)
+    @variant = variants(:v600e)
+    @clinvar_entry = clinvar_entries(:one)
+    @variant.clinvar_entries << @clinvar_entry
+  end
+
+  test 'raises exception if known good id fails' do
+    @job.stubs(:check_clinvar_id).yields(false)
+    assert_raises(StandardError) { @job.perform }
+  end
+
+  test 'skips clinvar ID with value N/A' do
+    @clinvar_entry.update(clinvar_id: 'N/A')
+    assert_nothing_raised { @job.perform }
+    assert_equal 0, Flag.count
+  end
+
+  test 'skips clinvar ID with value NONE FOUND' do
+    @clinvar_entry.update(clinvar_id: 'NONE FOUND')
+    assert_nothing_raised { @job.perform }
+    assert_equal 0, Flag.count
+  end
+
+  test 'skips if flag already exists' do
+    verbiage = @job.send(:flag_verbiage, @variant, @clinvar_entry.clinvar_id)
+
+    flag = Flag.new(
+      flaggable: @variant,
+      state: 'open',
+      flagging_user_id: Constants::CIVICBOT_USER_ID,
+      open_activity: FlagEntityActivity.create!(note: verbiage, subject: @variant, user: users(:curator))
+    )
+
+    flag.save!
+
+    @job.perform
+
+    assert_equal 1, Flag.count
+  end
+
+  test 'creates flag on invalid record' do
+    @job.perform
+    assert_equal 1, Flag.count
+  end
+
+  test 'does not create flag on valid record' do
+    @clinvar_entry.update(clinvar_id: FindInvalidClinvarRecords::KNOWN_GOOD_ID)
+    @job.perform
+    assert_equal 0, Flag.count
+  end
+end

server/test/test_helper.rb

@@ -1,6 +1,7 @@
 ENV["RAILS_ENV"] ||= "test"
 require_relative "../config/environment"
 require "rails/test_help"
+require 'mocha/minitest'
 
 class ActiveSupport::TestCase
   # Setup all fixtures in test/fixtures/*.yml for all tests in alphabetical order.