Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -9,20 +9,15 @@ module DelayedJob
class MetricsCollector < Judoscale::JobMetricsCollector
include ActiveRecordHelper

# Returns the oldest unlocked job and the count of locked (busy) jobs per queue in a single pass,
# since `delayed_jobs` has no index for either filter and each separate query scans the whole table.
METRICS_SQL = ActiveRecordHelper.cleanse_sql(<<~SQL)
SELECT COALESCE(queue, 'default'), min(run_at)
SELECT COALESCE(queue, 'default'),
min(CASE WHEN locked_at IS NULL THEN run_at END),
sum(CASE WHEN locked_at IS NOT NULL THEN 1 ELSE 0 END)
FROM delayed_jobs
WHERE locked_at IS NULL
AND failed_at IS NULL
GROUP BY queue
SQL

BUSY_METRICS_SQL = ActiveRecordHelper.cleanse_sql(<<~SQL)
SELECT COALESCE(queue, 'default'), count(*)
FROM delayed_jobs
WHERE locked_at IS NOT NULL
AND locked_by IS NOT NULL
AND failed_at IS NULL
WHERE failed_at IS NULL
AND (locked_at IS NULL OR locked_by IS NOT NULL)
GROUP BY 1
SQL

Expand All @@ -38,13 +33,10 @@ def collect
metrics = []
time = Time.now.utc

run_at_by_queue = select_rows_silently(METRICS_SQL).to_h
self.queues |= run_at_by_queue.keys

if track_busy_jobs?
busy_count_by_queue = select_rows_silently(BUSY_METRICS_SQL).to_h
self.queues |= busy_count_by_queue.keys
end
rows = select_rows_silently(METRICS_SQL)
run_at_by_queue = rows.to_h { |queue, run_at, _| [queue, run_at] }
busy_count_by_queue = rows.to_h { |queue, _, busy_count| [queue, busy_count.to_i] }
self.queues |= rows.map(&:first)

queues.each do |queue|
run_at = run_at_by_queue[queue]
Expand Down
40 changes: 40 additions & 0 deletions judoscale-delayed_job/test/metrics_collector_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,14 @@ def clear_enqueued_jobs
ActiveRecord::Base.connection.execute("DELETE FROM delayed_jobs")
end

# Counts the queries the block runs, ignoring schema lookups.
def count_queries(&block)
count = 0
counter = ->(*, payload) { count += 1 unless payload[:name] == "SCHEMA" }
ActiveSupport::Notifications.subscribed(counter, "sql.active_record", &block)
count
end

describe "#collect" do
after {
clear_enqueued_jobs
Expand Down Expand Up @@ -117,6 +125,38 @@ def clear_enqueued_jobs
end
end

it "tracks busy jobs without running extra queries" do
Delayable.new.delay(queue: "default").perform
Delayable.new.delay(queue: "default").perform
Delayed::Worker.new.send(:reserve_job)

query_counts = [false, true].map { |track_busy_jobs|
use_adapter_config :delayed_job, track_busy_jobs: track_busy_jobs do
count_queries { DelayedJob::MetricsCollector.new.collect }
end
}

_(query_counts[1]).must_equal query_counts[0]
end

it "excludes busy jobs from latency when tracking busy jobs" do
use_adapter_config :delayed_job, track_busy_jobs: true do
# A job that has been running for an hour shouldn't count as waiting in the queue.
Delayable.new.delay(queue: "default", run_at: Time.now.utc - 3600).perform
Delayed::Worker.new.send(:reserve_job)
Delayable.new.delay(queue: "default").perform
Delayable.new.delay(queue: "low", run_at: Time.now.utc - 3600).perform
Delayed::Worker.new.tap { |w| w.name = "dj_worker_low" }.send(:reserve_job)

metrics = subject.collect.to_h { |m| [[m.queue_name, m.identifier], m.value] }

_(metrics[["default", :qt]]).must_be :<, 1000
_(metrics[["default", :busy]]).must_equal 1
_(metrics[["low", :qt]]).must_equal 0
_(metrics[["low", :busy]]).must_equal 1
end
end

it "logs debug information about busy jobs being collected" do
use_config log_level: :debug do
use_adapter_config :delayed_job, track_busy_jobs: true do
Expand Down
27 changes: 14 additions & 13 deletions judoscale-good_job/lib/judoscale/good_job/metrics_collector.rb
Original file line number Diff line number Diff line change
Expand Up @@ -38,22 +38,23 @@ def collect
self.queues |= run_silently { @good_job_base_class.distinct.pluck(:queue_name) }
end

# logically we don't need the finished_at condition, but it lets postgres use the indexes
oldest_execution_time_by_queue = run_silently do
# Read the oldest pending job and the count of running jobs per queue in one scan of unfinished jobs, so
# busy job tracking adds no queries. Running means performed but not finished, matching GoodJob's
# `running` scope on v3's Execution and v4.2+'s Job (v4.0-4.1 checked advisory locks).
# The finished_at condition lets postgres use the indexes.
rows = run_silently do
@good_job_base_class
.where(performed_at: nil, finished_at: nil)
.where(finished_at: nil)
.group(:queue_name)
.pluck(:queue_name, Arel.sql("min(coalesce(scheduled_at, created_at))"))
.to_h
end
self.queues |= oldest_execution_time_by_queue.keys

if track_busy_jobs?
busy_count_by_queue = run_silently do
@good_job_base_class.running.group(:queue_name).count
end
self.queues |= busy_count_by_queue.keys
.pluck(
:queue_name,
Arel.sql("min(coalesce(scheduled_at, created_at)) FILTER (WHERE performed_at IS NULL)"),
Arel.sql("count(*) FILTER (WHERE performed_at IS NOT NULL)")
)
end
oldest_execution_time_by_queue = rows.to_h { |queue, run_at, _| [queue, run_at] }
busy_count_by_queue = rows.to_h { |queue, _, busy_count| [queue, busy_count] }
self.queues |= rows.map(&:first)

queues.each do |queue|
run_at = oldest_execution_time_by_queue[queue]
Expand Down
41 changes: 41 additions & 0 deletions judoscale-good_job/test/metrics_collector_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,14 @@ def clear_enqueued_jobs
ActiveRecord::Base.connection.execute("DELETE FROM good_jobs")
end

# Counts the queries the block runs, ignoring schema lookups.
def count_queries(&block)
count = 0
counter = ->(*, payload) { count += 1 unless payload[:name] == "SCHEMA" }
ActiveSupport::Notifications.subscribed(counter, "sql.active_record", &block)
count
end

describe "#collect" do
after {
clear_enqueued_jobs
Expand Down Expand Up @@ -157,6 +165,39 @@ def clear_enqueued_jobs
end
end

it "tracks busy jobs without running extra queries" do
Delayable.set(queue: "default").perform_later
Delayable.set(queue: "default").perform_later
::GoodJob::Job.limit(1).update_all(performed_at: Time.now.utc)

query_counts = [false, true].map { |track_busy_jobs|
use_adapter_config :good_job, track_busy_jobs: track_busy_jobs do
count_queries { GoodJob::MetricsCollector.new.collect }
end
}

_(query_counts[1]).must_equal query_counts[0]
end

it "excludes busy jobs from latency when tracking busy jobs" do
use_adapter_config :good_job, track_busy_jobs: true do
# A job that has been running for an hour shouldn't count as waiting in the queue.
hour_ago = Time.now.utc - 3600
%w[default low].each do |queue|
Delayable.set(queue: queue).perform_later
::GoodJob::Job.where(queue_name: queue).update_all(scheduled_at: hour_ago, created_at: hour_ago, performed_at: hour_ago)
end
Delayable.set(queue: "default").perform_later

metrics = subject.collect.to_h { |m| [[m.queue_name, m.identifier], m.value] }

_(metrics[["default", :qt]]).must_be :<, 1000
_(metrics[["default", :busy]]).must_equal 1
_(metrics[["low", :qt]]).must_equal 0
_(metrics[["low", :busy]]).must_equal 1
end
end

it "logs debug information about busy jobs being collected" do
use_config log_level: :debug do
use_adapter_config :good_job, track_busy_jobs: true do
Expand Down
6 changes: 3 additions & 3 deletions judoscale-resque/lib/judoscale/resque/metrics_collector.rb
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@ def collect
if track_busy_jobs?
busy_counts = Hash.new { |h, k| h[k] = 0 }

# `Resque.working` reads every worker's job in one call and skips idle workers. `job(false)` uses
# that job instead of fetching it again, which would cost a Redis round trip per busy worker.
::Resque.working.each do |worker|
if !worker.idle? && (job = worker.job)
busy_counts[job["queue"]] += 1
end
busy_counts[worker.job(false)["queue"]] += 1
end
end

Expand Down
18 changes: 9 additions & 9 deletions judoscale-resque/test/metrics_collector_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,10 @@
require "securerandom"

module Judoscale
ResqueWorkerStub = Struct.new(:job) do
def idle?
job.nil?
ResqueWorkerStub = Struct.new(:queue) do
def job(reload = true)
raise "the collector should reuse the job Resque.working loaded" if reload
{"queue" => queue}
end
end

Expand Down Expand Up @@ -104,17 +105,16 @@ def idle?
end
end

it "tracks busy jobs when the configuration is enabled, ignoring idle workers" do
it "tracks busy jobs when the configuration is enabled" do
use_adapter_config :resque, track_busy_jobs: true do
queues = ["default", "high"]
size = 2
latency = 1.0

workers = [
ResqueWorkerStub.new({"queue" => "default"}),
ResqueWorkerStub.new({"queue" => "default"}),
ResqueWorkerStub.new({"queue" => "high"}),
ResqueWorkerStub.new # idle, shouldn't be tracked
ResqueWorkerStub.new("default"),
ResqueWorkerStub.new("default"),
ResqueWorkerStub.new("high")
]

metrics = ::Resque.stub(:queues, queues) {
Expand Down Expand Up @@ -143,7 +143,7 @@ def idle?
queues = ["default"]
size = 2
latency = 1.0
workers = [ResqueWorkerStub.new({"queue" => "default"})]
workers = [ResqueWorkerStub.new("default")]

::Resque.stub(:queues, queues) {
::Resque.stub(:size, size) {
Expand Down
Loading