diff --git a/generator/apache/apache.go b/generator/apache/apache.go index a52f6fc..f427db9 100644 --- a/generator/apache/apache.go +++ b/generator/apache/apache.go @@ -134,20 +134,24 @@ func (g *ApacheLogGenerator) worker(workerID int) { backoffConfig.MaxInterval = 5 * time.Second backoffConfig.MaxElapsedTime = 0 // Never stop retrying - backoffTicker := backoff.NewTicker(backoffConfig) - defer backoffTicker.Stop() + // Drive the timer from this goroutine only. backoff.ExponentialBackOff is + // not safe for concurrent use, so we never hand it to backoff.NewTicker's + // internal goroutine; instead we own every NextBackOff/Reset call here. + timer := time.NewTimer(backoffConfig.NextBackOff()) + defer timer.Stop() for { select { case <-g.stopCh: g.logger.Debug("Worker stopping", zap.Int("worker_id", workerID)) return - case <-backoffTicker.C: + case <-timer.C: if g.tracker != nil && !g.tracker.Acquire() { select { case <-g.stopCh: return case <-g.tracker.ResumeC(): + timer.Reset(backoffConfig.NextBackOff()) continue } } @@ -156,9 +160,11 @@ func (g *ApacheLogGenerator) worker(workerID int) { g.logger.Error("Failed to write log", zap.Int("worker_id", workerID), zap.Error(err)) + timer.Reset(backoffConfig.NextBackOff()) continue } backoffConfig.Reset() + timer.Reset(backoffConfig.NextBackOff()) } } } diff --git a/generator/apache_combined/apache_combined.go b/generator/apache_combined/apache_combined.go index 0deb344..4cc3a5c 100644 --- a/generator/apache_combined/apache_combined.go +++ b/generator/apache_combined/apache_combined.go @@ -131,20 +131,24 @@ func (g *ApacheCombinedLogGenerator) worker(workerID int) { backoffConfig.MaxInterval = 5 * time.Second backoffConfig.MaxElapsedTime = 0 // Never stop retrying - backoffTicker := backoff.NewTicker(backoffConfig) - defer backoffTicker.Stop() + // Drive the timer from this goroutine only. backoff.ExponentialBackOff is + // not safe for concurrent use, so we never hand it to backoff.NewTicker's + // internal goroutine; instead we own every NextBackOff/Reset call here. + timer := time.NewTimer(backoffConfig.NextBackOff()) + defer timer.Stop() for { select { case <-g.stopCh: g.logger.Debug("Worker stopping", zap.Int("worker_id", workerID)) return - case <-backoffTicker.C: + case <-timer.C: if g.tracker != nil && !g.tracker.Acquire() { select { case <-g.stopCh: return case <-g.tracker.ResumeC(): + timer.Reset(backoffConfig.NextBackOff()) continue } } @@ -153,9 +157,11 @@ func (g *ApacheCombinedLogGenerator) worker(workerID int) { g.logger.Error("Failed to write log", zap.Int("worker_id", workerID), zap.Error(err)) + timer.Reset(backoffConfig.NextBackOff()) continue } backoffConfig.Reset() + timer.Reset(backoffConfig.NextBackOff()) } } } diff --git a/generator/apache_error/apache_error.go b/generator/apache_error/apache_error.go index 9150324..87a9f78 100644 --- a/generator/apache_error/apache_error.go +++ b/generator/apache_error/apache_error.go @@ -128,20 +128,24 @@ func (g *ApacheErrorLogGenerator) worker(workerID int) { backoffConfig.MaxInterval = 5 * time.Second backoffConfig.MaxElapsedTime = 0 // Never stop retrying - backoffTicker := backoff.NewTicker(backoffConfig) - defer backoffTicker.Stop() + // Drive the timer from this goroutine only. backoff.ExponentialBackOff is + // not safe for concurrent use, so we never hand it to backoff.NewTicker's + // internal goroutine; instead we own every NextBackOff/Reset call here. + timer := time.NewTimer(backoffConfig.NextBackOff()) + defer timer.Stop() for { select { case <-g.stopCh: g.logger.Debug("Worker stopping", zap.Int("worker_id", workerID)) return - case <-backoffTicker.C: + case <-timer.C: if g.tracker != nil && !g.tracker.Acquire() { select { case <-g.stopCh: return case <-g.tracker.ResumeC(): + timer.Reset(backoffConfig.NextBackOff()) continue } } @@ -150,9 +154,11 @@ func (g *ApacheErrorLogGenerator) worker(workerID int) { g.logger.Error("Failed to write log", zap.Int("worker_id", workerID), zap.Error(err)) + timer.Reset(backoffConfig.NextBackOff()) continue } backoffConfig.Reset() + timer.Reset(backoffConfig.NextBackOff()) } } } diff --git a/generator/filegen/filegen.go b/generator/filegen/filegen.go index 932701a..c39ed95 100644 --- a/generator/filegen/filegen.go +++ b/generator/filegen/filegen.go @@ -443,20 +443,24 @@ func (g *FileLogGenerator) worker(id int, files []string) { backoffConfig.MaxInterval = 5 * time.Second backoffConfig.MaxElapsedTime = 0 // Never stop retrying - backoffTicker := backoff.NewTicker(backoffConfig) - defer backoffTicker.Stop() + // Drive the timer from this goroutine only. backoff.ExponentialBackOff is + // not safe for concurrent use, so we never hand it to backoff.NewTicker's + // internal goroutine; instead we own every NextBackOff/Reset call here. + timer := time.NewTimer(backoffConfig.NextBackOff()) + defer timer.Stop() for { select { case <-g.stopCh: g.logger.Debug("Worker received stop signal", zap.Int("id", id)) return - case <-backoffTicker.C: + case <-timer.C: if g.tracker != nil && !g.tracker.Acquire() { select { case <-g.stopCh: return case <-g.tracker.ResumeC(): + timer.Reset(backoffConfig.NextBackOff()) continue } } @@ -471,11 +475,13 @@ func (g *FileLogGenerator) worker(id int, files []string) { g.logger.Error("Error reading file", zap.String("file", file), zap.Error(err)) generator.BlitzGeneratorWriteErrorsCounter.Add(context.Background(), 1, componentName) // On error, backoff will automatically handle retry timing + timer.Reset(backoffConfig.NextBackOff()) continue } // On success, reset backoff to configured rate backoffConfig.Reset() + timer.Reset(backoffConfig.NextBackOff()) fileIdx += g.workers } } diff --git a/generator/json/json.go b/generator/json/json.go index f54549b..db56f6b 100644 --- a/generator/json/json.go +++ b/generator/json/json.go @@ -183,20 +183,24 @@ func (g *JSONLogGenerator) worker(workerID int) { backoffConfig.MaxInterval = 5 * time.Second backoffConfig.MaxElapsedTime = 0 // Never stop retrying - backoffTicker := backoff.NewTicker(backoffConfig) - defer backoffTicker.Stop() + // Drive the timer from this goroutine only. backoff.ExponentialBackOff is + // not safe for concurrent use, so we never hand it to backoff.NewTicker's + // internal goroutine; instead we own every NextBackOff/Reset call here. + timer := time.NewTimer(backoffConfig.NextBackOff()) + defer timer.Stop() for { select { case <-g.stopCh: g.logger.Debug("Worker stopping", zap.Int("worker_id", workerID)) return - case <-backoffTicker.C: + case <-timer.C: if g.tracker != nil && !g.tracker.Acquire() { select { case <-g.stopCh: return case <-g.tracker.ResumeC(): + timer.Reset(backoffConfig.NextBackOff()) continue } } @@ -205,9 +209,11 @@ func (g *JSONLogGenerator) worker(workerID int) { g.logger.Error("Failed to write log", zap.Int("worker_id", workerID), zap.Error(err)) + timer.Reset(backoffConfig.NextBackOff()) continue } backoffConfig.Reset() + timer.Reset(backoffConfig.NextBackOff()) } } } diff --git a/generator/kubernetes/kubernetes.go b/generator/kubernetes/kubernetes.go index 80bd1ad..da5c131 100644 --- a/generator/kubernetes/kubernetes.go +++ b/generator/kubernetes/kubernetes.go @@ -152,20 +152,24 @@ func (g *Generator) worker(workerID int) { backoffConfig.MaxInterval = 5 * time.Second backoffConfig.MaxElapsedTime = 0 - backoffTicker := backoff.NewTicker(backoffConfig) - defer backoffTicker.Stop() + // Drive the timer from this goroutine only. backoff.ExponentialBackOff is + // not safe for concurrent use, so we never hand it to backoff.NewTicker's + // internal goroutine; instead we own every NextBackOff/Reset call here. + timer := time.NewTimer(backoffConfig.NextBackOff()) + defer timer.Stop() for { select { case <-g.stopCh: g.logger.Debug("Worker stopping", zap.Int("worker_id", workerID)) return - case <-backoffTicker.C: + case <-timer.C: if g.tracker != nil && !g.tracker.Acquire() { select { case <-g.stopCh: return case <-g.tracker.ResumeC(): + timer.Reset(backoffConfig.NextBackOff()) continue } } @@ -174,9 +178,11 @@ func (g *Generator) worker(workerID int) { g.logger.Error("Failed to write log", zap.Int("worker_id", workerID), zap.Error(err)) + timer.Reset(backoffConfig.NextBackOff()) continue } backoffConfig.Reset() + timer.Reset(backoffConfig.NextBackOff()) } } } diff --git a/generator/nginx/nginx.go b/generator/nginx/nginx.go index 58226a2..56bc391 100644 --- a/generator/nginx/nginx.go +++ b/generator/nginx/nginx.go @@ -155,20 +155,24 @@ func (g *Generator) worker(workerID int) { backoffConfig.MaxInterval = 5 * time.Second backoffConfig.MaxElapsedTime = 0 - backoffTicker := backoff.NewTicker(backoffConfig) - defer backoffTicker.Stop() + // Drive the timer from this goroutine only. backoff.ExponentialBackOff is + // not safe for concurrent use, so we never hand it to backoff.NewTicker's + // internal goroutine; instead we own every NextBackOff/Reset call here. + timer := time.NewTimer(backoffConfig.NextBackOff()) + defer timer.Stop() for { select { case <-g.stopCh: g.logger.Debug("Worker stopping", zap.Int("worker_id", workerID)) return - case <-backoffTicker.C: + case <-timer.C: if g.tracker != nil && !g.tracker.Acquire() { select { case <-g.stopCh: return case <-g.tracker.ResumeC(): + timer.Reset(backoffConfig.NextBackOff()) continue } } @@ -177,9 +181,11 @@ func (g *Generator) worker(workerID int) { g.logger.Error("Failed to write log", zap.Int("worker_id", workerID), zap.Error(err)) + timer.Reset(backoffConfig.NextBackOff()) continue } backoffConfig.Reset() + timer.Reset(backoffConfig.NextBackOff()) } } } diff --git a/generator/okta/okta.go b/generator/okta/okta.go index f313a84..dd2a603 100644 --- a/generator/okta/okta.go +++ b/generator/okta/okta.go @@ -269,20 +269,24 @@ func (g *Generator) worker(workerID int) { backoffConfig.MaxInterval = 5 * time.Second backoffConfig.MaxElapsedTime = 0 - backoffTicker := backoff.NewTicker(backoffConfig) - defer backoffTicker.Stop() + // Drive the timer from this goroutine only. backoff.ExponentialBackOff is + // not safe for concurrent use, so we never hand it to backoff.NewTicker's + // internal goroutine; instead we own every NextBackOff/Reset call here. + timer := time.NewTimer(backoffConfig.NextBackOff()) + defer timer.Stop() for { select { case <-g.stopCh: g.logger.Debug("Worker stopping", zap.Int("worker_id", workerID)) return - case <-backoffTicker.C: + case <-timer.C: if g.tracker != nil && !g.tracker.Acquire() { select { case <-g.stopCh: return case <-g.tracker.ResumeC(): + timer.Reset(backoffConfig.NextBackOff()) continue } } @@ -291,9 +295,11 @@ func (g *Generator) worker(workerID int) { g.logger.Error("Failed to write log", zap.Int("worker_id", workerID), zap.Error(err)) + timer.Reset(backoffConfig.NextBackOff()) continue } backoffConfig.Reset() + timer.Reset(backoffConfig.NextBackOff()) } } } diff --git a/generator/paloalto/paloalto.go b/generator/paloalto/paloalto.go index 37a1f98..b7a2b44 100644 --- a/generator/paloalto/paloalto.go +++ b/generator/paloalto/paloalto.go @@ -118,28 +118,34 @@ func (g *Generator) worker(workerID int) { backoffConfig.MaxInterval = 5 * time.Second backoffConfig.MaxElapsedTime = 0 - backoffTicker := backoff.NewTicker(backoffConfig) - defer backoffTicker.Stop() + // Drive the timer from this goroutine only. backoff.ExponentialBackOff is + // not safe for concurrent use, so we never hand it to backoff.NewTicker's + // internal goroutine; instead we own every NextBackOff/Reset call here. + timer := time.NewTimer(backoffConfig.NextBackOff()) + defer timer.Stop() for { select { case <-g.stopCh: g.logger.Debug("Worker stopping", zap.Int("worker_id", workerID)) return - case <-backoffTicker.C: + case <-timer.C: if g.tracker != nil && !g.tracker.Acquire() { select { case <-g.stopCh: return case <-g.tracker.ResumeC(): + timer.Reset(backoffConfig.NextBackOff()) continue } } if err := g.generateAndWrite(workerID); err != nil { g.logger.Error("Failed to write log", zap.Int("worker_id", workerID), zap.Error(err)) + timer.Reset(backoffConfig.NextBackOff()) continue } backoffConfig.Reset() + timer.Reset(backoffConfig.NextBackOff()) } } } diff --git a/generator/postgres/postgres.go b/generator/postgres/postgres.go index 9eba378..6d61560 100644 --- a/generator/postgres/postgres.go +++ b/generator/postgres/postgres.go @@ -307,20 +307,24 @@ func (g *Generator) worker(workerID int) { backoffConfig.MaxInterval = 5 * time.Second backoffConfig.MaxElapsedTime = 0 - backoffTicker := backoff.NewTicker(backoffConfig) - defer backoffTicker.Stop() + // Drive the timer from this goroutine only. backoff.ExponentialBackOff is + // not safe for concurrent use, so we never hand it to backoff.NewTicker's + // internal goroutine; instead we own every NextBackOff/Reset call here. + timer := time.NewTimer(backoffConfig.NextBackOff()) + defer timer.Stop() for { select { case <-g.stopCh: g.logger.Debug("Worker stopping", zap.Int("worker_id", workerID)) return - case <-backoffTicker.C: + case <-timer.C: if g.tracker != nil && !g.tracker.Acquire() { select { case <-g.stopCh: return case <-g.tracker.ResumeC(): + timer.Reset(backoffConfig.NextBackOff()) continue } } @@ -329,9 +333,11 @@ func (g *Generator) worker(workerID int) { g.logger.Error("Failed to write log", zap.Int("worker_id", workerID), zap.Error(err)) + timer.Reset(backoffConfig.NextBackOff()) continue } backoffConfig.Reset() + timer.Reset(backoffConfig.NextBackOff()) } } } diff --git a/generator/wel/wel.go b/generator/wel/wel.go index 6fc3ab9..5f5db95 100644 --- a/generator/wel/wel.go +++ b/generator/wel/wel.go @@ -193,23 +193,28 @@ func (g *Generator) worker(workerID int) { backoffConfig.MaxInterval = 5 * time.Second backoffConfig.MaxElapsedTime = 0 - backoffTicker := backoff.NewTicker(backoffConfig) - defer backoffTicker.Stop() + // Drive the timer from this goroutine only. backoff.ExponentialBackOff is + // not safe for concurrent use, so we never hand it to backoff.NewTicker's + // internal goroutine; instead we own every NextBackOff/Reset call here. + timer := time.NewTimer(backoffConfig.NextBackOff()) + defer timer.Stop() for { select { case <-g.stopCh: g.logger.Debug("WEL worker stopping", zap.Int("worker_id", workerID)) return - case <-backoffTicker.C: + case <-timer.C: if err := g.generateAndWrite(rng); err != nil { g.logger.Error("Failed to write WEL event", zap.Int("worker_id", workerID), zap.Error(err), ) + timer.Reset(backoffConfig.NextBackOff()) continue } backoffConfig.Reset() + timer.Reset(backoffConfig.NextBackOff()) } } } diff --git a/generator/winevt/winevt.go b/generator/winevt/winevt.go index dfc4bec..d7f064e 100644 --- a/generator/winevt/winevt.go +++ b/generator/winevt/winevt.go @@ -103,28 +103,34 @@ func (g *WinevtGenerator) worker(workerID int, writer output.Writer) { backoffConfig.MaxInterval = 5 * time.Second backoffConfig.MaxElapsedTime = 0 - backoffTicker := backoff.NewTicker(backoffConfig) - defer backoffTicker.Stop() + // Drive the timer from this goroutine only. backoff.ExponentialBackOff is + // not safe for concurrent use, so we never hand it to backoff.NewTicker's + // internal goroutine; instead we own every NextBackOff/Reset call here. + timer := time.NewTimer(backoffConfig.NextBackOff()) + defer timer.Stop() for { select { case <-g.stopCh: g.logger.Debug("Worker stopping", zap.Int("worker_id", workerID)) return - case <-backoffTicker.C: + case <-timer.C: if g.tracker != nil && !g.tracker.Acquire() { select { case <-g.stopCh: return case <-g.tracker.ResumeC(): + timer.Reset(backoffConfig.NextBackOff()) continue } } if err := g.generateAndWrite(writer, workerID); err != nil { g.logger.Error("Failed to write log", zap.Int("worker_id", workerID), zap.Error(err)) + timer.Reset(backoffConfig.NextBackOff()) continue } backoffConfig.Reset() + timer.Reset(backoffConfig.NextBackOff()) } } }