Skip to content

addressing issue 11, added support for exponential backoff #12

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

addressing issue 11, added support for exponential backoff #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0d9bf50
addressing issue 11, added support for exponential backoff
DrClick Sep 8, 2017
ff3eec3
changed 1 to 0 for exponent
DrClick Sep 8, 2017
d39dc4f
added exponentialRetryCap, tests and cleaned up tests in general
DrClick Sep 11, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 43 additions & 6 deletions src/main/scala/com/hootsuite/circuitbreaker/CircuitBreaker.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ import java.util.concurrent.atomic.{AtomicInteger, AtomicReference}
* @param name the name of the circuit breaker
* @param failLimit maximum number of consecutive failures before the circuit breaker is tripped (opened)
* @param retryDelay duration until an open/broken circuit breaker lets a call through to verify whether or not it should be reset
* @param isExponentialBackoff indicating the retry delayed should be increased exponential on consecutive failures
* @param exponentialRetryCap limits the number of times the retryDelay will be increased exponentially, ignored if not exponential backoff
* @param isResultFailure partial function to allow users to determine return cases which should be considered as failures
* @param isExceptionNotFailure partial function to allow users to determine exceptions which should not be considered failures
* @param stateChangeListeners listeners that will be notified when the circuit breaker changes state (open <--> closed)
Expand All @@ -26,6 +28,8 @@ class CircuitBreaker private[circuitbreaker] (
val name: String,
val failLimit: Int,
val retryDelay: FiniteDuration,
val isExponentialBackoff: Boolean = false,
val exponentialRetryCap: Option[Int],
val isResultFailure: PartialFunction[Any, Boolean] = { case _ => false },
val isExceptionNotFailure: PartialFunction[Throwable, Boolean] = { case _ => false },
val stateChangeListeners: List[CircuitBreakerStateChangeListener] = List(),
Expand All @@ -38,6 +42,8 @@ class CircuitBreaker private[circuitbreaker] (
builder.name,
builder.failLimit,
builder.retryDelay,
builder.isExponentialBackoff,
builder.exponentialRetryCap,
builder.isResultFailure,
builder.isExceptionNotFailure,
builder.stateChangeListeners,
Expand Down Expand Up @@ -180,9 +186,9 @@ class CircuitBreaker private[circuitbreaker] (
* @param currentState the expected current state
* @return true when the state was changed, false when the given state was not the current state
*/
def attemptResetBrokenState(currentState: BrokenState): Boolean = {
def attemptResetBrokenState(currentState: BrokenState, retryCount: Int): Boolean = {
logger.debug(s"Circuit breaker \'$name\', attempting to reset open/broken state")
state.compareAndSet(currentState, new BrokenState(this))
state.compareAndSet(currentState, new BrokenState(this, retryCount))
}

/**
Expand Down Expand Up @@ -286,7 +292,7 @@ private object CircuitBreaker {
override def onFailure(): Unit =
incrementFailure()

private[this] def incrementFailure() = {
private[this] def incrementFailure(): Unit = {
val currentCount = failureCount.incrementAndGet
logger.debug(
s"Circuit breaker ${cb.name} increment failure count to $currentCount; fail limit is ${cb.failLimit}"
Expand All @@ -298,8 +304,35 @@ private object CircuitBreaker {
/**
* CircuitBreaker is opened/broken. Invocations fail immediately.
*/
class BrokenState(cb: CircuitBreaker) extends State {
val retryAt: Long = System.currentTimeMillis() + cb.retryDelay.toMillis
class BrokenState(cb: CircuitBreaker, retryCount: Int = 0) extends State {
val retryAt: Long = System.currentTimeMillis() + calcRetryDelay(cb)

def calcRetryDelay(cb: CircuitBreaker): Long = {

//calc jitter up to 1/10 the current retryDelay
//for exponential backoff
val jitter: Long = if (cb.isExponentialBackoff) {
(scala.util.Random.nextFloat() * cb.retryDelay.toMillis / 10).toLong
} else {
0
}

val retryCap = cb.exponentialRetryCap match {
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Consider the following: val retryCap = cb.exponentialRetryCap.getOrElse(Int.MaxValue)

case Some(x) => x
case None => Int.MaxValue
}

val exponent: Int = if (cb.isExponentialBackoff) {
Math.min(retryCount, retryCap)
} else 0

val result = (cb.retryDelay.toMillis + jitter) * Math.pow(2, exponent).toLong

logger.debug(s"CB details retry delay details: jitter $jitter, " +
s"retryCap $retryCap, exponent $exponent delay $result")

result
}

override def preInvoke(): Unit = {
cb.invocationListeners.foreach { listener =>
Expand All @@ -311,7 +344,7 @@ private object CircuitBreaker {
}

val retry = System.currentTimeMillis > retryAt
if (!(retry && cb.attemptResetBrokenState(this))) {
if (!(retry && cb.attemptResetBrokenState(this, this.retryCount + 1))) {
throw new CircuitBreakerBrokenException(
cb.name,
s"Making ${cb.name} unavailable after ${cb.failLimit} errors"
Expand All @@ -336,6 +369,8 @@ private object CircuitBreaker {
* @param name the name of the circuit breaker
* @param failLimit maximum number of consecutive failures before the circuit breaker is tripped (opened)
* @param retryDelay duration until an open/broken circuit breaker lets a call through to verify whether or not it should be reset
* @param isExponentialBackoff indicating the retry delayed should be increased exponential on consecutive failures
* @param exponentialRetryCap limits the number of times the retryDelay will be increased exponentially, ignored if not exponential backoff
* @param isResultFailure partial function to allow users to determine return cases which should be considered as failures
* @param isExceptionNotFailure partial function to allow users to determine exceptions which should not be considered failures
* @param stateChangeListeners listeners that will be notified when the circuit breaker changes state (open <--> closed)
Expand All @@ -345,6 +380,8 @@ case class CircuitBreakerBuilder(
name: String,
failLimit: Int,
retryDelay: FiniteDuration,
isExponentialBackoff: Boolean = false,
exponentialRetryCap: Option[Int] = Some(10),
isResultFailure: PartialFunction[Any, Boolean] = { case _ => false },
isExceptionNotFailure: PartialFunction[Throwable, Boolean] = { case _ => false },
stateChangeListeners: List[CircuitBreakerStateChangeListener] = List(),
Expand Down
Loading