Merge pull request #461 from cleverhu/add-retry-for-bad-conn

internalstorage: add recoverable err

Author: Iceber

pkg/storage/internalstorage/errors.go

@@ -1,6 +1,7 @@
 package internalstorage
 
 import (
+	"database/sql/driver"
 	"errors"
 	"fmt"
 	"io"
@@ -17,6 +18,14 @@ import (
 	"github.com/clusterpedia-io/clusterpedia/pkg/storage"
 )
 
+var recoverableErrors = []error{
+	io.ErrClosedPipe,
+	io.ErrUnexpectedEOF,
+	os.ErrDeadlineExceeded,
+	syscall.ECONNREFUSED,
+	driver.ErrBadConn,
+}
+
 func InterpretResourceDBError(cluster, name string, err error) error {
 	if err == nil {
 		return nil
@@ -34,15 +43,20 @@ func InterpretDBError(key string, err error) error {
 		return genericstorage.NewKeyNotFoundError(key, 0)
 	}
 
-	if _, isNetError := err.(net.Error); isNetError ||
-		errors.Is(err, io.ErrClosedPipe) ||
-		errors.Is(err, io.ErrUnexpectedEOF) ||
-		os.IsTimeout(err) ||
-		errors.Is(err, os.ErrDeadlineExceeded) ||
-		errors.Is(err, syscall.ECONNREFUSED) {
+	if _, isNetError := err.(net.Error); isNetError {
 		return storage.NewRecoverableException(err)
 	}
 
+	if os.IsTimeout(err) {
+		return storage.NewRecoverableException(err)
+	}
+
+	for _, re := range recoverableErrors {
+		if errors.Is(err, re) {
+			return storage.NewRecoverableException(err)
+		}
+	}
+
 	// TODO(iceber): add dialector judgment
 	mysqlErr := InterpretMysqlError(key, err)
 	if mysqlErr != err {