sdk: retry if mount failed in case master is unavailable temporarily

Signed-off-by: Shuoran Liu <[email protected]>
dbpaul · Jun 11, 2019 · a4534ca · a4534ca
1 parent 002d678
commit a4534ca
Show file tree

Hide file tree

Showing 2 changed files with 38 additions and 2 deletions.
diff --git a/sdk/data/stream/extent_client.go b/sdk/data/stream/extent_client.go
@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"runtime"
 	"sync"
+	"time"
 
 	"github.com/chubaofs/chubaofs/util/errors"
 
@@ -31,6 +32,11 @@ type AppendExtentKeyFunc func(inode uint64, key proto.ExtentKey) error
 type GetExtentsFunc func(inode uint64) (uint64, uint64, []proto.ExtentKey, error)
 type TruncateFunc func(inode, size uint64) error
 
+const (
+	MaxMountRetryLimit = 5
+	MountRetryInterval = time.Second * 5
+)
+
 var (
 	gDataWrapper       *wrapper.Wrapper
 	openRequestPool    *sync.Pool
@@ -54,14 +60,26 @@ type ExtentClient struct {
 func NewExtentClient(volname, master string, appendExtentKey AppendExtentKeyFunc, getExtents GetExtentsFunc, truncate TruncateFunc) (client *ExtentClient, err error) {
 	runtime.GOMAXPROCS(runtime.NumCPU())
 	client = new(ExtentClient)
+
+	limit := MaxMountRetryLimit
+retry:
 	gDataWrapper, err = wrapper.NewDataPartitionWrapper(volname, master)
 	if err != nil {
-		return nil, errors.Trace(err, "Init dp wrapper failed!")
+		if limit <= 0 {
+			return nil, errors.Trace(err, "Init data wrapper failed!")
+		} else {
+			limit--
+			time.Sleep(MountRetryInterval)
+			goto retry
+		}
 	}
+
 	client.streamers = make(map[uint64]*Streamer)
 	client.appendExtentKey = appendExtentKey
 	client.getExtents = getExtents
 	client.truncate = truncate
+
+	// Init request pools
 	openRequestPool = &sync.Pool{New: func() interface{} {
 		return &OpenRequest{}
 	}}
@@ -80,6 +98,7 @@ func NewExtentClient(volname, master string, appendExtentKey AppendExtentKeyFunc
 	evictRequestPool = &sync.Pool{New: func() interface{} {
 		return &EvictRequest{}
 	}}
+
 	return
 }
 

diff --git a/sdk/meta/meta.go b/sdk/meta/meta.go
@@ -24,6 +24,7 @@ import (
 	"github.com/chubaofs/chubaofs/proto"
 	"github.com/chubaofs/chubaofs/util"
 	"github.com/chubaofs/chubaofs/util/btree"
+	"github.com/chubaofs/chubaofs/util/errors"
 )
 
 const (
@@ -43,6 +44,11 @@ const (
 	statusNotPerm
 )
 
+const (
+	MaxMountRetryLimit = 5
+	MountRetryInterval = time.Second * 5
+)
+
 type MetaWrapper struct {
 	sync.RWMutex
 	cluster string
@@ -83,9 +89,20 @@ func NewMetaWrapper(volname, owner, masterHosts string) (*MetaWrapper, error) {
 	mw.rwPartitions = make([]*MetaPartition, 0)
 	mw.updateClusterInfo()
 	mw.updateVolStatInfo()
+
+	limit := MaxMountRetryLimit
+retry:
 	if err := mw.updateMetaPartitions(); err != nil {
-		return nil, err
+		if limit <= 0 {
+			return nil, errors.Trace(err, "Init meta wrapper failed!")
+		} else {
+			limit--
+			time.Sleep(MountRetryInterval)
+			goto retry
+		}
+
 	}
+
 	go mw.refresh()
 	return mw, nil
 }