Merge branch '2.3' into merge-2.3-20180822

hatched · Aug 22, 2018 · 3039f52 · 3039f52
2 parents 2b83588 + be83905
commit 3039f52
Show file tree

Hide file tree

Showing 9 changed files with 171 additions and 17 deletions.
diff --git a/.gitignore b/.gitignore
@@ -11,6 +11,8 @@ TAGS
 parts/
 prime/
 stage/
+vendor/
+_vendor/
 *.snap
 .emacs.desktop
 .emacs.desktop.lock

diff --git a/api/apiclient.go b/api/apiclient.go
@@ -840,6 +840,7 @@ type dialer struct {
 // when appropriate.
 func (d dialer) dial(_ <-chan struct{}) (io.Closer, error) {
 	a := retry.StartWithCancel(d.openAttempt, d.opts.Clock, d.ctx.Done())
+	var lastErr error = nil
 	for a.Next() {
 		conn, tlsConfig, err := d.dial1()
 		if err == nil {
@@ -853,11 +854,15 @@ func (d dialer) dial(_ <-chan struct{}) (io.Closer, error) {
 		}
 		if isX509Error(err) || !a.More() {
 			// certificate errors don't improve with retries.
-			logger.Debugf("error dialing websocket: %v", err)
 			return nil, errors.Annotatef(err, "unable to connect to API")
 		}
+		lastErr = err
 	}
-	return nil, parallel.ErrStopped
+	if lastErr == nil {
+		logger.Debugf("no error, but not connected, probably cancelled before we started")
+		return nil, parallel.ErrStopped
+	}
+	return nil, errors.Trace(lastErr)
 }
 
 // dial1 makes a single dial attempt.

diff --git a/api/apiclient_whitebox_test.go b/api/apiclient_whitebox_test.go
@@ -0,0 +1,81 @@
+// Copyright 2018 Canonical Ltd.
+// Licensed under the AGPLv3, see LICENCE file for details.
+
+package api
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"regexp"
+	"time"
+
+	"github.com/juju/testing"
+	jc "github.com/juju/testing/checkers"
+	gc "gopkg.in/check.v1"
+
+	jtesting "github.com/juju/juju/testing"
+)
+
+type apiclientWhiteboxSuite struct {
+	testing.IsolationSuite
+}
+
+var _ = gc.Suite(&apiclientWhiteboxSuite{})
+
+func (s *apiclientWhiteboxSuite) TestDialWebsocketMultiCancelled(c *gc.C) {
+	ctx := context.TODO()
+	ctx, cancel := context.WithCancel(ctx)
+	started := make(chan struct{})
+	go func() {
+		select {
+		case <-started:
+		case <-time.After(jtesting.LongWait):
+			c.Fatalf("timed out waiting %s for started", jtesting.LongWait)
+		}
+		<-time.After(10 * time.Millisecond)
+		if cancel != nil {
+			c.Logf("cancelling")
+			cancel()
+		}
+	}()
+	listen, err := net.Listen("tcp4", ":0")
+	c.Assert(err, jc.ErrorIsNil)
+	addr := listen.Addr().String()
+	c.Logf("listening at: %s", addr)
+	// Note that we Listen, but we never Accept
+	close(started)
+	info := &Info{
+		Addrs: []string{addr},
+	}
+	opts := DialOpts{
+		DialAddressInterval: 50 * time.Millisecond,
+		RetryDelay:          40 * time.Millisecond,
+		Timeout:             100 * time.Millisecond,
+		DialTimeout:         100 * time.Millisecond,
+	}
+	// Close before we connect
+	listen.Close()
+	_, err = dialAPI(ctx, info, opts)
+	c.Check(err, gc.ErrorMatches, fmt.Sprintf("dial tcp %s:.*", regexp.QuoteMeta(addr)))
+}
+
+func (s *apiclientWhiteboxSuite) TestDialWebsocketMultiClosed(c *gc.C) {
+	listen, err := net.Listen("tcp4", ":0")
+	c.Assert(err, jc.ErrorIsNil)
+	addr := listen.Addr().String()
+	c.Logf("listening at: %s", addr)
+	// Note that we Listen, but we never Accept
+	info := &Info{
+		Addrs: []string{addr},
+	}
+	opts := DialOpts{
+		DialAddressInterval: 1 * time.Second,
+		RetryDelay:          1 * time.Second,
+		Timeout:             2 * time.Second,
+		DialTimeout:         3 * time.Second,
+	}
+	listen.Close()
+	_, _, err = DialAPI(info, opts)
+	c.Check(err, gc.ErrorMatches, fmt.Sprintf("unable to connect to API: dial tcp %s:.*", regexp.QuoteMeta(addr)))
+}
diff --git a/controller/config.go b/controller/config.go
@@ -112,6 +112,16 @@ const (
 	// MaxTxnLogSize is the maximum size the of capped txn log collection, eg "10M"
 	MaxTxnLogSize = "max-txn-log-size"
 
+	// MaxPruneTxnBatchSize is the maximum number of transactions we will evaluate in one go when pruning
+	// Default is 1M transactions. A value <= 0 indicates to do all transactions at once.
+	MaxPruneTxnBatchSize = "max-prune-txn-batch-size"
+
+	// MaxPruneTxnPasses is the maximum number of batches that we will process. So total number of transactions that can
+	// be processed is MaxPruneTxnBatchSize * MaxPruneTxnPasses.
+	// A value <= 0 implies 'do a single pass'. If both MaxPruneTxnBatchSize and MaxPruneTxnPasses are 0, then the
+	// default value of 1M BatchSize and 100 passes will be used instead.
+	MaxPruneTxnPasses = "max-prune-txn-passes"
+
 	// Attribute Defaults
 
 	// DefaultAuditingEnabled contains the default value for the
@@ -153,6 +163,12 @@ const (
 	// DefaultMaxTxnLogCollectionMB is the maximum size the txn log collection.
 	DefaultMaxTxnLogCollectionMB = 10 // 10 MB
 
+	// DefaultMaxPruneTxnBatchSize is the normal number of transaction we will prune in a given pass (1M)
+	DefaultMaxPruneTxnBatchSize = 1 * 1000 * 1000
+
+	// DefaultMaxPruneTxnPasses is the default number of batches we will process
+	DefaultMaxPruneTxnPasses = 100
+
 	// JujuHASpace is the network space within which the MongoDB replica-set
 	// should communicate.
 	JujuHASpace = "juju-ha-space"
@@ -187,6 +203,8 @@ var (
 		MaxLogsSize,
 		MaxLogsAge,
 		MaxTxnLogSize,
+		MaxPruneTxnBatchSize,
+		MaxPruneTxnPasses,
 		JujuHASpace,
 		JujuManagementSpace,
 		AuditingEnabled,
@@ -205,6 +223,8 @@ var (
 		AuditingEnabled,
 		AuditLogCaptureArgs,
 		AuditLogExcludeMethods,
+		MaxPruneTxnBatchSize,
+		MaxPruneTxnPasses,
 		JujuHASpace,
 		JujuManagementSpace,
 		CAASOperatorImagePath,
@@ -274,6 +294,13 @@ func (c Config) mustInt(name string) int {
 	return value
 }
 
+func (c Config) intOrDefault(name string, defaultVal int) int {
+	if _, ok := c[name]; ok {
+		return c.mustInt(name)
+	}
+	return defaultVal
+}
+
 // asString is a private helper method to keep the ugly string casting
 // in once place. It returns the given named attribute as a string,
 // returning "" if it isn't found.
@@ -331,14 +358,7 @@ func (c Config) AuditLogMaxSizeMB() int {
 // AuditLogMaxBackups returns the maximum number of backup audit log
 // files to keep.
 func (c Config) AuditLogMaxBackups() int {
-	if value, ok := c[AuditLogMaxBackups]; ok {
-		// Values obtained over the API are encoded as float64.
-		if floatValue, ok := value.(float64); ok {
-			return int(floatValue)
-		}
-		return value.(int)
-	}
-	return DefaultAuditLogMaxBackups
+	return c.intOrDefault(AuditLogMaxBackups, DefaultAuditLogMaxBackups)
 }
 
 // AuditLogExcludeMethods returns the set of method names that are
@@ -465,6 +485,16 @@ func (c Config) MaxTxnLogSizeMB() int {
 	return int(val)
 }
 
+// MaxPruneTxnBatchSize is the maximum size of the txn log collection.
+func (c Config) MaxPruneTxnBatchSize() int {
+	return c.intOrDefault(MaxPruneTxnBatchSize, DefaultMaxPruneTxnBatchSize)
+}
+
+// MaxPruneTxnPasses is the maximum number of batches of the txn log collection we will process at a time.
+func (c Config) MaxPruneTxnPasses() int {
+	return c.intOrDefault(MaxPruneTxnPasses, DefaultMaxPruneTxnPasses)
+}
+
 // JujuHASpace is the network space within which the MongoDB replica-set
 // should communicate.
 func (c Config) JujuHASpace() string {
@@ -671,6 +701,8 @@ var configChecker = schema.FieldMap(schema.Fields{
 	MaxLogsAge:              schema.String(),
 	MaxLogsSize:             schema.String(),
 	MaxTxnLogSize:           schema.String(),
+	MaxPruneTxnBatchSize:    schema.ForceInt(),
+	MaxPruneTxnPasses:       schema.ForceInt(),
 	JujuHASpace:             schema.String(),
 	JujuManagementSpace:     schema.String(),
 	CAASOperatorImagePath:   schema.String(),
@@ -693,6 +725,8 @@ var configChecker = schema.FieldMap(schema.Fields{
 	MaxLogsAge:              fmt.Sprintf("%vh", DefaultMaxLogsAgeDays*24),
 	MaxLogsSize:             fmt.Sprintf("%vM", DefaultMaxLogCollectionMB),
 	MaxTxnLogSize:           fmt.Sprintf("%vM", DefaultMaxTxnLogCollectionMB),
+	MaxPruneTxnBatchSize:    DefaultMaxPruneTxnBatchSize,
+	MaxPruneTxnPasses:       DefaultMaxPruneTxnPasses,
 	JujuHASpace:             schema.Omit,
 	JujuManagementSpace:     schema.Omit,
 	CAASOperatorImagePath:   schema.Omit,

diff --git a/controller/config_test.go b/controller/config_test.go
@@ -281,6 +281,27 @@ func (s *ConfigSuite) TestTxnLogConfigValue(c *gc.C) {
 	c.Assert(cfg.MaxTxnLogSizeMB(), gc.Equals, 8192)
 }
 
+func (s *ConfigSuite) TestMaxPruneTxnConfigDefault(c *gc.C) {
+	cfg, err := controller.NewConfig(testing.ControllerTag.Id(), testing.CACert, nil)
+	c.Assert(err, jc.ErrorIsNil)
+	c.Check(cfg.MaxPruneTxnBatchSize(), gc.Equals, 1*1000*1000)
+	c.Check(cfg.MaxPruneTxnPasses(), gc.Equals, 100)
+}
+
+func (s *ConfigSuite) TestMaxPruneTxnConfigValue(c *gc.C) {
+	cfg, err := controller.NewConfig(
+		testing.ControllerTag.Id(),
+		testing.CACert,
+		map[string]interface{}{
+			"max-prune-txn-batch-size": "12345678",
+			"max-prune-txn-passes":     "10",
+		},
+	)
+	c.Assert(err, jc.ErrorIsNil)
+	c.Check(cfg.MaxPruneTxnBatchSize(), gc.Equals, 12345678)
+	c.Check(cfg.MaxPruneTxnPasses(), gc.Equals, 10)
+}
+
 func (s *ConfigSuite) TestNetworkSpaceConfigValues(c *gc.C) {
 	haSpace := "space1"
 	managementSpace := "space2"

diff --git a/dependencies.tsv b/dependencies.tsv
@@ -62,7 +62,7 @@ github.com/juju/romulus	git	5b6f449d5c36ed6927c417c68379ab5acb7d7b46	2018-03-27T
 github.com/juju/schema	git	075de04f9b7d7580d60a1e12a0b3f50bb18e6998	2016-04-20T04:42:03Z
 github.com/juju/terms-client	git	9b925afd677234e4146dde3cb1a11e187cbed64e	2016-08-09T13:19:00Z
 github.com/juju/testing	git	c84dd6ba038a9012fb7830d206a006915b6e7937	2018-08-07T04:45:55Z
-github.com/juju/txn	git	f50b17d1ff3c31b7ea1c70e0d38a83f19af6d334	2018-04-06T04:58:51Z
+github.com/juju/txn	git	43be63df1fb67abe617877d8fbe7238344f1f820	2018-08-20T12:57:51Z
 github.com/juju/usso	git	68a59c96c178fbbad65926e7f93db50a2cd14f33	2016-04-01T10:44:24Z
 github.com/juju/utils	git	2000ea4ff0431598aec2b7e1d11d5d49b5384d63	2018-04-24T09:41:59Z
 github.com/juju/version	git	1f41e27e54f21acccf9b2dddae063a782a8a7ceb	2016-10-31T05:19:06Z

diff --git a/featuretests/cmd_juju_deploy_test.go b/featuretests/cmd_juju_deploy_test.go
@@ -6,11 +6,12 @@ package featuretests
 import (
 	"github.com/juju/cmd/cmdtesting"
 	"github.com/juju/errors"
-	"github.com/juju/juju/juju/testing"
-	"github.com/juju/juju/testcharms"
 	jc "github.com/juju/testing/checkers"
 	gc "gopkg.in/check.v1"
 	"gopkg.in/juju/charm.v6"
+
+	"github.com/juju/juju/juju/testing"
+	"github.com/juju/juju/testcharms"
 )
 
 type cmdDeploySuite struct {

diff --git a/state/controller_test.go b/state/controller_test.go
@@ -36,6 +36,8 @@ func (s *ControllerSuite) TestControllerAndModelConfigInitialisation(c *gc.C) {
 		controller.JujuHASpace,
 		controller.JujuManagementSpace,
 		controller.AuditLogExcludeMethods,
+		controller.MaxPruneTxnBatchSize,
+		controller.MaxPruneTxnPasses,
 		controller.CAASOperatorImagePath,
 		controller.Features,
 	)

diff --git a/state/txns.go b/state/txns.go
@@ -40,12 +40,20 @@ func (st *State) ResumeTransactions() error {
 func (st *State) MaybePruneTransactions() error {
 	runner, closer := st.database.TransactionRunner()
 	defer closer()
+	cfg, err := st.ControllerConfig()
+	if err != nil {
+		return errors.Trace(err)
+	}
+	maxBatchSize := cfg.MaxPruneTxnBatchSize()
+	maxPasses := cfg.MaxPruneTxnPasses()
 	// Prune txns when txn count has increased by 10% since last prune.
 	return runner.MaybePruneTransactions(jujutxn.PruneOptions{
-		PruneFactor:        1.1,
-		MinNewTransactions: 1000,
-		MaxNewTransactions: 100000,
-		MaxTime:            time.Now().Add(-time.Hour),
+		PruneFactor:          1.1,
+		MinNewTransactions:   1000,
+		MaxNewTransactions:   100000,
+		MaxTime:              time.Now().Add(-time.Hour),
+		MaxBatchTransactions: maxBatchSize,
+		MaxBatches:           maxPasses,
 	})
 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -11,6 +11,8 @@ TAGS @@
     parts/
     prime/
     stage/
+    vendor/
+    _vendor/
     *.snap
     .emacs.desktop
     .emacs.desktop.lock
@@ Expand Down @@