Skip to content

Commit 82e3447

Browse files
anastasiamacanastasiamac
committed
Added support ro run numactl when needed. Bug 1350337.
1 parent 3b189d8 commit 82e3447

File tree

10 files changed

+169
-45
lines changed

10 files changed

+169
-45
lines changed

‎agent/agent.go

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -50,14 +50,15 @@ var DefaultDataDir = dataDir
5050
const SystemIdentity = "system-identity"
5151

5252
const (
53-
LxcBridge = "LXC_BRIDGE"
54-
ProviderType = "PROVIDER_TYPE"
55-
ContainerType = "CONTAINER_TYPE"
56-
Namespace = "NAMESPACE"
57-
StorageDir = "STORAGE_DIR"
58-
StorageAddr = "STORAGE_ADDR"
59-
AgentServiceName = "AGENT_SERVICE_NAME"
60-
MongoOplogSize = "MONGO_OPLOG_SIZE"
53+
LxcBridge = "LXC_BRIDGE"
54+
ProviderType = "PROVIDER_TYPE"
55+
ContainerType = "CONTAINER_TYPE"
56+
Namespace = "NAMESPACE"
57+
StorageDir = "STORAGE_DIR"
58+
StorageAddr = "STORAGE_ADDR"
59+
AgentServiceName = "AGENT_SERVICE_NAME"
60+
MongoOplogSize = "MONGO_OPLOG_SIZE"
61+
NumaCtlPreference = "NUMA_CTL_PREFERENCE"
6162
)
6263

6364
// The Config interface is the sole way that the agent gets access to the

‎cmd/jujud/bootstrap.go

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,17 @@ func newEnsureServerParams(agentConfig agent.Config) (mongo.EnsureServerParams,
243243
}
244244
}
245245

246+
// If numa ctl preference is specified in the agent configuration, use that.
247+
// Otherwise leave the default false value to indicate to EnsureServer
248+
// that numactl should not be used.
249+
var wantNumaCTL bool
250+
if numaCtlString := agentConfig.Value(agent.NumaCtlPreference); numaCtlString != "" {
251+
var err error
252+
if wantNumaCTL, err = strconv.ParseBool(numaCtlString); err != nil {
253+
return mongo.EnsureServerParams{}, fmt.Errorf("invalid numactl preference: %q", numaCtlString)
254+
}
255+
}
256+
246257
si, ok := agentConfig.StateServingInfo()
247258
if !ok {
248259
return mongo.EnsureServerParams{}, fmt.Errorf("agent config has no state serving info")
@@ -256,9 +267,10 @@ func newEnsureServerParams(agentConfig agent.Config) (mongo.EnsureServerParams,
256267
SharedSecret: si.SharedSecret,
257268
SystemIdentity: si.SystemIdentity,
258269

259-
DataDir: agentConfig.DataDir(),
260-
Namespace: agentConfig.Value(agent.Namespace),
261-
OplogSize: oplogSize,
270+
DataDir: agentConfig.DataDir(),
271+
Namespace: agentConfig.Value(agent.Namespace),
272+
OplogSize: oplogSize,
273+
WantNumaCTL: wantNumaCTL,
262274
}
263275
return params, nil
264276
}

‎environs/cloudinit.go

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,12 @@ func FinishMachineConfig(mcfg *cloudinit.MachineConfig, cfg *config.Config) (err
159159
return err
160160
}
161161

162+
if isStateMachineConfig(mcfg) {
163+
// Add NUMACTL preference. Needed to work for both bootstrap and high availability
164+
// Only makes sens for state server
165+
logger.Debugf("Setting numa ctl preference to %q", cfg.NumaCtlPreference())
166+
mcfg.AgentEnvironment[agent.NumaCtlPreference] = cfg.NumaCtlPreference()
167+
}
162168
// The following settings are only appropriate at bootstrap time. At the
163169
// moment, the only state server is the bootstrap node, but this
164170
// will probably change.
@@ -200,6 +206,18 @@ func FinishMachineConfig(mcfg *cloudinit.MachineConfig, cfg *config.Config) (err
200206
return nil
201207
}
202208

209+
// isStateMachineConfig determines if given machine configuration
210+
// is for State Server by iterating over machine's jobs.
211+
// If JobManageEnviron is present, this is a state server.
212+
func isStateMachineConfig(mcfg *cloudinit.MachineConfig) bool {
213+
for _, aJob := range mcfg.Jobs {
214+
if aJob == params.JobManageEnviron {
215+
return true
216+
}
217+
}
218+
return false
219+
}
220+
203221
func configureCloudinit(mcfg *cloudinit.MachineConfig, cloudcfg *coreCloudinit.Config) (cloudinit.UserdataConfig, error) {
204222
// When bootstrapping, we only want to apt-get update/upgrade
205223
// and setup the SSH keys. The rest we leave to cloudinit/sshinit.

‎environs/config/config.go

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,9 @@ const (
6868
// fallbackLtsSeries is the latest LTS series we'll use, if we fail to
6969
// obtain this information from the system.
7070
fallbackLtsSeries string = "trusty"
71+
72+
// Only use numactl if user specifically requests it
73+
DefaultNumaCtlPreference = "false"
7174
)
7275

7376
// TODO(katco-): Please grow this over time.
@@ -111,6 +114,9 @@ const (
111114
// LxcClone stores the value for this setting.
112115
LxcClone = "lxc-clone"
113116

117+
// NumaCtlPreference stores the value for this setting
118+
NumaCtlPreferenceKey = "use-numa-ctl"
119+
114120
//
115121
// Deprecated Settings Attributes
116122
//
@@ -702,6 +708,14 @@ func (c *Config) SyslogPort() int {
702708
return c.mustInt("syslog-port")
703709
}
704710

711+
// NumaCtlPreference returns if numactl is preferred.
712+
func (c *Config) NumaCtlPreference() string {
713+
if numa, ok := c.defined[NumaCtlPreferenceKey]; ok {
714+
return numa.(string)
715+
}
716+
return DefaultNumaCtlPreference
717+
}
718+
705719
// RsyslogCACert returns the certificate of the CA that signed the
706720
// rsyslog certificate, in PEM format, or nil if one hasn't been
707721
// generated yet.
@@ -1081,6 +1095,7 @@ var fields = schema.Fields{
10811095
"enable-os-refresh-update": schema.Bool(),
10821096
"enable-os-upgrade": schema.Bool(),
10831097
"disable-network-management": schema.Bool(),
1098+
NumaCtlPreferenceKey: schema.String(),
10841099

10851100
// Deprecated fields, retain for backwards compatibility.
10861101
ToolsMetadataURLKey: schema.String(),
@@ -1121,6 +1136,7 @@ var alwaysOptional = schema.Defaults{
11211136
LxcClone: schema.Omit,
11221137
"disable-network-management": schema.Omit,
11231138
AgentStreamKey: schema.Omit,
1139+
NumaCtlPreferenceKey: DefaultNumaCtlPreference,
11241140

11251141
// Deprecated fields, retain for backwards compatibility.
11261142
ToolsMetadataURLKey: "",
@@ -1183,6 +1199,7 @@ func allDefaults() schema.Defaults {
11831199
"proxy-ssh": true,
11841200
"prefer-ipv6": false,
11851201
"disable-network-management": false,
1202+
NumaCtlPreferenceKey: DefaultNumaCtlPreference,
11861203
}
11871204
for attr, val := range alwaysOptional {
11881205
if _, ok := d[attr]; !ok {

‎environs/config/config_test.go

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -343,6 +343,22 @@ var configTests = []configTest{
343343
"name": "my-name",
344344
"disable-network-management": true,
345345
},
346+
}, {
347+
about: "use-numa-ctl on",
348+
useDefaults: config.UseDefaults,
349+
attrs: testing.Attrs{
350+
"type": "my-type",
351+
"name": "my-name",
352+
"use-numa-ctl": "true",
353+
},
354+
}, {
355+
about: "use-numa-ctl off",
356+
useDefaults: config.UseDefaults,
357+
attrs: testing.Attrs{
358+
"type": "my-type",
359+
"name": "my-name",
360+
"use-numa-ctl": "false",
361+
},
346362
}, {
347363
about: "Invalid prefer-ipv6 flag",
348364
useDefaults: config.UseDefaults,
@@ -1264,6 +1280,7 @@ func (s *ConfigSuite) TestConfigAttrs(c *gc.C) {
12641280
attrs["proxy-ssh"] = false
12651281
attrs["lxc-clone-aufs"] = false
12661282
attrs["prefer-ipv6"] = false
1283+
attrs["use-numa-ctl"] = "false"
12671284

12681285
// Default firewall mode is instance
12691286
attrs["firewall-mode"] = string(config.FwInstance)

‎mongo/mongo.go

Lines changed: 40 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,22 @@ var (
5555
upstartServiceStopAndRemove = (*upstart.Service).StopAndRemove
5656
upstartServiceStop = (*upstart.Service).Stop
5757
upstartServiceStart = (*upstart.Service).Start
58+
59+
// This is NUMACTL package name for apt-get
60+
numaCtlPkg = "numactl"
61+
// This is the name of the variable to use in ExtraScript
62+
// fragment to substitu into upstart template.
63+
multinodeVarName = "MULTI_NODE"
64+
// This value will be used to wrap desired mongo cmd in numactl if wanted/needed
65+
numaCtlWrap = "$%v"
66+
// Extra shell script fragment for upstart template.
67+
// This determines if we are dealing with multi-node environment
68+
detectMultiNodeScript = `%v=""
69+
if [ $(find /sys/devices/system/node/ -maxdepth 1 -mindepth 1 -type d -name node\* | wc -l ) -gt 1 ]
70+
then
71+
%v=" numactl --interleave=all "
72+
fi
73+
`
5874
)
5975

6076
// WithAddresses represents an entity that has a set of
@@ -131,26 +147,6 @@ func Path() (string, error) {
131147
return path, nil
132148
}
133149

134-
// constructCmdWrap returns a wrapper to be used when running mongoDB command.
135-
// This should elevate operational difficulties with db on some machines, eg. NUMA.
136-
// If wrapper is not needed, return empty string.
137-
func constructCmdWrap() string {
138-
// Atm, this only needs to be done for NUMA machines.
139-
desiredWrap := "numactl"
140-
// Determine the executable path to run numactl on this machine.
141-
path, err := exec.LookPath(desiredWrap)
142-
if err != nil {
143-
// This will happen for non-NUMA machines. No wrap is needed.
144-
return ""
145-
}
146-
// When running MongoDB servers and clients on NUMA hardware,
147-
// a memory interleave policy needs to be configured
148-
// so that the host behaves in a non-NUMA fashion.
149-
// Fix for Bug#1350337.
150-
logger.Debugf("using %q to wrap the command", desiredWrap)
151-
return fmt.Sprintf(" %v --interleave=all ", path)
152-
}
153-
154150
// RemoveService removes the mongoDB upstart service from this machine.
155151
func RemoveService(namespace string) error {
156152
svc := upstart.NewService(ServiceName(namespace), common.Conf{})
@@ -189,6 +185,9 @@ type EnsureServerParams struct {
189185
// calculate a default size according to the
190186
// algorithm defined in Mongo.
191187
OplogSize int
188+
189+
// Numactl preference - whether the user wants to run numactl.
190+
WantNumaCTL bool
192191
}
193192

194193
// EnsureServer ensures that the correct mongo upstart script is installed
@@ -219,7 +218,7 @@ func EnsureServer(args EnsureServerParams) error {
219218
}
220219
}
221220

222-
if err := aptGetInstallMongod(); err != nil {
221+
if err := aptGetInstallMongod(args.WantNumaCTL); err != nil {
223222
return fmt.Errorf("cannot install mongod: %v", err)
224223
}
225224
mongoPath, err := Path()
@@ -228,7 +227,7 @@ func EnsureServer(args EnsureServerParams) error {
228227
}
229228
logVersion(mongoPath)
230229

231-
svc, err := upstartService(args.Namespace, args.DataDir, dbDir, mongoPath, args.StatePort, oplogSizeMB)
230+
svc, err := upstartService(args.Namespace, args.DataDir, dbDir, mongoPath, args.StatePort, oplogSizeMB, args.WantNumaCTL)
232231
if err != nil {
233232
return err
234233
}
@@ -321,7 +320,7 @@ func sharedSecretPath(dataDir string) string {
321320
// upstartService returns the upstart config for the mongo state service.
322321
// It also returns the path to the mongod executable that the upstart config
323322
// will be using.
324-
func upstartService(namespace, dataDir, dbDir, mongoPath string, port, oplogSizeMB int) (*upstart.Service, error) {
323+
func upstartService(namespace, dataDir, dbDir, mongoPath string, port, oplogSizeMB int, wantNumaCtl bool) (*upstart.Service, error) {
325324
mongoCmd := mongoPath + " --auth" +
326325
" --dbpath=" + utils.ShQuote(dbDir) +
327326
" --sslOnNormalPorts" +
@@ -336,28 +335,41 @@ func upstartService(namespace, dataDir, dbDir, mongoPath string, port, oplogSize
336335
" --replSet " + ReplicaSetName +
337336
" --ipv6 " +
338337
" --oplogSize " + strconv.Itoa(oplogSizeMB)
338+
extraScript := ""
339+
if wantNumaCtl {
340+
extraScript = fmt.Sprintf(detectMultiNodeScript, multinodeVarName, multinodeVarName)
341+
mongoCmd = fmt.Sprintf(numaCtlWrap, multinodeVarName) + mongoCmd
342+
}
339343
conf := common.Conf{
340344
Desc: "juju state database",
341345
Limit: map[string]string{
342346
"nofile": fmt.Sprintf("%d %d", maxFiles, maxFiles),
343347
"nproc": fmt.Sprintf("%d %d", maxProcs, maxProcs),
344348
},
345-
Cmd: constructCmdWrap() + mongoCmd,
349+
ExtraScript: extraScript,
350+
Cmd: mongoCmd,
346351
}
347352
svc := upstart.NewService(ServiceName(namespace), conf)
348353
return svc, nil
349354
}
350355

351-
func aptGetInstallMongod() error {
356+
func aptGetInstallMongod(numaCtl bool) error {
352357
// Only Quantal requires the PPA.
353358
if version.Current.Series == "quantal" {
354359
if err := addAptRepository("ppa:juju/stable"); err != nil {
355360
return err
356361
}
357362
}
358-
pkg := packageForSeries(version.Current.Series)
359-
cmds := apt.GetPreparePackages([]string{pkg}, version.Current.Series)
360-
logger.Infof("installing %s", pkg)
363+
mongoPkg := packageForSeries(version.Current.Series)
364+
365+
aptPkgs := []string{mongoPkg}
366+
if numaCtl {
367+
aptPkgs = []string{mongoPkg, numaCtlPkg}
368+
logger.Infof("installing %s and %s", mongoPkg, numaCtlPkg)
369+
} else {
370+
logger.Infof("installing %s", mongoPkg)
371+
}
372+
cmds := apt.GetPreparePackages(aptPkgs, version.Current.Series)
361373
for _, cmd := range cmds {
362374
if err := apt.GetInstall(cmd...); err != nil {
363375
return err

‎mongo/mongo_test.go

Lines changed: 40 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,7 @@ func (s *MongoSuite) TestEnsureServer(c *gc.C) {
170170
c.Assert(service.Conf.InitDir, gc.Equals, "/etc/init")
171171
c.Assert(service.Conf.Desc, gc.Equals, "juju state database")
172172
c.Assert(service.Conf.Cmd, gc.Matches, regexp.QuoteMeta(s.mongodPath)+".*")
173+
c.Assert(service.Conf.ExtraScript, gc.Matches, "")
173174
// TODO(nate) set Out so that mongod output goes somewhere useful?
174175
c.Assert(service.Conf.Out, gc.Equals, "")
175176
}
@@ -262,6 +263,34 @@ func (s *MongoSuite) TestEnsureServerServerExistsNotRunningStartError(c *gc.C) {
262263
c.Assert(s.installed, gc.HasLen, 0)
263264
}
264265

266+
func (s *MongoSuite) TestEnsureServerNumaCtl(c *gc.C) {
267+
dataDir := c.MkDir()
268+
dbDir := filepath.Join(dataDir, "db")
269+
namespace := "namespace"
270+
271+
mockShellCommand(c, &s.CleanupSuite, "apt-get")
272+
273+
testParams := makeEnsureServerParams(dataDir, namespace)
274+
testParams.WantNumaCTL = true
275+
err := mongo.EnsureServer(testParams)
276+
c.Assert(err, gc.IsNil)
277+
278+
testJournalDirs(dbDir, c)
279+
280+
assertInstalled := func() {
281+
c.Assert(s.installed, gc.HasLen, 1)
282+
service := s.installed[0]
283+
c.Assert(service.Name, gc.Equals, "juju-db-namespace")
284+
c.Assert(service.Conf.InitDir, gc.Equals, "/etc/init")
285+
c.Assert(service.Conf.Desc, gc.Equals, "juju state database")
286+
c.Assert(service.Conf.ExtraScript, gc.Not(gc.Matches), "")
287+
c.Assert(service.Conf.Cmd, gc.Matches, ".*"+regexp.QuoteMeta(s.mongodPath)+".*")
288+
// TODO(nate) set Out so that mongod output goes somewhere useful?
289+
c.Assert(service.Conf.Out, gc.Equals, "")
290+
}
291+
assertInstalled()
292+
}
293+
265294
func (s *MongoSuite) TestInstallMongod(c *gc.C) {
266295
type installs struct {
267296
series string
@@ -330,23 +359,31 @@ func (s *MongoSuite) TestInstallMongodServiceExists(c *gc.C) {
330359
func (s *MongoSuite) TestUpstartServiceWithReplSet(c *gc.C) {
331360
dataDir := c.MkDir()
332361

333-
svc, err := mongo.UpstartService("", dataDir, dataDir, mongo.JujuMongodPath, 1234, 1024)
362+
svc, err := mongo.UpstartService("", dataDir, dataDir, mongo.JujuMongodPath, 1234, 1024, false)
334363
c.Assert(err, gc.IsNil)
335364
c.Assert(strings.Contains(svc.Conf.Cmd, "--replSet"), jc.IsTrue)
336365
}
337366

367+
func (s *MongoSuite) TestUpstartServiceWithNumCtl(c *gc.C) {
368+
dataDir := c.MkDir()
369+
370+
svc, err := mongo.UpstartService("", dataDir, dataDir, mongo.JujuMongodPath, 1234, 1024, true)
371+
c.Assert(err, gc.IsNil)
372+
c.Assert(svc.Conf.ExtraScript, gc.Not(gc.Matches), "")
373+
}
374+
338375
func (s *MongoSuite) TestUpstartServiceIPv6(c *gc.C) {
339376
dataDir := c.MkDir()
340377

341-
svc, err := mongo.UpstartService("", dataDir, dataDir, mongo.JujuMongodPath, 1234, 1024)
378+
svc, err := mongo.UpstartService("", dataDir, dataDir, mongo.JujuMongodPath, 1234, 1024, false)
342379
c.Assert(err, gc.IsNil)
343380
c.Assert(strings.Contains(svc.Conf.Cmd, "--ipv6"), jc.IsTrue)
344381
}
345382

346383
func (s *MongoSuite) TestUpstartServiceWithJournal(c *gc.C) {
347384
dataDir := c.MkDir()
348385

349-
svc, err := mongo.UpstartService("", dataDir, dataDir, mongo.JujuMongodPath, 1234, 1024)
386+
svc, err := mongo.UpstartService("", dataDir, dataDir, mongo.JujuMongodPath, 1234, 1024, false)
350387
c.Assert(err, gc.IsNil)
351388
journalPresent := strings.Contains(svc.Conf.Cmd, " --journal ") || strings.HasSuffix(svc.Conf.Cmd, " --journal")
352389
c.Assert(journalPresent, jc.IsTrue)

‎service/common/common.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ type Conf struct {
1919
// InitDir is the folder in which the init/upstart script should be written
2020
// defaults to "/etc/init" on Ubuntu
2121
// Currently not used on Windows
22-
2322
InitDir string
23+
// ExtraScript allows to insert script before command execution
24+
ExtraScript string
2425
}

0 commit comments

Comments
 (0)