Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pkg/controlplane/apiserver/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ func BuildGenericConfig(
ctx := wait.ContextForChannel(genericConfig.DrainedNotify())

// Authentication.ApplyTo requires already applied OpenAPIConfig and EgressSelector if present
if lastErr = s.Authentication.ApplyTo(ctx, &genericConfig.Authentication, genericConfig.SecureServing, genericConfig.EgressSelector, genericConfig.OpenAPIConfig, genericConfig.OpenAPIV3Config, clientgoExternalClient, versionedInformers); lastErr != nil {
if lastErr = s.Authentication.ApplyTo(ctx, &genericConfig.Authentication, genericConfig.SecureServing, genericConfig.EgressSelector, genericConfig.OpenAPIConfig, genericConfig.OpenAPIV3Config, clientgoExternalClient, versionedInformers, genericConfig.APIServerID); lastErr != nil {
return
}

Expand Down
20 changes: 18 additions & 2 deletions pkg/kubeapiserver/options/authentication.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ import (
genericapiserver "k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/server/egressselector"
genericoptions "k8s.io/apiserver/pkg/server/options"
authenticationconfigmetrics "k8s.io/apiserver/pkg/server/options/authenticationconfig/metrics"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/apiserver/plugin/pkg/authenticator/token/oidc"
"k8s.io/client-go/informers"
Expand Down Expand Up @@ -588,7 +589,16 @@ func (o *BuiltInAuthenticationOptions) ToAuthenticationConfig() (kubeauthenticat

// ApplyTo requires already applied OpenAPIConfig and EgressSelector if present.
// The input context controls the lifecycle of background goroutines started to reload the authentication config file.
func (o *BuiltInAuthenticationOptions) ApplyTo(ctx context.Context, authInfo *genericapiserver.AuthenticationInfo, secureServing *genericapiserver.SecureServingInfo, egressSelector *egressselector.EgressSelector, openAPIConfig *openapicommon.Config, openAPIV3Config *openapicommon.OpenAPIV3Config, extclient kubernetes.Interface, versionedInformer informers.SharedInformerFactory) error {
func (o *BuiltInAuthenticationOptions) ApplyTo(
ctx context.Context,
authInfo *genericapiserver.AuthenticationInfo,
secureServing *genericapiserver.SecureServingInfo,
egressSelector *egressselector.EgressSelector,
openAPIConfig *openapicommon.Config,
openAPIV3Config *openapicommon.OpenAPIV3Config,
extclient kubernetes.Interface,
versionedInformer informers.SharedInformerFactory,
apiServerID string) error {
if o == nil {
return nil
}
Expand Down Expand Up @@ -654,14 +664,14 @@ func (o *BuiltInAuthenticationOptions) ApplyTo(ctx context.Context, authInfo *ge
authInfo.Authenticator = authenticator

if len(o.AuthenticationConfigFile) > 0 {
authenticationconfigmetrics.RegisterMetrics()
trackedAuthenticationConfigData := authenticatorConfig.AuthenticationConfigData
var mu sync.Mutex
go filesystem.WatchUntil(
ctx,
time.Minute,
o.AuthenticationConfigFile,
func() {
// TODO add metrics
// TODO collapse onto shared logic with DynamicEncryptionConfigContent controller

mu.Lock()
Expand All @@ -670,6 +680,7 @@ func (o *BuiltInAuthenticationOptions) ApplyTo(ctx context.Context, authInfo *ge
authConfigBytes, err := os.ReadFile(o.AuthenticationConfigFile)
if err != nil {
klog.ErrorS(err, "failed to read authentication config file")
authenticationconfigmetrics.RecordAuthenticationConfigAutomaticReloadFailure(apiServerID)
// we do not update the tracker here because this error could eventually resolve as we keep retrying
return
}
Expand All @@ -683,13 +694,15 @@ func (o *BuiltInAuthenticationOptions) ApplyTo(ctx context.Context, authInfo *ge
authConfig, err := loadAuthenticationConfigFromData(authConfigBytes)
if err != nil {
klog.ErrorS(err, "failed to load authentication config")
authenticationconfigmetrics.RecordAuthenticationConfigAutomaticReloadFailure(apiServerID)
// this config is not structurally valid and never will be, update the tracker so we stop retrying
trackedAuthenticationConfigData = authConfigData
return
}

if err := apiservervalidation.ValidateAuthenticationConfiguration(authConfig, authenticatorConfig.ServiceAccountIssuers).ToAggregate(); err != nil {
klog.ErrorS(err, "failed to validate authentication config")
authenticationconfigmetrics.RecordAuthenticationConfigAutomaticReloadFailure(apiServerID)
// this config is not semantically valid and never will be, update the tracker so we stop retrying
trackedAuthenticationConfigData = authConfigData
return
Expand All @@ -699,11 +712,14 @@ func (o *BuiltInAuthenticationOptions) ApplyTo(ctx context.Context, authInfo *ge
defer timeoutCancel()
if err := updateAuthenticationConfig(timeoutCtx, authConfig); err != nil {
klog.ErrorS(err, "failed to update authentication config")
authenticationconfigmetrics.RecordAuthenticationConfigAutomaticReloadFailure(apiServerID)
// we do not update the tracker here because this error could eventually resolve as we keep retrying
return
}

trackedAuthenticationConfigData = authConfigData
klog.InfoS("reloaded authentication config")
authenticationconfigmetrics.RecordAuthenticationConfigAutomaticReloadSuccess(apiServerID)
},
func(err error) { klog.ErrorS(err, "watching authentication config file") },
)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
/*
Copyright 2024 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package metrics

import (
"crypto/sha256"
"fmt"
"sync"

"k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
)

const (
namespace = "apiserver"
subsystem = "authentication_config_controller"
)

var (
authenticationConfigAutomaticReloadsTotal = metrics.NewCounterVec(
&metrics.CounterOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "automatic_reloads_total",
Help: "Total number of automatic reloads of authentication configuration split by status and apiserver identity.",
StabilityLevel: metrics.ALPHA,
},
[]string{"status", "apiserver_id_hash"},
)

authenticationConfigAutomaticReloadLastTimestampSeconds = metrics.NewGaugeVec(
&metrics.GaugeOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "automatic_reload_last_timestamp_seconds",
Help: "Timestamp of the last automatic reload of authentication configuration split by status and apiserver identity.",
StabilityLevel: metrics.ALPHA,
},
[]string{"status", "apiserver_id_hash"},
)
)

var registerMetrics sync.Once

func RegisterMetrics() {
registerMetrics.Do(func() {
legacyregistry.MustRegister(authenticationConfigAutomaticReloadsTotal)
legacyregistry.MustRegister(authenticationConfigAutomaticReloadLastTimestampSeconds)
})
}

func ResetMetricsForTest() {
authenticationConfigAutomaticReloadsTotal.Reset()
authenticationConfigAutomaticReloadLastTimestampSeconds.Reset()
}

func RecordAuthenticationConfigAutomaticReloadFailure(apiServerID string) {
apiServerIDHash := getHash(apiServerID)
authenticationConfigAutomaticReloadsTotal.WithLabelValues("failure", apiServerIDHash).Inc()
authenticationConfigAutomaticReloadLastTimestampSeconds.WithLabelValues("failure", apiServerIDHash).SetToCurrentTime()
}

func RecordAuthenticationConfigAutomaticReloadSuccess(apiServerID string) {
apiServerIDHash := getHash(apiServerID)
authenticationConfigAutomaticReloadsTotal.WithLabelValues("success", apiServerIDHash).Inc()
authenticationConfigAutomaticReloadLastTimestampSeconds.WithLabelValues("success", apiServerIDHash).SetToCurrentTime()
}

func getHash(data string) string {
if len(data) == 0 {
return ""
}
return fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(data)))
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
/*
Copyright 2024 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package metrics

import (
"strings"
"testing"

"k8s.io/component-base/metrics/legacyregistry"
"k8s.io/component-base/metrics/testutil"
)

const (
testAPIServerID = "testAPIServerID"
testAPIServerIDHash = "sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37"
)

func TestRecordAuthenticationConfigAutomaticReloadFailure(t *testing.T) {
expectedValue := `
# HELP apiserver_authentication_config_controller_automatic_reloads_total [ALPHA] Total number of automatic reloads of authentication configuration split by status and apiserver identity.
# TYPE apiserver_authentication_config_controller_automatic_reloads_total counter
apiserver_authentication_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="failure"} 1
`
metrics := []string{
namespace + "_" + subsystem + "_automatic_reloads_total",
}

authenticationConfigAutomaticReloadsTotal.Reset()
RegisterMetrics()

RecordAuthenticationConfigAutomaticReloadFailure(testAPIServerID)
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
t.Fatal(err)
}
}

func TestRecordAuthenticationConfigAutomaticReloadSuccess(t *testing.T) {
expectedValue := `
# HELP apiserver_authentication_config_controller_automatic_reloads_total [ALPHA] Total number of automatic reloads of authentication configuration split by status and apiserver identity.
# TYPE apiserver_authentication_config_controller_automatic_reloads_total counter
apiserver_authentication_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 1
`
metrics := []string{
namespace + "_" + subsystem + "_automatic_reloads_total",
}

authenticationConfigAutomaticReloadsTotal.Reset()
RegisterMetrics()

RecordAuthenticationConfigAutomaticReloadSuccess(testAPIServerID)
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
t.Fatal(err)
}
}

func TestAuthenticationConfigAutomaticReloadLastTimestampSeconds(t *testing.T) {
testCases := []struct {
expectedValue string
resultLabel string
timestamp int64
}{
{
expectedValue: `
# HELP apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds [ALPHA] Timestamp of the last automatic reload of authentication configuration split by status and apiserver identity.
# TYPE apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds gauge
apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="failure"} 1.689101941e+09
`,
resultLabel: "failure",
timestamp: 1689101941,
},
{
expectedValue: `
# HELP apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds [ALPHA] Timestamp of the last automatic reload of authentication configuration split by status and apiserver identity.
# TYPE apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds gauge
apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 1.689101941e+09
`,
resultLabel: "success",
timestamp: 1689101941,
},
}

metrics := []string{
namespace + "_" + subsystem + "_automatic_reload_last_timestamp_seconds",
}
RegisterMetrics()

for _, tc := range testCases {
authenticationConfigAutomaticReloadLastTimestampSeconds.Reset()
authenticationConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(tc.resultLabel, testAPIServerIDHash).Set(float64(tc.timestamp))

if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tc.expectedValue), metrics...); err != nil {
t.Fatal(err)
}
}
}
Loading