Skip to content

Refactor Layer2 Speaker #380

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ks-ci-bot merged 4 commits into from
Jan 25, 2024
Merged

Refactor Layer2 Speaker #380

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
82bb452
handle reconcile error requeue
renyunkang Nov 1, 2023
c2e77b1
handle reconcile error requeue
renyunkang Nov 1, 2023
c7d0168
Merge branch 'refactor' of github.com:renyunkang/openelb into refactor
renyunkang Jan 3, 2024
d2c43e9
refactor(speaker): refactor layer2 speaker
renyunkang Jan 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 1 addition & 7 deletions api/v1alpha2/eip_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,13 +55,7 @@ func (e Eip) GetSpeakerName() string {
return constant.OpenELBProtocolDummy
}

if e.Spec.Protocol == constant.OpenELBProtocolLayer2 {
return e.Spec.Interface
}
if e.Spec.Protocol == constant.OpenELBProtocolVip {
return constant.OpenELBProtocolVip
}
return constant.OpenELBProtocolBGP
return e.GetProtocol()
}

func (e Eip) GetProtocol() string {
Expand Down
6 changes: 4 additions & 2 deletions build/speaker/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,11 @@ WORKDIR /workspace
ADD . /workspace/

RUN make speaker
RUN go build -o bin/cmd/gobgp github.com/osrg/gobgp/cmd/gobgp
RUN mv /workspace/bin/cmd/speaker ${OUTDIR}/usr/local/bin/openelb-speaker
RUN mv /workspace/bin/cmd/gobgp ${OUTDIR}/usr/local/bin/gobgp

RUN GO111MODULE=on CGO_ENABLED=0 go install -ldflags '-w -s' github.com/osrg/gobgp/cmd/gobgp
RUN mv /go/bin/gobgp ${OUTDIR}/usr/local/bin/gobgp



##############
Expand Down
17 changes: 17 additions & 0 deletions cmd/speaker/app/options/options.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,21 @@
package options

import (
"flag"
"strings"

"github.com/openelb/openelb/pkg/log"
"github.com/openelb/openelb/pkg/speaker/bgp"
"github.com/openelb/openelb/pkg/speaker/layer2"
"github.com/openelb/openelb/pkg/speaker/vip"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/klog/v2"
)

type OpenELBSpeakerOptions struct {
MetricsAddr string
Bgp *bgp.BgpOptions
Layer2 *layer2.Options
Vip *vip.VipOptions
LogOptions *log.Options
}
Expand All @@ -18,25 +24,36 @@ func NewOpenELBSpeakerOptions() *OpenELBSpeakerOptions {
return &OpenELBSpeakerOptions{
MetricsAddr: ":50053",
Bgp: bgp.NewBgpOptions(),
Layer2: layer2.NewOptions(),
Vip: vip.NewVipOptions(),
LogOptions: log.NewOptions(),
}
}

func (s *OpenELBSpeakerOptions) Validate() []error {
var errs []error

return errs
}

func (s *OpenELBSpeakerOptions) Flags() cliflag.NamedFlagSets {
fss := cliflag.NamedFlagSets{}

s.Bgp.AddFlags(fss.FlagSet("bgp"))
s.Layer2.AddFlags(fss.FlagSet("layer2"))
s.Vip.AddFlags(fss.FlagSet("vip"))
s.LogOptions.AddFlags(fss.FlagSet("log"))

fs := fss.FlagSet("generic")
fs.StringVar(&s.MetricsAddr, "metrics-addr", s.MetricsAddr, "The address the metric endpoint binds to.")

kfs := fss.FlagSet("klog")
local := flag.NewFlagSet("klog", flag.ExitOnError)
klog.InitFlags(local)
local.VisitAll(func(fl *flag.Flag) {
fl.Name = strings.Replace(fl.Name, "_", "-", -1)
kfs.AddGoFlag(fl)
})

return fss
}
19 changes: 18 additions & 1 deletion cmd/speaker/app/speaker.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
_ "github.com/openelb/openelb/pkg/metrics"
"github.com/openelb/openelb/pkg/speaker"
bgpd "github.com/openelb/openelb/pkg/speaker/bgp"
"github.com/openelb/openelb/pkg/speaker/layer2"
"github.com/openelb/openelb/pkg/speaker/vip"
"github.com/openelb/openelb/pkg/util"
"github.com/openelb/openelb/pkg/version"
Expand All @@ -24,6 +25,7 @@ import (
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/term"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
)

Expand Down Expand Up @@ -136,12 +138,27 @@ func Run(opt *options.OpenELBSpeakerOptions) error {
vip.Clean(k8sClient)
}

// TODO: for layer2 mode
// for layer2 mode
reloadChan := make(chan event.GenericEvent)
if opt.Layer2.EnableLayer2 {
layer2speaker, err := layer2.NewSpeaker(k8sClient, opt.Layer2, reloadChan, spmanager.Queue)
if err != nil {
setupLog.Error(err, "unable to new layer2 speaker")
return err
}

if err := spmanager.RegisterSpeaker(constant.OpenELBProtocolLayer2, layer2speaker); err != nil {
setupLog.Error(err, "unable to register keepalive speaker")
return err
}
}

if err := (&speaker.LBReconciler{
Handler: spmanager.HandleService,
Reloader: spmanager.ResyncServices,
Client: mgr.GetClient(),
EventRecorder: mgr.GetEventRecorderFor("lb"),
Reload: reloadChan,
}).SetupWithManager(mgr); err != nil {
setupLog.Error(err, "unable to setup lb")
return err
Expand Down
8 changes: 8 additions & 0 deletions config/rbac/role.yaml
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion config/webhook/kustomization.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
resources:
- service.yaml
#- manifests.yaml
- webhook.yaml
2 changes: 1 addition & 1 deletion config/webhook/manifests.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@

---
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingWebhookConfiguration
metadata:
Expand Down
7 changes: 7 additions & 0 deletions config/workloads/cert.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,11 @@
apiVersion: v1
data:
key: b3BlbmVsYi1zcGVha2Vycw==
kind: Secret
metadata:
name: memberlist
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: openelb-admission
Expand Down
17 changes: 17 additions & 0 deletions config/workloads/speaker.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,20 @@ spec:
- --api-hosts=:50051
- --zap-log-level=info
- --enable-keepalived-vip=false
- --enable-layer2=false
image: speaker:latest
imagePullPolicy: IfNotPresent
readinessProbe:
exec:
command:
- sh
- -c
- |
gobgp -p 50051 global
failureThreshold: 3
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
name: openelb-speaker
securityContext:
capabilities:
Expand All @@ -48,6 +60,11 @@ spec:
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: MEMBER_LIST_SECRET
valueFrom:
secretKeyRef:
name: memberlist
key: key
- name: OPENELB_DSNAME
valueFrom:
fieldRef:
Expand Down
25 changes: 25 additions & 0 deletions deploy/openelb.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1122,6 +1122,14 @@ subjects:
namespace: openelb-system
---
apiVersion: v1
data:
key: b3BlbmVsYi1zcGVha2Vycw==
Copy link
Contributor

@chaunceyjiang chaunceyjiang Jan 25, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just a question, where did the b3BlbmVsYi1zcGVha2Vycw== of this key come from?

Copy link
Member Author

@renyunkang renyunkang Jan 25, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is the base64 encoding of "openelb-speakers". As you can see, I use "openelb-speakers" as the default secret for the memberlist in OpenELB. Of course, you can also customize the secret by modifying the Secret resource.

kind: Secret
metadata:
name: memberlist
namespace: openelb-system
---
apiVersion: v1
kind: Service
metadata:
name: openelb-admission
Expand Down Expand Up @@ -1248,6 +1256,7 @@ spec:
- --api-hosts=:50051
- --zap-log-level=info
- --enable-keepalived-vip=false
- --enable-layer2=false
command:
- openelb-speaker
env:
Expand All @@ -1259,13 +1268,29 @@ spec:
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: MEMBER_LIST_SECRET
valueFrom:
secretKeyRef:
key: key
name: memberlist
- name: OPENELB_DSNAME
valueFrom:
fieldRef:
fieldPath: metadata.name
image: kubesphere/openelb-speaker:v0.6.0
imagePullPolicy: IfNotPresent
name: openelb-speaker
readinessProbe:
exec:
command:
- sh
- -c
- |
gobgp -p 50051 global
failureThreshold: 3
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
resources:
limits:
cpu: 100m
Expand Down
11 changes: 11 additions & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ require (
github.com/NYTimes/gziphandler v1.1.1 // indirect
github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e // indirect
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da // indirect
github.com/aws/aws-sdk-go v1.44.116 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/blang/semver/v4 v4.0.0 // indirect
Expand Down Expand Up @@ -94,6 +95,7 @@ require (
github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/mock v1.6.0 // indirect
github.com/google/btree v1.0.1 // indirect
github.com/google/cadvisor v0.46.0 // indirect
github.com/google/cel-go v0.12.6 // indirect
github.com/google/gnostic v0.5.7-v3refs // indirect
Expand All @@ -104,7 +106,14 @@ require (
github.com/googleapis/gax-go/v2 v2.1.1 // indirect
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
github.com/hashicorp/errwrap v1.0.0 // indirect
github.com/hashicorp/go-immutable-radix v1.0.0 // indirect
github.com/hashicorp/go-msgpack v0.5.3 // indirect
github.com/hashicorp/go-multierror v1.0.0 // indirect
github.com/hashicorp/go-sockaddr v1.0.0 // indirect
github.com/hashicorp/golang-lru v0.5.1 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/hashicorp/memberlist v0.5.0
github.com/imdario/mergo v0.3.6 // indirect
github.com/inconshreveable/mousetrap v1.0.1 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
Expand All @@ -117,6 +126,7 @@ require (
github.com/magiconair/properties v1.8.1 // indirect
github.com/mailru/easyjson v0.7.6 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect
github.com/miekg/dns v1.1.26 // indirect
github.com/mindprince/gonvml v0.0.0-20190828220739-9ebdce4bb989 // indirect
github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible // indirect
github.com/mitchellh/mapstructure v1.4.1 // indirect
Expand All @@ -140,6 +150,7 @@ require (
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
github.com/rubiojr/go-vhd v0.0.0-20200706105327-02e210299021 // indirect
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 // indirect
github.com/spf13/afero v1.2.2 // indirect
github.com/spf13/cast v1.3.0 // indirect
Expand Down
Loading