Skip to content

Commit

Permalink
✨ use iptables to let traffic in (#66)
Browse files Browse the repository at this point in the history
  • Loading branch information
Xuetao Song authored and ks-ci-bot committed Sep 17, 2019
1 parent bc7c58a commit 0996766
Show file tree
Hide file tree
Showing 17 changed files with 288 additions and 214 deletions.
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

## Principle

The following figure desribes the principle of Porter. Suppose there is a distributed service on node1 (192.168.0.2) and node2 (192.168.0.6). The service needs to be accessed through EIP `1.1.1.1`. After deploying the [Example Service](https://github.com/kubesphere/porter/blob/master/config/samples/service.yaml), Porter will automatically synchronize routing information to the leaf switch, and then synchronize to the spine and border switch, thus external users can access the service through EIP `1.1.1.1`.
The following figure desribes the principle of Porter. Suppose there is a distributed service on node1 (192.168.0.2) and node2 (192.168.0.6). The service needs to be accessed through EIP `1.1.1.1`. After deploying the [Example Service](https://github.com/kubesphere/porter/blob/master/test/samples/test.yaml), Porter will automatically synchronize routing information to the leaf switch, and then synchronize to the spine and border switch, thus external users can access the service through EIP `1.1.1.1`.

![node architecture](doc/img/node-arch.png)

Expand Down
2 changes: 1 addition & 1 deletion README_zh.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ Porter 是 [KubeSphere](https://kubesphere.io/) 的一个子项目。

## 物理部署架构

下图是物理部署架构图,假设有一个服务部署在 node1 (192.168.0.2) 和 node2 (192.168.0.6) 上,需要通过公网 IP 1.1.1.1 访问该服务,服务部署人员按照[示例](config/sample/service.yaml)部署该服务后,Porter 会自动同步路由信息到 leaf 交换机,进而同步到 spine,border 交换机,互联网用户就可以通过 EIP 1.1.1.1 直接访问该服务了。
下图是物理部署架构图,假设有一个服务部署在 node1 (192.168.0.2) 和 node2 (192.168.0.6) 上,需要通过公网 IP 1.1.1.1 访问该服务,服务部署人员按照[示例](test/samples/test.yaml)部署该服务后,Porter 会自动同步路由信息到 leaf 交换机,进而同步到 spine,border 交换机,互联网用户就可以通过 EIP 1.1.1.1 直接访问该服务了。

![node architecture](doc/img/node-arch.png)

Expand Down
2 changes: 2 additions & 0 deletions cmd/manager/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"github.com/kubesphere/porter/controllers/lb"
bgpserver "github.com/kubesphere/porter/pkg/bgp/serverd"
"github.com/kubesphere/porter/pkg/ipam"
"github.com/kubesphere/porter/pkg/nettool/iptables"
"github.com/kubesphere/porter/pkg/route"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
Expand Down Expand Up @@ -62,6 +63,7 @@ func main() {
//starting bgp server
setupLog.Info("starting bgp server")
ready := make(chan interface{})
bgpStartOption.IptablesIface = iptables.NewIPTables()
go bgpserver.Run(bgpStartOption, ready)
<-ready
setupLog.Info("bgp server started successfully")
Expand Down
2 changes: 1 addition & 1 deletion config/dev/agent_image_patch.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,6 @@ spec:
spec:
containers:
# Change the value of image field below to your controller image URL
- image: kubespheredev/porter-agent:cea56a86
- image: kubespheredev/porter-agent:bc7c58a5
name: porter-agent
imagePullPolicy: Always
2 changes: 1 addition & 1 deletion config/dev/kustomization.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
namespace: porter-test-cea56a86
namespace: porter-test-bc7c58a5

bases:
- ../default
Expand Down
2 changes: 1 addition & 1 deletion config/dev/manager_image_patch.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,6 @@ spec:
spec:
containers:
# Change the value of image field below to your controller image URL
- image: kubespheredev/porter:cea56a86
- image: kubespheredev/porter:bc7c58a5
name: manager
imagePullPolicy: Always
18 changes: 9 additions & 9 deletions controllers/eip/eip_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,15 @@ import (
"os"
"time"

"k8s.io/client-go/util/retry"

"github.com/go-logr/logr"
"github.com/kiali/kiali/log"
networkv1alpha1 "github.com/kubesphere/porter/api/v1alpha1"
"github.com/kubesphere/porter/pkg/constant"
"github.com/kubesphere/porter/pkg/nettool/iptables"
"github.com/kubesphere/porter/pkg/util"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/client-go/tools/record"
"k8s.io/client-go/util/retry"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/event"
Expand All @@ -44,6 +44,7 @@ type EipReconciler struct {
client.Client
Log logr.Logger
record.EventRecorder
iptableExec iptables.IptablesIface
}

func (r *EipReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
Expand All @@ -54,12 +55,9 @@ func (r *EipReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
err := r.Client.Get(ctx, req.NamespacedName, instance)
if err != nil {
if errors.IsNotFound(err) {
// Object not found, return. Created objects are automatically garbage collected.
// For additional cleanup logic use finalizers.
r.Log.Info("EIP is deleted safely", "name", instance.GetName(), "namespace", instance.GetNamespace())
return ctrl.Result{}, nil
}
// Error reading the object - requeue the request.
return ctrl.Result{}, err
}
var deleted bool
Expand All @@ -77,9 +75,10 @@ func (r *EipReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
return ctrl.Result{RequeueAfter: time.Second * 10}, err
}
if !deleted {
err = r.AddRule(instance)
r.Log.Info("open forward for eip")
err = r.OpenEIPForward(instance.Spec.Address)
if err != nil {
r.Log.Info("Failed to add rule of eip, try again later")
r.Log.Info("Failed to open forward chain, try again later")
return ctrl.Result{RequeueAfter: time.Second * 10}, err
}
}
Expand All @@ -89,6 +88,7 @@ func (r *EipReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
func (r *EipReconciler) SetupWithManager(mgr ctrl.Manager) error {
r.Client = mgr.GetClient()
r.EventRecorder = mgr.GetEventRecorderFor("eip")
r.iptableExec = iptables.NewIPTables()
return ctrl.NewControllerManagedBy(mgr).
For(&networkv1alpha1.Eip{}).
WithEventFilter(predicate.Funcs{
Expand Down Expand Up @@ -126,8 +126,8 @@ func (r *EipReconciler) useFinalizerIfNeeded(eip *networkv1alpha1.Eip) (bool, er
if util.ContainsString(eip.ObjectMeta.Finalizers, agentFinalizer) {
r.Log.Info("Begin to remove finalizer")
// our finalizer is present, so lets handle our external dependency
if err := r.DeleteRule(eip); err != nil {
log.Error(nil, "Failed to delete route", "name", eip.GetName(), "namespace", eip.GetNamespace())
if err := r.CloseEIPForward(eip.Spec.Address); err != nil {
log.Error(nil, "Failed to delete rule in forward chain", "name", eip.GetName(), "namespace", eip.GetNamespace())
return true, err
}
// remove our finalizer from the list and update it.
Expand Down
104 changes: 54 additions & 50 deletions controllers/eip/netutil.go
Original file line number Diff line number Diff line change
@@ -1,59 +1,63 @@
package eip

import (
"fmt"
"os"

networkv1alpha1 "github.com/kubesphere/porter/api/v1alpha1"
"github.com/kubesphere/porter/pkg/nettool"
)

func (r *EipReconciler) AddRule(instance *networkv1alpha1.Eip) error {
if instance.Spec.Address != "" {
rule, err := nettool.NewEIPRule(instance.Spec.Address)
if err != nil {
r.Log.Info("Failed to initialize ip rule", "eip", instance.Spec.Address)
return err
}
nodeName := os.Getenv("MY_NODE_NAME")
if ok, err := rule.IsExist(); err != nil {
return err
} else {
if !ok {
err = rule.Add()
if err != nil {
return err
}
r.Event(instance, "Normal", "Rule Created", fmt.Sprintf("Created ip rule for EIP %s in agent %s", instance.Spec.Address, nodeName))
} else {
r.Log.Info("Detect rule in node")
r.Event(instance, "Normal", "Detect rule in node", fmt.Sprintf("Skipped Creating ip rule for EIP %s in agent %s", instance.Spec.Address, nodeName))
}
}
}
return nil
// func (r *EipReconciler) AddRule(instance *networkv1alpha1.Eip) error {
// if instance.Spec.Address != "" {
// rule, err := nettool.NewEIPRule(instance.Spec.Address)
// if err != nil {
// r.Log.Info("Failed to initialize ip rule", "eip", instance.Spec.Address)
// return err
// }
// nodeName := os.Getenv("MY_NODE_NAME")
// if ok, err := rule.IsExist(); err != nil {
// return err
// } else {
// if !ok {
// err = rule.Add()
// if err != nil {
// return err
// }
// r.Event(instance, "Normal", "Rule Created", fmt.Sprintf("Created ip rule for EIP %s in agent %s", instance.Spec.Address, nodeName))
// } else {
// r.Log.Info("Detect rule in node")
// r.Event(instance, "Normal", "Detect rule in node", fmt.Sprintf("Skipped Creating ip rule for EIP %s in agent %s", instance.Spec.Address, nodeName))
// }
// }
// }
// return nil
// }

// func (r *EipReconciler) DeleteRule(instance *networkv1alpha1.Eip) error {
// if instance.Spec.Address != "" {
// rule, err := nettool.NewEIPRule(instance.Spec.Address)
// if err != nil {
// r.Log.Info("Failed to initialize ip rule", "eip", instance.Spec.Address)
// return err
// }
// if ok, err := rule.IsExist(); err != nil {
// return err
// } else {
// if ok {
// err = rule.Delete()
// if err != nil {
// return err
// }
// r.Log.Info("Rule is deleted successfully", "rule", rule.ToAgentRule().String())
// } else {
// r.Log.Info("Try to delete a non-exist rule", "rule", rule.ToAgentRule().String())
// }
// }
// }
// return nil
// }

func (r *EipReconciler) OpenEIPForward(eip string) error {
return nettool.OpenForwardForEIP(r.iptableExec, eip)
}

func (r *EipReconciler) DeleteRule(instance *networkv1alpha1.Eip) error {
if instance.Spec.Address != "" {
rule, err := nettool.NewEIPRule(instance.Spec.Address)
if err != nil {
r.Log.Info("Failed to initialize ip rule", "eip", instance.Spec.Address)
return err
}
if ok, err := rule.IsExist(); err != nil {
return err
} else {
if ok {
err = rule.Delete()
if err != nil {
return err
}
r.Log.Info("Rule is deleted successfully", "rule", rule.ToAgentRule().String())
} else {
r.Log.Info("Try to delete a non-exist rule", "rule", rule.ToAgentRule().String())
}
}
}
return nil
func (r *EipReconciler) CloseEIPForward(eip string) error {
return nettool.CloseForwardForEIP(r.iptableExec, eip)
}
1 change: 1 addition & 0 deletions deploy/agent/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
FROM alpine
WORKDIR /
RUN apk add iptables
COPY agent .
ENTRYPOINT ["/agent"]
6 changes: 2 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ require (
cloud.google.com/go v0.44.3 // indirect
github.com/Microsoft/go-winio v0.4.14 // indirect
github.com/armon/go-radix v1.0.0
github.com/coreos/go-iptables v0.4.2
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2
github.com/docker/distribution v2.7.1+incompatible // indirect
github.com/docker/docker v1.13.1
Expand All @@ -14,7 +15,6 @@ require (
github.com/eapache/queue v1.1.0 // indirect
github.com/fsnotify/fsnotify v1.4.7
github.com/go-logr/logr v0.1.0
github.com/godbus/dbus v4.1.0+incompatible // indirect
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect
github.com/golang/protobuf v1.3.2
github.com/hashicorp/golang-lru v0.5.3 // indirect
Expand All @@ -37,13 +37,11 @@ require (
github.com/vishvananda/netns v0.0.0-20190625233234-7109fa855b0f // indirect
golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a // indirect
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 // indirect
google.golang.org/grpc v1.23.0
k8s.io/api v0.0.0-20190409021203-6e4e0e4f393b
k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d
k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible
k8s.io/kubernetes v1.14.6
k8s.io/utils v0.0.0-20190506122338-8fab8cb257d5
sigs.k8s.io/controller-runtime v0.2.0-beta.5
)
6 changes: 2 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-iptables v0.4.2 h1:KH0EwId05JwWIfb96gWvkiT2cbuOu8ygqUaB+yPAwIg=
github.com/coreos/go-iptables v0.4.2/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
Expand Down Expand Up @@ -71,8 +73,6 @@ github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7
github.com/go-logr/zapr v0.1.0 h1:h+WVe9j6HAA01niTJPA/kKH0i7e0rLZBCwauQFcRE54=
github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/godbus/dbus v4.1.0+incompatible h1:WqqLRTsQic3apZUK9qC5sGNfXthmPXzUZ7nQPrNITa4=
github.com/godbus/dbus v4.1.0+incompatible/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
Expand Down Expand Up @@ -447,8 +447,6 @@ k8s.io/klog v0.3.0 h1:0VPpR+sizsiivjIfIAQH/rl8tan6jvWkS7lU+0di3lE=
k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
k8s.io/kube-openapi v0.0.0-20180731170545-e3762e86a74c h1:3KSCztE7gPitlZmWbNwue/2U0YruD65DqX3INopDAQM=
k8s.io/kube-openapi v0.0.0-20180731170545-e3762e86a74c/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc=
k8s.io/kubernetes v1.14.6 h1:t8Q3aaWanmiariBBr3qYIcAL9o0pv4MB5tZtfbILJGk=
k8s.io/kubernetes v1.14.6/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=
k8s.io/utils v0.0.0-20190506122338-8fab8cb257d5 h1:VBM/0P5TWxwk+Nw6Z+lAw3DKgO76g90ETOiA6rfLV1Y=
k8s.io/utils v0.0.0-20190506122338-8fab8cb257d5/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
Expand Down
9 changes: 6 additions & 3 deletions pkg/bgp/serverd/serverd.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"github.com/kubesphere/porter/pkg/bgp/config"
"github.com/kubesphere/porter/pkg/bgp/table"
"github.com/kubesphere/porter/pkg/nettool"
"github.com/kubesphere/porter/pkg/nettool/iptables"
"github.com/kubesphere/porter/pkg/util"
api "github.com/osrg/gobgp/api"
"github.com/osrg/gobgp/pkg/packet/bgp"
Expand All @@ -39,6 +40,8 @@ type StartOption struct {
ConfigType string `short:"t" long:"config-type" description:"specifying config type (toml, yaml, json)" default:"toml"`
GrpcHosts string `long:"api-hosts" description:"specify the hosts that gobgpd listens on" default:":50051"`
GracefulRestart bool `short:"r" long:"graceful-restart" description:"flag restart-state in graceful-restart capability"`

iptables.IptablesIface
}

var bgpServer *server.BgpServer
Expand Down Expand Up @@ -100,7 +103,7 @@ func Run(opts *StartOption, ready chan<- interface{}) {
}
localip := util.GetOutboundIP()
for _, nei := range c.Neighbors {
err := nettool.DeletePortForwardOfBGP(nei.Config.NeighborAddress, localip, c.Global.Config.Port)
err := nettool.DeletePortForwardOfBGP(opts.IptablesIface, nei.Config.NeighborAddress, localip, c.Global.Config.Port)
if err != nil {
log.Fatalf("Error in deleting iptables, %s", err.Error())
}
Expand All @@ -121,7 +124,7 @@ func Run(opts *StartOption, ready chan<- interface{}) {
}
localip := util.GetOutboundIP()
for _, nei := range c.Neighbors {
err := nettool.AddPortForwardOfBGP(nei.Config.NeighborAddress, localip, c.Global.Config.Port)
err := nettool.AddPortForwardOfBGP(opts.IptablesIface, nei.Config.NeighborAddress, localip, c.Global.Config.Port)
if err != nil {
log.Fatalf("Error in creating iptables, %s", err.Error())
}
Expand Down Expand Up @@ -198,7 +201,7 @@ func Run(opts *StartOption, ready chan<- interface{}) {
}
localip := util.GetOutboundIP()
for _, nei := range newConfig.Neighbors {
err := nettool.AddPortForwardOfBGP(nei.Config.NeighborAddress, localip, c.Global.Config.Port)
err := nettool.AddPortForwardOfBGP(opts.IptablesIface, nei.Config.NeighborAddress, localip, c.Global.Config.Port)
if err != nil {
log.Fatalf("Error in creating iptables, %s", err.Error())
}
Expand Down
Loading

0 comments on commit 0996766

Please sign in to comment.