Skip to content

Commit

Permalink
RFE-2962: Set interface name from node-ip-configuration to configure-ovs
Browse files Browse the repository at this point in the history 
node-ip-configuration will set env file for configure-ovs when it will
set kubelet ip, this will allow to configure ovs with the bridge on the
same interface that kubelet will use
  • Loading branch information
@tsorya
tsorya committed Jul 5, 2022
1 parent 70d770d commit 3e3ae1e
Show file tree
Hide file tree
Showing 3 changed files with 65 additions and 23 deletions.
44 changes: 35 additions & 9 deletions cmd/runtimecfg/node-ip.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,9 @@ import (
)

const (
kubeletSvcOverridePath = "/etc/systemd/system/kubelet.service.d/20-nodenet.conf"
crioSvcOverridePath = "/etc/systemd/system/crio.service.d/20-nodenet.conf"
kubeletSvcOverridePath = "/etc/systemd/system/kubelet.service.d/20-nodenet.conf"
crioSvcOverridePath = "/etc/systemd/system/crio.service.d/20-nodenet.conf"
ovsConfigurationSvcOverridePath = "/etc/systemd/system/ovs-configuration.service.d//20-nodenet.conf"
)

var retry, preferIPv6 bool
Expand Down Expand Up @@ -104,10 +105,10 @@ func set(cmd *cobra.Command, args []string) error {
}
defer kOverride.Close()

nodeIP := chosenAddresses[0].String()
nodeIP := chosenAddresses[0].Address.String()
nodeIPs := nodeIP
if len(chosenAddresses) > 1 {
nodeIPs += "," + chosenAddresses[1].String()
nodeIPs += "," + chosenAddresses[1].Address.String()
}
kOverrideContent := fmt.Sprintf("[Service]\nEnvironment=\"KUBELET_NODE_IP=%s\" \"KUBELET_NODE_IPS=%s\"\n", nodeIP, nodeIPs)
log.Infof("Writing Kubelet service override with content %s", kOverrideContent)
Expand All @@ -129,27 +130,52 @@ func set(cmd *cobra.Command, args []string) error {
}
defer cOverride.Close()

cOverrideContent := fmt.Sprintf("[Service]\nEnvironment=\"CONTAINER_STREAM_ADDRESS=%s\"\n", chosenAddresses[0])
cOverrideContent := fmt.Sprintf("[Service]\nEnvironment=\"CONTAINER_STREAM_ADDRESS=%s\"\n", chosenAddresses[0].Address)
log.Infof("Writing CRI-O service override with content %s", cOverrideContent)
_, err = cOverride.WriteString(cOverrideContent)
if err != nil {
return err
}

// OVN
// Set kubelet interface in case it is not br-ex that means that it was already configured and node is rebooting
// in that case don't change the previous written interface
if chosenAddresses[0].LinkName != "br-ex" {
ovsConfigurationDir := filepath.Dir(ovsConfigurationSvcOverridePath)
err = os.MkdirAll(ovsConfigurationDir, 0755)
if err != nil {
return err
}
log.Infof("Opening OVS configuration service override path %s", ovsConfigurationSvcOverridePath)
ovsOverride, err := os.Create(ovsConfigurationSvcOverridePath)
if err != nil {
return err
}
defer ovsOverride.Close()
ovsOverrideContent := fmt.Sprintf("[Service]\nEnvironment=\"KUBELET_INTERFACE=%s\"\n", chosenAddresses[0].LinkName)
log.Infof("Writing OVS-O service override with content %s", ovsOverrideContent)
_, err = ovsOverride.WriteString(ovsOverrideContent)
if err != nil {
return err
}
} else {
log.Infof("Skipping set ovs env file as bridge was already configured on the expected interface")
}

return nil
}

func getSuitableIPs(retry bool, vips []net.IP, preferIPv6 bool) (chosen []net.IP, err error) {
func getSuitableIPs(retry bool, vips []net.IP, preferIPv6 bool) (chosen []utils.FoundAddress, err error) {
// Enable debug logging in utils package
utils.SetDebugLogLevel()
for {
if len(vips) > 0 {
chosen, err = utils.AddressesRouting(vips, utils.ValidNodeAddress)
if len(chosen) > 0 || err != nil {

// If using IPv6, verify that the choosen address isn't tentative
// i.e. we can actually bind to it
if len(chosen) > 0 && net.IPv6len == len(chosen[0]) {
_, err := net.Listen("tcp", "["+chosen[0].String()+"]:")
if len(chosen) > 0 && net.IPv6len == len(chosen[0].Address) {
_, err := net.Listen("tcp", "["+chosen[0].Address.String()+"]:")
if err != nil {
log.Errorf("Chosen node IP is not usable")
if !retry {
Expand Down
4 changes: 2 additions & 2 deletions pkg/config/net.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ func getInterfaceAndNonVIPAddr(vips []net.IP) (vipIface net.Interface, nonVipAdd
// To make sure that the correct interface being chosen for cases like:
// 2 interfaces , subnetA: 1001:db8::/120 , subnetB: 1001:db8::f00/120 and VIP address 1001:db8::64
nodeAddrs, err := utils.AddressesRouting(vips, utils.ValidNodeAddress)
if err == nil && len(nodeAddrs) > 0 && n.IP.Equal(nodeAddrs[0]) {
if err == nil && len(nodeAddrs) > 0 && n.IP.Equal(nodeAddrs[0].Address) {
return iface, n, nil
}
}
Expand All @@ -66,7 +66,7 @@ func getInterfaceAndNonVIPAddr(vips []net.IP) (vipIface net.Interface, nonVipAdd
for _, addr := range addrs {
switch n := addr.(type) {
case *net.IPNet:
if n.IP.String() == nodeAddrs[0].String() {
if n.IP.String() == nodeAddrs[0].Address.String() {
return iface, n, nil
}
default:
Expand Down
40 changes: 28 additions & 12 deletions pkg/utils/addresses.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,19 +119,19 @@ func isIPv6(ip net.IP) bool {
return ip.To4() == nil
}

// AddressesRouting takes a slice of Virtual IPs and returns a configured address in the current network namespace that directly routes to at least one of those vips. If the interface containing that address is dual-stack, it will also return a single address of the opposite IP family. You can optionally pass an AddressFilter to further filter down which addresses are considered
func AddressesRouting(vips []net.IP, af AddressFilter) ([]net.IP, error) {
// AddressesRouting takes a slice of Virtual IPs and returns a FoundAddress address in the current network namespace that directly routes to at least one of those vips. If the interface containing that address is dual-stack, it will also return a single address of the opposite IP family. You can optionally pass an AddressFilter to further filter down which addresses are considered
func AddressesRouting(vips []net.IP, af AddressFilter) ([]FoundAddress, error) {
return addressesRoutingInternal(vips, af, getAddrs, getRouteMap)
}

func addressesRoutingInternal(vips []net.IP, af AddressFilter, getAddrs addressMapFunc, getRouteMap routeMapFunc) ([]net.IP, error) {
func addressesRoutingInternal(vips []net.IP, af AddressFilter, getAddrs addressMapFunc, getRouteMap routeMapFunc) ([]FoundAddress, error) {
addrMap, err := getAddrs(af)
if err != nil {
return nil, err
}

var routeMap map[int][]netlink.Route
matches := make([]net.IP, 0)
matches := make([]FoundAddress, 0)
for link, addresses := range addrMap {
addrLoop:
for _, address := range addresses {
Expand Down Expand Up @@ -161,7 +161,12 @@ func addressesRoutingInternal(vips []net.IP, af AddressFilter, getAddrs addressM
log.Debugf("Checking whether address %s with route %s contains VIP %s", address, route, vip)
if containmentNet.Contains(vip) {
log.Debugf("Address %s with route %s contains VIP %s", address, route, vip)
matches = append(matches, address.IP)
matches = append(matches, FoundAddress{
Address: address.IP,
Priority: route.Priority,
LinkIndex: link.Attrs().Index,
LinkName: link.Attrs().Name,
})
break addrLoop
}
}
Expand All @@ -172,7 +177,11 @@ func addressesRoutingInternal(vips []net.IP, af AddressFilter, getAddrs addressM
log.Debugf("Checking whether address %s contains VIP %s", address, vip)
if address.Contains(vip) {
log.Debugf("Address %s contains VIP %s", address, vip)
matches = append(matches, address.IP)
matches = append(matches, FoundAddress{
Address: address.IP,
LinkIndex: link.Attrs().Index,
LinkName: link.Attrs().Name,
})
break addrLoop
}
}
Expand All @@ -182,8 +191,13 @@ func addressesRoutingInternal(vips []net.IP, af AddressFilter, getAddrs addressM
if len(matches) > 0 {
// Find an address of the opposite IP family on the same interface
for _, address := range addresses {
if isIPv6(address.IP) != isIPv6(matches[0]) {
matches = append(matches, address.IP)
if isIPv6(address.IP) != isIPv6(matches[0].Address) {
matches = append(matches, FoundAddress{
Address: address.IP,
Priority: matches[0].Priority,
LinkIndex: matches[0].LinkIndex,
LinkName: matches[0].LinkName,
})
break
}
}
Expand All @@ -199,17 +213,18 @@ func defaultRoute(route netlink.Route) bool {
}

// AddressesDefault returns a slice of configured addresses in the current network namespace associated with default routes; IPv4 first (if any), then IPv6 (if any). You can optionally pass an AddressFilter to further filter down which addresses are considered
func AddressesDefault(preferIPv6 bool, af AddressFilter) ([]net.IP, error) {
func AddressesDefault(preferIPv6 bool, af AddressFilter) ([]FoundAddress, error) {
return addressesDefaultInternal(preferIPv6, af, getAddrs, getRouteMap)
}

type FoundAddress struct {
Address net.IP
Priority int
LinkIndex int
LinkName string
}

func addressesDefaultInternal(preferIPv6 bool, af AddressFilter, getAddrs addressMapFunc, getRouteMap routeMapFunc) ([]net.IP, error) {
func addressesDefaultInternal(preferIPv6 bool, af AddressFilter, getAddrs addressMapFunc, getRouteMap routeMapFunc) ([]FoundAddress, error) {
addrMap, err := getAddrs(af)
if err != nil {
return nil, err
Expand All @@ -219,7 +234,7 @@ func addressesDefaultInternal(preferIPv6 bool, af AddressFilter, getAddrs addres
return nil, err
}

matches := make([]net.IP, 0)
matches := make([]FoundAddress, 0)
addrs := make([]FoundAddress, 0)
for link, addresses := range addrMap {
linkIndex := link.Attrs().Index
Expand All @@ -233,6 +248,7 @@ func addressesDefaultInternal(preferIPv6 bool, af AddressFilter, getAddrs addres
Address: address.IP,
Priority: routeMap[linkIndex][0].Priority,
LinkIndex: linkIndex,
LinkName: link.Attrs().Name,
})
}
}
Expand All @@ -256,7 +272,7 @@ func addressesDefaultInternal(preferIPv6 bool, af AddressFilter, getAddrs addres
if (isIPv6(addr.Address) && foundv6) || (!isIPv6(addr.Address) && foundv4) {
continue
}
matches = append(matches, addr.Address)
matches = append(matches, addr)
if isIPv6(addr.Address) {
foundv6 = true
} else {
Expand Down

0 comments on commit 3e3ae1e

Please sign in to comment.