Skip to content

Commit 2cf54b9

Browse files
Merge pull request #18911 from pravisankar/sdn-minimize-live-apicalls
Minimize direct API calls in SDN master by using informer cache
2 parents 9251284 + ae0f3e1 commit 2cf54b9

File tree

5 files changed

+398
-348
lines changed

5 files changed

+398
-348
lines changed

pkg/network/master/master.go

+50-24
Original file line numberDiff line numberDiff line change
@@ -11,16 +11,19 @@ import (
1111
ktypes "k8s.io/apimachinery/pkg/types"
1212
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
1313
"k8s.io/apimachinery/pkg/util/wait"
14+
"k8s.io/client-go/tools/cache"
1415
kapi "k8s.io/kubernetes/pkg/apis/core"
1516
kclientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
1617
kinternalinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion"
18+
kcoreinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion/core/internalversion"
1719

1820
osconfigapi "github.com/openshift/origin/pkg/cmd/server/apis/config"
1921
"github.com/openshift/origin/pkg/network"
2022
networkapi "github.com/openshift/origin/pkg/network/apis/network"
2123
osapivalidation "github.com/openshift/origin/pkg/network/apis/network/validation"
2224
"github.com/openshift/origin/pkg/network/common"
23-
networkinformers "github.com/openshift/origin/pkg/network/generated/informers/internalversion"
25+
networkinternalinformers "github.com/openshift/origin/pkg/network/generated/informers/internalversion"
26+
networkinformers "github.com/openshift/origin/pkg/network/generated/informers/internalversion/network/internalversion"
2427
networkclient "github.com/openshift/origin/pkg/network/generated/internalclientset"
2528
"github.com/openshift/origin/pkg/util/netutils"
2629
)
@@ -30,30 +33,41 @@ const (
3033
)
3134

3235
type OsdnMaster struct {
33-
kClient kclientset.Interface
34-
networkClient networkclient.Interface
35-
networkInfo *common.NetworkInfo
36-
subnetAllocatorList []*SubnetAllocator
37-
vnids *masterVNIDMap
36+
kClient kclientset.Interface
37+
networkClient networkclient.Interface
38+
networkInfo *common.NetworkInfo
39+
vnids *masterVNIDMap
40+
41+
nodeInformer kcoreinformers.NodeInformer
42+
namespaceInformer kcoreinformers.NamespaceInformer
43+
hostSubnetInformer networkinformers.HostSubnetInformer
44+
netNamespaceInformer networkinformers.NetNamespaceInformer
3845

39-
kubeInformers kinternalinformers.SharedInformerFactory
40-
networkInformers networkinformers.SharedInformerFactory
46+
// Used for allocating subnets in order
47+
subnetAllocatorList []*SubnetAllocator
48+
// Used for clusterNetwork --> subnetAllocator lookup
49+
subnetAllocatorMap map[common.ClusterNetwork]*SubnetAllocator
4150

4251
// Holds Node IP used in creating host subnet for a node
4352
hostSubnetNodeIPs map[ktypes.UID]string
4453
}
4554

4655
func Start(networkConfig osconfigapi.MasterNetworkConfig, networkClient networkclient.Interface,
4756
kClient kclientset.Interface, kubeInformers kinternalinformers.SharedInformerFactory,
48-
networkInformers networkinformers.SharedInformerFactory) error {
57+
networkInformers networkinternalinformers.SharedInformerFactory) error {
4958
glog.Infof("Initializing SDN master of type %q", networkConfig.NetworkPluginName)
5059

5160
master := &OsdnMaster{
52-
kClient: kClient,
53-
networkClient: networkClient,
54-
kubeInformers: kubeInformers,
55-
networkInformers: networkInformers,
56-
hostSubnetNodeIPs: map[ktypes.UID]string{},
61+
kClient: kClient,
62+
networkClient: networkClient,
63+
64+
nodeInformer: kubeInformers.Core().InternalVersion().Nodes(),
65+
namespaceInformer: kubeInformers.Core().InternalVersion().Namespaces(),
66+
hostSubnetInformer: networkInformers.Network().InternalVersion().HostSubnets(),
67+
netNamespaceInformer: networkInformers.Network().InternalVersion().NetNamespaces(),
68+
69+
subnetAllocatorMap: map[common.ClusterNetwork]*SubnetAllocator{},
70+
hostSubnetNodeIPs: map[ktypes.UID]string{},
5771
}
5872

5973
var err error
@@ -142,24 +156,36 @@ func Start(networkConfig osconfigapi.MasterNetworkConfig, networkClient networkc
142156
return err
143157
}
144158

145-
if err = master.SubnetStartMaster(master.networkInfo.ClusterNetworks); err != nil {
146-
return err
159+
go master.startSubSystems(networkConfig.NetworkPluginName)
160+
161+
return nil
162+
}
163+
164+
func (master *OsdnMaster) startSubSystems(pluginName string) {
165+
// Wait for informer sync
166+
if !cache.WaitForCacheSync(wait.NeverStop,
167+
master.nodeInformer.Informer().GetController().HasSynced,
168+
master.namespaceInformer.Informer().GetController().HasSynced,
169+
master.hostSubnetInformer.Informer().GetController().HasSynced,
170+
master.netNamespaceInformer.Informer().GetController().HasSynced) {
171+
glog.Fatalf("failed to sync SDN master informers")
172+
}
173+
174+
if err := master.startSubnetMaster(); err != nil {
175+
glog.Fatalf("failed to start subnet master: %v", err)
147176
}
148177

149-
switch networkConfig.NetworkPluginName {
178+
switch pluginName {
150179
case network.MultiTenantPluginName:
151180
master.vnids = newMasterVNIDMap(true)
152-
if err = master.VnidStartMaster(); err != nil {
153-
return err
154-
}
155181
case network.NetworkPolicyPluginName:
156182
master.vnids = newMasterVNIDMap(false)
157-
if err = master.VnidStartMaster(); err != nil {
158-
return err
183+
}
184+
if master.vnids != nil {
185+
if err := master.startVNIDMaster(); err != nil {
186+
glog.Fatalf("failed to start VNID master: %v", err)
159187
}
160188
}
161-
162-
return nil
163189
}
164190

165191
func (master *OsdnMaster) checkClusterNetworkAgainstLocalNetworks() error {

pkg/network/master/subnet_allocator.go

+92-18
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"net"
77
"sync"
88

9+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
910
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
1011
)
1112

@@ -23,7 +24,7 @@ type SubnetAllocator struct {
2324
mutex sync.Mutex
2425
}
2526

26-
func NewSubnetAllocator(network string, hostBits uint32, inUse []string) (*SubnetAllocator, error) {
27+
func newSubnetAllocator(network string, hostBits uint32) (*SubnetAllocator, error) {
2728
_, netIP, err := net.ParseCIDR(network)
2829
if err != nil {
2930
return nil, fmt.Errorf("failed to parse network address: %q", network)
@@ -64,7 +65,7 @@ func NewSubnetAllocator(network string, hostBits uint32, inUse []string) (*Subne
6465
rightMask = 0
6566
}
6667

67-
sa := &SubnetAllocator{
68+
return &SubnetAllocator{
6869
network: netIP,
6970
hostBits: hostBits,
7071
leftShift: leftShift,
@@ -73,22 +74,10 @@ func NewSubnetAllocator(network string, hostBits uint32, inUse []string) (*Subne
7374
rightMask: rightMask,
7475
next: 0,
7576
allocMap: make(map[string]bool),
76-
}
77-
for _, netStr := range inUse {
78-
_, ipNet, err := net.ParseCIDR(netStr)
79-
if err != nil {
80-
utilruntime.HandleError(fmt.Errorf("failed to parse network address: %s", netStr))
81-
continue
82-
}
83-
if err = sa.AllocateNetwork(ipNet); err != nil {
84-
utilruntime.HandleError(err)
85-
continue
86-
}
87-
}
88-
return sa, nil
77+
}, nil
8978
}
9079

91-
func (sna *SubnetAllocator) AllocateNetwork(ipNet *net.IPNet) error {
80+
func (sna *SubnetAllocator) markAllocatedNetwork(ipNet *net.IPNet) error {
9281
sna.mutex.Lock()
9382
defer sna.mutex.Unlock()
9483

@@ -101,7 +90,7 @@ func (sna *SubnetAllocator) AllocateNetwork(ipNet *net.IPNet) error {
10190
return nil
10291
}
10392

104-
func (sna *SubnetAllocator) GetNetwork() (*net.IPNet, error) {
93+
func (sna *SubnetAllocator) allocateNetwork() (*net.IPNet, error) {
10594
var (
10695
numSubnets uint32
10796
numSubnetBits uint32
@@ -132,7 +121,7 @@ func (sna *SubnetAllocator) GetNetwork() (*net.IPNet, error) {
132121
return nil, ErrSubnetAllocatorFull
133122
}
134123

135-
func (sna *SubnetAllocator) ReleaseNetwork(ipnet *net.IPNet) error {
124+
func (sna *SubnetAllocator) releaseNetwork(ipnet *net.IPNet) error {
136125
sna.mutex.Lock()
137126
defer sna.mutex.Unlock()
138127

@@ -158,3 +147,88 @@ func Uint32ToIP(u uint32) net.IP {
158147
binary.BigEndian.PutUint32(ip, u)
159148
return net.IPv4(ip[0], ip[1], ip[2], ip[3])
160149
}
150+
151+
//--------------------- Master methods ----------------------
152+
153+
func (master *OsdnMaster) initSubnetAllocators() error {
154+
for _, cn := range master.networkInfo.ClusterNetworks {
155+
sa, err := newSubnetAllocator(cn.ClusterCIDR.String(), cn.HostSubnetLength)
156+
if err != nil {
157+
return err
158+
}
159+
master.subnetAllocatorList = append(master.subnetAllocatorList, sa)
160+
master.subnetAllocatorMap[cn] = sa
161+
}
162+
163+
// Populate subnet allocator
164+
subnets, err := master.networkClient.Network().HostSubnets().List(metav1.ListOptions{})
165+
if err != nil {
166+
return err
167+
}
168+
for _, sn := range subnets.Items {
169+
if err := master.markAllocatedNetwork(sn.Subnet); err != nil {
170+
utilruntime.HandleError(err)
171+
}
172+
}
173+
174+
return nil
175+
}
176+
177+
func (master *OsdnMaster) markAllocatedNetwork(subnet string) error {
178+
sa, ipnet, err := master.getSubnetAllocator(subnet)
179+
if err != nil {
180+
return err
181+
}
182+
if err = sa.markAllocatedNetwork(ipnet); err != nil {
183+
return err
184+
}
185+
return nil
186+
}
187+
188+
func (master *OsdnMaster) allocateNetwork(nodeName string) (string, error) {
189+
var sn *net.IPNet
190+
var err error
191+
192+
for _, possibleSubnet := range master.subnetAllocatorList {
193+
sn, err = possibleSubnet.allocateNetwork()
194+
if err == ErrSubnetAllocatorFull {
195+
// Current subnet exhausted, check the next one
196+
continue
197+
} else if err != nil {
198+
utilruntime.HandleError(fmt.Errorf("Error allocating network from subnet: %v", possibleSubnet))
199+
continue
200+
} else {
201+
return sn.String(), nil
202+
}
203+
}
204+
return "", fmt.Errorf("error allocating network for node %s: %v", nodeName, err)
205+
}
206+
207+
func (master *OsdnMaster) releaseNetwork(subnet string) error {
208+
sa, ipnet, err := master.getSubnetAllocator(subnet)
209+
if err != nil {
210+
return err
211+
}
212+
if err = sa.releaseNetwork(ipnet); err != nil {
213+
return err
214+
}
215+
return nil
216+
}
217+
218+
func (master *OsdnMaster) getSubnetAllocator(subnet string) (*SubnetAllocator, *net.IPNet, error) {
219+
_, ipnet, err := net.ParseCIDR(subnet)
220+
if err != nil {
221+
return nil, nil, fmt.Errorf("error parsing subnet %q: %v", subnet, err)
222+
}
223+
224+
for _, cn := range master.networkInfo.ClusterNetworks {
225+
if cn.ClusterCIDR.Contains(ipnet.IP) {
226+
sa, ok := master.subnetAllocatorMap[cn]
227+
if !ok || sa == nil {
228+
return nil, nil, fmt.Errorf("subnet allocator not found for cluster network: %v", cn)
229+
}
230+
return sa, ipnet, nil
231+
}
232+
}
233+
return nil, nil, fmt.Errorf("subnet %q not found in the cluster networks: %v", subnet, master.networkInfo.ClusterNetworks)
234+
}

0 commit comments

Comments
 (0)