Skip to content

Commit 5d62e09

Browse files
danwinshipdanwinship
committed
Move remaining pod-namespace operations out of the main process
1 parent a241b05 commit 5d62e09

File tree

4 files changed

+162
-85
lines changed

4 files changed

+162
-85
lines changed

‎pkg/network/node/cniserver/cniserver.go

+11
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,10 @@ const CNIServerRunDir string = "/var/run/openshift-sdn"
5252
const CNIServerSocketName string = "cni-server.sock"
5353
const CNIServerSocketPath string = CNIServerRunDir + "/" + CNIServerSocketName
5454

55+
// Config file containing MTU, and default full path
56+
const CNIMTUFileName string = "mtu"
57+
const CNIMTUFilePath string = CNIServerRunDir + "/" + CNIMTUFileName
58+
5559
// Explicit type for CNI commands the server handles
5660
type CNICommand string
5761

@@ -80,6 +84,8 @@ type PodRequest struct {
8084
SandboxID string
8185
// kernel network namespace path
8286
Netns string
87+
// for an ADD request, the host side of the created veth
88+
HostVeth string
8389
// Channel for returning the operation result to the CNIServer
8490
Result chan *PodResult
8591
}
@@ -207,6 +213,11 @@ func cniRequestToPodRequest(r *http.Request) (*PodRequest, error) {
207213
return nil, fmt.Errorf("missing CNI_NETNS")
208214
}
209215

216+
req.HostVeth, ok = cr.Env["OSDN_HOSTVETH"]
217+
if !ok && req.Command == CNI_ADD {
218+
return nil, fmt.Errorf("missing OSDN_HOSTVETH")
219+
}
220+
210221
cniArgs, err := gatherCNIArgs(cr.Env)
211222
if err != nil {
212223
return nil, err

‎pkg/network/node/pod.go

+12-71
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@ package node
55
import (
66
"encoding/json"
77
"fmt"
8+
"io/ioutil"
89
"net"
10+
"os"
911
"path/filepath"
1012
"strconv"
1113
"sync"
@@ -36,9 +38,6 @@ import (
3638
"github.com/containernetworking/cni/pkg/invoke"
3739
cnitypes "github.com/containernetworking/cni/pkg/types"
3840
cni020 "github.com/containernetworking/cni/pkg/types/020"
39-
cnicurrent "github.com/containernetworking/cni/pkg/types/current"
40-
"github.com/containernetworking/plugins/pkg/ip"
41-
"github.com/containernetworking/plugins/pkg/ipam"
4241
"github.com/containernetworking/plugins/pkg/ns"
4342

4443
"github.com/vishvananda/netlink"
@@ -179,6 +178,11 @@ func (m *podManager) Start(rundir string, localSubnetCIDR string, clusterNetwork
179178
return err
180179
}
181180

181+
err = ioutil.WriteFile(filepath.Join(rundir, cniserver.CNIMTUFileName), []byte(fmt.Sprintf("%d", m.mtu)), os.FileMode(0444))
182+
if err != nil {
183+
return fmt.Errorf("could not write MTU file: %v", err)
184+
}
185+
182186
go m.processCNIRequests()
183187

184188
m.cniServer = cniserver.NewCNIServer(rundir)
@@ -385,6 +389,8 @@ func maybeAddMacvlan(pod *kapi.Pod, netns string) error {
385389
}
386390
}
387391

392+
// Note that this use of ns is safe because it doesn't call Do() or WithNetNSPath()
393+
388394
podNs, err := ns.GetNS(netns)
389395
if err != nil {
390396
return fmt.Errorf("could not open netns %q", netns)
@@ -403,17 +409,7 @@ func maybeAddMacvlan(pod *kapi.Pod, netns string) error {
403409
if err != nil {
404410
return fmt.Errorf("failed to create macvlan interface: %v", err)
405411
}
406-
return podNs.Do(func(netns ns.NetNS) error {
407-
l, err := netlink.LinkByName("macvlan0")
408-
if err != nil {
409-
return fmt.Errorf("failed to find macvlan interface: %v", err)
410-
}
411-
err = netlink.LinkSetUp(l)
412-
if err != nil {
413-
return fmt.Errorf("failed to set macvlan interface up: %v", err)
414-
}
415-
return nil
416-
})
412+
return nil
417413
}
418414

419415
func createIPAMArgs(netnsPath string, action cniserver.CNICommand, id string) *invoke.Args {
@@ -540,61 +536,6 @@ func (m *podManager) setup(req *cniserver.PodRequest) (cnitypes.Result, *running
540536
}
541537
}
542538

543-
var hostVethName string
544-
err = ns.WithNetNSPath(req.Netns, func(hostNS ns.NetNS) error {
545-
hostVeth, contVeth, err := ip.SetupVeth(podInterfaceName, int(m.mtu), hostNS)
546-
if err != nil {
547-
return fmt.Errorf("failed to create container veth: %v", err)
548-
}
549-
// Force a consistent MAC address based on the IP address
550-
if err := ip.SetHWAddrByIP(podInterfaceName, podIP, nil); err != nil {
551-
return fmt.Errorf("failed to set pod interface MAC address: %v", err)
552-
}
553-
// refetch to get hardware address and other properties
554-
tmp, err := net.InterfaceByIndex(contVeth.Index)
555-
if err != nil {
556-
return fmt.Errorf("failed to fetch container veth: %v", err)
557-
}
558-
contVeth = *tmp
559-
560-
// Clear out gateway to prevent ConfigureIface from adding the cluster
561-
// subnet via the gateway
562-
ipamResult.IP4.Gateway = nil
563-
result030, err := cnicurrent.NewResultFromResult(ipamResult)
564-
if err != nil {
565-
return fmt.Errorf("failed to convert IPAM: %v", err)
566-
}
567-
// Add a sandbox interface record which ConfigureInterface expects.
568-
// The only interface we report is the pod interface.
569-
result030.Interfaces = []*cnicurrent.Interface{
570-
{
571-
Name: podInterfaceName,
572-
Mac: contVeth.HardwareAddr.String(),
573-
Sandbox: req.Netns,
574-
},
575-
}
576-
intPtr := 0
577-
result030.IPs[0].Interface = &intPtr
578-
579-
if err = ipam.ConfigureIface(podInterfaceName, result030); err != nil {
580-
return fmt.Errorf("failed to configure container IPAM: %v", err)
581-
}
582-
583-
lo, err := netlink.LinkByName("lo")
584-
if err == nil {
585-
err = netlink.LinkSetUp(lo)
586-
}
587-
if err != nil {
588-
return fmt.Errorf("failed to configure container loopback: %v", err)
589-
}
590-
591-
hostVethName = hostVeth.Name
592-
return nil
593-
})
594-
if err != nil {
595-
return nil, nil, err
596-
}
597-
598539
vnid, err := m.policy.GetVNID(req.PodNamespace)
599540
if err != nil {
600541
return nil, nil, err
@@ -604,11 +545,11 @@ func (m *podManager) setup(req *cniserver.PodRequest) (cnitypes.Result, *running
604545
return nil, nil, err
605546
}
606547

607-
ofport, err := m.ovs.SetUpPod(req.SandboxID, hostVethName, podIP, vnid)
548+
ofport, err := m.ovs.SetUpPod(req.SandboxID, req.HostVeth, podIP, vnid)
608549
if err != nil {
609550
return nil, nil, err
610551
}
611-
if err := setupPodBandwidth(m.ovs, pod, hostVethName, req.SandboxID); err != nil {
552+
if err := setupPodBandwidth(m.ovs, pod, req.HostVeth, req.SandboxID); err != nil {
612553
return nil, nil, err
613554
}
614555

‎pkg/network/sdn-cni-plugin/openshift-sdn.go

+128-12
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"net"
1212
"net/http"
1313
"os"
14+
"strconv"
1415
"strings"
1516
"time"
1617

@@ -19,15 +20,22 @@ import (
1920
"github.com/containernetworking/cni/pkg/skel"
2021
"github.com/containernetworking/cni/pkg/types"
2122
"github.com/containernetworking/cni/pkg/types/020"
23+
"github.com/containernetworking/cni/pkg/types/current"
2224
"github.com/containernetworking/cni/pkg/version"
25+
"github.com/containernetworking/plugins/pkg/ip"
26+
"github.com/containernetworking/plugins/pkg/ipam"
27+
"github.com/containernetworking/plugins/pkg/ns"
28+
29+
"github.com/vishvananda/netlink"
2330
)
2431

2532
type cniPlugin struct {
2633
socketPath string
34+
hostNS ns.NetNS
2735
}
2836

29-
func NewCNIPlugin(socketPath string) *cniPlugin {
30-
return &cniPlugin{socketPath: socketPath}
37+
func NewCNIPlugin(socketPath string, hostNS ns.NetNS) *cniPlugin {
38+
return &cniPlugin{socketPath: socketPath, hostNS: hostNS}
3139
}
3240

3341
// Create and fill a CNIRequest with this plugin's environment and stdin which
@@ -63,7 +71,11 @@ func (p *cniPlugin) doCNI(url string, req *cniserver.CNIRequest) ([]byte, error)
6371
},
6472
}
6573

66-
resp, err := client.Post(url, "application/json", bytes.NewReader(data))
74+
var resp *http.Response
75+
err = p.hostNS.Do(func(ns.NetNS) error {
76+
resp, err = client.Post(url, "application/json", bytes.NewReader(data))
77+
return err
78+
})
6779
if err != nil {
6880
return nil, fmt.Errorf("failed to send CNI request: %v", err)
6981
}
@@ -81,10 +93,27 @@ func (p *cniPlugin) doCNI(url string, req *cniserver.CNIRequest) ([]byte, error)
8193
return body, nil
8294
}
8395

96+
func readMTU() (int, error) {
97+
bytes, err := ioutil.ReadFile(cniserver.CNIMTUFilePath)
98+
if err != nil {
99+
if os.IsNotExist(err) {
100+
return -1, fmt.Errorf("OpenShift SDN network process is not (yet?) available")
101+
} else {
102+
return -1, fmt.Errorf("could not read MTU file: %v", err)
103+
}
104+
}
105+
mtu, err := strconv.ParseUint(string(bytes), 10, 0)
106+
if err != nil {
107+
return -1, fmt.Errorf("could not parse MTU value %q", string(bytes))
108+
}
109+
return int(mtu), nil
110+
}
111+
84112
// Send the ADD command environment and config to the CNI server, returning
85113
// the IPAM result to the caller
86-
func (p *cniPlugin) CmdAdd(args *skel.CmdArgs) (types.Result, error) {
87-
body, err := p.doCNI("http://dummy/", newCNIRequest(args))
114+
func (p *cniPlugin) doCNIServerAdd(req *cniserver.CNIRequest, hostVeth string) (types.Result, error) {
115+
req.Env["OSDN_HOSTVETH"] = hostVeth
116+
body, err := p.doCNI("http://dummy/", req)
88117
if err != nil {
89118
return nil, err
90119
}
@@ -99,24 +128,111 @@ func (p *cniPlugin) CmdAdd(args *skel.CmdArgs) (types.Result, error) {
99128
return result, nil
100129
}
101130

102-
// Send the ADD command environment and config to the CNI server, printing
103-
// the IPAM result to stdout when called as a CNI plugin
104-
func (p *cniPlugin) skelCmdAdd(args *skel.CmdArgs) error {
105-
result, err := p.CmdAdd(args)
131+
func (p *cniPlugin) testCmdAdd(args *skel.CmdArgs) (types.Result, error) {
132+
return p.doCNIServerAdd(newCNIRequest(args), "dummy0")
133+
}
134+
135+
func (p *cniPlugin) CmdAdd(args *skel.CmdArgs) error {
136+
req := newCNIRequest(args)
137+
ifname := req.Env["CNI_IFNAME"]
138+
netns := req.Env["CNI_NETNS"]
139+
if ifname == "" || netns == "" {
140+
return fmt.Errorf("CNI request did not include required environment variables")
141+
}
142+
143+
mtu, err := readMTU()
144+
if err != nil {
145+
return err
146+
}
147+
148+
var hostVeth, contVeth net.Interface
149+
err = ns.WithNetNSPath(netns, func(hostNS ns.NetNS) error {
150+
hostVeth, contVeth, err = ip.SetupVeth(ifname, mtu, hostNS)
151+
if err != nil {
152+
return fmt.Errorf("failed to create container veth: %v", err)
153+
}
154+
return nil
155+
})
156+
if err != nil {
157+
return err
158+
}
159+
result, err := p.doCNIServerAdd(req, hostVeth.Name)
160+
if err != nil {
161+
return err
162+
}
163+
164+
result030, err := current.NewResultFromResult(result)
165+
if err != nil {
166+
return fmt.Errorf("failed to convert IPAM result: %v", err)
167+
}
168+
169+
if len(result030.IPs) != 1 || result030.IPs[0].Version != "4" {
170+
return fmt.Errorf("unexpected IPAM result from CNIServer")
171+
}
172+
173+
// Clear out gateway to prevent ConfigureIface from adding the cluster
174+
// subnet via the gateway
175+
result030.IPs[0].Gateway = nil
176+
// Add a sandbox interface record which ConfigureInterface expects.
177+
// The only interface we report is the pod interface.
178+
result030.Interfaces = []*current.Interface{
179+
{
180+
Name: ifname,
181+
Mac: contVeth.HardwareAddr.String(),
182+
Sandbox: netns,
183+
},
184+
}
185+
index := 0
186+
result030.IPs[0].Interface = &index
187+
188+
err = ns.WithNetNSPath(netns, func(ns.NetNS) error {
189+
// Set up eth0
190+
if err := ip.SetHWAddrByIP(ifname, result030.IPs[0].Address.IP, nil); err != nil {
191+
return fmt.Errorf("failed to set pod interface MAC address: %v", err)
192+
}
193+
if err := ipam.ConfigureIface(ifname, result030); err != nil {
194+
return fmt.Errorf("failed to configure container IPAM: %v", err)
195+
}
196+
197+
// Set up lo
198+
link, err := netlink.LinkByName("lo")
199+
if err == nil {
200+
err = netlink.LinkSetUp(link)
201+
}
202+
if err != nil {
203+
return fmt.Errorf("failed to configure container loopback: %v", err)
204+
}
205+
206+
// Set up macvlan0 (if it exists)
207+
link, err = netlink.LinkByName("macvlan0")
208+
if err == nil {
209+
err = netlink.LinkSetUp(link)
210+
if err != nil {
211+
return fmt.Errorf("failed to configure macvlan device: %v", err)
212+
}
213+
}
214+
215+
return nil
216+
})
106217
if err != nil {
107218
return err
108219
}
220+
109221
return result.Print()
110222
}
111223

112-
// Send the DEL command environment and config to the CNI server
113224
func (p *cniPlugin) CmdDel(args *skel.CmdArgs) error {
114225
_, err := p.doCNI("http://dummy/", newCNIRequest(args))
115226
return err
116227
}
117228

118229
func main() {
119230
rand.Seed(time.Now().UTC().UnixNano())
120-
p := NewCNIPlugin(cniserver.CNIServerSocketPath)
121-
skel.PluginMain(p.skelCmdAdd, p.CmdDel, version.Legacy)
231+
hostNS, err := ns.GetCurrentNS()
232+
if err != nil {
233+
panic(fmt.Sprintf("could not get current kernel netns: %v", err))
234+
}
235+
defer hostNS.Close()
236+
p := NewCNIPlugin(cniserver.CNIServerSocketPath, hostNS)
237+
skel.PluginMain(p.CmdAdd, p.CmdDel, version.Legacy)
122238
}

0 commit comments

Comments
 (0)