cni-plugins之ipam-host-local

host-local是一种IP地址管理(IPAM)插件。

它从一组地址范围中分配IP地址,也可以从主机上的resolv.conf文件中获取DNS配置。它在主机文件系统上存储本地状态,因此可以确保单个主机上IP地址的唯一性。
该分配器可以分配多个地址范围,并支持多个(不相交的)子网集。分配策略是在每个范围集中松散的轮询。
在 CNI 调用链中,每个插件都会返回一个 JSON 格式的结果,其中会包含网络配置信息,如 IP 地址、子网掩码、网关等。当一个插件执行完毕,将其返回的结果传递给下一个插件,下一个插件将使用上一个插件返回的网络配置信息来配置网络。当最后一个插件执行完毕后,kubelet 会检查所有插件返回的结果,只有当所有插件都成功设置了 IP 地址,才会认为整个 CNI 调用链执行成功。如果其中一个插件失败,后续的插件将不会被执行,同时整个 CNI 调用链也会被标记为失败。

配置示例(多个地址范围)

ranges表示一个列表,元素是rangeSet,下面的ranges包含两个rangeSetranges的长度表示返回多少iprangeSet表示ip地址的可选范围。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
{
"ipam": {
"type": "host-local",
"ranges": [
[
{
"subnet": "10.10.0.0/16",
"rangeStart": "10.10.1.20",
"rangeEnd": "10.10.3.50",
"gateway": "10.10.0.254"
},
{
"subnet": "172.16.5.0/24"
}
],
[
{
"subnet": "3ffe:ffff:0:01ff::/64",
"rangeStart": "3ffe:ffff:0:01ff::0010",
"rangeEnd": "3ffe:ffff:0:01ff::0020"
}
]
],
"routes": [
{ "dst": "0.0.0.0/0" },
{ "dst": "192.168.0.0/16", "gw": "10.10.5.1" },
{ "dst": "3ffe:ffff:0:01ff::1/64" }
],
"dataDir": "/run/my-orchestrator/container-ipam-state"
}
}

ip地址分配

对于每一个请求的自定义 IP,host-local 分配器会在它所管理的地址范围中进行请求。因此可以指定多个自定义 IP 和多个地址范围。如果一个IP在使用中或者不在范围内将会分配失败。

文件存储

host-local将已分配的 IP 地址作为文件存储在 /var/lib/cni/networks/$NETWORK_NAME 目录下。

源码分析
host-local入口
1
2
3
4
5
6
7
8
9
10
11
12
13
14
// PluginMain 是plugin的main函数,自动进行了错误处理
func main() {
skel.PluginMain(cmdAdd, cmdCheck, cmdDel, version.All, bv.BuildString("host-local"))
}

// vendor/github.com/containernetworking/cni/pkg/skel/skel.go
func PluginMain(cmdAdd, cmdCheck, cmdDel func(_ *CmdArgs) error, versionInfo version.PluginInfo, about string) {
if e := PluginMainWithError(cmdAdd, cmdCheck, cmdDel, versionInfo, about); e != nil {
if err := e.Print(); err != nil {
log.Print("Error writing error JSON to stdout: ", err)
}
os.Exit(1)
}
}
三个方法cmdCheck,cmdAdd, cmdDel。
cmdCheck
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
// 根据提供的ContainerID和IfName,判断ip是否存在。如不存在则返回一个错误。
func cmdCheck(args *skel.CmdArgs) error {
// 通过传过来的参数(也就是第二步中的json)构建一个IPAMConfig
ipamConf, _, err := allocator.LoadIPAMConfig(args.StdinData, args.Args)
if err != nil {
return err
}

// Look to see if there is at least one IP address allocated to the container
// in the data dir, irrespective of what that address actually is
// 创建一个store对象,包含一个FileLock和目录.用于文件的操作
store, err := disk.New(ipamConf.Name, ipamConf.DataDir)
if err != nil {
return err
}
defer store.Close()

// 通过容器ID和网络接口的名称(如eth0)
containerIPFound := store.FindByID(args.ContainerID, args.IfName)
if !containerIPFound {
return fmt.Errorf("host-local: Failed to find address added by container %v", args.ContainerID)
}

return nil
}
cmdAdd
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
// 从可选分配一个ip
func cmdAdd(args *skel.CmdArgs) error {
// 通过传过来的参数(也就是第二步中的json)构建一个IPAMConfig
ipamConf, confVersion, err := allocator.LoadIPAMConfig(args.StdinData, args.Args)
if err != nil {
return err
}

result := &current.Result{CNIVersion: current.ImplementedSpecVersion}

if ipamConf.ResolvConf != "" {
dns, err := parseResolvConf(ipamConf.ResolvConf)
if err != nil {
return err
}
result.DNS = *dns
}
   // 创建一个store对象,包含一个FileLock和目录.用于文件的操作
store, err := disk.New(ipamConf.Name, ipamConf.DataDir)
if err != nil {
return err
}
defer store.Close()

// Keep the allocators we used, so we can release all IPs if an error
// occurs after we start allocating
// 保存分配的ip,如果在分配过程中出现错误可以进行释放
allocs := []*allocator.IPAllocator{}

// Store all requested IPs in a map, so we can easily remove ones we use
// and error if some remain
requestedIPs := map[string]net.IP{} // net.IP cannot be a key

// ipamConf.IPArgs为请求的ip列表
for _, ip := range ipamConf.IPArgs {
requestedIPs[ip.String()] = ip
}
// 遍历Ranges
for idx, rangeset := range ipamConf.Ranges {
// 返回IPAllocator指针
// IPAllocator实现了Get方法获取一个IP
// Release方法释放为容器分配的IP
allocator := allocator.NewIPAllocator(&rangeset, store, idx)

// Check to see if there are any custom IPs requested in this range.
var requestedIP net.IP
for k, ip := range requestedIPs {
if rangeset.Contains(ip) {
requestedIP = ip
// 在一个rangeset找到了ip,则删除requestedIPs中的记录
delete(requestedIPs, k)
break
}
}
// 根据容器ID,网络接口的名称和IP返回IPConfig
ipConf, err := allocator.Get(args.ContainerID, args.IfName, requestedIP)
if err != nil {
// Deallocate all already allocated IPs
// 如果获取IP出错,释放对应的IP
for _, alloc := range allocs {
_ = alloc.Release(args.ContainerID, args.IfName)
}
return fmt.Errorf("failed to allocate for range %d: %v", idx, err)
}

allocs = append(allocs, allocator)

result.IPs = append(result.IPs, ipConf)
}

// If an IP was requested that wasn't fulfilled, fail
// 如果requestedIPs长度不为0,表示有些IP不符合要求
// 进行回退操作,释放已分配IP
if len(requestedIPs) != 0 {
for _, alloc := range allocs {
_ = alloc.Release(args.ContainerID, args.IfName)
}
errstr := "failed to allocate all requested IPs:"
for _, ip := range requestedIPs {
errstr = errstr + " " + ip.String()
}
return fmt.Errorf(errstr)
}

result.Routes = ipamConf.Routes

return types.PrintResult(result, confVersion)
}
cmdDel
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
// 释放ip
// 通过对cmdAdd的分析,这里应该比较清晰了
func cmdDel(args *skel.CmdArgs) error {
ipamConf, _, err := allocator.LoadIPAMConfig(args.StdinData, args.Args)
if err != nil {
return err
}

store, err := disk.New(ipamConf.Name, ipamConf.DataDir)
if err != nil {
return err
}
defer store.Close()

// Loop through all ranges, releasing all IPs, even if an error occurs
var errors []string
for idx, rangeset := range ipamConf.Ranges {
ipAllocator := allocator.NewIPAllocator(&rangeset, store, idx)

err := ipAllocator.Release(args.ContainerID, args.IfName)
if err != nil {
errors = append(errors, err.Error())
}
}

if errors != nil {
return fmt.Errorf(strings.Join(errors, ";"))
}
return nil
}
对应的一些结构体
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62

// plugins/ipam/host-local/backend/allocator/allocator.go
type IPAllocator struct {
rangeset *RangeSet
store backend.Store
rangeID string // Used for tracking last reserved ip
}

// plugins/ipam/host-local/backend/disk/backend.go
type Store struct {
*FileLock
dataDir string
}

// plugins/ipam/host-local/backend/allocator/config.go
// The top-level network config - IPAM plugins are passed the full configuration
// of the calling plugin, not just the IPAM section.
type Net struct {
Name string `json:"name"`
CNIVersion string `json:"cniVersion"`
IPAM *IPAMConfig `json:"ipam"`
RuntimeConfig struct {
// The capability arg
IPRanges []RangeSet `json:"ipRanges,omitempty"`
IPs []*ip.IP `json:"ips,omitempty"`
} `json:"runtimeConfig,omitempty"`
Args *struct {
A *IPAMArgs `json:"cni"`
} `json:"args"`
}

// IPAMConfig represents the IP related network configuration.
// This nests Range because we initially only supported a single
// range directly, and wish to preserve backwards compatibility
type IPAMConfig struct {
*Range
Name string
Type string `json:"type"`
Routes []*types.Route `json:"routes"`
DataDir string `json:"dataDir"`
ResolvConf string `json:"resolvConf"`
Ranges []RangeSet `json:"ranges"`
IPArgs []net.IP `json:"-"` // Requested IPs from CNI_ARGS, args and capabilities
}

type IPAMEnvArgs struct {
types.CommonArgs
IP ip.IP `json:"ip,omitempty"`
}

type IPAMArgs struct {
IPs []*ip.IP `json:"ips"`
}

type RangeSet []Range

type Range struct {
RangeStart net.IP `json:"rangeStart,omitempty"` // The first ip, inclusive
RangeEnd net.IP `json:"rangeEnd,omitempty"` // The last ip, inclusive
Subnet types.IPNet `json:"subnet"`
Gateway net.IP `json:"gateway,omitempty"`
}

REF:
1.https://www.cni.dev/plugins/current/ipam/host-local/
2.plugins/ipam/host-local/main.go
3.plugins/ipam/host-local/backend/allocator/allocator.go
4.plugins/ipam/host-local/backend/disk/backend.go
5. plugins/ipam/host-local/backend/allocator/config.go