cni-plugins之ipam-host-local

发表于
host-local是一种IP地址管理(IPAM)插件。

它从一组地址范围中分配IP地址,也可以从主机上的resolv.conf文件中获取DNS配置。它在主机文件系统上存储本地状态，因此可以确保单个主机上IP地址的唯一性。
该分配器可以分配多个地址范围，并支持多个(不相交的)子网集。分配策略是在每个范围集中松散的轮询。
在 CNI 调用链中，每个插件都会返回一个 JSON 格式的结果，其中会包含网络配置信息，如 IP 地址、子网掩码、网关等。当一个插件执行完毕，将其返回的结果传递给下一个插件，下一个插件将使用上一个插件返回的网络配置信息来配置网络。当最后一个插件执行完毕后，kubelet 会检查所有插件返回的结果，只有当所有插件都成功设置了 IP 地址，才会认为整个 CNI 调用链执行成功。如果其中一个插件失败，后续的插件将不会被执行，同时整个 CNI 调用链也会被标记为失败。

配置示例(多个地址范围)

ranges表示一个列表，元素是rangeSet,下面的ranges包含两个rangeSet。ranges的长度表示返回多少ip。rangeSet表示ip地址的可选范围。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
{
	"ipam": {
		"type": "host-local",
		"ranges": [
			[
				{
					"subnet": "10.10.0.0/16",
					"rangeStart": "10.10.1.20",
					"rangeEnd": "10.10.3.50",
					"gateway": "10.10.0.254"
				},
				{
					"subnet": "172.16.5.0/24"
				}
			],
			[
				{
					"subnet": "3ffe:ffff:0:01ff::/64",
					"rangeStart": "3ffe:ffff:0:01ff::0010",
					"rangeEnd": "3ffe:ffff:0:01ff::0020"
				}
			]
		],
		"routes": [
			{ "dst": "0.0.0.0/0" },
			{ "dst": "192.168.0.0/16", "gw": "10.10.5.1" },
			{ "dst": "3ffe:ffff:0:01ff::1/64" }
		],
		"dataDir": "/run/my-orchestrator/container-ipam-state"
	}
}

ip地址分配

对于每一个请求的自定义 IP，host-local 分配器会在它所管理的地址范围中进行请求。因此可以指定多个自定义 IP 和多个地址范围。如果一个IP在使用中或者不在范围内将会分配失败。

文件存储

host-local将已分配的 IP 地址作为文件存储在 /var/lib/cni/networks/$NETWORK_NAME 目录下。

源码分析
host-local入口
1
2
3
4
5
6
7
8
9
10
11
12
13
14
// PluginMain 是plugin的main函数，自动进行了错误处理
func main() {
	skel.PluginMain(cmdAdd, cmdCheck, cmdDel, version.All, bv.BuildString("host-local"))
}

// vendor/github.com/containernetworking/cni/pkg/skel/skel.go
func PluginMain(cmdAdd, cmdCheck, cmdDel func(_ *CmdArgs) error, versionInfo version.PluginInfo, about string) {
	if e := PluginMainWithError(cmdAdd, cmdCheck, cmdDel, versionInfo, about); e != nil {
		if err := e.Print(); err != nil {
			log.Print("Error writing error JSON to stdout: ", err)
		}
		os.Exit(1)
	}
}
三个方法cmdCheck,cmdAdd, cmdDel。
cmdCheck
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
// 根据提供的ContainerID和IfName，判断ip是否存在。如不存在则返回一个错误。
func cmdCheck(args *skel.CmdArgs) error {
    // 通过传过来的参数(也就是第二步中的json)构建一个IPAMConfig
	ipamConf, _, err := allocator.LoadIPAMConfig(args.StdinData, args.Args)
	if err != nil {
		return err
	}

	// Look to see if there is at least one IP address allocated to the container
	// in the data dir, irrespective of what that address actually is
    // 创建一个store对象，包含一个FileLock和目录.用于文件的操作
	store, err := disk.New(ipamConf.Name, ipamConf.DataDir)
	if err != nil {
		return err
	}
	defer store.Close()

    // 通过容器ID和网络接口的名称(如eth0)
	containerIPFound := store.FindByID(args.ContainerID, args.IfName)
	if !containerIPFound {
		return fmt.Errorf("host-local: Failed to find address added by container %v", args.ContainerID)
	}

	return nil
}
cmdAdd
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
// 从可选分配一个ip
func cmdAdd(args *skel.CmdArgs) error {
    // 通过传过来的参数(也就是第二步中的json)构建一个IPAMConfig
	ipamConf, confVersion, err := allocator.LoadIPAMConfig(args.StdinData, args.Args)
	if err != nil {
		return err
	}

	result := &current.Result{CNIVersion: current.ImplementedSpecVersion}

	if ipamConf.ResolvConf != "" {
		dns, err := parseResolvConf(ipamConf.ResolvConf)
		if err != nil {
			return err
		}
		result.DNS = *dns
	}
　　 // 创建一个store对象，包含一个FileLock和目录.用于文件的操作
	store, err := disk.New(ipamConf.Name, ipamConf.DataDir)
	if err != nil {
		return err
	}
	defer store.Close()

	// Keep the allocators we used, so we can release all IPs if an error
	// occurs after we start allocating
    // 保存分配的ip,如果在分配过程中出现错误可以进行释放
	allocs := []*allocator.IPAllocator{}

	// Store all requested IPs in a map, so we can easily remove ones we use
	// and error if some remain
	requestedIPs := map[string]net.IP{} // net.IP cannot be a key

    // ipamConf.IPArgs为请求的ip列表
	for _, ip := range ipamConf.IPArgs {
		requestedIPs[ip.String()] = ip
	}
    // 遍历Ranges
	for idx, rangeset := range ipamConf.Ranges {
        // 返回IPAllocator指针
        // IPAllocator实现了Get方法获取一个IP
        // Release方法释放为容器分配的IP
		allocator := allocator.NewIPAllocator(&rangeset, store, idx)

		// Check to see if there are any custom IPs requested in this range.
		var requestedIP net.IP
		for k, ip := range requestedIPs {
			if rangeset.Contains(ip) {
				requestedIP = ip
                // 在一个rangeset找到了ip,则删除requestedIPs中的记录
				delete(requestedIPs, k)
				break
			}
		}
        // 根据容器ID,网络接口的名称和IP返回IPConfig
		ipConf, err := allocator.Get(args.ContainerID, args.IfName, requestedIP)
		if err != nil {
			// Deallocate all already allocated IPs
            // 如果获取IP出错，释放对应的IP
			for _, alloc := range allocs {
				_ = alloc.Release(args.ContainerID, args.IfName)
			}
			return fmt.Errorf("failed to allocate for range %d: %v", idx, err)
		}

		allocs = append(allocs, allocator)

		result.IPs = append(result.IPs, ipConf)
	}

	// If an IP was requested that wasn't fulfilled, fail
    // 如果requestedIPs长度不为0,表示有些IP不符合要求
    // 进行回退操作，释放已分配IP
	if len(requestedIPs) != 0 {
		for _, alloc := range allocs {
			_ = alloc.Release(args.ContainerID, args.IfName)
		}
		errstr := "failed to allocate all requested IPs:"
		for _, ip := range requestedIPs {
			errstr = errstr + " " + ip.String()
		}
		return fmt.Errorf(errstr)
	}

	result.Routes = ipamConf.Routes

	return types.PrintResult(result, confVersion)
}
cmdDel
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
// 释放ip
// 通过对cmdAdd的分析，这里应该比较清晰了
func cmdDel(args *skel.CmdArgs) error {
	ipamConf, _, err := allocator.LoadIPAMConfig(args.StdinData, args.Args)
	if err != nil {
		return err
	}

	store, err := disk.New(ipamConf.Name, ipamConf.DataDir)
	if err != nil {
		return err
	}
	defer store.Close()

	// Loop through all ranges, releasing all IPs, even if an error occurs
	var errors []string
	for idx, rangeset := range ipamConf.Ranges {
		ipAllocator := allocator.NewIPAllocator(&rangeset, store, idx)

		err := ipAllocator.Release(args.ContainerID, args.IfName)
		if err != nil {
			errors = append(errors, err.Error())
		}
	}

	if errors != nil {
		return fmt.Errorf(strings.Join(errors, ";"))
	}
	return nil
}
对应的一些结构体
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62

// plugins/ipam/host-local/backend/allocator/allocator.go
type IPAllocator struct {
	rangeset *RangeSet
	store    backend.Store
	rangeID  string // Used for tracking last reserved ip
}

// plugins/ipam/host-local/backend/disk/backend.go
type Store struct {
	*FileLock
	dataDir string
}

// plugins/ipam/host-local/backend/allocator/config.go
// The top-level network config - IPAM plugins are passed the full configuration
// of the calling plugin, not just the IPAM section.
type Net struct {
	Name          string      `json:"name"`
	CNIVersion    string      `json:"cniVersion"`
	IPAM          *IPAMConfig `json:"ipam"`
	RuntimeConfig struct {
		// The capability arg
		IPRanges []RangeSet `json:"ipRanges,omitempty"`
		IPs      []*ip.IP   `json:"ips,omitempty"`
	} `json:"runtimeConfig,omitempty"`
	Args *struct {
		A *IPAMArgs `json:"cni"`
	} `json:"args"`
}

// IPAMConfig represents the IP related network configuration.
// This nests Range because we initially only supported a single
// range directly, and wish to preserve backwards compatibility
type IPAMConfig struct {
	*Range
	Name       string
	Type       string         `json:"type"`
	Routes     []*types.Route `json:"routes"`
	DataDir    string         `json:"dataDir"`
	ResolvConf string         `json:"resolvConf"`
	Ranges     []RangeSet     `json:"ranges"`
	IPArgs     []net.IP       `json:"-"` // Requested IPs from CNI_ARGS, args and capabilities
}

type IPAMEnvArgs struct {
	types.CommonArgs
	IP ip.IP `json:"ip,omitempty"`
}

type IPAMArgs struct {
	IPs []*ip.IP `json:"ips"`
}

type RangeSet []Range

type Range struct {
	RangeStart net.IP      `json:"rangeStart,omitempty"` // The first ip, inclusive
	RangeEnd   net.IP      `json:"rangeEnd,omitempty"`   // The last ip, inclusive
	Subnet     types.IPNet `json:"subnet"`
	Gateway    net.IP      `json:"gateway,omitempty"`
}

REF:
1.https://www.cni.dev/plugins/current/ipam/host-local/
2.plugins/ipam/host-local/main.go
3.plugins/ipam/host-local/backend/allocator/allocator.go
4.plugins/ipam/host-local/backend/disk/backend.go
5. plugins/ipam/host-local/backend/allocator/config.go