【kubernetes/k8s概念】CNI plugin calico源码分析

版权声明：本文为博主原创文章，未经博主允许不得转载。 https://blog.csdn.net/zhonglinzhang/article/details/82757166

https://github.com/projectcalico/cni-plugin

       calico解决不同物理机上容器之间的通信，而calico-plugin是在k8s创建Pod时为Pod设置虚拟网卡(容器中的eth0和lo网卡)，calico-plugin是由两个静态的二进制文件组成，由kubelet以命令行的形式调用，这两个二进制的作用如下：

• calico-ipam：分配维护IP，依赖etcd
• calico：系统调用API来修改namespace中的网卡信息

calico插件配置

# cat /etc/cni/net.d/10-calico.conf 
{
    "name": "calico-k8s-network",
    "cniVersion": "0.1.0",
    "type": "calico",
    "etcd_endpoints": "https://node1:2379,https://node2:2379,https://node3:2379",
    "etcd_key_file": "/etc/calico/ssl/calico-key.pem",
    "etcd_cert_file": "/etc/calico/ssl/calico.pem",
    "etcd_ca_cert_file": "/etc/calico/ssl/ca.pem",
    "log_level": "info",
    "mtu": 1500,
    "ipam": {
        "type": "calico-ipam"
    },
    "policy": {
        "type": "k8s"
    },
    "kubernetes": {
        "kubeconfig": "/root/.kube/config"
    }
}

calico-plugin工作原理

       kubelet在创建一个Pod，首先启动pause容器，然后为pause容器添加设置网络，也就是添加网卡，这里会通过CNI调起文件系统中的/opt/cni/bin/calico，并将Pod信息通过标准输入(stdin)传递给calico进程，calico通过修改系统中Namespace

1、main函数

• 与其他plugin代码一样的套路，主要调用skel.PluginMain函数，主要分析cmdAdd函数

func main() {
	// Set up logging formatting.
	logrus.SetFormatter(&logutils.Formatter{})

	// Install a hook that adds file/line no information.
	logrus.AddHook(&logutils.ContextHook{})

	// Display the version on "-v", otherwise just delegate to the skel code.
	// Use a new flag set so as not to conflict with existing libraries which use "flag"
	flagSet := flag.NewFlagSet("Calico", flag.ExitOnError)

	version := flagSet.Bool("v", false, "Display version")
	err := flagSet.Parse(os.Args[1:])
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	if *version {
		fmt.Println(VERSION)
		os.Exit(0)
	}

	if err := utils.AddIgnoreUnknownArgs(); err != nil {
		os.Exit(1)
	}

	skel.PluginMain(cmdAdd, cmdDel, cniSpecVersion.All)
}

2、cmdAdd函数

   cmdAdd函数写了一大沱，分析其中主要函数

 （1）、k8s.CmdAddK8s函数

// If running under Kubernetes then branch off into the kubernetes code, otherwise handle everything in this
	// function.
	if wepIDs.Orchestrator == api.OrchestratorKubernetes {
		if result, err = k8s.CmdAddK8s(ctx, args, conf, *wepIDs, calicoClient, endpoint); err != nil {
			return err
		}
	}

  （2）、根据IPAM类型为calico-ipam

if conf.IPAM.Type == "calico-ipam" {

			v4pools := annot["cni.projectcalico.org/ipv4pools"]
			v6pools := annot["cni.projectcalico.org/ipv6pools"]

			if len(v4pools) != 0 || len(v6pools) != 0 {
				var stdinData map[string]interface{}
				if err := json.Unmarshal(args.StdinData, &stdinData); err != nil {
					return nil, err
				}
				var v4PoolSlice, v6PoolSlice []string

				if len(v4pools) > 0 {
					if err := json.Unmarshal([]byte(v4pools), &v4PoolSlice); err != nil {
						logger.WithField("IPv4Pool", v4pools).Error("Error parsing IPv4 IPPools")
						return nil, err
					}

					if _, ok := stdinData["ipam"].(map[string]interface{}); !ok {
						logger.Fatal("Error asserting stdinData type")
						os.Exit(0)
					}
					stdinData["ipam"].(map[string]interface{})["ipv4_pools"] = v4PoolSlice
					logger.WithField("ipv4_pools", v4pools).Debug("Setting IPv4 Pools")
				}
				if len(v6pools) > 0 {
					if err := json.Unmarshal([]byte(v6pools), &v6PoolSlice); err != nil {
						logger.WithField("IPv6Pool", v6pools).Error("Error parsing IPv6 IPPools")
						return nil, err
					}

					if _, ok := stdinData["ipam"].(map[string]interface{}); !ok {
						logger.Fatal("Error asserting stdinData type")
						os.Exit(0)
					}
					stdinData["ipam"].(map[string]interface{})["ipv6_pools"] = v6PoolSlice
					logger.WithField("ipv6_pools", v6pools).Debug("Setting IPv6 Pools")
				}

				newData, err := json.Marshal(stdinData)
				if err != nil {
					logger.WithField("stdinData", stdinData).Error("Error Marshaling data")
					return nil, err
				}
				args.StdinData = newData
				logger.WithField("stdin", string(args.StdinData)).Debug("Updated stdin data")
			}
		}

  （3）、

• 调用ipam.ExecAdd(conf.IPAM.Type, args.StdinData)申请一个IP

// Switch based on which annotations are passed or not passed.
	switch {
	case ipAddrs == "" && ipAddrsNoIpam == "":
		// Call IPAM plugin if ipAddrsNoIpam or ipAddrs annotation is not present.
		logger.Debugf("Calling IPAM plugin %s", conf.IPAM.Type)
		ipamResult, err := ipam.ExecAdd(conf.IPAM.Type, args.StdinData)
		if err != nil {
			return nil, err
		}
		logger.Debugf("IPAM plugin returned: %+v", ipamResult)

		// Convert IPAM result into current Result.
		// IPAM result has a bunch of fields that are optional for an IPAM plugin
		// but required for a CNI plugin, so this is to populate those fields.
		// See CNI Spec doc for more details.
		result, err = current.NewResultFromResult(ipamResult)
		if err != nil {
			utils.ReleaseIPAllocation(logger, conf.IPAM.Type, args.StdinData)
			return nil, err
		}

		if len(result.IPs) == 0 {
			utils.ReleaseIPAllocation(logger, conf.IPAM.Type, args.StdinData)
			return nil, errors.New("IPAM plugin returned missing IP config")
		}

	case ipAddrs != "" && ipAddrsNoIpam != "":
		
	}

  （4）、utils.DoNetworking函数

• 调用netlink.LinkAdd(veth)  netlink.LinkSetUp(hostVeth) 创建一个网卡对veth，主机端一cali开通，后面11位是容器的id开头。然后就是把网卡插入容器内设置IP和路由。一个在Linux的物机机上，一个在容器中，用于容器与物理机之间的通信

• ip.AddRoute(r, gw, contVeth)添加路由

• netlink.LinkSetNsFd(hostVeth, int(hostNS.Fd()))将host端veth加入加入到namespace，相当于命令ip link set $link netns $ns

• SetupRoutes(hostVeth, result)建立host端路由

• utils.CreateOrUpdate(ctx, calicoClient, endpoint)记录到etcd中

// Whether the endpoint existed or not, the veth needs (re)creating.
	hostVethName := k8sconversion.VethNameForWorkload(epIDs.Namespace, epIDs.Pod)
	_, contVethMac, err := utils.DoNetworking(args, conf, result, logger, hostVethName, routes)
	if err != nil {
		logger.WithError(err).Error("Error setting up networking")
		releaseIPAM()
		return nil, err
	}