· cloud

ClusterAutoScaler无法从零扩容

背景

最近在 AWS k8s 集群部署一个多 AZ 应用时，发现cluster-autoscaler无法正常scale up。

通过反复测试发现，当NodeGroup的初始容量为 0 时（minSize=0）无法扩容，报错信息如下，当NodeGrop初始容量为 1 时（minSize=1）可以正常扩容。

Normal   NotTriggerScaleUp  18m                   cluster-autoscaler  pod didn't trigger scale-up (it wouldn't fit if a new node is added): 3 Insufficient cpu, 1 node(s) didn't match node selector, 1 max node group size reached, 12 Insufficient memory, 1 node(s) didn't match pod affinity/anti-affinity, 1 node(s) didn't match pod anti-affinity rules, 3 node(s) had taint, that the pod didn't tolerate

这个现象很有意思，跟着代码查看下究竟是哪里出问题了。

探究

ClusterAutoScaler主要扩容逻辑如下：

  • 定期获取（默认 10s）所有Pending Pod，过滤出由于资源不足调度失败的 Pod
  • 根据NodeGroup生成新 Node 模拟调度，如果可以调度则将新节点加入集群

根据现象猜测与生成NodeTemplate有关，否则不会造成两次情况调度结果不一致。 主要代码位于utils.go#L42

// nodes 指当前集群中所有node
func GetNodeInfosForGroups(nodes []*apiv1.Node, nodeInfoCache map[string]*schedulerframework.NodeInfo, cloudProvider cloudprovider.CloudProvider, listers kube_util.ListerRegistry,
	daemonsets []*appsv1.DaemonSet, predicateChecker simulator.PredicateChecker, ignoredTaints taints.TaintKeySet) (map[string]*schedulerframework.NodeInfo, errors.AutoscalerError) {
	result := make(map[string]*schedulerframework.NodeInfo)
	seenGroups := make(map[string]bool)

  // 构建node与pod的映射
	podsForNodes, err := getPodsForNodes(listers)
	if err != nil {
		return map[string]*schedulerframework.NodeInfo{}, err
	}

	// processNode returns information whether the nodeTemplate was generated and if there was an error.
    // processNode 函数通过提供的node生成node模板
	processNode := func(node *apiv1.Node) (bool, string, errors.AutoscalerError) {
		nodeGroup, err := cloudProvider.NodeGroupForNode(node)
		if err != nil {
			return false, "", errors.ToAutoscalerError(errors.CloudProviderError, err)
		}
		if nodeGroup == nil || reflect.ValueOf(nodeGroup).IsNil() {
			return false, "", nil
		}
		id := nodeGroup.Id()
		// nodeGroup id不存在则将其添加到result中
		if _, found := result[id]; !found {
			// Build nodeInfo.
			// 根据当前node生成模板
			nodeInfo, err := simulator.BuildNodeInfoForNode(node, podsForNodes)
			if err != nil {
				return false, "", err
			}
			sanitizedNodeInfo, err := sanitizeNodeInfo(nodeInfo, id, ignoredTaints)
			if err != nil {
				return false, "", err
			}
			result[id] = sanitizedNodeInfo
			return true, id, nil
		}
		return false, "", nil
	}

	// 遍历所有node，如果通过node能获取到对应nodeGroup的模板，则添加到nodeInfoCache中
	for _, node := range nodes {
		// Broken nodes might have some stuff missing. Skipping.
		if !kube_util.IsNodeReadyAndSchedulable(node) {
			continue
		}
		added, id, typedErr := processNode(node)
		if typedErr != nil {
			return map[string]*schedulerframework.NodeInfo{}, typedErr
		}
		if added && nodeInfoCache != nil {
			if nodeInfoCopy, err := deepCopyNodeInfo(result[id]); err == nil {
				nodeInfoCache[id] = nodeInfoCopy
			}
		}
	}

	//如果不在cahce中，则通过nodeGroup生成模板
	for _, nodeGroup := range cloudProvider.NodeGroups() {
		id := nodeGroup.Id()
		seenGroups[id] = true
		if _, found := result[id]; found {
			continue
		}

		// No good template, check cache of previously running nodes.
		if nodeInfoCache != nil {
			if nodeInfo, found := nodeInfoCache[id]; found {
				if nodeInfoCopy, err := deepCopyNodeInfo(nodeInfo); err == nil {
					result[id] = nodeInfoCopy
					continue
				}
			}
		}

		// No good template, trying to generate one. This is called only if there are no
		// working nodes in the node groups. By default CA tries to use a real-world example.
		nodeInfo, err := GetNodeInfoFromTemplate(nodeGroup, daemonsets, predicateChecker, ignoredTaints)
		if err != nil {
			if err == cloudprovider.ErrNotImplemented {
				continue
			} else {
				klog.Errorf("Unable to build proper template node for %s: %v", id, err)
				return map[string]*schedulerframework.NodeInfo{}, errors.ToAutoscalerError(errors.CloudProviderError, err)
			}
		}
		result[id] = nodeInfo
	}

	// Remove invalid node groups from cache
	for id := range nodeInfoCache {
		if _, ok := seenGroups[id]; !ok {
			delete(nodeInfoCache, id)
		}
	}

	// 处理unready/unschedulable的节点
	for _, node := range nodes {
		// Allowing broken nodes
		if !kube_util.IsNodeReadyAndSchedulable(node) {
			added, _, typedErr := processNode(node)
			if typedErr != nil {
				return map[string]*schedulerframework.NodeInfo{}, typedErr
			}
			nodeGroup, err := cloudProvider.NodeGroupForNode(node)
			if err != nil {
				return map[string]*schedulerframework.NodeInfo{}, errors.ToAutoscalerError(
					errors.CloudProviderError, err)
			}
			if added {
				klog.Warningf("Built template for %s based on unready/unschedulable node %s", nodeGroup.Id(), node.Name)
			}
		}
	}

	return result, nil
}

方法GetNodeInfosForGroups生成模板的主要逻辑如下：

  • 遍历集群中所有节点，节点有对应nodeGroup，则根据节点生成模板
  • 其他nodeGroup则根据nodeGroup配置与daemonset信息生成模板

节点存在nodeGroup则根据sanitizeNodeInfo方法生成模板

func BuildNodeInfoForNode(node *apiv1.Node, podsForNodes map[string][]*apiv1.Pod) (*schedulerframework.NodeInfo, errors.AutoscalerError) {
	// 获取当前节点daemonset与mirror pod
	requiredPods, err := getRequiredPodsForNode(node.Name, podsForNodes)
	if err != nil {
		return nil, err
	}
	// 生成node模板
	result := schedulerframework.NewNodeInfo(requiredPods...)
	if err := result.SetNode(node); err != nil {
		return nil, errors.ToAutoscalerError(errors.InternalError, err)
	}
	return result, nil
}

func filterRequiredPodsForNode(allPods []*apiv1.Pod) []*apiv1.Pod {
	var selectedPods []*apiv1.Pod

	for id, pod := range allPods {
		// Ignore pod in deletion phase
		if pod.DeletionTimestamp != nil {
			continue
		}

		if pod_util.IsMirrorPod(pod) || pod_util.IsDaemonSetPod(pod) {
			selectedPods = append(selectedPods, allPods[id])
		}
	}

	return selectedPods
}

根据 nodeGroup 生成模板调用GetNodeInfoFromTemplate方法，首先获取 noGroup 模板信息，再将 daemonset pod 信息加入 node

func GetNodeInfoFromTemplate(nodeGroup cloudprovider.NodeGroup, daemonsets []*appsv1.DaemonSet, predicateChecker simulator.PredicateChecker, ignoredTaints taints.TaintKeySet) (*schedulerframework.NodeInfo, errors.AutoscalerError) {
	id := nodeGroup.Id()
	baseNodeInfo, err := nodeGroup.TemplateNodeInfo()
	if err != nil {
		return nil, errors.ToAutoscalerError(errors.CloudProviderError, err)
	}

	pods, err := daemonset.GetDaemonSetPodsForNode(baseNodeInfo, daemonsets, predicateChecker)
	if err != nil {
		return nil, errors.ToAutoscalerError(errors.InternalError, err)
	}
	for _, podInfo := range baseNodeInfo.Pods {
		pods = append(pods, podInfo.Pod)
	}
	fullNodeInfo := schedulerframework.NewNodeInfo(pods...)
	fullNodeInfo.SetNode(baseNodeInfo.Node())
	sanitizedNodeInfo, typedErr := sanitizeNodeInfo(fullNodeInfo, id, ignoredTaints)
	if typedErr != nil {
		return nil, typedErr
	}
	return sanitizedNodeInfo, nil
}

其中nodeGroup.TemplateNodeInfo方法是每个云供应商提供的，AWS相关代码位于aws_cloud_provider.go#L100

// TemplateNodeInfo returns a node template for this node group.
func (ng *AwsNodeGroup) TemplateNodeInfo() (*schedulerframework.NodeInfo, error) {
	// 获取模板
	template, err := ng.awsManager.getAsgTemplate(ng.asg)
	if err != nil {
		return nil, err
	}

	// 生成node
	node, err := ng.awsManager.buildNodeFromTemplate(ng.asg, template)
	if err != nil {
		return nil, err
	}

	nodeInfo := schedulerframework.NewNodeInfo(cloudprovider.BuildKubeProxy(ng.asg.Name))
	nodeInfo.SetNode(node)
	return nodeInfo, nil
}

func (m *AwsManager) getAsgTemplate(asg *asg) (*asgTemplate, error) {
	...
	instanceTypeName, err := m.buildInstanceType(asg)
	if err != nil {
		return nil, err
	}
	// 根据instanceTypeName获取模板
	if t, ok := m.instanceTypes[instanceTypeName]; ok {
		return &asgTemplate{
			InstanceType: t,
			Region:       region,
			Zone:         az,
			Tags:         asg.Tags,
		}, nil
	}
	return nil, fmt.Errorf("ASG %q uses the unknown EC2 instance type %q", asg.Name, instanceTypeName)
}

在AWS通过nodeGroup生成节点模板，是根据ec2_instance_types.go文件中的信息来生成的，如下c5d.large为 ec2 类型名称，CPU 为 2，内存为4Gi

	"c5d.large": {
		InstanceType: "c5d.large",
		VCPU:         2,
		MemoryMb:     4096,
		GPU:          0,
	},

而在我们的环境中，由于成本原因实际使用的c5d.large内存是16Gi。当 nodeGroup 初始节点为 0 时，根据 ec2 类型生成的节点是4Gi内存；而当初始节点为 1 时，集群中已经存在对应 nodeGroup 的节点，则通过已存在节点生成模板的内存为16Gi，内存不足才导致调度失败，我们也验证了调小 pod 申请的内存确实能够正常扩容。

一个简单的解决办法是将初始节点设置为 1，另外可以将特殊类型添加到配置文件中，则可以正常工作。但根本原因是在AWS中无法正常感知节点资源的动态变化。

总结

在AWS中通过静态文件来获取节点资源信息，当实际生产中与文件不一致时，会造成ClusterAutoScaler无法按照预期工作。ClusterAutoScaler大大减少了运维工作，无需时时关心资源申请量，但只有了解其内部逻辑，才能更好的应用于生产。