Safely drain a node for maintenance with pod eviction and validation
Safely evicts all pods from a Kubernetes node for maintenance, with pre-drain validation and verification checks. Use this before node upgrades, repairs, or decommissioning to ensure graceful pod migration without downtime.
/plugin marketplace add kcns008/cluster-code/plugin install kcns008-cluster-core-plugins-cluster-core@kcns008/cluster-codeSafely evict all pods from a node for maintenance, upgrades, or decommissioning.
Verify node exists:
kubectl get node $NODE_NAME || {
echo "❌ Node not found: $NODE_NAME"
exit 1
}
Get node information:
NODE_INFO=$(kubectl get node $NODE_NAME -o json)
NODE_STATUS=$(echo $NODE_INFO | jq -r '.status.conditions[] | select(.type=="Ready") | .status')
NODE_ROLE=$(echo $NODE_INFO | jq -r '.metadata.labels."node-role.kubernetes.io/control-plane" // "worker"')
KUBELET_VERSION=$(echo $NODE_INFO | jq -r '.status.nodeInfo.kubeletVersion')
echo "Node: $NODE_NAME"
echo "Status: $NODE_STATUS"
echo "Role: $NODE_ROLE"
echo "Kubelet Version: $KUBELET_VERSION"
echo ""
Check pods on node:
PODS=$(kubectl get pods --all-namespaces --field-selector spec.nodeName=$NODE_NAME -o json)
POD_COUNT=$(echo $PODS | jq '.items | length')
echo "Pods on node: $POD_COUNT"
echo ""
if [[ $POD_COUNT -eq 0 ]]; then
echo "ℹ️ No pods on node - drain not needed"
exit 0
fi
# Show pod breakdown
echo "Pod breakdown:"
echo $PODS | jq -r '.items[] | "\(.metadata.namespace)/\(.metadata.name)"' | \
awk -F/ '{ns[$1]++} END {for (n in ns) printf " %s: %d pods\n", n, ns[n]}'
echo ""
# Check for DaemonSets
DAEMONSET_PODS=$(echo $PODS | jq '[.items[] | select(.metadata.ownerReferences[]?.kind=="DaemonSet")] | length')
if [[ $DAEMONSET_PODS -gt 0 ]]; then
echo "ℹ️ $DAEMONSET_PODS DaemonSet pods (will be ignored with --ignore-daemonsets)"
fi
# Check for standalone pods
STANDALONE_PODS=$(echo $PODS | jq '[.items[] | select(.metadata.ownerReferences | length == 0)] | length')
if [[ $STANDALONE_PODS -gt 0 ]]; then
echo "⚠️ $STANDALONE_PODS standalone pods (require --force to drain)"
echo $PODS | jq -r '.items[] | select(.metadata.ownerReferences | length == 0) | " - \(.metadata.namespace)/\(.metadata.name)"'
fi
echo ""
echo "🔒 Cordoning node..."
kubectl cordon $NODE_NAME
if [[ $? -eq 0 ]]; then
echo "✅ Node cordoned (no new pods will be scheduled)"
else
echo "❌ Failed to cordon node"
exit 1
fi
echo ""
echo "🚰 Draining node..."
echo ""
# Build drain command
DRAIN_CMD="kubectl drain $NODE_NAME"
[[ "$IGNORE_DAEMONSETS" == "true" ]] && DRAIN_CMD="$DRAIN_CMD --ignore-daemonsets"
[[ "$FORCE" == "true" ]] && DRAIN_CMD="$DRAIN_CMD --force"
[[ "$DELETE_EMPTYDIR" == "true" ]] && DRAIN_CMD="$DRAIN_CMD --delete-emptydir-data"
DRAIN_CMD="$DRAIN_CMD --grace-period=$GRACE_PERIOD --timeout=$TIMEOUT"
# Execute drain
$DRAIN_CMD
DRAIN_EXIT=$?
if [[ $DRAIN_EXIT -eq 0 ]]; then
echo ""
echo "✅ Node drained successfully"
else
echo ""
echo "❌ Drain failed"
echo ""
echo "Common issues:"
echo "- Standalone pods (use --force)"
echo "- PodDisruptionBudgets preventing eviction"
echo "- Pods using emptyDir (use --delete-emptydir-data)"
exit 1
fi
echo ""
echo "Verifying drain..."
REMAINING_PODS=$(kubectl get pods --all-namespaces --field-selector spec.nodeName=$NODE_NAME --no-headers 2>/dev/null | grep -v DaemonSet | wc -l)
if [[ $REMAINING_PODS -eq 0 ]]; then
echo "✅ All non-DaemonSet pods evicted"
else
echo "⚠️ $REMAINING_PODS pods remaining"
kubectl get pods --all-namespaces --field-selector spec.nodeName=$NODE_NAME
fi
echo ""
echo "Node is ready for maintenance"
echo ""
echo "To uncordon node when ready:"
echo " kubectl uncordon $NODE_NAME"