We encountered some pretty interesting behavior when working with the Headless service in Kubernetes. In our case, the problem occurred with mongos, but it is relevant for any Headless service. I invite you to read our story and try to play around with this problem locally.
On one of the projects, we are using MongoDB and Kubernetes. MongoDB has a component: mongos. Through it, queries are executed in a sharded MongoDB cluster (you can assume that this is just a tricky proxy). Before moving to Kubernetes, mongos services were installed directly on each host.
When moving services to Kubernetes, we settled the mongos pool into a Headless service with automatic Deployment scaling through HPA (Horizontal Pod Autoscaler).
After a while, it turned out that the application was not doing very well with the decrease in the number of Pods from mongos.
Through debugging, it turned out that the application hangs exactly when it tries to establish a connection with mongos ( net.Dial
in terms of Go) and coincides in time with stopping any Pod.
, Headless-: , IP- (ClusterIP: None
). DNS- IP Pod, .
Headless- , , Pod , :
mongodb- IP , , , ( «» mongos).
ClusterIP
«» .
gRPC- , .
ClusterIP
Pod .
, Pod , , IP- Pod. :
Pod DNS, DNS ;
DNS .
, Pod?
. , .
, Pod Out of Memory, , “connection refused” . , .
, .
SIGTERM
Pod mongos. 45 DNS ( Pod ). mongos 15 ( IP “connection refused”, ).
terminationGracePeriodSeconds
, Pod .
minReadySeconds
Pod .
, , IP- ( Pod , ).
minReadySeconds
. , : IP Pod.
minReadySeconds
- , Pod Terminating
. x2 Pod.
, IP- , DNS minReadySeconds
.
, minReadySeconds
gRPC-: , .
?
MiniKube nginx.
headless Service (service.yml
):
---
apiVersion: v1
kind: Service
metadata:
name: nginx
spec:
clusterIP: None
selector:
app: nginx
ports:
- protocol: TCP
port: 80
targetPort: 80
(dialer.go
):
package main
import (
"fmt"
"net"
"os"
"time"
)
const timeFormat = "15:04:05.999"
func main() {
address := os.Args[1]
last := ""
ticker := time.NewTicker(time.Millisecond * 100)
t := time.Now()
fmt.Printf("%s: === %s\n", t.Format(timeFormat), address)
for {
conn, err := net.DialTimeout("tcp", address, time.Millisecond*100)
var msg string
if conn != nil {
msg = fmt.Sprintf("connected (%s)", conn.RemoteAddr())
_ = conn.Close()
}
if err != nil {
msg = err.Error()
}
if last != msg {
now := time.Now()
if last != "" {
fmt.Printf("%s: --- %s: %v\n", now.Format(timeFormat), last, now.Sub(t))
}
last = msg
fmt.Printf("%s: +++ %s\n", now.Format(timeFormat), last)
t = now
}
<-ticker.C
}
}
nginx 80- . ( , ):
#!/bin/bash
echo "
tee dialer.go << EEOF
$(cat dialer.go)
EEOF
go run dialer.go nginx:80
" | kubectl --context=minikube run -i --rm "debug-$(date +'%s')" \
--image=golang:1.16 --restart=Never --
- :
16:57:19.986: === nginx:80
16:57:19.988: +++ dial tcp: lookup nginx on 10.96.0.10:53: server misbehaving
.
Deployment
Deployment (nginx.yml
):
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx
spec:
replicas: 1
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: nginx:1.14.2
ports:
- containerPort: 80
replicas
, IP-.
Deployment livenessProbe
readinessProbe
. .
«» Deployment:
#!/bin/bash
kubectl --context minikube rollout restart deployment/nginx
Deployment. , : Pod Pod. Pod.
( ):
# Deployment
#
17:04:08.288: +++ connected (172.17.0.10:80)
17:07:32.187: --- connected (172.17.0.10:80): 3m23.899438044s
# nginx Pod,
# IP.
# Pod , "connection refused"
17:07:32.187: +++ dial tcp 172.17.0.10:80: connect: connection refused
17:07:32.488: --- dial tcp 172.17.0.10:80: connect: connection refused: 301.155902ms
# Pod , IP.
# IP- , .
17:07:32.488: +++ dial tcp 172.17.0.10:80: i/o timeout
17:07:38.448: --- dial tcp 172.17.0.10:80: i/o timeout: 5.960150161s
# IP Pod.
17:07:38.448: +++ connected (172.17.0.7:80)
Pod
Deployment , “connection refused”:
#!/bin/bash
kubectl --context minikube patch deployment nginx --output yaml --patch '
---
spec:
template:
spec:
containers:
- name: nginx
command: [ "sh" ]
# nginx
args:
- "-c"
- "nginx -g \"daemon off;\" && sleep 60"
# , sh SIGTERM
lifecycle:
preStop:
exec:
command: ["sh", "-c", "nginx -s stop"]
# , Pod-
#
terminationGracePeriodSeconds: 180
'
Pod ( SIGTERM
). , , Out Of Memory Segmentation fault, .
«» Deployment:
#!/bin/bash
kubectl --context minikube rollout restart deployment/nginx
( ):
# Deployment
#
17:58:10.389: +++ connected (172.17.0.7:80)
18:00:53.687: --- connected (172.17.0.7:80): 2m43.29763747s
# nginx Pod,
# IP.
# Pod , "connection refused".
# Pod sleep nginx.
18:00:53.687: +++ dial tcp 172.17.0.7:80: connect: connection refused
18:01:10.491: --- dial tcp 172.17.0.7:80: connect: connection refused: 16.804114254s
# IP Pod.
18:01:10.491: +++ connected (172.17.0.10:80)
Pod
Deployment , , Pod :
#!/bin/bash
kubectl --context minikube patch deployment nginx --output yaml --patch '
---
spec:
template:
spec:
containers:
- name: nginx
# nginx
lifecycle:
preStop:
exec:
command: ["sh", "-c", "sleep 60 && nginx -s stop"]
# , Pod
#
terminationGracePeriodSeconds: 180
'
«» Deployment:
#!/bin/bash
kubectl --context minikube rollout restart deployment/nginx
( ):
# Deployment
#
18:05:10.589: +++ connected (172.17.0.7:80)
18:07:10.689: --- connected (172.17.0.7:80): 2m0.099149168s
# IP Pod.
# Pod - .
18:07:10.689: +++ connected (172.17.0.10:80)
?
: .
SIGTERM
— DNS- Pod .
, DNS-.
, DNS- .
IP- ,
SIGTERM
minReadySeconds
.
Pod, / Pod “connection refused”, .
,
SIGTERM
Pod DNS .
, , .
.