istio微服务实验之监控日志与可视化

简介

本文是istio微服务实验的后续文章,实验前请先参考之前文章。

分布式调用链追踪

安装

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
# 下载yml文件
mkdir jaeger && cd jaeger
wget https://raw.githubusercontent.com/jaegertracing/jaeger-kubernetes/master/all-in-one/jaeger-all-in-one-template.yml

# 实验环境不支持 LoadBalancer
# 可以修改jaeger-all-in-one-template.yml使用nodeport
# 也可以不修改,这样的会使用随机的nodeport

# 启动
kubectl apply -n istio-system -f jaeger-all-in-one-template.yml

# 查看
kubectl get pods -n istio-system
kubectl get svc -n istio-system

# 多次访问之前的vue react界面并点击发射按钮

# 访问
jaegerNodePort=$(kubectl get svc -n istio-system | grep jaeger-query | awk '{print $5}' | cut -d '/' -f 1 | cut -d ':' -f 2)
nodeName=$(kubectl get no | grep '<none>' | head -1 | awk '{print $1}')
nodeIP=$(ping -c 1 $nodeName | grep PING | awk '{print $3}' | tr -d '()')
echo "http://$nodeIP:"$jaegerNodePort

# 选择 istio-ingress 可以方便查看整个调用链

# 清理
cd jaeger
kubectl delete -n istio-system -f jaeger-all-in-one-template.yml

jaeger的dashboard界面

调用链

服务树展示

日志与指标收集

安装

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
# 安装prometheus
cd /usr/local/istio

# 修改支持nodeport
cp install/kubernetes/addons/prometheus.yaml install/kubernetes/addons/prometheus.yaml.ori
vim install/kubernetes/addons/prometheus.yaml
...
apiVersion: v1
kind: Service
metadata:
annotations:
prometheus.io/scrape: 'true'
labels:
name: prometheus
name: prometheus
namespace: istio-system
spec:
selector:
app: prometheus
ports:
- name: prometheus
protocol: TCP
port: 9090
# 设置使用 nodeport
type: NodePort
...

# 部署
kubectl apply -f install/kubernetes/addons/prometheus.yaml

# 配置收集
istioctl create -f istio/new_telemetry.yml

# 多次访问之前的vue react界面并点击发射按钮

# 访问web测试
prometheusNodePort=$(kubectl get svc -n istio-system | grep prometheus | awk '{print $5}' | cut -d '/' -f 1 | cut -d ':' -f 2)
nodeName=$(kubectl get no | grep '<none>' | head -1 | awk '{print $1}')
nodeIP=$(ping -c 1 $nodeName | grep PING | awk '{print $3}' | tr -d '()')
echo "http://$nodeIP:"$prometheusNodePort

# 使用 istio_double_request_count 关键字查询

# 查看日志
kubectl -n istio-system logs $(kubectl -n istio-system get pods -l istio=mixer -o jsonpath='{.items[0].metadata.name}') mixer | grep \"instance\":\"newlog.logentry.istio-system\"

# 清理
kubectl delete -f install/kubernetes/addons/prometheus.yaml
istioctl delete -f istio/new_telemetry.yml

收集TCP服务的指标

安装

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
# 安装prometheus
cd /usr/local/istio

# 修改支持nodeport
cp install/kubernetes/addons/prometheus.yaml install/kubernetes/addons/prometheus.yaml.ori
vim install/kubernetes/addons/prometheus.yaml
...
apiVersion: v1
kind: Service
metadata:
annotations:
prometheus.io/scrape: 'true'
labels:
name: prometheus
name: prometheus
namespace: istio-system
spec:
selector:
app: prometheus
ports:
- name: prometheus
protocol: TCP
port: 9090
# 设置使用 nodeport
type: NodePort
...

# 部署
kubectl apply -f install/kubernetes/addons/prometheus.yaml

# 配置收集
istioctl create -f istio/tcp_telemetry.yml

# 部署使用mongodb应用测试

# 访问web测试
prometheusNodePort=$(kubectl get svc -n istio-system | grep prometheus | awk '{print $5}' | cut -d '/' -f 1 | cut -d ':' -f 2)
nodeName=$(kubectl get no | grep '<none>' | head -1 | awk '{print $1}')
nodeIP=$(ping -c 1 $nodeName | grep PING | awk '{print $3}' | tr -d '()')
echo "http://$nodeIP:"$prometheusNodePort

# 使用 istio_mongo_received_bytes 关键字查询

# 清理
kubectl delete -f install/kubernetes/addons/prometheus.yaml
istioctl delete -f istio/tcp_telemetry.yml

TCP数据流图

使用grafana可视化指标

安装

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
# 安装prometheus
cd /usr/local/istio

# 修改支持nodeport
cp install/kubernetes/addons/prometheus.yaml install/kubernetes/addons/prometheus.yaml.ori
vim install/kubernetes/addons/prometheus.yaml
...
apiVersion: v1
kind: Service
metadata:
annotations:
prometheus.io/scrape: 'true'
labels:
name: prometheus
name: prometheus
namespace: istio-system
spec:
selector:
app: prometheus
ports:
- name: prometheus
protocol: TCP
port: 9090
# 设置使用 nodeport
type: NodePort
...
cp install/kubernetes/addons/grafana.yaml install/kubernetes/addons/grafana.yaml.ori
vim install/kubernetes/addons/grafana.yaml
kind: Service
metadata:
name: grafana
namespace: istio-system
spec:
# 设置使用 nodeport
type: NodePort
ports:
- port: 3000
protocol: TCP
name: http
selector:
app: grafana

# 部署
kubectl apply -f install/kubernetes/addons/prometheus.yaml
kubectl apply -f install/kubernetes/addons/grafana.yaml

# 访问web测试
grafanaNodePort=$(kubectl get svc -n istio-system | grep grafana | awk '{print $5}' | cut -d '/' -f 1 | cut -d ':' -f 2)
nodeName=$(kubectl get no | grep '<none>' | head -1 | awk '{print $1}')
nodeIP=$(ping -c 1 $nodeName | grep PING | awk '{print $3}' | tr -d '()')
echo "http://$nodeIP:"$grafanaNodePort

# 压力测试查看图表
# 创建测试用的fortio
kubectl apply -f <(istioctl kube-inject -f istio/fortio-deploy.yaml)

# 正常访问测试
FORTIO_POD=$(kubectl get pod | grep fortio | awk '{ print $1 }')
kubectl exec -it $FORTIO_POD -c fortio /usr/local/bin/fortio -- load -curl http://service-python/env

# 加大压力测试
kubectl exec -it $FORTIO_POD -c fortio /usr/local/bin/fortio -- load -qps 20 -t 100s -loglevel Warning http://service-python/env
kubectl exec -it $FORTIO_POD -c fortio /usr/local/bin/fortio -- load -qps 50 -t 100s -loglevel Warning http://service-go/env

# 清理
kubectl delete -f install/kubernetes/addons/prometheus.yaml
kubectl delete -f install/kubernetes/addons/grafana.yaml
kubectl delete -f istio/fortio-deploy.yaml

service mesh 数据监控展示

pilot数据监控展示

生成服务树

安装

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
# 修改使用nodeport
cd /usr/local/istio
cp install/kubernetes/addons/servicegraph.yaml install/kubernetes/addons/servicegraph.yaml.ori
vim install/kubernetes/addons/servicegraph.yaml
...
apiVersion: v1
kind: Service
metadata:
name: servicegraph
namespace: istio-system
spec:
# 设置使用 nodeport
type: NodePort
ports:
- name: http
port: 8088
selector:
app: servicegraph
...

# 安装prometheus
cd /usr/local/istio

# 修改支持nodeport
cp install/kubernetes/addons/prometheus.yaml install/kubernetes/addons/prometheus.yaml.ori
vim install/kubernetes/addons/prometheus.yaml
...
apiVersion: v1
kind: Service
metadata:
annotations:
prometheus.io/scrape: 'true'
labels:
name: prometheus
name: prometheus
namespace: istio-system
spec:
selector:
app: prometheus
ports:
- name: prometheus
protocol: TCP
port: 9090
# 设置使用 nodeport
type: NodePort
...

# 部署
kubectl apply -f install/kubernetes/addons/prometheus.yaml
kubectl apply -f install/kubernetes/addons/servicegraph.yaml

# 多次访问之前的vue react界面并点击发射按钮

# 访问web测试
servicegraphNodePort=$(kubectl get svc -n istio-system | grep servicegraph | awk '{print $5}' | cut -d '/' -f 1 | cut -d ':' -f 2)
nodeName=$(kubectl get no | grep '<none>' | head -1 | awk '{print $1}')
nodeIP=$(ping -c 1 $nodeName | grep PING | awk '{print $3}' | tr -d '()')
echo "http://$nodeIP:"$servicegraphNodePort/force/forcegraph.html

# 可使用url
# /force/forcegraph.html
# /dotviz
# /dotgraph
# /d3graph
# /graph

# 清理
kubectl delete -f install/kubernetes/addons/prometheus.yaml
kubectl delete -f install/kubernetes/addons/servicegraph.yaml

服务树

使用Fluentd收集日志

安装

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
# 安装efk
kubectl apply -f istio/logging-stack.yml

# 配置istio使用efk
istioctl create -f istio/fluentd-istio.yml

# 多次访问之前的vue react界面并点击发射按钮

# 访问web测试
kibanaNodePort=$(kubectl get svc -n istio-system | grep kibana | awk '{print $5}' | cut -d '/' -f 1 | cut -d ':' -f 2)
nodeName=$(kubectl get no | grep '<none>' | head -1 | awk '{print $1}')
nodeIP=$(ping -c 1 $nodeName | grep PING | awk '{print $3}' | tr -d '()')
echo "http://$nodeIP:"$kibanaNodePort

# 清理
kubectl delete -f istio/logging-stack.yml
istio delete -f istio/fluentd-istio.yml