00：文章简介
介绍k8s的综合实践及相关用法。
01：nginx反代tomcat实践
构建nginx镜像和tomcat镜像的基础镜像
# 拉取nginx镜像
docker pull nginx
docker tag nginx harbor.linux98.com/mykubernetes/nginx:1.21.3
# 构建tomcat镜像
cd tomcat-web/
tar zcf ROOT.tar.gz ROOT/
docker build -t tomcat-web:v0.1
docker run --rm -it -P tomcat-web:v0.1
docker tag tomcat-web:v0.1 harbor.linux98.com/mykubernetes/tomcat-web:v0.1
# 登录仓库
docker login harbor.linux98.com
docker push harbor.linux98.com/mykubernetes/nginx:1.21.3
docker push harbor.linux98.com/mykubernetes/tomcat-web:v0.1
编写nginx镜像的yaml文件
# cat nginx.yml
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx-deployment
labels:
app: nginx
spec:
replicas: 1
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: harbor.linux98.com/mykubernetes/nginx:1.21.3
ports:
- containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
name: nginx-web-service
labels:
app: nginx-web-service
spec:
type: NodePort
selector:
app: nginx
ports:
- protocol: TCP
name: http
port: 80
targetPort: 80
nodePort: 30086
应用nginx的yaml文件
kubectl apply -f nginx.yaml
查看nginx的pod状态
# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 11h
nginx-web-service NodePort 10.101.47.126 <none> 80:30086/TCP 6m39s
# kubectl describe svc nginx-web-service
Name: nginx-web-service
Namespace: default
Labels: app=nginx-web-service
Annotations: <none>
Selector: app=nginx
Type: NodePort
IP Family Policy: SingleStack
IP Families: IPv4
IP: 10.101.47.126
IPs: 10.101.47.126
Port: http 80/TCP
TargetPort: 80/TCP
NodePort: http 30086/TCP
Endpoints: 10.244.5.2:80
Session Affinity: None
External Traffic Policy: Cluster
Events: <none>
# kubectl get pod -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
nginx-deployment-7d5c79cd8d-wkgnq 1/1 Running 0 5m14s 10.244.5.2 node03 <none> <none>
从上面可以看到，service自动将nginx-web-service与pod的ip地址绑定到一起了也就是Endpoints
这里我们使用curl对svc的地址进行访问，就可以直接映射到nginx 的pod内部
# curl 10.101.47.126
Welcome to Nginx !! Nginx Version: 1.21.3
我们再继续使用tomcat的yaml文件进行部署tomcat
# tomcat.yml
apiVersion: apps/v1
kind: Deployment
metadata:
name: tomcat-deployment
labels:
app: tomcat
spec:
replicas: 1
selector:
matchLabels:
app: tomcat
template:
metadata:
labels:
app: tomcat
spec:
containers:
- name: tomcat
image: harbor.linux98.com/mykubernetes/tomcat-web:v0.1
ports:
- containerPort: 8080
---
apiVersion: v1
kind: Service
metadata:
name: tomcat-web-service
labels:
app: tomcat-web-service
spec:
type: NodePort
selector:
app: tomcat
ports:
- protocol: TCP
name: http
port: 80
targetPort: 8080
nodePort: 30087
kubectl apply -f tomcat.yml
应用后，查看pod和svc的状态
# kubectl describe svc tomcat-web-service
Name: tomcat-web-service
Namespace: default
Labels: app=tomcat-web-service
Annotations: <none>
Selector: app=tomcat
Type: NodePort
IP Family Policy: SingleStack
IP Families: IPv4
IP: 10.106.253.160
IPs: 10.106.253.160
Port: http 80/TCP
TargetPort: 8080/TCP
NodePort: http 30087/TCP
Endpoints: 10.244.3.3:8080
Session Affinity: None
External Traffic Policy: Cluster
Events: <none>
# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 11h
nginx-web-service NodePort 10.101.47.126 <none> 80:30086/TCP 10m
tomcat-web-service NodePort 10.106.253.160 <none> 80:30087/TCP 34s
# kubectl get pod -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
nginx-deployment-7d5c79cd8d-wkgnq 1/1 Running 0 10m 10.244.5.2 node03 <none> <none>
tomcat-deployment-5bbff76f57-9rc8b 1/1 Running 0 48s 10.244.3.3 node01 <none> <none>
测试效果
root@master01:~/mykube/shijian# curl 10.106.253.160
<html>
<head>
<title>电商网站</title>
</head>
<body>
<h1 style="color:red ; font-size:30px ; display:inline;"> 开业大酬宾： </h1>
买 1 送 2
</body>
</html>
kubectl get svc中的PORTS，前部分是service的端口，后部分是Nodeport，也就是节点主机端口。
# curl localhost:30087
<html>
<head>
<title>电商网站</title>
</head>
<body>
<h1 style="color:red ; font-size:30px ; display:inline;"> 开业大酬宾： </h1>
买 1 送 2
</body>
</html>
接下来，我们使用nginx来反向代理tomcat-web，当我们访问nginx的/tomcat url时，跳转到tomcat页面。
进入nginx的pod内部
# kubectl get pods
NAME READY STATUS RESTARTS AGE
nginx-deployment-7d5c79cd8d-wkgnq 1/1 Running 0 20m
tomcat-deployment-5bbff76f57-9rc8b 1/1 Running 0 11m
# kubectl exec -it nginx-deployment-7d5c79cd8d-wkgnq /bin/bash
由于pod内部没有nsloopup和vim工具，需要在pod中安装
apt update
apt install dnsutils vim -y
先测试能否通过tomcat的service的name获取ip
# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 11h
nginx-web-service NodePort 10.101.47.126 <none> 80:30086/TCP 26m
tomcat-web-service NodePort 10.106.253.160 <none> 80:30087/TCP 16m
root@nginx-deployment-7d5c79cd8d-wkgnq:~# nslookup tomcat-web-service
Server: 10.96.0.10
Address: 10.96.0.10#53
Name: tomcat-web-service.default.svc.cluster.local
Address: 10.106.253.160
root@nginx-deployment-7d5c79cd8d-wkgnq:~# curl tomcat-web-service
<html>
<head>
<title>电商网站</title>
</head>
<body>
<h1 style="color:red ; font-size:30px ; display:inline;"> 开业大酬宾： </h1>
买 1 送 2
</body>
</html>
测试可以获取ip，那么修改nginx配置文件
# vim /etc/nginx/conf.d/default.conf
...
location /tomcat {
proxy_pass http://tomcat-web-service.default.svc.cluster.local/;
}
...
重载nginx
# nginx -t
nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
nginx: configuration file /etc/nginx/nginx.conf test is successful
# nginx -s reload
此时，我们从外部的节点，访问nginx的service ip + /tomcat查看，是否能访问到tomcat的页面
root@master01:~/mykube/init/dashboard# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 11h
nginx-web-service NodePort 10.101.47.126 <none> 80:30086/TCP 26m
tomcat-web-service NodePort 10.106.253.160 <none> 80:30087/TCP 16m
root@master01:~/mykube/init/dashboard# curl 10.101.47.126/tomcat
<html>
<head>
<title>电商网站</title>
</head>
<body>
<h1 style="color:red ; font-size:30px ; display:inline;"> 开业大酬宾： </h1>
买 1 送 2
</body>
</html>
这时，我们退出pod，在mster01上，重新build一下nginx的镜像，使代理tomcat的配置直接存储在镜像中。
root@master01:~/mykube/init/dashboard# exit
root@master01:~/mykube/shijian# vim default.conf
server {
listen 80;
listen [::]:80;
server_name localhost;
location / {
root /usr/share/nginx/html;
index index.html index.htm;
}
location /tomcat {
proxy_pass http://tomcat-web-service.default.svc.cluster.local/;
}
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root /usr/share/nginx/html;
}
}
# mkdir nginx-proxy
# cp default.conf nginx-proxy
# cd nginx-proxy/
# vim Dockerfile
FROM harbor.linux98.com/mykubernetes/nginx:1.21.3
MAINTAINER linux98 000000@qq.com
ADD default.conf /etc/nginx/conf.d/default.conf
构建并提交镜像到harbor
# 构建
docker build -t harbor.linux98.com/mykubernetes/nginx-proxy:1.21.3 .
# 登陆harbor
expect -c "
spawn docker login harbor.linux98.com
expect {
\"*Username:*\" {send \"cpli\r\"; exp_continue}
\"*Password:*\" {send \"A12345678a\r\"; exp_continue}
} "
# 镜像提交
docker push harbor.linux98.com/mykubernetes/nginx-proxy:1.21.3
重新修改nginx-proxy.yml文件中的image路径
# grep proxy nginx-proxy.yml
image: harbor.linux98.com/mykubernetes/nginx-proxy:1.21.3
重新应用nginx-proxy的配置文件
# kubectl apply -f nginx-proxy.yml
查看nginx-proxy的详细构建信息
# kubectl describe pod nginx-deployment-b66b75495-r5klj
Containers:
nginx:
Container ID: docker://9c1113b742cdaa72a7ec6c5ebb35b59cb88fbbd2d74dd0ba57b72f264754bee1
Image: harbor.linux98.com/mykubernetes/nginx-proxy:1.21.3
Image ID: docker-pullable://harbor.linux98.com/mykubernetes/nginx-proxy@sha256:25f23677fee755bc5ff9ea279c36dedd3c3b188404ee81dc3708f69b67b1d7e3
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Scheduled 46s default-scheduler Successfully assigned default/nginx-deployment-b66b75495-r5klj to node03
Normal Pulling 45s kubelet Pulling image "harbor.linux98.com/mykubernetes/nginx-proxy:1.21.3"
Normal Pulled 44s kubelet Successfully pulled image "harbor.linux98.com/mykubernetes/nginx-proxy:1.21.3" in 626.340202ms
Normal Created 44s kubelet Created container nginx
Normal Started 44s kubelet Started container nginx
直接使用nginx-proxy的service地址访问/tomcat
# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 12h
nginx-web-service NodePort 10.104.170.250 <none> 80:30086/TCP 33s
tomcat-web-service NodePort 10.106.253.160 <none> 80:30087/TCP 59m
# curl 10.104.170.250
Welcome to Nginx !! Nginx Version: 1.21.3
# curl 10.104.170.250/tomcat
<html>
<head>
<title>电商网站</title>
</head>
<body>
<h1 style="color:red ; font-size:30px ; display:inline;"> 开业大酬宾： </h1>
买 1 送 2
</body>
</html>
# curl localhost:30086/tomcat
<html>
<head>
<title>电商网站</title>
</head>
<body>
<h1 style="color:red ; font-size:30px ; display:inline;"> 开业大酬宾： </h1>
买 1 送 2
</body>
</html>
接下来，我们将nginx的端口，通过haproxy代理到外部，使外部可以通过80端口直接访问到tomcat app
# keepalived
global_defs {
router_id ha01
}
vrrp_script chk_haproxy {
script "/etc/keepalived/check_haproxy.sh"
interval 2
weight 2
}
vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 51
priority 100
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.31.105 dev eth0 label eth0:1
192.168.31.106 dev eth0 label eth0:2
}
}
# haproxy
listen cluster-nginx-app-30086
bind 192.168.31.106:80
mode tcp
server master1 192.168.31.11:30086 check inter 3s fall 3 rise 5
server master2 192.168.31.12:30086 check inter 3s fall 3 rise 5
server master3 192.168.31.13:30086 check inter 3s fall 3 rise 5
重启keepalived和haproxy
root@ha01:/etc/keepalived# systemctl restart keepalived
root@ha01:/etc/keepalived# systemctl restart haproxy
root@ha01:/etc/keepalived# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:50:56:2e:34:b3 brd ff:ff:ff:ff:ff:ff
inet 192.168.31.101/24 brd 192.168.31.255 scope global eth0
valid_lft forever preferred_lft forever
inet 192.168.31.105/32 scope global eth0:1
valid_lft forever preferred_lft forever
inet 192.168.31.106/32 scope global eth0:2
valid_lft forever preferred_lft forever
inet6 fe80::250:56ff:fe2e:34b3/64 scope link
valid_lft forever preferred_lft forever
root@ha01:/etc/keepalived# ss -nutlp
Netid State Recv-Q Send-Q Local Address:Port Peer Address:Port
tcp LISTEN 0 486 192.168.31.106:80 0.0.0.0:* users:(("haproxy",pid=24633,fd=11))
访问haproxy代理的80端口
root@ha01:/etc/keepalived# curl 192.168.31.106/tomcat
<html>
<head>
<title>电商网站</title>
</head>
<body>
<h1 style="color:red ; font-size:30px ; display:inline;"> 开业大酬宾： </h1>
买 1 送 2
</body>
</html>
正常业务情况下，将haproxy的vip地址解析为域名，例如linux98.com，那么用户在浏览器中访问linux98.com/tomcat，即可访问我们发布的app了。
实践流程总结：
1.分析业务需求场景
2.梳理需要的资源对象
3.定制资源对象配置文件
4.执行资源配置文件
5.调试环境，测试效果
6.如果需要改进，更改资源配置文件
7.成型
评论区