Jenkinsforkubernetes实现slave动态伸缩思路和部署
本⽂章案例可⽤于参考Jenkins for kubernetes部署。因每个公司的架构和环境不⼀样，需要改变⼀些部署的⽅式。
Jenkins for kubernetes的好处：
Jenkins-master的⾼可⽤。k8s 的rc或deployment可以监控副本的存活状态（通过探针）和副本数量，如果master出现⽆法提供服务的情况，就会重启或者迁移到其他节点。
Jenkins-slave的动态伸缩。 每次构建都会启动⼀个pod⽤于部署slave，构建完成后就会释放掉。那么pod在创建的时候，k8s就会选择集内资源剩余较多的节点创建slave的pod，构建完成后pod会⾃动删除。
扩展性好。 因为可以同时拥有很多个slave，可以配置Jenkins同时执⾏很多构建操作，减少排队等待构建的时间。
部署思路：
⾸先在k8s中部署Jenkins-master然后使⽤插件进⾏slave的动态伸缩。并且使⽤NFS作为后端存储的PersistentVolume来挂载Jenkins-master的jenkins_home⽬录、构建时slave的maven缓存m2⽬录（可以利⽤缓存加快每次构建的速度）、保留slave每次构建产⽣的数据（workspace⽬录中的每个job）。
使⽤PersistentVolume的原因是k8s任何节点都可以访问到挂载的⽬录，不会因为master迁移节点导致数据丢失。NFS⽅便部署⽽且性能也满⾜Jenkins的使⽤需求所以选择了NFS，也可以使⽤其他的后端存储。
部署
部署⽅式可以⾃定义也可以使⽤提供的部署yml。⾃定义使⽤deployment也是可以的，但是官⽹的部署⽅式使⽤了StatefulSet。Jenkins 是⼀个有状态的应⽤，我感觉使⽤StatefulSet部署更加严谨⼀点。我这⾥使⽤了官⽹提供的⽂档进⾏部署的，但是也根据实际情况修改了⼀些东西。
⾸先需要在k8s所有节点部署NFS客户端：
yum -y install nfs-utils
systemctl start nfs-utils
systemctl enable nfs-utils
rpcinfo -p
NFS服务端配置⽂件增加配置：
/data/dev_jenkins      10.0.0.0/24(rw,sync,no_root_squash,no_subtree_check)
#dev环境Jenkins slave节点挂载workspace
/data/dev_jenkins/workspace  0.0.0.0/0(rw,sync,no_root_squash,no_subtree_check)
#dev环境Jenkins slave节点挂载m2 maven缓存⽬录
/data/dev_jenkins/m2 0.0.0.0/0(rw,sync,no_root_squash,no_subtree_check)
共享⽬录⼀定要给777权限。不然容器内部会报错没有写⼊权限。
# In GKE need to get RBAC permissions first with
# kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin [--user=<user-name>|--group=<group-name>]
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: jenkins
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: jenkins
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["create","delete","get","list","patch","update","watch"]
- apiGroups: [""]
resources: ["pods/exec"]
verbs: ["create","delete","get","list","patch","update","watch"]
- apiGroups: [""]
resources: ["pods/log"]
verbs: ["get","list","watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["watch"]
-
apiGroups: [""]
resources: ["secrets"]
verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
nodeselectormetadata:
name: jenkins            #与l中的serviceAccountName: jenkins相对应
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: jenkins
subjects:
- kind: ServiceAccount
name: jenkins
[root@dev-master1 kubernetes]# l
apiVersion: v1
kind: PersistentVolume
metadata:
name: jenkins-home
spec:
capacity:  #指定容量
storage: 20Gi
accessModes:
- ReadWriteOnce  #访问模式，还有ReadOnlyMany ##ReadOnlymany
#  persistenVolumeReclaimPolicy: Recycle
#  storageClassName: nfs  ##指定存储的类型
nfs:
path: /data/dev_jenkins  #指明nfs的路径
server: 10.0.0.250  #指明nfs的ip
[root@dev-master1 kubernetes]# l
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
namespace: kubernetes-plugin
name: jenkins-home
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 20Gi
创建Jenkins的master，可以根据实际情况限制Jenkins的资源使⽤
[root@dev-master1 kubernetes]# l
# jenkins
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: jenkins
labels:
name: jenkins
spec:
selector:
matchLabels:
name: jenkins
serviceName: jenkins
replicas: 1
updateStrategy:
type: RollingUpdate
template:
metadata:
name: jenkins
labels:
name: jenkins
spec:
terminationGracePeriodSeconds: 10
serviceAccountName: jenkins
containers:
- name: jenkins
image: 10.0.0.59/jenkins/jenkins:lts-alpine #官⽅镜像为jenkins/jenkins:lts-alpine，为了节省下载时间已经push到⾃⼰到harbor仓库          imagePullPolicy: Always
ports:
- containerPort: 8080
- containerPort: 50000
resources:
limits:
cpu: 1
memory: 1Gi
requests:
cpu: 0.5
memory: 500Mi
env:
- name: LIMITS_MEMORY
valueFrom:
resourceFieldRef:
resource:
divisor: 1Mi
-
name: JAVA_OPTS
# value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:MaxRAMFraction=1 -XshowSettings:vm -Dhudson.slave s.NodeProvisioner.initialDelay=0 -Dhudson.slaves.NodeProvisioner.MARGIN=50 -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85
value: -Xmx$(LIMITS_MEMORY)m -XshowSettings:vm -Dhudson.slaves.NodeProvisioner.initialDelay=0 -Dhudson.slaves.NodeProvisioner.MARGI N=50 -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85
volumeMounts:        #挂载pvc存储到Jenkins容器的/var/jenkins_home
- name: jenkinshome
mountPath: /var/jenkins_home
livenessProbe:
httpGet:
path: /login
port: 8080
initialDelaySeconds: 600  #存活探针时间改为600s，如果服务器配置低，Jenkins还没有启动成功就被重启了。
timeoutSeconds: 5
failureThreshold: 12 # ~2 minutes
readinessProbe:
httpGet:
path: /login
port: 8080
initialDelaySeconds: 60
timeoutSeconds: 5
failureThreshold: 12 # ~2 minutes
securityContext:
fsGroup: 1000
volumes:    #此处声明Jenkins的pvc存储
- name: jenkinshome
persistentVolumeClaim:
claimName: jenkins-home
#      imagePullSecrets:                        如果使⽤私有仓库，并且仓库对镜像设置了访问权限，需要在k8s master创建⼀个secret
#        - name: registry-secret
[root@dev-master1 kubernetes]# l
apiVersion: v1
kind: Service
metadata:
name: jenkins
spec:
sessionAffinity: "ClientIP"
type: NodePort
selector:
name: jenkins
ports:
-
name: http
port: 80
nodePort: 31006
protocol: TCP
-
name: agent
port: 50000
nodePort: 31007
protocol: TCP
挂载maven缓存⽬录
[root@dev-master1 kubernetes]# l
#m2是maven的缓存，挂载以提⾼build速度
apiVersion: v1
kind: PersistentVolume
metadata:
name: maven-m2
spec:
capacity:  #指定容量
storage: 200Gi
accessModes:
- ReadWriteOnce  #访问模式，还有ReadOnlyMany ##ReadOnlymany #  persistenVolumeReclaimPolicy: Recycle
#  storageClassName: nfs  ##指定存储的类型
nfs:
path: /data/dev_jenkins/m2  #指明nfs的路径
server: 10.0.0.250  #指明nfs的ip
[root@dev-master1 kubernetes]# l
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
namespace: kubernetes-plugin
name: maven-m2
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 200Gi
挂载slave节点保存构建结果的⽬录
[root@dev-master1 kubernetes]# l
#m2是maven的缓存，挂载以提⾼build速度
apiVersion: v1
kind: PersistentVolume
metadata:
name: workspace
spec:
capacity:  #指定容量
storage: 200Gi
accessModes:
- ReadWriteOnce  #访问模式，还有ReadOnlyMany ##ReadOnlymany #  persistenVolumeReclaimPolicy: Recycle
#  storageClassName: nfs  ##指定存储的类型
nfs:
path: /data/dev_jenkins/workspace  #指明nfs的路径
server: 10.0.0.250  #指明nfs的ip
[root@dev-master1 kubernetes]# l
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
namespace: kubernetes-plugin
name: workspace
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 200Gi

版权声明：本站内容均来自互联网，仅供演示用，请勿用于商业和其他非法用途。如果侵犯了您的权益请与我们联系QQ:729038198，我们将在24小时内删除。