k8s使⽤filebeat收集所有容器标准输出的⽇志k8s-filebeat收集所有容器标准输出的⽇志
1. k8s-收集所有容器标准输出的⽇志
filebeat-kubernetes.yaml # 采集所有容器标准输出
app-log-stdout.yaml # 标准输出测试应⽤
app-log-logfile.yaml # ⽇志⽂件测试应⽤
1.1 filebeat-kubernetes 配置⽂件
filebeat-kubernetes采集⽰意图
针对标准输出：以DaemonSet⽅式在每个Node上部署⼀个⽇志收集程序，采集/var/lib/docker/containers/⽬录下所有容器⽇志⽰例filebeat-kubernetes.yaml配置⽂件
---
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config
namespace: ops
labels:
k8s-app: filebeat
data:
inputs:
# Mounted `filebeat-inputs` configmap:
path: ${fig}/inputs.d/*.yml
# Reload inputs configs as they change:
modules:
path: ${fig}/modules.d/*.yml
# Reload module configs as they change:
output.elasticsearch:
hosts: ['49.65.125.91:9200']
---
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-inputs
namespace: ops
labels:
k8s-app: filebeat
data:
-
type: docker
containers.ids:
- "*"
processors:
- add_kubernetes_metadata:
in_cluster: true
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: filebeat
namespace: ops
labels:
k8s-app: filebeat
spec:
selector:
matchLabels:
k8s-app: filebeat
template:
metadata:
labels:
k8s-app: filebeat
spec:
serviceAccountName: filebeat
terminationGracePeriodSeconds: 30
containers:
- name: filebeat
image: elastic/filebeat:7.9.2
args: [
"-c", "/l",
"-e",
]
securityContext:
runAsUser: 0
# If using Red Hat OpenShift uncomment this:
#privileged: true
resources:
limits:
memory: 200Mi
requests:
cpu: 100m
memory: 100Mi
volumeMounts:
- name: config
mountPath: /l
readOnly: true
subPath: l
- name: inputs
mountPath: /usr/share/filebeat/inputs.d
readOnly: true
- name: data
mountPath: /usr/share/filebeat/data
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
volumes:
- name: config
configMap:
defaultMode: 0600
name: filebeat-config
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: inputs
configMap:
defaultMode: 0600
name: filebeat-inputs
# data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart      - name: data
hostPath:
path: /var/lib/filebeat-data
type: DirectoryOrCreate
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: filebeat
subjects:
- kind: ServiceAccount
name: filebeat
namespace: ops
roleRef:
kind: ClusterRole
name: filebeat
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: filebeat
labels:
k8s-app: filebeat
rules:
- apiGroups: [""] # "" indicates the core API group
resources:
- namespaces
- pods
verbs:
- get
- watch
-
list
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: filebeat
namespace: ops
labels:
k8s-app: filebeat
可视化展⽰⽇志：
1.查看索引（⽇志记录集合）：Management -> Stack Management -> 索引管理
2.将索引关联到Kibana：索引模式-> 创建-> 匹配模式-> 选择时间戳
3.在Discover选择索引模式查看⽇志
图⽰
1.2 ⽇志⽂件输出
⽇志⽂件输出架构图解
针对容器中⽇志⽂件：在Pod中增加⼀个容器运⾏⽇志采集器，使⽤emtyDir共享⽇志⽬录让⽇志采集器读取到⽇志⽂件2. 操作案例
编写filebeat-kubernetes.yaml配置⽂件
[root@k8s-master elk]# vim filebeat-kubernetes.yaml
[root@k8s-master elk]# cat filebeat-kubernetes.yaml
---
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config
namespace: ops
labels:
k8s-app: filebeat
data:
inputs:
# Mounted `filebeat-inputs` configmap:
path: ${fig}/inputs.d/*.yml
# Reload inputs configs as they change:
modules:
path: ${fig}/modules.d/*.yml
# Reload module configs as they change:
output.elasticsearch:
hosts: ['127.0.0.1:9200']
username: "admin"
password: "12345678"
---
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-inputs
namespace: ops
labels:
k8s-app: filebeat
data:
- type: docker
containers.ids:
- "*"
processors:
- add_kubernetes_metadata:
in_cluster: true
-
--
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: filebeat
namespace: ops
labels:
k8s-app: filebeat
spec:
selector:
matchLabels:
k8s-app: filebeat
template:
metadata:
labels:
k8s-app: filebeat
spec:
serviceAccountName: filebeat
terminationGracePeriodSeconds: 30
containers:
- name: filebeat
image: elastic/filebeat:7.10.1
args: [
"-c", "/l",
"-e",
]
securityContext:
runAsUser: 0
# If using Red Hat OpenShift uncomment this:
#privileged: true
resources:
limits:nodeselector
memory: 200Mi
requests:
cpu: 100m
memory: 100Mi
volumeMounts:
- name: config
mountPath: /l
readOnly: true
subPath: l
- name: inputs
mountPath: /usr/share/filebeat/inputs.d
readOnly: true
-
name: data
mountPath: /usr/share/filebeat/data
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
volumes:
- name: config
configMap:
defaultMode: 0600
name: filebeat-config
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: inputs
configMap:
defaultMode: 0600
name: filebeat-inputs
# data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart      - name: data
hostPath:
path: /var/lib/filebeat-data
type: DirectoryOrCreate
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: filebeat
subjects:
- kind: ServiceAccount
name: filebeat
namespace: ops
roleRef:
kind: ClusterRole
name: filebeat
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: filebeat
labels:
k8s-app: filebeat
rules:
- apiGroups: [""] # "" indicates the core API group
resources:
-
namespaces
- pods
verbs:
- get
- watch
- list
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: filebeat
namespace: ops
labels:
k8s-app: filebeat
运⾏配置
[root@k8s-master elk]# kubectl create namespace ops
namespace/ops created
[root@k8s-master elk]# kubectl apply -f filebeat-kubernetes.yaml
configmap/filebeat-config created
configmap/filebeat-inputs created
daemonset.apps/filebeat created
clusterrolebinding.rbac.authorization.k8s.io/filebeat unchanged clusterrole.rbac.authorization.k8s.io/filebeat unchanged
serviceaccount/filebeat created
查看运⾏配置
[root@k8s-master elk]# kubectl get pods -n ops
NAME            READY  STATUS    RESTARTS  AGE
filebeat-dmbzg  1/1    Running  0          24m
[root@k8s-master elk]# kubectl logs  -f filebeat-dmbzg -n ops
查看kibana是否有索引
3. 可视化展⽰数据
可视化展⽰数据

版权声明：本站内容均来自互联网，仅供演示用，请勿用于商业和其他非法用途。如果侵犯了您的权益请与我们联系QQ:729038198，我们将在24小时内删除。