附014.Kubernetes Prometheus+Grafana+EFK+Kibana+Glusterfs整合解决方案

1 { 2 "_port_comment": "Heketi Server Port Number", 3 "port": "8080", 4 5 "_use_auth": "Enable JWT authorization. Please enable for deployment", 6 "use_auth": true, 7 8 "_jwt": "Private keys for access", 9 "jwt": { 10 "_admin": "Admin has access to all APIs", 11 "admin": { 12 "key": "admin123" 13 }, 14 "_user": "User only has access to /volumes endpoint", 15 "user": { 16 "key": "xianghy" 17 } 18 }, 19 20 "_glusterfs_comment": "GlusterFS Configuration", 21 "glusterfs": { 22 "_executor_comment": [ 23 "Execute plugin. Possible choices: mock, ssh", 24 "mock: This setting is used for testing and development.", 25 " It will not send commands to any node.", 26 "ssh: This setting will notify Heketi to ssh to the nodes.", 27 " It will need the values in sshexec to be configured.", 28 "kubernetes: Communicate with GlusterFS containers over", 29 " Kubernetes exec api." 30 ], 31 "executor": "ssh", 32 33 "_sshexec_comment": "SSH username and private key file information", 34 "sshexec": { 35 "keyfile": "/etc/heketi/heketi_key", 36 "user": "root", 37 "port": "22", 38 "fstab": "/etc/fstab" 39 }, 40 41 "_db_comment": "Database file name", 42 "db": "/var/lib/heketi/heketi.db", 43 44 "_loglevel_comment": [ 45 "Set log level. Choices are:", 46 " none, critical, error, warning, info, debug", 47 "Default is warning" 48 ], 49 "loglevel" : "warning" 50 } 51 }

1 { 2 "clusters": [ 3 { 4 "nodes": [ 5 { 6 "node": { 7 "hostnames": { 8 "manage": [ 9 "k8snode01" 10 ], 11 "storage": [ 12 "172.24.8.74" 13 ] 14 }, 15 "zone": 1 16 }, 17 "devices": [ 18 "https://img.qb5200.com/download-x/dev/sdb" 19 ] 20 }, 21 { 22 "node": { 23 "hostnames": { 24 "manage": [ 25 "k8snode02" 26 ], 27 "storage": [ 28 "172.24.8.75" 29 ] 30 }, 31 "zone": 1 32 }, 33 "devices": [ 34 "https://img.qb5200.com/download-x/dev/sdb" 35 ] 36 }, 37 { 38 "node": { 39 "hostnames": { 40 "manage": [ 41 "k8snode03" 42 ], 43 "storage": [ 44 "172.24.8.76" 45 ] 46 }, 47 "zone": 1 48 }, 49 "devices": [ 50 "https://img.qb5200.com/download-x/dev/sdb" 51 ] 52 } 53 ] 54 } 55 ] 56 }

1 apiVersion: v1 2 kind: Secret 3 metadata: 4 name: heketi-secret 5 namespace: heketi 6 data: 7 key: YWRtaW4xMjM= 8 type: kubernetes.io/glusterfs

1 StorageClass 2 apiVersion: storage.k8s.io/v1 3 kind: StorageClass 4 metadata: 5 name: ghstorageclass 6 parameters: 7 resturl: "http://172.24.8.71:8080" 8 clusterid: "ad0f81f75f01d01ebd6a21834a2caa30" 9 restauthenabled: "true" 10 restuser: "admin" 11 secretName: "heketi-secret" 12 secretNamespace: "heketi" 13 volumetype: "replicate:3" 14 provisioner: kubernetes.io/glusterfs 15 reclaimPolicy: Delete

1 …… 2 image: mirrorgooglecontainers/metrics-server-amd64:v0.3.6 #修改为国内源 3 command: 4 - /metrics-server 5 - --metric-resolution=30s 6 - --kubelet-insecure-tls 7 - --kubelet-preferred-address-types=InternalIP,Hostname,InternalDNS,ExternalDNS,ExternalIP #添加如上command 8 ……

1 apiVersion: v1 2 kind: Namespace 3 metadata: 4 name: monitoring 5

1 apiVersion: rbac.authorization.k8s.io/v1beta1 2 kind: ClusterRole 3 metadata: 4 name: prometheus 5 rules: 6 - apiGroups: [""] 7 resources: 8 - nodes 9 - nodes/proxy 10 - services 11 - endpoints 12 - pods 13 verbs: ["get", "list", "watch"] 14 - apiGroups: 15 - extensions 16 resources: 17 - ingresses 18 verbs: ["get", "list", "watch"] 19 - nonResourceURLs: ["/metrics"] 20 verbs: ["get"] 21 --- 22 apiVersion: v1 23 kind: ServiceAccount 24 metadata: 25 name: prometheus 26 namespace: monitoring #仅需修改命名空间 27 --- 28 apiVersion: rbac.authorization.k8s.io/v1beta1 29 kind: ClusterRoleBinding 30 metadata: 31 name: prometheus 32 roleRef: 33 apiGroup: rbac.authorization.k8s.io 34 kind: ClusterRole 35 name: prometheus 36 subjects: 37 - kind: ServiceAccount 38 name: prometheus 39 namespace: monitoring #仅需修改命名空间

1 apiVersion: v1 2 kind: ConfigMap 3 metadata: 4 name: prometheus-server-conf 5 labels: 6 name: prometheus-server-conf 7 namespace: monitoring #修改命名空间 8 data: 9 prometheus.yml: |- 10 global: 11 scrape_interval: 10s 12 evaluation_interval: 10s 13 14 scrape_configs: 15 - job_name: 'kubernetes-apiservers' 16 kubernetes_sd_configs: 17 - role: endpoints 18 scheme: https 19 tls_config: 20 ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt 21 bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token 22 relabel_configs: 23 - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] 24 action: keep 25 regex: default;kubernetes;https 26 27 - job_name: 'kubernetes-nodes' 28 scheme: https 29 tls_config: 30 ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt 31 bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token 32 kubernetes_sd_configs: 33 - role: node 34 relabel_configs: 35 - action: labelmap 36 regex: __meta_kubernetes_node_label_(.+) 37 - target_label: __address__ 38 replacement: kubernetes.default.svc:443 39 - source_labels: [__meta_kubernetes_node_name] 40 regex: (.+) 41 target_label: __metrics_path__ 42 replacement: /api/v1/nodes/${1}/proxy/metrics 43 44 - job_name: 'kubernetes-cadvisor' 45 scheme: https 46 tls_config: 47 ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt 48 bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token 49 kubernetes_sd_configs: 50 - role: node 51 relabel_configs: 52 - action: labelmap 53 regex: __meta_kubernetes_node_label_(.+) 54 - target_label: __address__ 55 replacement: kubernetes.default.svc:443 56 - source_labels: [__meta_kubernetes_node_name] 57 regex: (.+) 58 target_label: __metrics_path__ 59 replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor 60 61 - job_name: 'kubernetes-service-endpoints' 62 kubernetes_sd_configs: 63 - role: endpoints 64 relabel_configs: 65 - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape] 66 action: keep 67 regex: true 68 - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] 69 action: replace 70 target_label: __scheme__ 71 regex: (https?) 72 - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] 73 action: replace 74 target_label: __metrics_path__ 75 regex: (.+) 76 - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] 77 action: replace 78 target_label: __address__ 79 regex: ([^:]+)(?::\d+)?;(\d+) 80 replacement: $1:$2 81 - action: labelmap 82 regex: __meta_kubernetes_service_label_(.+) 83 - source_labels: [__meta_kubernetes_namespace] 84 action: replace 85 target_label: kubernetes_namespace 86 - source_labels: [__meta_kubernetes_service_name] 87 action: replace 88 target_label: kubernetes_name 89 90 - job_name: 'kubernetes-services' 91 metrics_path: /probe 92 params: 93 module: [http_2xx] 94 kubernetes_sd_configs: 95 - role: service 96 relabel_configs: 97 - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe] 98 action: keep 99 regex: true 100 - source_labels: [__address__] 101 target_label: __param_target 102 - target_label: __address__ 103 replacement: blackbox-exporter.example.com:9115 104 - source_labels: [__param_target] 105 target_label: instance 106 - action: labelmap 107 regex: __meta_kubernetes_service_label_(.+) 108 - source_labels: [__meta_kubernetes_namespace] 109 target_label: kubernetes_namespace 110 - source_labels: [__meta_kubernetes_service_name] 111 target_label: kubernetes_name 112 113 - job_name: 'kubernetes-ingresses' 114 kubernetes_sd_configs: 115 - role: ingress 116 relabel_configs: 117 - source_labels: [__meta_kubernetes_ingress_annotation_prometheus_io_probe] 118 action: keep 119 regex: true 120 - source_labels: [__meta_kubernetes_ingress_scheme,__address__,__meta_kubernetes_ingress_path] 121 regex: (.+);(.+);(.+) 122 replacement: ${1}://${2}${3} 123 target_label: __param_target 124 - target_label: __address__ 125 replacement: blackbox-exporter.example.com:9115 126 - source_labels: [__param_target] 127 target_label: instance 128 - action: labelmap 129 regex: __meta_kubernetes_ingress_label_(.+) 130 - source_labels: [__meta_kubernetes_namespace] 131 target_label: kubernetes_namespace 132 - source_labels: [__meta_kubernetes_ingress_name] 133 target_label: kubernetes_name 134 135 - job_name: 'kubernetes-pods' 136 kubernetes_sd_configs: 137 - role: pod 138 relabel_configs: 139 - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] 140 action: keep 141 regex: true 142 - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] 143 action: replace 144 target_label: __metrics_path__ 145 regex: (.+) 146 - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] 147 action: replace 148 regex: ([^:]+)(?::\d+)?;(\d+) 149 replacement: $1:$2 150 target_label: __address__ 151 - action: labelmap 152 regex: __meta_kubernetes_pod_label_(.+) 153 - source_labels: [__meta_kubernetes_namespace] 154 action: replace 155 target_label: kubernetes_namespace 156 - source_labels: [__meta_kubernetes_pod_name] 157 action: replace 158 target_label: kubernetes_pod_name

1 apiVersion: v1 2 kind: PersistentVolumeClaim 3 metadata: 4 name: prometheus-pvc 5 namespace: monitoring 6 annotations: 7 volume.beta.kubernetes.io/storage-class: ghstorageclass 8 spec: 9 accessModes: 10 - ReadWriteMany 11 resources: 12 requests: 13 storage: 5Gi

1 apiVersion: apps/v1beta2 2 kind: Deployment 3 metadata: 4 labels: 5 name: prometheus-deployment 6 name: prometheus-server 7 namespace: monitoring 8 spec: 9 replicas: 1 10 selector: 11 matchLabels: 12 app: prometheus-server 13 template: 14 metadata: 15 labels: 16 app: prometheus-server 17 spec: 18 containers: 19 - name: prometheus-server 20 image: prom/prometheus:v2.14.0 21 command: 22 - "/bin/prometheus" 23 args: 24 - "--config.file=/etc/prometheus/prometheus.yml" 25 - "--storage.tsdb.path=/prometheus/" 26 - "--storage.tsdb.retention=72h" 27 ports: 28 - containerPort: 9090 29 protocol: TCP 30 volumeMounts: 31 - name: prometheus-config-volume 32 mountPath: /etc/prometheus/ 33 - name: prometheus-storage-volume 34 mountPath: /prometheus/ 35 serviceAccountName: prometheus 36 imagePullSecrets: 37 - name: regsecret 38 volumes: 39 - name: prometheus-config-volume 40 configMap: 41 defaultMode: 420 42 name: prometheus-server-conf 43 - name: prometheus-storage-volume 44 persistentVolumeClaim: 45 claimName: prometheus-pvc

1 apiVersion: v1 2 kind: Service 3 metadata: 4 labels: 5 app: prometheus-service 6 name: prometheus-service 7 namespace: monitoring 8 spec: 9 type: NodePort 10 selector: 11 app: prometheus-server 12 ports: 13 - port: 9090 14 targetPort: 9090 15 nodePort: 30001

1 apiVersion: v1 2 kind: PersistentVolumeClaim 3 metadata: 4 name: grafana-data-pvc 5 namespace: monitoring 6 annotations: 7 volume.beta.kubernetes.io/storage-class: ghstorageclass 8 spec: 9 accessModes: 10 - ReadWriteOnce 11 resources: 12 requests: 13 storage: 5Gi

1 apiVersion: extensions/v1beta1 2 kind: Deployment 3 metadata: 4 name: monitoring-grafana 5 namespace: monitoring 6 spec: 7 replicas: 1 8 template: 9 metadata: 10 labels: 11 task: monitoring 12 k8s-app: grafana 13 spec: 14 containers: 15 - name: grafana 16 image: grafana/grafana:6.5.0 17 imagePullPolicy: IfNotPresent 18 ports: 19 - containerPort: 3000 20 protocol: TCP 21 volumeMounts: 22 - mountPath: /var/lib/grafana 23 name: grafana-storage 24 env: 25 - name: INFLUXDB_HOST 26 value: monitoring-influxdb 27 - name: GF_SERVER_HTTP_PORT 28 value: "3000" 29 - name: GF_AUTH_BASIC_ENABLED 30 value: "false" 31 - name: GF_AUTH_ANONYMOUS_ENABLED 32 value: "true" 33 - name: GF_AUTH_ANONYMOUS_ORG_ROLE 34 value: Admin 35 - name: GF_SERVER_ROOT_URL 36 value: / 37 readinessProbe: 38 httpGet: 39 path: /login 40 port: 3000 41 volumes: 42 - name: grafana-storage 43 persistentVolumeClaim: 44 claimName: grafana-data-pvc 45 nodeSelector: 46 node-role.kubernetes.io/master: "true" 47 tolerations: 48 - key: "node-role.kubernetes.io/master" 49 effect: "NoSchedule" 50 --- 51 apiVersion: v1 52 kind: Service 53 metadata: 54 labels: 55 kubernetes.io/cluster-service: 'true' 56 kubernetes.io/name: monitoring-grafana 57 annotations: 58 prometheus.io/scrape: 'true' 59 prometheus.io/tcp-probe: 'true' 60 prometheus.io/tcp-probe-port: '80' 61 name: monitoring-grafana 62 namespace: monitoring 63 spec: 64 type: NodePort 65 ports: 66 - port: 80 67 targetPort: 3000 68 nodePort: 30002 69 selector: 70 k8s-app: grafana

1 …… 2 - image: quay-mirror.qiniu.com/fluentd_elasticsearch/elasticsearch:v7.3.2 3 name: elasticsearch-logging 4 imagePullPolicy: IfNotPresent #添加镜像策略 5 ……

1 …… 2 image: quay-mirror.qiniu.com/fluentd_elasticsearch/fluentd:v2.7.0 3 imagePullPolicy: IfNotPresent #添加镜像策略 4 ……

1 …… 2 image: docker.elastic.co/kibana/kibana-oss:7.3.2 #修改为匹配版本 3 imagePullPolicy: IfNotPresent #添加镜像策略 4 ……

1 apiVersion: v1 2 kind: PersistentVolumeClaim 3 metadata: 4 name: elasticsearch-pvc 5 namespace: kube-system 6 annotations: 7 volume.beta.kubernetes.io/storage-class: ghstorageclass 8 spec: 9 accessModes: 10 - ReadWriteMany 11 resources: 12 requests: 13 storage: 5Gi

  1 apiVersion: v1
  2 kind: ServiceAccount
  3 metadata:
  4   name: elasticsearch-logging
  5   namespace: kube-system
  6   labels:
  7     k8s-app: elasticsearch-logging
  8     addonmanager.kubernetes.io/mode: Reconcile
  9 ---
 10 kind: ClusterRole
 11 apiVersion: rbac.authorization.k8s.io/v1
 12 metadata:
 13   name: elasticsearch-logging
 14   labels:
 15     k8s-app: elasticsearch-logging
 16     addonmanager.kubernetes.io/mode: Reconcile
 17 rules:
 18 - apiGroups:
 19   - ""
 20   resources:
 21   - "services"
 22   - "namespaces"
 23   - "endpoints"
 24   verbs:
 25   - "get"
 26 ---
 27 kind: ClusterRoleBinding
 28 apiVersion: rbac.authorization.k8s.io/v1
 29 metadata:
 30   namespace: kube-system
 31   name: elasticsearch-logging
 32   labels:
 33     k8s-app: elasticsearch-logging
 34     addonmanager.kubernetes.io/mode: Reconcile
 35 subjects:
 36 - kind: ServiceAccount
 37   name: elasticsearch-logging
 38   namespace: kube-system
 39   apiGroup: ""
 40 roleRef:
 41   kind: ClusterRole
 42   name: elasticsearch-logging
 43   apiGroup: ""
 44 ---
 45 apiVersion: apps/v1
 46 kind: StatefulSet
 47 metadata:
 48   name: elasticsearch-logging
 49   namespace: kube-system
 50   labels:
 51     k8s-app: elasticsearch-logging
 52     version: v7.3.2
 53     addonmanager.kubernetes.io/mode: Reconcile
 54 spec:
 55   serviceName: elasticsearch-logging
 56   replicas: 1
 57   selector:
 58     matchLabels:
 59       k8s-app: elasticsearch-logging
 60       version: v7.3.2
 61   template:
 62     metadata:
 63       labels:
 64         k8s-app: elasticsearch-logging
 65         version: v7.3.2
 66     spec:
 67       serviceAccountName: elasticsearch-logging
 68       containers:
 69       - image: quay-mirror.qiniu.com/fluentd_elasticsearch/elasticsearch:v7.3.2
 70         name: elasticsearch-logging
 71         imagePullPolicy: IfNotPresent
 72         resources:
 73           limits:
 74             cpu: 1000m
 75             memory: 3Gi
 76           requests:
 77             cpu: 100m
 78             memory: 3Gi
 79         ports:
 80         - containerPort: 9200
 81           name: db
 82           protocol: TCP
 83         - containerPort: 9300
 84           name: transport
 85           protocol: TCP
 86         volumeMounts:
 87         - name: elasticsearch-logging
 88           mountPath: https://img.qb5200.com/download-x/data
 89         env:
 90         - name: "NAMESPACE"
 91           valueFrom:
 92             fieldRef:
 93               fieldPath: metadata.namespace
 94       volumes:
 95       - name: elasticsearch-logging			#挂载永久存储PVC
 96         persistentVolumeClaim:
 97           claimName: elasticsearch-pvc
 98       initContainers:
 99       - image: alpine:3.6
100         command: ["/sbin/sysctl", "-w", "vm.max_map_count=262144"]
101         name: elasticsearch-logging-init
102         securityContext:
103           privileged: true

  1 apiVersion: v1
  2 kind: Service
  3 metadata:
  4   name: elasticsearch-logging
  5   namespace: kube-system
  6   labels:
  7     k8s-app: elasticsearch-logging
  8     kubernetes.io/cluster-service: "true"
  9     addonmanager.kubernetes.io/mode: Reconcile
 10     kubernetes.io/name: "Elasticsearch"
 11 spec:
 12   ports:
 13   - port: 9200
 14     protocol: TCP
 15     targetPort: db
 16   selector:
 17     k8s-app: elasticsearch-logging

  1 apiVersion: apps/v1
  2 kind: Deployment
  3 metadata:
  4   name: kibana-logging
  5   namespace: kube-system
  6   labels:
  7     k8s-app: kibana-logging
  8     addonmanager.kubernetes.io/mode: Reconcile
  9 spec:
 10   replicas: 1
 11   selector:
 12     matchLabels:
 13       k8s-app: kibana-logging
 14   template:
 15     metadata:
 16       labels:
 17         k8s-app: kibana-logging
 18       annotations:
 19         seccomp.security.alpha.kubernetes.io/pod: 'dockerhttps://img.qb5200.com/download-x/default'
 20     spec:
 21       containers:
 22       - name: kibana-logging
 23         image: docker.elastic.co/kibana/kibana-oss:7.3.2
 24         imagePullPolicy: IfNotPresent
 25         resources:
 26           limits:
 27             cpu: 1000m
 28           requests:
 29             cpu: 100m
 30         env:
 31           - name: ELASTICSEARCH_HOSTS
 32             value: http://elasticsearch-logging:9200
 33         ports:
 34         - containerPort: 5601
 35           name: ui
 36           protocol: TCP

  1 apiVersion: v1
  2 kind: Service
  3 metadata:
  4   name: kibana-logging
  5   namespace: kube-system
  6   labels:
  7     k8s-app: kibana-logging
  8     kubernetes.io/cluster-service: "true"
  9     addonmanager.kubernetes.io/mode: Reconcile
 10     kubernetes.io/name: "Kibana"
 11 spec:
 12   type: NodePort
 13   ports:
 14   - port: 5601
 15     protocol: TCP
 16     nodePort: 30003
 17     targetPort: ui
 18   selector:
 19     k8s-app: kibana-logging