From 45ac5fbe0f664dd572155c76fb51e9061e473ba7 Mon Sep 17 00:00:00 2001 From: Steve Wilkerson Date: Wed, 18 Dec 2019 13:44:23 -0600 Subject: [PATCH] Update Elastic Beats versions to 7.1.0 This updates the Elastic Beats charts to 7.1.0 to keep them aligned with the Kibana and Elasticsearch chart versions, which is required for compatibility This also updates the experimental job to use the single node minikube deployment as opposed to the standard 5 node multinode deployment Change-Id: I4baba6ca2ea2f3785f11905138b67979a4501caa Signed-off-by: Steve Wilkerson --- elastic-filebeat/templates/configmap-etc.yaml | 4 +- elastic-filebeat/templates/daemonset.yaml | 13 ++ elastic-filebeat/values.yaml | 181 +++++++++++------- .../templates/daemonset-node-metrics.yaml | 10 +- .../templates/deployment-modules.yaml | 55 +++++- elastic-metricbeat/values.yaml | 88 +++++---- elastic-packetbeat/templates/daemonset.yaml | 5 +- elastic-packetbeat/values.yaml | 20 +- .../elastic-beats/005-deploy-k8s.sh | 1 + .../010-deploy-docker-registry.sh | 1 - .../elastic-beats/050-elasticsearch.sh | 29 ++- tools/deployment/elastic-beats/060-kibana.sh | 60 ++++++ ...etricbeat.sh => 080-elastic-metricbeat.sh} | 19 +- ...ic-filebeat.sh => 090-elastic-filebeat.sh} | 18 +- .../elastic-beats/100-elastic-apm-server.sh | 31 --- ...acketbeat.sh => 100-elastic-packetbeat.sh} | 19 +- tools/deployment/elastic-beats/110-kibana.sh | 1 - zuul.d/jobs.yaml | 19 +- zuul.d/project.yaml | 6 +- 19 files changed, 401 insertions(+), 179 deletions(-) create mode 120000 tools/deployment/elastic-beats/005-deploy-k8s.sh delete mode 120000 tools/deployment/elastic-beats/010-deploy-docker-registry.sh create mode 100755 tools/deployment/elastic-beats/060-kibana.sh rename tools/deployment/elastic-beats/{060-elastic-metricbeat.sh => 080-elastic-metricbeat.sh} (76%) rename tools/deployment/elastic-beats/{080-elastic-filebeat.sh => 090-elastic-filebeat.sh} (76%) delete mode 100755 tools/deployment/elastic-beats/100-elastic-apm-server.sh rename tools/deployment/elastic-beats/{090-elastic-packetbeat.sh => 100-elastic-packetbeat.sh} (76%) delete mode 120000 tools/deployment/elastic-beats/110-kibana.sh diff --git a/elastic-filebeat/templates/configmap-etc.yaml b/elastic-filebeat/templates/configmap-etc.yaml index 2e2fc12328..29e448a3fe 100644 --- a/elastic-filebeat/templates/configmap-etc.yaml +++ b/elastic-filebeat/templates/configmap-etc.yaml @@ -23,5 +23,7 @@ metadata: name: filebeat-etc data: filebeat.yml: | -{{ toYaml .Values.conf | indent 4 }} +{{ toYaml .Values.conf.filebeat | indent 4 }} + system.yml: | +{{ toYaml .Values.conf.modules.system | indent 4 }} {{- end }} diff --git a/elastic-filebeat/templates/daemonset.yaml b/elastic-filebeat/templates/daemonset.yaml index 26f524136d..e146c23bbc 100644 --- a/elastic-filebeat/templates/daemonset.yaml +++ b/elastic-filebeat/templates/daemonset.yaml @@ -48,6 +48,7 @@ rules: - nodes - pods - services + - endpoints - replicationcontrollers - limitranges verbs: @@ -105,10 +106,18 @@ spec: - name: filebeat containerPort: {{ tuple "filebeat" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName - name: ELASTICSEARCH_HOST value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} - name: ELASTICSEARCH_PORT value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: KIBANA_HOST + value: {{ tuple "kibana" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: KIBANA_PORT + value: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} - name: ELASTICSEARCH_USERNAME valueFrom: secretKeyRef: @@ -133,6 +142,10 @@ spec: mountPath: /usr/share/filebeat/filebeat.yml readOnly: true subPath: filebeat.yml + - name: filebeat-etc + mountPath: /usr/share/filebeat/modules.d/system.yml + subPath: system.yml + readOnly: true {{ if $mounts_filebeat.volumeMounts }}{{ toYaml $mounts_filebeat.volumeMounts | indent 8 }}{{ end }} volumes: - name: pod-tmp diff --git a/elastic-filebeat/values.yaml b/elastic-filebeat/values.yaml index 92289fd1d2..c0c1b76f4d 100644 --- a/elastic-filebeat/values.yaml +++ b/elastic-filebeat/values.yaml @@ -28,7 +28,7 @@ labels: images: tags: - filebeat: docker.elastic.co/beats/filebeat:6.2.3 + filebeat: docker.elastic.co/beats/filebeat-oss:7.1.0 dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0 image_repo_sync: docker.io/docker:17.07.0 pull_policy: IfNotPresent @@ -53,82 +53,106 @@ dependencies: service: local_image_registry static: filebeat: - services: null + services: + - endpoint: internal + service: kibana image_repo_sync: services: - endpoint: internal service: local_image_registry conf: - setup: - dashboards: - enabled: true - path: - logs: /var/log/ - output: - elasticsearch: - hosts: ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}"] - username: "${ELASTICSEARCH_USERNAME}" - password: "${ELASTICSEARCH_PASSWORD}" filebeat: - config: - prospectors: - path: ${path.config}/prospectors.d/*.yml - reload: - enabled: false - modules: - path: ${path.config}/modules.d/*.yml - reload: - enabled: false - autodiscover: - providers: - - type: kubernetes - templates: - - condition: - equals: - kubernetes.namespace: kube-system - config: - - type: docker - containers.ids: - - "${data.kubernetes.container.id}" - exclude_lines: ["^\\s+[\\-`('.|_]"] - - type: kubernetes - templates: - - condition: - equals: - kubernetes.namespace: ceph - config: - - type: docker - containers.ids: - - "${data.kubernetes.container.id}" - exclude_lines: ["^\\s+[\\-`('.|_]"] - - type: kubernetes - templates: - - condition: - equals: - kubernetes.namespace: openstack - config: - - type: docker - containers.ids: - - "${data.kubernetes.container.id}" - exclude_lines: ["^\\s+[\\-`('.|_]"] - prospectors: - - type: docker - containers.ids: - - "*" - multiline: - pattern: '^Traceback' - match: after - negate: true - processors: - - add_kubernetes_metadata: - in_cluster: true - - drop_event: - when: - equals: - kubernetes: - container: - name: "filebeat" + setup: + dashboards: + enabled: true + index: "filebeat-*" + retry: + enabled: true + interval: 5 + kibana: + host: "${KIBANA_HOST}:${KIBANA_PORT}" + username: "${ELASTICSEARCH_USERNAME}" + password: "${ELASTICSEARCH_PASSWORD}" + path: + logs: /var/log/ + output: + elasticsearch: + hosts: ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/"] + username: "${ELASTICSEARCH_USERNAME}" + password: "${ELASTICSEARCH_PASSWORD}" + filebeat: + config: + modules: + path: ${path.config}/modules.d/*.yml + reload: + enabled: true + autodiscover: + providers: + - type: kubernetes + templates: + - condition: + equals: + kubernetes.namespace: kube-system + config: + - type: docker + containers.ids: + - "${data.kubernetes.container.id}" + exclude_lines: ["^\\s+[\\-`('.|_]"] + - type: kubernetes + templates: + - condition: + equals: + kubernetes.namespace: ceph + config: + - type: docker + containers.ids: + - "${data.kubernetes.container.id}" + exclude_lines: ["^\\s+[\\-`('.|_]"] + - type: kubernetes + templates: + - condition: + equals: + kubernetes.namespace: openstack + config: + - type: docker + containers.ids: + - "${data.kubernetes.container.id}" + exclude_lines: ["^\\s+[\\-`('.|_]"] + - type: kubernetes + templates: + - condition: + equals: + kubernetes.namespace: osh-infra + config: + - type: docker + containers.ids: + - "${data.kubernetes.container.id}" + exclude_lines: ["^\\s+[\\-`('.|_]"] + processors: + - add_kubernetes_metadata: + in_cluster: true + - drop_event: + when: + equals: + kubernetes: + container: + name: "filebeat" + modules: + system: + - module: system + syslog: + enabled: true + var.paths: ["/var/log/syslog*"] + fields: + host: + name: "${NODE_NAME}" + auth: + enabled: true + var.paths: ["/var/log/auth.log"] + fields: + host: + name: "${NODE_NAME}" endpoints: cluster_domain_suffix: cluster.local @@ -165,6 +189,23 @@ endpoints: port: http: default: 80 + kibana: + name: kibana + namespace: null + hosts: + default: kibana-dash + public: kibana + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + kibana: + default: 5601 + http: + default: 80 filebeat: namespace: null name: filebeat diff --git a/elastic-metricbeat/templates/daemonset-node-metrics.yaml b/elastic-metricbeat/templates/daemonset-node-metrics.yaml index 51595e2f33..9f47a8393b 100644 --- a/elastic-metricbeat/templates/daemonset-node-metrics.yaml +++ b/elastic-metricbeat/templates/daemonset-node-metrics.yaml @@ -48,8 +48,10 @@ rules: - nodes - pods - services + - endpoints - replicationcontrollers - limitranges + - events verbs: - get - list @@ -104,7 +106,7 @@ spec: {{ tuple $envAll $envAll.Values.pod.resources.metricbeat | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} args: - "-c" - - "/etc/metricbeat.yml" + - "/usr/share/metricbeat/metricbeat.yml" - "-e" - "-system.hostfs=/hostfs" env: @@ -112,6 +114,10 @@ spec: value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} - name: ELASTICSEARCH_PORT value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: KIBANA_HOST + value: {{ tuple "kibana" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: KIBANA_PORT + value: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} - name: ELASTICSEARCH_USERNAME valueFrom: secretKeyRef: @@ -130,7 +136,7 @@ spec: - name: pod-tmp mountPath: /tmp - name: metricbeat-etc - mountPath: /etc/metricbeat.yml + mountPath: /usr/share/metricbeat/metricbeat.yml subPath: metricbeat.yml readOnly: true - name: metricbeat-etc diff --git a/elastic-metricbeat/templates/deployment-modules.yaml b/elastic-metricbeat/templates/deployment-modules.yaml index 4e1c602360..7ffadb6745 100644 --- a/elastic-metricbeat/templates/deployment-modules.yaml +++ b/elastic-metricbeat/templates/deployment-modules.yaml @@ -19,9 +19,54 @@ limitations under the License. {{- $esUserSecret := .Values.secrets.elasticsearch.user }} -{{- $serviceAccountName := "metricbeat" }} +{{- $serviceAccountName := printf "%s-%s" .Release.Name "metricbeat-deployments" }} {{ tuple $envAll "metricbeat" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} --- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ $serviceAccountName }} + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - namespaces + - nodes + - pods + - services + - endpoints + - replicationcontrollers + - limitranges + - events + verbs: + - get + - list + - watch + - apiGroups: + - apps + resources: + - statefulsets + - daemonsets + - deployments + - replicasets + verbs: + - get + - list + - watch +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -57,7 +102,7 @@ spec: {{ tuple $envAll $envAll.Values.pod.resources.metricbeat | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} args: - "-c" - - "/etc/metricbeat.yml" + - "/usr/share/metricbeat/metricbeat.yml" - "-e" env: - name: ELASTICSEARCH_HOST @@ -68,6 +113,10 @@ spec: value: {{ tuple "kube_state_metrics" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} - name: KUBE_STATE_METRICS_PORT value: {{ tuple "kube_state_metrics" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: KIBANA_HOST + value: {{ tuple "kibana" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: KIBANA_PORT + value: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} - name: ELASTICSEARCH_USERNAME valueFrom: secretKeyRef: @@ -86,7 +135,7 @@ spec: - name: pod-tmp mountPath: /tmp - name: metricbeat-etc - mountPath: /etc/metricbeat.yml + mountPath: /usr/share/metricbeat/metricbeat.yml subPath: metricbeat.yml readOnly: true - name: metricbeat-etc diff --git a/elastic-metricbeat/values.yaml b/elastic-metricbeat/values.yaml index c8a4a3e3cb..636636268a 100644 --- a/elastic-metricbeat/values.yaml +++ b/elastic-metricbeat/values.yaml @@ -28,7 +28,7 @@ labels: images: tags: - metricbeat: docker.elastic.co/beats/metricbeat:6.3.1 + metricbeat: docker.elastic.co/beats/metricbeat-oss:7.1.0 dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0 image_repo_sync: docker.io/docker:17.07.0 pull_policy: IfNotPresent @@ -53,7 +53,9 @@ dependencies: service: local_image_registry static: metricbeat: - services: null + services: + - endpoint: internal + service: kibana image_repo_sync: services: - endpoint: internal @@ -64,9 +66,14 @@ conf: setup: dashboards: enabled: true + index: metricbeat-* retry: enabled: true interval: 5 + kibana: + host: "${KIBANA_HOST}:${KIBANA_PORT}" + username: "${ELASTICSEARCH_USERNAME}" + password: "${ELASTICSEARCH_PASSWORD}" metricbeat: config: modules: @@ -79,35 +86,20 @@ conf: username: ${ELASTICSEARCH_USERNAME} password: ${ELASTICSEARCH_PASSWORD} modules: - mysql: - - module: mysql - metricsets: ["status"] - hosts: ["tcp(mariadb.openstack.svc.cluster.local:3306)/"] - username: root - password: password - rabbitmq: - - module: rabbitmq - metricsets: ["node", "queue", "connection"] + docker: + - module: docker + metricsets: + - "container" + - "cpu" + - "diskio" + - "healthcheck" + - "info" + - "image" + - "memory" + - "network" + hosts: ["unix:///var/run/docker.sock"] + period: 10s enabled: true - period: 30s - hosts: ["rabbitmq.openstack.svc.cluster.local:15672"] - username: rabbitmq - password: password - # docker: - # - module: docker - # metricsets: - # - "container" - # - "cpu" - # - "diskio" - # - "healthcheck" - # - "info" - # - "image" - # - "memory" - # - "network" - # hosts: ["unix:///var/run/docker.sock"] - # period: 10s - # enabled: false - # labels.dedot: true system: - module: system period: 10s @@ -121,18 +113,15 @@ conf: - core - diskio - socket + - filesystem + - fsstat processes: ['.*'] + cpu.metrics: ["percentages"] + core.metrics: ["percentages"] process.include_top_n: by_cpu: 5 by_memory: 5 - - module: system - period: 1m - metricsets: - - filesystem - - fsstat - processors: - - drop_event.when.regexp: - system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)' + enabled: true daemonset_kubernetes: - module: kubernetes metricsets: @@ -143,6 +132,9 @@ conf: - volume period: 10s hosts: ["localhost:10255"] + add_metadata: true + in_cluster: true + enabled: true deployment_kubernetes: - module: kubernetes metricsets: @@ -154,6 +146,9 @@ conf: - event period: 10s hosts: ['${KUBE_STATE_METRICS_HOST}:${KUBE_STATE_METRICS_PORT}'] + add_metadata: true + in_cluster: true + enabled: true endpoints: cluster_domain_suffix: cluster.local @@ -203,6 +198,23 @@ endpoints: port: http: default: 80 + kibana: + name: kibana + namespace: osh-infra + hosts: + default: kibana-dash + public: kibana + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + kibana: + default: 5601 + http: + default: 80 pod: affinity: diff --git a/elastic-packetbeat/templates/daemonset.yaml b/elastic-packetbeat/templates/daemonset.yaml index 6db4cdac45..7978b652c8 100644 --- a/elastic-packetbeat/templates/daemonset.yaml +++ b/elastic-packetbeat/templates/daemonset.yaml @@ -48,6 +48,7 @@ rules: - nodes - pods - services + - endpoints - replicationcontrollers - limitranges verbs: @@ -102,7 +103,7 @@ spec: - NET_ADMIN args: - "-c" - - "/etc/packetbeat/packetbeat.yml" + - "/usr/share/packetbeat/packetbeat.yml" - "-e" env: - name: ELASTICSEARCH_HOST @@ -131,7 +132,7 @@ spec: - name: pod-tmp mountPath: /tmp - name: packetbeat-etc - mountPath: /etc/packetbeat/packetbeat.yml + mountPath: /usr/share/packetbeat/packetbeat.yml subPath: packetbeat.yml readOnly: true {{ if $mounts_packetbeat.volumeMounts }}{{ toYaml $mounts_packetbeat.volumeMounts | indent 12 }}{{ end }} diff --git a/elastic-packetbeat/values.yaml b/elastic-packetbeat/values.yaml index 78955e4291..2b163ed8f5 100644 --- a/elastic-packetbeat/values.yaml +++ b/elastic-packetbeat/values.yaml @@ -28,7 +28,7 @@ labels: images: tags: - packetbeat: docker.elastic.co/beats/packetbeat:6.3.1 + packetbeat: docker.elastic.co/beats/packetbeat-oss:7.1.0 dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0 image_repo_sync: docker.io/docker:17.07.0 pull_policy: IfNotPresent @@ -63,28 +63,30 @@ conf: packetbeat: setup: kibana: - host: ['${KIBANA_HOST}:${KIBANA_PORT}'] + host: "${KIBANA_HOST}:${KIBANA_PORT}" + username: "${ELASTICSEARCH_USERNAME}" + password: "${ELASTICSEARCH_PASSWORD}" dashboards: enabled: true + index: "packetbeat-*" retry: enabled: true interval: 5 packetbeat: + flows: + timeout: 30s + period: 10s interfaces: device: any protocols: + - type: dhcpv4 + ports: [67, 68] - type: dns ports: [53] include_authorities: true include_additionals: true - type: http - ports: [80, 8080, 8081, 5000, 8002] - - type: memcache - ports: [11211] - - type: mysql - ports: [3306] - - type: rabbit - ports: [5672, 15672] + ports: [80, 8080, 8081, 5000, 8002, 6666, 3000, 5601, 9100, 9090, 44134] output: elasticsearch: hosts: ['${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}'] diff --git a/tools/deployment/elastic-beats/005-deploy-k8s.sh b/tools/deployment/elastic-beats/005-deploy-k8s.sh new file mode 120000 index 0000000000..257a39f7a3 --- /dev/null +++ b/tools/deployment/elastic-beats/005-deploy-k8s.sh @@ -0,0 +1 @@ +../common/005-deploy-k8s.sh \ No newline at end of file diff --git a/tools/deployment/elastic-beats/010-deploy-docker-registry.sh b/tools/deployment/elastic-beats/010-deploy-docker-registry.sh deleted file mode 120000 index 69d244e73f..0000000000 --- a/tools/deployment/elastic-beats/010-deploy-docker-registry.sh +++ /dev/null @@ -1 +0,0 @@ -../multinode/010-deploy-docker-registry.sh \ No newline at end of file diff --git a/tools/deployment/elastic-beats/050-elasticsearch.sh b/tools/deployment/elastic-beats/050-elasticsearch.sh index a0a2273947..0313c64e7d 100755 --- a/tools/deployment/elastic-beats/050-elasticsearch.sh +++ b/tools/deployment/elastic-beats/050-elasticsearch.sh @@ -21,12 +21,39 @@ make elasticsearch #NOTE: Deploy command tee /tmp/elasticsearch.yaml << EOF +manifests: + cron_curator: false + configmap_bin_curator: false + configmap_etc_curator: false +images: + tags: + elasticsearch: docker.io/openstackhelm/elasticsearch-s3:7_1_0-20191115 +storage: + data: + requests: + storage: 20Gi + master: + requests: + storage: 5Gi +jobs: + verify_repositories: + cron: "*/10 * * * *" monitoring: prometheus: - enabled: true + enabled: false pod: replicas: + client: 1 data: 1 + master: 2 +conf: + elasticsearch: + config: + xpack: + security: + enabled: false + ilm: + enabled: false EOF helm upgrade --install elasticsearch ./elasticsearch \ diff --git a/tools/deployment/elastic-beats/060-kibana.sh b/tools/deployment/elastic-beats/060-kibana.sh new file mode 100755 index 0000000000..ac955075b7 --- /dev/null +++ b/tools/deployment/elastic-beats/060-kibana.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Lint and package chart +make kibana + +: ${OSH_INFRA_EXTRA_HELM_ARGS_KIBANA:="$(./tools/deployment/common/get-values-overrides.sh kibana)"} + +tee /tmp/kibana.yaml << EOF +images: + tags: + kibana: docker.elastic.co/kibana/kibana:7.1.0 +conf: + kibana: + xpack: + security: + enabled: false + spaces: + enabled: false + apm: + enabled: false + graph: + enabled: false + ml: + enabled: false + monitoring: + enabled: false + reporting: + enabled: false + canvas: + enabled: false +EOF + +#NOTE: Deploy command +helm upgrade --install kibana ./kibana \ + --namespace=osh-infra \ + --values=/tmp/kibana.yaml + ${OSH_INFRA_EXTRA_HELM_ARGS} \ + ${OSH_INFRA_EXTRA_HELM_ARGS_KIBANA} + +#NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh osh-infra + +#NOTE: Validate Deployment info +helm status kibana diff --git a/tools/deployment/elastic-beats/060-elastic-metricbeat.sh b/tools/deployment/elastic-beats/080-elastic-metricbeat.sh similarity index 76% rename from tools/deployment/elastic-beats/060-elastic-metricbeat.sh rename to tools/deployment/elastic-beats/080-elastic-metricbeat.sh index 1fe6cda397..cf3c73400c 100755 --- a/tools/deployment/elastic-beats/060-elastic-metricbeat.sh +++ b/tools/deployment/elastic-beats/080-elastic-metricbeat.sh @@ -19,11 +19,26 @@ set -xe #NOTE: Lint and package chart make elastic-metricbeat +tee /tmp/metricbeat.yaml << EOF +images: + tags: + metricbeat: docker.elastic.co/beats/metricbeat:7.1.0 +conf: + metricbeat: + setup: + ilm: + enabled: false +endpoints: + elasticsearch: + namespace: osh-infra + kibana: + namespace: osh-infra +EOF + #NOTE: Deploy command helm upgrade --install elastic-metricbeat ./elastic-metricbeat \ --namespace=kube-system \ - --set endpoints.kube_state_metrics.namespace=kube-system \ - --set endpoints.elasticsearch.namespace=osh-infra + --values=/tmp/metricbeat.yaml #NOTE: Wait for deploy ./tools/deployment/common/wait-for-pods.sh kube-system diff --git a/tools/deployment/elastic-beats/080-elastic-filebeat.sh b/tools/deployment/elastic-beats/090-elastic-filebeat.sh similarity index 76% rename from tools/deployment/elastic-beats/080-elastic-filebeat.sh rename to tools/deployment/elastic-beats/090-elastic-filebeat.sh index cb36c71c2d..f738480952 100755 --- a/tools/deployment/elastic-beats/080-elastic-filebeat.sh +++ b/tools/deployment/elastic-beats/090-elastic-filebeat.sh @@ -19,10 +19,26 @@ set -xe #NOTE: Lint and package chart make elastic-filebeat +tee /tmp/filebeat.yaml << EOF +images: + tags: + filebeat: docker.elastic.co/beats/filebeat:7.1.0 +conf: + filebeat: + setup: + ilm: + enabled: false +endpoints: + elasticsearch: + namespace: osh-infra + kibana: + namespace: osh-infra +EOF + #NOTE: Deploy command helm upgrade --install elastic-filebeat ./elastic-filebeat \ --namespace=kube-system \ - --set endpoints.elasticsearch.namespace=osh-infra + --values=/tmp/filebeat.yaml #NOTE: Wait for deploy ./tools/deployment/common/wait-for-pods.sh kube-system diff --git a/tools/deployment/elastic-beats/100-elastic-apm-server.sh b/tools/deployment/elastic-beats/100-elastic-apm-server.sh deleted file mode 100755 index 1e4f705133..0000000000 --- a/tools/deployment/elastic-beats/100-elastic-apm-server.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -#NOTE: Lint and package chart -make elastic-apm-server - -#NOTE: Deploy command -helm upgrade --install elastic-apm-server ./elastic-apm-server \ - --namespace=kube-system \ - --set endpoints.elasticsearch.namespace=osh-infra - -#NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh kube-system - -#NOTE: Validate Deployment info -helm status elastic-apm-server diff --git a/tools/deployment/elastic-beats/090-elastic-packetbeat.sh b/tools/deployment/elastic-beats/100-elastic-packetbeat.sh similarity index 76% rename from tools/deployment/elastic-beats/090-elastic-packetbeat.sh rename to tools/deployment/elastic-beats/100-elastic-packetbeat.sh index 0b596ef403..c0fc08672a 100755 --- a/tools/deployment/elastic-beats/090-elastic-packetbeat.sh +++ b/tools/deployment/elastic-beats/100-elastic-packetbeat.sh @@ -19,11 +19,26 @@ set -xe #NOTE: Lint and package chart make elastic-packetbeat +tee /tmp/packetbeat.yaml << EOF +images: + tags: + filebeat: docker.elastic.co/beats/packetbeat:7.1.0 +conf: + packetbeat: + setup: + ilm: + enabled: false +endpoints: + elasticsearch: + namespace: osh-infra + kibana: + namespace: osh-infra +EOF + #NOTE: Deploy command helm upgrade --install elastic-packetbeat ./elastic-packetbeat \ --namespace=kube-system \ - --set endpoints.elasticsearch.namespace=osh-infra \ - --set endpoints.kibana.namespace=osh-infra + --values=/tmp/packetbeat.yaml #NOTE: Wait for deploy ./tools/deployment/common/wait-for-pods.sh kube-system diff --git a/tools/deployment/elastic-beats/110-kibana.sh b/tools/deployment/elastic-beats/110-kibana.sh deleted file mode 120000 index 03852eb062..0000000000 --- a/tools/deployment/elastic-beats/110-kibana.sh +++ /dev/null @@ -1 +0,0 @@ -../osh-infra-logging/075-kibana.sh \ No newline at end of file diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 8385a59e85..1026c9e00d 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -348,29 +348,24 @@ - job: name: openstack-helm-infra-elastic-beats parent: openstack-helm-infra-functional - nodeset: openstack-helm-five-node-ubuntu timeout: 7200 - pre-run: - - playbooks/osh-infra-upgrade-host.yaml - - playbooks/osh-infra-deploy-docker.yaml - - playbooks/osh-infra-build.yaml - - playbooks/osh-infra-deploy-k8s.yaml + pre-run: playbooks/osh-infra-upgrade-host.yaml run: playbooks/osh-infra-gate-runner.yaml post-run: playbooks/osh-infra-collect-logs.yaml + nodeset: openstack-helm-single-node vars: gate_scripts: - - ./tools/deployment/elastic-beats/010-deploy-docker-registry.sh + - ./tools/deployment/elastic-beats/005-deploy-k8s.sh - ./tools/deployment/elastic-beats/020-ingress.sh - ./tools/deployment/elastic-beats/030-ceph.sh - ./tools/deployment/elastic-beats/035-ceph-ns-activate.sh - ./tools/deployment/elastic-beats/040-ldap.sh - ./tools/deployment/elastic-beats/050-elasticsearch.sh - - ./tools/deployment/elastic-beats/060-elastic-metricbeat.sh + - ./tools/deployment/elastic-beats/060-kibana.sh - ./tools/deployment/elastic-beats/070-kube-state-metrics.sh - - ./tools/deployment/elastic-beats/080-elastic-filebeat.sh - - ./tools/deployment/elastic-beats/090-elastic-packetbeat.sh - - ./tools/deployment/elastic-beats/100-elastic-apm-server.sh - - ./tools/deployment/elastic-beats/110-kibana.sh + - ./tools/deployment/elastic-beats/080-elastic-metricbeat.sh + - ./tools/deployment/elastic-beats/090-elastic-filebeat.sh + - ./tools/deployment/elastic-beats/100-elastic-packetbeat.sh - job: name: openstack-helm-infra-armada-deploy diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index ec3e29410b..5b95b3ec6c 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -57,15 +57,15 @@ # - openstack-helm-infra-armada-deploy # - openstack-helm-infra-armada-update-uuid # - openstack-helm-infra-armada-update-passwords - # experimental: - # jobs: + experimental: + jobs: # NOTE(srwilkers): Disable fedora experimental jobs until issues resolved #- openstack-helm-infra-five-fedora # NOTE(srwilkers): Disable centos experimental jobs until issues resolved #- openstack-helm-infra-five-centos # - openstack-helm-infra-five-ubuntu + - openstack-helm-infra-elastic-beats # - openstack-helm-infra-tenant-ceph - # - openstack-helm-infra-elastic-beats # - openstack-helm-infra-armada-deploy # - openstack-helm-infra-armada-update-uuid # - openstack-helm-infra-armada-update-passwords