From e4c84aa28d76b88080f718d8898e4c606a8aeb25 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Mon, 13 Aug 2018 10:57:20 -0500 Subject: [PATCH] Add Redhat to the ELK deployment capabilities Change-Id: Id34e046a546f8d0878843596f53e400165e37c6e Signed-off-by: Kevin Carter --- .../roles/elastic_apm_server/vars/redhat.yml | 17 ++++++ .../roles/elastic_auditbeat/vars/redhat.yml | 18 ++++++ .../roles/elastic_curator/vars/redhat.yml | 17 ++++++ .../roles/elastic_dependencies/tasks/main.yml | 38 +++++++++--- .../elastic_dependencies/vars/redhat.yml | 17 ++++++ .../roles/elastic_filebeat/vars/redhat.yml | 17 ++++++ .../roles/elastic_heartbeat/vars/redhat.yml | 17 ++++++ .../roles/elastic_journalbeat/vars/redhat.yml | 19 ++++++ .../roles/elastic_kibana/vars/redhat.yml | 20 +++++++ .../roles/elastic_logstash/vars/redhat.yml | 18 ++++++ .../roles/elastic_metricbeat/vars/redhat.yml | 18 ++++++ .../roles/elastic_packetbeat/vars/redhat.yml | 18 ++++++ .../tasks/elastic_yum_repos.yml | 58 +++++++++++++++++++ .../elastic_repositories/vars/redhat.yml | 24 ++++++++ .../roles/elasticsearch/vars/redhat.yml | 18 ++++++ elk_metrics_6x/vars/variables.yml | 5 ++ zuul.d/jobs.yaml | 21 +++++-- zuul.d/project.yaml | 2 + 18 files changed, 347 insertions(+), 15 deletions(-) create mode 100644 elk_metrics_6x/roles/elastic_apm_server/vars/redhat.yml create mode 100644 elk_metrics_6x/roles/elastic_auditbeat/vars/redhat.yml create mode 100644 elk_metrics_6x/roles/elastic_curator/vars/redhat.yml create mode 100644 elk_metrics_6x/roles/elastic_dependencies/vars/redhat.yml create mode 100644 elk_metrics_6x/roles/elastic_filebeat/vars/redhat.yml create mode 100644 elk_metrics_6x/roles/elastic_heartbeat/vars/redhat.yml create mode 100644 elk_metrics_6x/roles/elastic_journalbeat/vars/redhat.yml create mode 100644 elk_metrics_6x/roles/elastic_kibana/vars/redhat.yml create mode 100644 elk_metrics_6x/roles/elastic_logstash/vars/redhat.yml create mode 100644 elk_metrics_6x/roles/elastic_metricbeat/vars/redhat.yml create mode 100644 elk_metrics_6x/roles/elastic_packetbeat/vars/redhat.yml create mode 100644 elk_metrics_6x/roles/elastic_repositories/tasks/elastic_yum_repos.yml create mode 100644 elk_metrics_6x/roles/elastic_repositories/vars/redhat.yml create mode 100644 elk_metrics_6x/roles/elasticsearch/vars/redhat.yml diff --git a/elk_metrics_6x/roles/elastic_apm_server/vars/redhat.yml b/elk_metrics_6x/roles/elastic_apm_server/vars/redhat.yml new file mode 100644 index 00000000..3fac8c52 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_apm_server/vars/redhat.yml @@ -0,0 +1,17 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apm_server_distro_packages: + - apm-server diff --git a/elk_metrics_6x/roles/elastic_auditbeat/vars/redhat.yml b/elk_metrics_6x/roles/elastic_auditbeat/vars/redhat.yml new file mode 100644 index 00000000..59cc129c --- /dev/null +++ b/elk_metrics_6x/roles/elastic_auditbeat/vars/redhat.yml @@ -0,0 +1,18 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +auditbeat_distro_packages: + - audispd-plugins + - auditbeat diff --git a/elk_metrics_6x/roles/elastic_curator/vars/redhat.yml b/elk_metrics_6x/roles/elastic_curator/vars/redhat.yml new file mode 100644 index 00000000..a3eb703e --- /dev/null +++ b/elk_metrics_6x/roles/elastic_curator/vars/redhat.yml @@ -0,0 +1,17 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +curator_distro_packages: + - python-virtualenv diff --git a/elk_metrics_6x/roles/elastic_dependencies/tasks/main.yml b/elk_metrics_6x/roles/elastic_dependencies/tasks/main.yml index a49b2a96..1b0f25d8 100644 --- a/elk_metrics_6x/roles/elastic_dependencies/tasks/main.yml +++ b/elk_metrics_6x/roles/elastic_dependencies/tasks/main.yml @@ -105,17 +105,26 @@ when: - physical_host == inventory_hostname -- name: elasticsearch datapath bind mount - lxc_container: - name: "{{ inventory_hostname }}" - container_command: | - [[ ! -d "/var/lib/{{ service_name }}" ]] && mkdir -p "/var/lib/{{ service_name }}" - container_config: - - "lxc.mount.entry=/openstack/{{ inventory_hostname }}/{{ service_name }} var/lib/{{ service_name }} none bind 0 0" - delegate_to: "{{ physical_host }}" +- name: Container block + block: + - name: Ensure service directories data-path exists + file: + path: "/openstack/{{ inventory_hostname }}/{{ service_name }}" + state: "directory" + delegate_to: "{{ physical_host }}" + + - name: elasticsearch datapath bind mount + lxc_container: + name: "{{ inventory_hostname }}" + container_command: | + [[ ! -d "/var/lib/{{ service_name }}" ]] && mkdir -p "/var/lib/{{ service_name }}" + container_config: + - "lxc.mount.entry=/openstack/{{ inventory_hostname }}/{{ service_name }} var/lib/{{ service_name }} none bind 0 0" + delegate_to: "{{ physical_host }}" + when: + - container_tech | default('lxc') == 'lxc' when: - physical_host != inventory_hostname - - container_tech | default('lxc') == 'lxc' - name: Ensure Java is installed package: @@ -149,6 +158,17 @@ createhome: "yes" home: "/var/lib/{{ service_name }}" rescue: + - name: Check for system user + debug: + msg: >- + The general user creation task failed. This typically means that the + user already exists and something in the user configuration provided + is changing the system user in way that is simply not possible at this + time. The playbooks will now simply ensure the user exists and before + carrying on to the next task. While it's not required, it may be + benificial to schedule a maintenance where the elastic services are + stopped. + - name: Ensure the system user exists user: name: "{{ service_owner }}" diff --git a/elk_metrics_6x/roles/elastic_dependencies/vars/redhat.yml b/elk_metrics_6x/roles/elastic_dependencies/vars/redhat.yml new file mode 100644 index 00000000..b31d349c --- /dev/null +++ b/elk_metrics_6x/roles/elastic_dependencies/vars/redhat.yml @@ -0,0 +1,17 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +elastic_distro_packages: + - java-1.8.0-openjdk diff --git a/elk_metrics_6x/roles/elastic_filebeat/vars/redhat.yml b/elk_metrics_6x/roles/elastic_filebeat/vars/redhat.yml new file mode 100644 index 00000000..2f055205 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_filebeat/vars/redhat.yml @@ -0,0 +1,17 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +filebeat_distro_packages: + - filebeat diff --git a/elk_metrics_6x/roles/elastic_heartbeat/vars/redhat.yml b/elk_metrics_6x/roles/elastic_heartbeat/vars/redhat.yml new file mode 100644 index 00000000..4a32d6b7 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_heartbeat/vars/redhat.yml @@ -0,0 +1,17 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +heartbeat_distro_packages: + - heartbeat-elastic diff --git a/elk_metrics_6x/roles/elastic_journalbeat/vars/redhat.yml b/elk_metrics_6x/roles/elastic_journalbeat/vars/redhat.yml new file mode 100644 index 00000000..77c33669 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_journalbeat/vars/redhat.yml @@ -0,0 +1,19 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +journalbeat_distro_packages: + - gcc + - git + - systemd-devel diff --git a/elk_metrics_6x/roles/elastic_kibana/vars/redhat.yml b/elk_metrics_6x/roles/elastic_kibana/vars/redhat.yml new file mode 100644 index 00000000..6d5b7c26 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_kibana/vars/redhat.yml @@ -0,0 +1,20 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kibana_nginx_vhost_path: /etc/nginx/conf.d/ +kibana_distro_packages: + - kibana + - nginx + - python-passlib diff --git a/elk_metrics_6x/roles/elastic_logstash/vars/redhat.yml b/elk_metrics_6x/roles/elastic_logstash/vars/redhat.yml new file mode 100644 index 00000000..716310e6 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_logstash/vars/redhat.yml @@ -0,0 +1,18 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +logstash_distro_packages: + - logrotate + - logstash diff --git a/elk_metrics_6x/roles/elastic_metricbeat/vars/redhat.yml b/elk_metrics_6x/roles/elastic_metricbeat/vars/redhat.yml new file mode 100644 index 00000000..f24dbcf3 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_metricbeat/vars/redhat.yml @@ -0,0 +1,18 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +metricbeat_nginx_vhost_path: /etc/nginx/conf.d/ +metricbeat_distro_packages: + - metricbeat diff --git a/elk_metrics_6x/roles/elastic_packetbeat/vars/redhat.yml b/elk_metrics_6x/roles/elastic_packetbeat/vars/redhat.yml new file mode 100644 index 00000000..e949478a --- /dev/null +++ b/elk_metrics_6x/roles/elastic_packetbeat/vars/redhat.yml @@ -0,0 +1,18 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +packetbeat_distro_packages: + - tcpdump + - packetbeat diff --git a/elk_metrics_6x/roles/elastic_repositories/tasks/elastic_yum_repos.yml b/elk_metrics_6x/roles/elastic_repositories/tasks/elastic_yum_repos.yml new file mode 100644 index 00000000..0a11e7e6 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_repositories/tasks/elastic_yum_repos.yml @@ -0,0 +1,58 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Add Elastic search public GPG key + rpm_key: + state: "{{ elastic_repo.state }}" + key: "{{ elastic_repo.key_url }}" + register: _zypp_task + until: _zypp_task is success + retries: 3 + delay: 2 + tags: + - package_install + +# Force refresh of a repository +- name: Add elk repo to yum sources list + yum_repository: + name: "elastic" + file: "elastic" + description: "Elastic repositories, you know, for ELK." + baseurl: "{{ elastic_repo.repo }}" + state: "{{ elastic_repo.state }}" + enabled: yes + register: _yum_task + until: _yum_task is success + retries: 3 + delay: 2 + tags: + - package_install + +# Force refresh of a repository +- name: Add nginx repo to yum sources list + yum_repository: + name: "nginx" + file: "nginx" + description: "NGINX repo" + baseurl: "{{ elastic_nginx_repo.repo }}" + state: "{{ elastic_nginx_repo.state }}" + enabled: yes + gpgcheck: no + register: _yum_task + until: _yum_task is success + retries: 3 + delay: 2 + tags: + - package_install diff --git a/elk_metrics_6x/roles/elastic_repositories/vars/redhat.yml b/elk_metrics_6x/roles/elastic_repositories/vars/redhat.yml new file mode 100644 index 00000000..888a7428 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_repositories/vars/redhat.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# elk apt repo +elastic_repo: + repo: 'https://artifacts.elastic.co/packages/6.x/yum' + state: "{{ ((elk_package_state | default('present')) == 'absent') | ternary('absent', 'present') }}" + key_url: "https://artifacts.elastic.co/GPG-KEY-elasticsearch" + +elastic_nginx_repo: + repo: 'http://nginx.org/packages/centos/$releasever/$basearch/' + state: "{{ ((elk_package_state | default('present')) == 'absent') | ternary('absent', 'present') }}" diff --git a/elk_metrics_6x/roles/elasticsearch/vars/redhat.yml b/elk_metrics_6x/roles/elasticsearch/vars/redhat.yml new file mode 100644 index 00000000..a5948d12 --- /dev/null +++ b/elk_metrics_6x/roles/elasticsearch/vars/redhat.yml @@ -0,0 +1,18 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +elasticsearch_distro_packages: + - logrotate + - elasticsearch diff --git a/elk_metrics_6x/vars/variables.yml b/elk_metrics_6x/vars/variables.yml index 2fe2f95d..d2449a68 100644 --- a/elk_metrics_6x/vars/variables.yml +++ b/elk_metrics_6x/vars/variables.yml @@ -1,4 +1,9 @@ --- +# Option to define third memory +q_mem: "{{ (ansible_memtotal_mb | int) // 3 }}" + +# Option to define half memory +h_mem: "{{ (ansible_memtotal_mb | int) // 2 }}" apm_port: 8200 elastic_port: 9200 diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 7d3d4ec4..8c4a99af 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -25,17 +25,26 @@ osa_test_repo: "openstack/openstack-ansible-ops" test_clustered_elk: false -# - job: -# name: "openstack-ansible-ops:elk_metrics_6x-centos-7" -# parent: "openstack-ansible-ops:elk_metrics_6x-ubuntu-xenial" -# nodeset: centos-7 -# voting: false +- job: + name: "openstack-ansible-ops:elk_metrics_6x-centos-7" + parent: "openstack-ansible-ops:elk_metrics_6x-ubuntu-xenial" + nodeset: centos-7 + voting: true + +- job: + name: "openstack-ansible-ops:elk_metrics_6x-centos-7-clustered" + parent: "openstack-ansible-ops:elk_metrics_6x-ubuntu-xenial" + nodeset: centos-7 + voting: false + vars: + osa_test_repo: "openstack/openstack-ansible-ops" + test_clustered_elk: true - job: name: "openstack-ansible-ops:elk_metrics_6x-opensuse-423" parent: "openstack-ansible-ops:elk_metrics_6x-ubuntu-xenial" nodeset: opensuse-423 - voting: false + voting: true - job: name: "openstack-ansible-ops:elk_metrics_6x-opensuse-423-clustered" diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 0445cf71..2337f1ec 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -17,6 +17,8 @@ - openstack-ansible-role-jobs check: jobs: + - "openstack-ansible-ops:elk_metrics_6x-centos-7" + - "openstack-ansible-ops:elk_metrics_6x-centos-7-clustered" - "openstack-ansible-ops:elk_metrics_6x-opensuse-423" - "openstack-ansible-ops:elk_metrics_6x-opensuse-423-clustered" - "openstack-ansible-ops:elk_metrics_6x-ubuntu-trusty"