diff --git a/elk_metrics_6x/calculate_index_retention_default.yml b/elk_metrics_6x/calculate_index_retention_default.yml deleted file mode 100644 index 058cfbf2..00000000 --- a/elk_metrics_6x/calculate_index_retention_default.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- - -- name: Set assumed buffer storage fact - set_fact: - es_assumed_buffer_storage: "{{ ((es_total_available_storage | int) * 0.25) | round | int }}" - -- name: Set usable buffer storage fact(s) - set_fact: - es_usable_buffer_storage: "{{ (es_total_available_storage | int) - (es_assumed_buffer_storage | int) }}" - es_expected_storage: "{{ ((elastic_beat_retention_policy_hosts.values() | map('int') | list) | sum) * (elastic_beat_storage_constant | int) }}" - -- name: Set buffer storage fact - set_fact: - es_assumed_usable_storage_per_node: "{{ (es_usable_buffer_storage | int) // (data_nodes | length | int) }}" - -- name: Set storage the mulitplier - set_fact: - es_storage_multiplier: "{{ ((es_usable_buffer_storage | int) < (es_expected_storage | int)) | ternary(((elastic_beat_storage_constant | int) * 2), elastic_beat_storage_constant | int) }}" - -- name: Set retention facts - set_fact: "elastic_{{ item.key }}_retention={{ (es_assumed_usable_storage_per_node | int) // ((item.value | int) * (es_storage_multiplier | int)) }}" - when: - - hostvars[inventory_hostname]["elastic_" + item.key + "_retention"] is undefined - with_dict: "{{ elastic_beat_retention_policy_hosts }}" diff --git a/elk_metrics_6x/common_task_data_node_hosts.yml b/elk_metrics_6x/common_task_data_node_hosts.yml deleted file mode 100644 index 98f87508..00000000 --- a/elk_metrics_6x/common_task_data_node_hosts.yml +++ /dev/null @@ -1,131 +0,0 @@ ---- - -- name: Refresh minimal facts - setup: - gather_subset: '!all,!any,network,virtual' - -# storage node count is equal to the cluster size -- name: Node count fact - set_fact: - storage_node_count: "{{ groups['elastic-logstash'] | length }}" - tags: - - always - -# the elasticserch cluster elects one master from all those which are marked as master-eligible -# 1 node cluster can only have one master -# 2 node clusters have 1 master-eligable nodes to avoid split-brain -# 3 node clusters have 3 master-eligable nodes -# >3 node clusters have (nodes // 2) eligable masters rounded up to the next odd number -- name: Master node count fact - set_fact: - master_node_count: |- - {% set masters = 0 %} - {% if (storage_node_count | int) < 3 %} - {% set masters = 1 %} - {% elif (storage_node_count | int) == 3 %} - {% set masters = 3 %} - {% else %} - {% set masters = (storage_node_count | int ) // 2 %} - {% if ((masters | int) % 2 == 0) %} - {% set masters = (masters | int) + 1 %} - {% endif %} - {% endif %} - {{ masters }} - tags: - - always - -# assign node roles -# the first 'master_node_count' hosts in groups['elastic-logstash'] become master-eligible nodes -# the first 'master_node_count' and subsequent alternate hosts in groups['elastic-logstash'] becomes data nodes -- name: Data nodes fact - set_fact: - data_nodes: "{{ (groups['elastic-logstash'][:master_node_count | int] + groups['elastic-logstash'][master_node_count | int::2]) }}" - master_nodes: "{{ groups['elastic-logstash'][:master_node_count | int] }}" - coordination_nodes: >- - {{ - (groups['kibana'] | map('extract', hostvars, 'ansible_host') | list) - | map('regex_replace', '(.*)' ,'\1:' ~ elastic_port) - | list - }} - zen_nodes: >- - {{ - (groups['elastic-logstash'] | union(groups['kibana'])) | map('extract', hostvars, 'ansible_host') | list - }} - elasticserch_interface_speed: |- - {% set default_interface_fact = hostvars[inventory_hostname]['ansible_' + (elastic_data_interface | replace('-', '_'))] %} - {% set speeds = [] %} - {% if default_interface_fact['type'] == 'bridge' %} - {% for interface in default_interface_fact['interfaces'] %} - {% set interface_fact = hostvars[inventory_hostname]['ansible_' + (interface | replace('-', '_'))] %} - {% if 'speed' in interface_fact %} - {% set speed = (interface_fact['speed'] | default(1000)) | string %} - {% if speed == "-1" %} - {% set _ = speeds.append(1000) %} - {% else %} - {% set _ = speeds.append(speed | int) %} - {% endif %} - {% if 'module' in interface_fact %} - {% set _ = speeds.append((interface_fact['speed'] | default(1000)) | int) %} - {% else %} - {% set _ = speeds.append(1000) %} - {% endif %} - {% endif %} - {% endfor %} - {% else %} - {% if ('module' in default_interface_fact) or (default_interface_fact['type'] == 'bond') %} - {% set speed = (default_interface_fact['speed'] | default(1000)) | string %} - {% if speed == "-1" %} - {% set _ = speeds.append(1000) %} - {% else %} - {% set _ = speeds.append(speed | int) %} - {% endif %} - {% else %} - {% set _ = speeds.append(1000) %} - {% endif %} - {% endif %} - {% set interface_speed = ((speeds | min) * 0.20) | int %} - {{ ((interface_speed | int) > 750) | ternary(750, interface_speed) }} - tags: - - always - -- name: Set data node details - set_fact: - elasticsearch_data_node_details: >- - {{ - (data_nodes | map('extract', hostvars, 'ansible_host') | list) | map('regex_replace', '(.*)' ,'\1:' ~ elastic_port) | list - }} - logstash_data_node_details: >- - {{ - (data_nodes | map('extract', hostvars, 'ansible_host') | list) | map('regex_replace', '(.*)' ,'\1:' ~ logstash_beat_input_port) | list - }} - -# based on the assignment of roles to hosts, set per host booleans -- name: Node enablement - set_fact: - master_node: "{{ (inventory_hostname in master_nodes) | ternary(true, false) }}" - data_node: "{{ (inventory_hostname in data_nodes) | ternary(true, false) }}" - elastic_thread_pool_size: "{{ ((ansible_processor_cores | int) >= 24) | ternary(24, ansible_processor_cores) }}" - tags: - - always - -# Set a data node facts. The data nodes, in the case of elasticsearch are also -# ingest nodes. -- name: Set data nodes - set_fact: - elasticsearch_number_of_replicas: "{{ ((data_nodes | length) > 2) | ternary('2', ((data_nodes | length) > 1) | ternary('1', '0')) }}" - elasticsearch_data_hosts: |- - {% set nodes = elasticsearch_data_node_details %} - {% if inventory_hostname in data_nodes %} - {% set _ = nodes.insert(0, '127.0.0.1:' ~ elastic_port) %} - {% endif %} - {% set data_hosts = nodes | shuffle(seed=inventory_hostname) %} - {{ data_hosts }} - logstash_data_hosts: |- - {% set nodes = logstash_data_node_details %} - {% if inventory_hostname in data_nodes %} - {% set _ = nodes.insert(0, '127.0.0.1:' ~ logstash_beat_input_port) %} - {% endif %} - {% set data_hosts = nodes | shuffle(seed=inventory_hostname) %} - {{ data_hosts }} - tags: - - always diff --git a/elk_metrics_6x/common_task_install_go1.10.1.yml b/elk_metrics_6x/common_task_install_go1.10.1.yml deleted file mode 100644 index 4d389438..00000000 --- a/elk_metrics_6x/common_task_install_go1.10.1.yml +++ /dev/null @@ -1,62 +0,0 @@ ---- -# Copyright 2018, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Check for go - stat: - path: /opt/go1.10.1/go/bin/go - register: go_path - -- name: Install go - block: - - name: GET go - get_url: - url: "https://dl.google.com/go/go1.10.1.linux-amd64.tar.gz" - dest: "/opt/go1.10.1.linux-amd64.tar.gz" - checksum: "sha256:72d820dec546752e5a8303b33b009079c15c2390ce76d67cf514991646c6127b" - register: _get_task - until: _get_task is success - retries: 3 - delay: 2 - tags: - - package_install - - - name: Create go directory - file: - path: "/opt/go1.10.1" - state: directory - - - name: Unarchive go - unarchive: - src: "/opt/go1.10.1.linux-amd64.tar.gz" - dest: "/opt/go1.10.1" - remote_src: yes - - - name: Create go defaults file - copy: - content: | - GOROOT=/opt/go1.10.1/go - GOPATH=/usr/local - PATH=${PATH}:${GOROOT}/bin - dest: /etc/default/go1.10.1 - when: - - (elk_package_state | default('present')) == 'present' - - not go_path.stat.exists | bool - -- name: Remove go - file: - path: "/opt/go1.10.1" - state: absent - when: - - (elk_package_state | default('present')) == 'absent' diff --git a/elk_metrics_6x/createElasticIndexes.yml b/elk_metrics_6x/createElasticIndexes.yml index b05f3ac8..2b7c45c1 100644 --- a/elk_metrics_6x/createElasticIndexes.yml +++ b/elk_metrics_6x/createElasticIndexes.yml @@ -1,4 +1,16 @@ --- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + - name: Create/Setup known indexes in Elasticsearch hosts: "elastic-logstash[0]" become: true @@ -8,12 +20,10 @@ environment: "{{ deployment_environment_variables | default({}) }}" - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml - tags: - - always + roles: + - role: elastic_retention - tasks: + post_tasks: - name: Create basic indexes uri: url: http://127.0.0.1:9200/{{ item.name }} diff --git a/elk_metrics_6x/installAPMserver.yml b/elk_metrics_6x/installAPMserver.yml index 137fc8d9..634c2038 100644 --- a/elk_metrics_6x/installAPMserver.yml +++ b/elk_metrics_6x/installAPMserver.yml @@ -1,4 +1,16 @@ --- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + - name: Install apm-server hosts: apm-server become: true @@ -8,76 +20,10 @@ vars_files: - vars/variables.yml - environment: "{{ deployment_environment_variables | default({}) }}" - - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml - tags: - - always - - tasks: - - include_tasks: common_task_install_elk_repo.yml - - - name: Ensure apm-server is installed - apt: - name: "{{ item }}" - state: "{{ elk_package_state | default('present') }}" - update_cache: true - with_items: - - apm-server - register: _apt_task - until: _apt_task is success - retries: 3 - delay: 2 - notify: - - Enable and restart apm server - tags: - - package_install - - - name: exit playbook after uninstall - meta: end_play - when: - - (elk_package_state | default('present')) == 'absent' - - post_tasks: - - name: Create apm-server systemd service config dir - file: - path: "/etc/systemd/system/apm-server.service.d" - state: "directory" - group: "root" - owner: "root" - mode: "0755" - - - name: Apply systemd options - template: - src: "{{ item.src }}" - dest: "/etc/systemd/system/apm-server.service.d/{{ item.dest }}" - mode: "0644" - with_items: - - { src: "systemd.general-overrides.conf.j2", dest: "apm-server-overrides.conf" } - notify: - - Enable and restart apm server - - - name: Drop apm-server conf file - template: - src: templates/apm-server.yml.j2 - dest: /etc/apm-server/apm-server.yml - notify: - - Enable and restart apm server - - handlers: - - name: Enable and restart apm server - systemd: - name: "apm-server" - enabled: true - state: restarted - daemon_reload: true - when: - - (elk_package_state | default('present')) != 'absent' - tags: - - config + roles: + - role: elastic_apm_server + - role: elastic_rollup + index_name: apm tags: - - server-install - -- import_playbook: setupAPMserver.yml + - apm-server diff --git a/elk_metrics_6x/installAuditbeat.yml b/elk_metrics_6x/installAuditbeat.yml index 6f69d25f..121e94f4 100644 --- a/elk_metrics_6x/installAuditbeat.yml +++ b/elk_metrics_6x/installAuditbeat.yml @@ -1,4 +1,16 @@ --- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + - name: Install Auditbeat hosts: hosts become: true @@ -10,75 +22,10 @@ environment: "{{ deployment_environment_variables | default({}) }}" - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml - tags: - - always - - tasks: - - include_tasks: common_task_install_elk_repo.yml - - - name: Ensure Auditbeat is installed - apt: - name: "{{ item }}" - state: "{{ elk_package_state | default('present') }}" - update_cache: true - with_items: - - audispd-plugins - - auditbeat - register: _apt_task - until: _apt_task is success - retries: 3 - delay: 2 - notify: - - Enable and restart auditbeat - tags: - - package_install - - - name: exit playbook after uninstall - meta: end_play - when: - - (elk_package_state | default('present')) == 'absent' - - post_tasks: - - name: Create auditbeat systemd service config dir - file: - path: "/etc/systemd/system/auditbeat.service.d" - state: "directory" - group: "root" - owner: "root" - mode: "0755" - - - name: Apply systemd options - template: - src: "{{ item.src }}" - dest: "/etc/systemd/system/auditbeat.service.d/{{ item.dest }}" - mode: "0644" - with_items: - - { src: "systemd.general-overrides.conf.j2", dest: "auditbeat-overrides.conf" } - notify: - - Enable and restart auditbeat - - - name: Drop auditbeat conf file - template: - src: templates/auditbeat.yml.j2 - dest: /etc/auditbeat/auditbeat.yml - notify: - - Enable and restart auditbeat - - handlers: - - name: Enable and restart auditbeat - systemd: - name: "auditbeat" - enabled: "true" - state: restarted - daemon_reload: true - when: - - (elk_package_state | default('present')) != 'absent' - tags: - - config + roles: + - role: elastic_auditbeat + - role: elastic_rollup + index_name: auditbeat tags: - beat-install - -- import_playbook: setupAuditbeat.yml diff --git a/elk_metrics_6x/installCurator.yml b/elk_metrics_6x/installCurator.yml index 221461b5..85746723 100644 --- a/elk_metrics_6x/installCurator.yml +++ b/elk_metrics_6x/installCurator.yml @@ -1,6 +1,4 @@ --- -# Copyright 2018, Rackspace US, Inc. -# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -25,128 +23,8 @@ environment: "{{ deployment_environment_variables | default({}) }}" - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml - tags: - - always - - - name: Query es storage - uri: - url: "http://127.0.0.1:9200/_nodes/{{ (data_nodes | map('extract', hostvars, 'ansible_host') | list) | join(',') }}/stats/fs" - method: GET - register: elk_data - until: elk_data is success - retries: 5 - delay: 5 - run_once: true - - - name: Set available storage fact - set_fact: - es_total_available_storage: "{{ ((elk_data['json']['nodes'].values() | list) | map(attribute='fs.total.total_in_bytes') | list | sum) // 1024 // 1024 }}" - - - name: Compute index retention - include_tasks: "calculate_index_retention_{{ elastic_index_retention_algorithm | default('default') }}.yml" - - - name: Ensure virtualenv is installed - apt: - name: "{{ item }}" - state: "{{ elk_package_state | default('present') }}" - update_cache: true - with_items: - - python-virtualenv - - virtualenv - tags: - - package_install - - - name: Ensure curator is installed - pip: - name: "elasticsearch-curator<6" - state: "{{ elk_package_state | default('present') }}" - extra_args: --isolated - virtualenv: /opt/elasticsearch-curator - register: _pip_task - until: _pip_task is success - retries: 3 - delay: 2 - tags: - - package_install - - - name: exit playbook after uninstall - meta: end_play - when: - - (elk_package_state | default('present')) == 'absent' - - tasks: - - name: create the system group - group: - name: "curator" - state: "present" - system: "yes" - - - name: Create the curator system user - user: - name: "curator" - group: "curator" - comment: "curator user" - shell: "/bin/false" - createhome: "yes" - home: "/var/lib/curator" - - - name: Create curator data path - file: - path: "{{ item }}" - state: directory - owner: "curator" - group: "curator" - mode: "0755" - recurse: true - with_items: - - "/var/lib/curator" - - "/var/log/curator" - - "/etc/curator" - - - name: Drop curator conf file - template: - src: templates/curator.yml.j2 - dest: /var/lib/curator/curator.yml - - - name: Drop curator action file - template: - src: templates/curator-actions.yml.j2 - dest: /var/lib/curator/actions.yml - - post_tasks: - - name: Run the systemd service role - include_role: - name: systemd_service - private: true - vars: - systemd_service_enabled: "{{ ((elk_package_state | default('present')) != 'absent') | ternary(true, false) }}" - systemd_service_restart_changed: false - systemd_user_name: curator - systemd_group_name: curator - systemd_services: - - service_name: "curator" - execstarts: - - /opt/elasticsearch-curator/bin/curator - --config /var/lib/curator/curator.yml - /var/lib/curator/actions.yml - timer: - state: "started" - options: - OnBootSec: 30min - OnUnitActiveSec: 24h - Persistent: true - - - name: Enable and restart curator.timer - systemd: - name: "curator.timer" - enabled: true - state: restarted - when: - - (elk_package_state | default('present')) != 'absent' - tags: - - config + roles: + - role: elastic_curator tags: - beat-install diff --git a/elk_metrics_6x/installElastic.yml b/elk_metrics_6x/installElastic.yml index 46f8b7f1..748834b7 100644 --- a/elk_metrics_6x/installElastic.yml +++ b/elk_metrics_6x/installElastic.yml @@ -1,241 +1,27 @@ --- - -- name: Run serialization detection - hosts: "elastic-logstash:kibana" - gather_facts: true - - vars_files: - - vars/variables.yml - - tasks: - - include_tasks: common_task_data_node_hosts.yml - - - name: Group by stand alone masters - group_by: - key: elastic_masters - parents: elastic-logstash - when: - - inventory_hostname in master_nodes - - - name: Group by non stand alone masters - group_by: - key: elastic_non_masters - parents: elastic-logstash - when: - - inventory_hostname in (data_nodes | difference(master_nodes)) - - - name: Group by coordinators - group_by: - key: elastic_coordinators - parents: elastic-logstash - when: - - inventory_hostname in groups['kibana'] - tags: - - always +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. - name: Install Elastic Search - hosts: "elastic_coordinators:elastic_masters:elastic_non_masters" - serial: "33%" + hosts: "elastic-logstash:kibana" become: true vars_files: - vars/variables.yml - vars: - temp_dir: /var/lib/elasticsearch/tmp - nfs_query: "[?fstype=='nfs' || fstype=='nfs4']" - environment: "{{ deployment_environment_variables | default({}) }}" - tasks: - - name: Set memory fact to half - set_fact: - h_mem: "{{ (ansible_memtotal_mb | int) // 2 }}" - when: - - h_mem is not defined - tags: - - always - - - name: Set elasticsearch facts - set_fact: - elastic_heap_size: "{{ ((h_mem | int) > 30720) | ternary(30720, h_mem) }}" - elastic_log_rotate_path: "/var/log/elasticsearch" - tags: - - always - - - name: Set kibana elasticsearch facts - block: - - name: Set kibana as elasticsearch coordinators - set_fact: - elasticsearch_node_master: false - elasticsearch_node_data: false - elasticsearch_node_ingest: false - elastic_coordination_node: true - elastic_heap_size: "{{ (elastic_heap_size | int) // 3 }}" - elastic_thread_pool_size: "{{ ((ansible_processor_cores | int) > 4) | ternary(4, (ansible_processor_cores // 2)) }}" - when: - - inventory_hostname in (groups['kibana'] | default([])) and - not inventory_hostname in (groups['elastic-logstash'] | default([])) - tags: - - always - - - name: Configure systcl vm.max_map_count=262144 on container hosts - sysctl: - name: "vm.max_map_count" - value: "262144" - state: "present" - reload: "yes" - delegate_to: "{{ physical_host }}" - tags: - - sysctl - - - name: Ensure mount directories exists - file: - path: "/openstack/{{ inventory_hostname }}/elasticsearch" - state: "directory" - delegate_to: "{{ physical_host }}" - - - name: elasticsearch datapath bind mount - lxc_container: - name: "{{ inventory_hostname }}" - container_command: | - [[ ! -d "/var/lib/elasticsearch" ]] && mkdir -p "/var/lib/elasticsearch" - container_config: - - "lxc.mount.entry=/openstack/{{ inventory_hostname }}/elasticsearch var/lib/elasticsearch none bind 0 0" - delegate_to: "{{ physical_host }}" - when: - - physical_host != inventory_hostname - - container_tech | default('lxc') == 'lxc' - - - name: Ensure Java is installed - apt: - name: openjdk-8-jre - state: "{{ elk_package_state | default('present') }}" - install_recommends: yes - update_cache: yes - register: _apt_task - until: _apt_task is success - retries: 3 - delay: 2 - tags: - - package_install - - - include_tasks: common_task_install_elk_repo.yml - - - name: Ensure elasticsearch is installed - apt: - name: "{{ item }}" - state: "{{ elk_package_state | default('present') }}" - update_cache: yes - with_items: - - logrotate - - elasticsearch - register: _apt_task - until: _apt_task is success - retries: 3 - delay: 2 - notify: - - Enable and restart elastic - tags: - - package_install - - - name: exit playbook after uninstall - meta: end_play - when: - - (elk_package_state | default('present')) == 'absent' - - post_tasks: - - name: Create elasticsearch systemd service config dir - file: - path: "/etc/systemd/system/elasticsearch.service.d" - state: "directory" - group: "root" - owner: "root" - mode: "0755" - - - name: Apply systemd options - template: - src: "{{ item.src }}" - dest: "/etc/systemd/system/elasticsearch.service.d/{{ item.dest }}" - mode: "0644" - with_items: - - { src: "systemd.elasticsearch-overrides.conf.j2", dest: "elasticsearch-overrides.conf" } - notify: - - Enable and restart elastic - - - name: Drop elasticsearch conf file - template: - src: "{{ item.src }}" - dest: "{{ item.dest }}" - with_items: - - src: templates/elasticsearch.yml.j2 - dest: /etc/elasticsearch/elasticsearch.yml - - src: templates/jvm.options.j2 - dest: /etc/elasticsearch/jvm.options - - src: templates/es-log4j2.properties.j2 - dest: /etc/elasticsearch/log4j2.properties - notify: - - Enable and restart elastic - tags: - - config - - - name: Ensure elasticsearch ownership - file: - path: /var/lib/elasticsearch - owner: elasticsearch - group: elasticsearch - recurse: true - register: e_perms - until: e_perms is success - retries: 3 - delay: 1 - tags: - - config - - - name: Ensure logstash tmp dir - file: - path: "/var/lib/elasticsearch/tmp" - state: directory - owner: "elasticsearch" - group: "elasticsearch" - mode: "0750" - - - name: Create logrotate config - template: - src: "templates/logrotate.j2" - dest: "/etc/logrotate.d/elasticsearch" - - - name: Ensure nfs client is available if necessary - package: - name: nfs-common - state: present - when: - - elastic_shared_fs_repos is defined - - (elastic_shared_fs_repos|json_query(nfs_query)) | length > 0 - - - name: Ensure backup filesystems are mounted - mount: - fstype: "{{ item.fstype }}" - src: "{{ item.src }}" - opts: "{{ item.opts }}" - path: "{{ item.path }}" - state: "{{ item.state }}" - with_items: - - "{{ elastic_shared_fs_repos }}" - when: - - elastic_shared_fs_repos is defined and elastic_shared_fs_repos | length > 0 - - handlers: - - name: Enable and restart elastic - systemd: - name: "elasticsearch" - enabled: true - state: restarted - daemon_reload: true - when: - - (elk_package_state | default('present')) != 'absent' - tags: - - config + roles: + - role: elasticsearch tags: - server-install diff --git a/elk_metrics_6x/installFilebeat.yml b/elk_metrics_6x/installFilebeat.yml index 16733fc3..5b349fde 100644 --- a/elk_metrics_6x/installFilebeat.yml +++ b/elk_metrics_6x/installFilebeat.yml @@ -1,4 +1,16 @@ --- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + - name: Install Filebeat hosts: hosts become: true @@ -10,184 +22,10 @@ environment: "{{ deployment_environment_variables | default({}) }}" - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml - tags: - - always - - tasks: - - include_tasks: common_task_install_elk_repo.yml - - - name: Ensure Filebeat is installed - apt: - name: "{{ item }}" - state: "{{ elk_package_state | default('present') }}" - update_cache: true - with_items: - - filebeat - register: _apt_task - until: _apt_task is success - retries: 3 - delay: 2 - notify: - - Enable and restart filebeat - tags: - - package_install - - - name: exit playbook after uninstall - meta: end_play - when: - - (elk_package_state | default('present')) == 'absent' - - post_tasks: - - name: Create filebeat systemd service config dir - file: - path: "/etc/systemd/system/filebeat.service.d" - state: "directory" - group: "root" - owner: "root" - mode: "0755" - - - name: Apply systemd options - template: - src: "{{ item.src }}" - dest: "/etc/systemd/system/filebeat.service.d/{{ item.dest }}" - mode: "0644" - with_items: - - { src: "systemd.general-overrides.conf.j2", dest: "filebeat-overrides.conf" } - notify: - - Enable and restart filebeat - - - name: Check for apache - stat: - path: /etc/apache2 - register: apache2 - - - name: Check for auditd - stat: - path: /etc/audit - register: audit - - - name: Check for ceph - stat: - path: /var/log/ceph - register: ceph - - - name: Check for cinder - stat: - path: /var/log/cinder - register: cinder - - - name: Check for glance - stat: - path: /var/log/glance - register: glance - - - name: Check for heat - stat: - path: /var/log/heat - register: heat - - - name: Check for horizon - stat: - path: /var/log/horizon - register: horizon - - - name: Check for httpd - stat: - path: /var/log/httpd - register: httpd - - - name: Check for keystone - stat: - path: /var/log/keystone - register: keystone - - - name: Check for mysql - stat: - path: /var/lib/mysql - register: mysql - - - name: Check for neutron - stat: - path: /var/log/neutron - register: neutron - - - name: Check for nginx - stat: - path: /var/log/nginx - register: nginx - - - name: Check for nova - stat: - path: /var/log/nova - register: nova - - - name: Check for octavia - stat: - path: /var/log/octavia - register: octavia - - - name: Check for swift - stat: - path: /var/log/swift - register: swift - - - name: Check for rabbitmq - stat: - path: /var/lib/rabbitmq - register: rabbitmq - - - name: Check for designate - stat: - path: /var/log/designate - register: designate - - - name: Check for osquery - stat: - path: /var/log/osquery/osqueryd.results.log - register: osquery - - - name: Set discovery facts - set_fact: - apache_enabled: "{{ (apache2.stat.exists | bool) or (httpd.stat.exists | bool) }}" - nginx_enabled: "{{ nginx.stat.exists | bool }}" - auditd_enabled: "{{ audit.stat.exists | bool }}" - mysql_enabled: "{{ (mysql.stat.exists | bool) or (inventory_hostname in groups['galera_all'] | default([])) }}" - ceph_enabled: "{{ (ceph.stat.exists | bool) or (inventory_hostname in groups['ceph_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*ceph.*') | list | length) > 0) }}" - cinder_enabled: "{{ (cinder.stat.exists | bool) or (inventory_hostname in groups['cinder_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*cinder.*') | list | length) > 0) }}" - glance_enabled: "{{ (glance.stat.exists | bool) or (inventory_hostname in groups['glance_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*glance.*') | list | length) > 0) }}" - heat_enabled: "{{ (heat.stat.exists | bool) or (inventory_hostname in groups['heat_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*heat.*') | list | length) > 0) }}" - horizon_enabled: "{{ (horizon.stat.exists | bool) or (inventory_hostname in groups['horizon_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*horizon.*') | list | length) > 0) }}" - keystone_enabled: "{{ (keystone.stat.exists | bool) or (inventory_hostname in groups['keystone_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*keystone.*') | list | length) > 0) }}" - neutron_enabled: "{{ (neutron.stat.exists | bool) or (inventory_hostname in groups['neutron_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*neutron.*') | list | length) > 0) }}" - nova_enabled: "{{ (nova.stat.exists | bool) or (inventory_hostname in groups['nova_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*nova.*') | list | length) > 0) }}" - octavia_enabled: "{{ (octavia.stat.exists | bool) or (inventory_hostname in groups['octavia_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*octavia.*') | list | length) > 0) }}" - swift_enabled: "{{ (swift.stat.exists | bool) or (inventory_hostname in groups['swift_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*swift.*') | list | length) > 0) }}" - rabbitmq_enabled: "{{ (rabbitmq.stat.exists | bool) or (inventory_hostname in groups['rabbitmq_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*rabbit.*') | list | length) > 0) }}" - designate_enabled: "{{ (designate.stat.exists | bool) or (inventory_hostname in groups['designate_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*designate.*') | list | length) > 0) }}" - osquery_enabled: "{{ osquery.stat.exists | bool }}" - - - name: Drop Filebeat conf file - template: - src: templates/filebeat.yml.j2 - dest: /etc/filebeat/filebeat.yml - notify: - - Enable and restart filebeat - - handlers: - - name: Enable and restart filebeat - systemd: - name: "filebeat" - enabled: true - state: restarted - daemon_reload: true - when: - - (elk_package_state | default('present')) != 'absent' - tags: - - config + roles: + - role: elastic_filebeat + - role: elastic_rollup + index_name: filebeat tags: - beat-install - -- import_playbook: setupFilebeat.yml diff --git a/elk_metrics_6x/installHeartbeat.yml b/elk_metrics_6x/installHeartbeat.yml index 4a4cf0bd..593458b3 100644 --- a/elk_metrics_6x/installHeartbeat.yml +++ b/elk_metrics_6x/installHeartbeat.yml @@ -1,4 +1,15 @@ --- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. - name: Set heartbeat host deployment group hosts: kibana @@ -11,7 +22,6 @@ parents: kibana when: - inventory_hostname in groups['kibana'][:3] - tags: - always @@ -26,70 +36,10 @@ environment: "{{ deployment_environment_variables | default({}) }}" - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml - tags: - - always - - tasks: - - include_tasks: common_task_install_elk_repo.yml - - - name: Ensure heartbeat is installed - apt: - name: "heartbeat-elastic" - state: "{{ elk_package_state | default('present') }}" - update_cache: true - register: _apt_task - until: _apt_task is success - retries: 3 - delay: 2 - tags: - - package_install - - - name: exit playbook after uninstall - meta: end_play - when: - - (elk_package_state | default('present')) == 'absent' - - post_tasks: - - name: Create heartbeat systemd service config dir - file: - path: "/etc/systemd/system/heartbeat.service.d" - state: "directory" - group: "root" - owner: "root" - mode: "0755" - - - name: Apply systemd options - template: - src: "{{ item.src }}" - dest: "/etc/systemd/system/heartbeat.service.d/{{ item.dest }}" - mode: "0644" - with_items: - - { src: "systemd.general-overrides.conf.j2", dest: "heartbeat-overrides.conf" } - notify: - - Enable and restart heartbeat - - - name: Drop heartbeat conf file - template: - src: templates/heartbeat.yml.j2 - dest: /etc/heartbeat/heartbeat.yml - notify: - - Enable and restart heartbeat - - handlers: - - name: Enable and restart heartbeat - systemd: - name: "heartbeat-elastic" - enabled: true - state: restarted - daemon_reload: true - when: - - (elk_package_state | default('present')) != 'absent' - tags: - - config + roles: + - role: elastic_heartbeat + - role: elastic_rollup + index_name: heartbeat tags: - beat-install - -- import_playbook: setupHeartbeat.yml diff --git a/elk_metrics_6x/installJournalbeat.yml b/elk_metrics_6x/installJournalbeat.yml index db71546b..836b3de4 100644 --- a/elk_metrics_6x/installJournalbeat.yml +++ b/elk_metrics_6x/installJournalbeat.yml @@ -1,6 +1,4 @@ --- -# Copyright 2018, Rackspace US, Inc. -# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -33,7 +31,7 @@ key: journalbeat_deployment_hosts parents: all_journalbeat_deployments when: - - physical_host is defined and + - physical_host is undefined or physical_host == inventory_hostname tags: @@ -50,143 +48,10 @@ environment: "{{ deployment_environment_variables | default({}) }}" - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml - tags: - - always - - tasks: - - include_tasks: common_task_install_go1.10.1.yml - - - name: Check for journal directory - stat: - path: /var/log/journal - register: journal_dir - - - name: Ensure libsystemd-dev is installed - apt: - name: "{{ item }}" - state: "{{ elk_package_state | default('present') }}" - update_cache: true - register: _apt_task - until: _apt_task is success - retries: 3 - delay: 2 - with_items: - - gcc - - git - - libsystemd-dev - tags: - - package_install - - - name: exit playbook after uninstall - meta: end_play - when: - - (elk_package_state | default('present')) == 'absent' - - post_tasks: - - name: Create journalbeat systemd service config dir - file: - path: "/etc/systemd/system/journalbeat.service.d" - state: "directory" - group: "root" - owner: "root" - mode: "0755" - - - name: Apply systemd options - template: - src: "{{ item.src }}" - dest: "/etc/systemd/system/journalbeat.service.d/{{ item.dest }}" - mode: "0644" - with_items: - - { src: "systemd.general-overrides.conf.j2", dest: "journalbeat-overrides.conf" } - notify: - - Enable and restart journalbeat - - - name: create the system group - group: - name: "journalbeat" - state: "present" - system: "yes" - - - name: Create the journalbeat user - user: - name: "journalbeat" - group: "journalbeat" - comment: "journalbeat user" - shell: "/bin/false" - createhome: "yes" - home: "/usr/share/journalbeat" - - - name: Create journalbeat data path - file: - path: "{{ item }}" - state: directory - owner: "journalbeat" - group: "journalbeat" - mode: "0755" - with_items: - - "/usr/share/journalbeat" - - "/var/lib/journalbeat" - - "/etc/journalbeat" - - - name: Install journalbeat - shell: /opt/go1.10.1/go/bin/go get -v github.com/mheese/journalbeat - args: - creates: /usr/local/bin/journalbeat - environment: - PATH: "{{ ansible_env.PATH }}:/opt/go1.10.1/go/bin" - GOPATH: /usr/local - notify: - - Enable and restart journalbeat - - - name: Drop journalbeat conf file - template: - src: "{{ item.src }}" - dest: "{{ item.dest }}" - with_items: - - src: templates/journalbeat.yml.j2 - dest: /etc/journalbeat/journalbeat.yml - notify: - - Enable and restart journalbeat - - - name: Run the systemd service role - include_role: - name: systemd_service - private: true - vars: - systemd_service_enabled: "{{ ((elk_package_state | default('present')) != 'absent') | ternary(true, false) }}" - systemd_service_restart_changed: false - systemd_services: - - service_name: "journalbeat" - execstarts: - - /usr/local/bin/journalbeat - -c /etc/journalbeat/journalbeat.yml - -path.home /usr/share/journalbeat - -path.config /etc/journalbeat - -path.data /var/lib/journalbeat - -path.logs /var/log/beats - config_overrides: - Service: - EnvironmentFile: "-/etc/default/go1.10.1" - Unit: - Documentation: https://github.com/mheese/journalbeat/blob/master/README.md - Wants: network-online.target - After: network-online.target - - handlers: - - name: Enable and restart journalbeat - systemd: - name: "journalbeat" - enabled: true - state: restarted - daemon_reload: yes - when: - - (elk_package_state | default('present')) != 'absent' - tags: - - config + roles: + - role: elastic_journalbeat + - role: elastic_rollup + index_name: journalbeat tags: - beat-install - -- import_playbook: setupJournalbeat.yml diff --git a/elk_metrics_6x/installKibana.yml b/elk_metrics_6x/installKibana.yml index 3dc1ee7c..087160c1 100644 --- a/elk_metrics_6x/installKibana.yml +++ b/elk_metrics_6x/installKibana.yml @@ -1,4 +1,16 @@ --- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + - name: Install Kibana hosts: kibana become: true @@ -7,109 +19,8 @@ environment: "{{ deployment_environment_variables | default({}) }}" - tasks: - - include_tasks: common_task_install_elk_repo.yml - - - name: Ensure Nginx is installed - apt: - name: "{{ item }}" - state: "{{ elk_package_state | default('present') }}" - update_cache: yes - with_items: - - nginx - - apache2-utils - - python-passlib - register: _apt_task - until: _apt_task is success - retries: 3 - delay: 2 - notify: - - Enable and restart nginx - tags: - - package_install - - - name: create kibana user to access web interface - htpasswd: - path: "/etc/nginx/htpasswd.users" - name: "{{ kibana_username }}" - password: "{{ kibana_password }}" - owner: root - mode: 0644 - - - name: Drop Nginx default conf file - template: - src: templates/nginx_default.j2 - dest: /etc/nginx/sites-available/default - notify: - - Enable and restart nginx - - - name: Ensure kibana is installed - apt: - name: kibana - state: "{{ elk_package_state | default('present') }}" - update_cache: yes - register: _apt_task - until: _apt_task is success - retries: 3 - delay: 2 - notify: - - Enable and restart kibana - tags: - - package_install - - - name: exit playbook after uninstall - meta: end_play - when: - - (elk_package_state | default('present')) == 'absent' - - post_tasks: - - name: Create kibana systemd service config dir - file: - path: "/etc/systemd/system/kibana.service.d" - state: "directory" - group: "root" - owner: "root" - mode: "0755" - - - name: Apply systemd options - template: - src: "{{ item.src }}" - dest: "/etc/systemd/system/kibana.service.d/{{ item.dest }}" - mode: "0644" - with_items: - - { src: "systemd.general-overrides.conf.j2", dest: "kibana-overrides.conf" } - notify: - - Enable and restart kibana - - - name: Drop kibana conf file - template: - src: templates/kibana.yml.j2 - dest: /etc/kibana/kibana.yml - mode: "0666" - notify: - - Enable and restart kibana - - handlers: - - name: Enable and restart kibana - systemd: - name: "kibana" - enabled: true - state: restarted - daemon_reload: true - when: - - (elk_package_state | default('present')) != 'absent' - tags: - - config - - - name: Enable and restart nginx - systemd: - name: "nginx" - enabled: true - state: restarted - when: - - (elk_package_state | default('present')) != 'absent' - tags: - - config + roles: + - role: elastic_kibana tags: - server-install diff --git a/elk_metrics_6x/installLogstash.yml b/elk_metrics_6x/installLogstash.yml index 1c1e128e..02a3e6d9 100644 --- a/elk_metrics_6x/installLogstash.yml +++ b/elk_metrics_6x/installLogstash.yml @@ -1,276 +1,26 @@ --- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + - name: Install Logstash hosts: elastic-logstash - serial: "50%" become: true vars_files: - vars/variables.yml - vars: - temp_dir: /var/lib/logstash/tmp - logstash_pipelines: "{{lookup('template', 'templates/logstash-pipelines.yml.j2') }}" - environment: "{{ deployment_environment_variables | default({}) }}" - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml - tags: - - always - - tasks: - - name: Set quarter memory fact - set_fact: - q_mem: "{{ (ansible_memtotal_mb | int) // 3 }}" - when: - - q_mem is not defined - tags: - - always - - - name: Set processor cores fact - set_fact: - q_storage: "{{ (ansible_processor_cores | int) * (ansible_processor_threads_per_core | int) * 2 }}" - when: - - q_storage is not defined - tags: - - always - - - name: Set logstash facts - set_fact: - elastic_heap_size: "{{ ((q_mem | int) > 30720) | ternary(30720, q_mem) }}" - logstash_queue_size: "{{ ((((q_storage | int) >= 2) | ternary(q_storage, 2) | int) * 1024) // ((logstash_pipelines | from_yaml) | length) }}" - elastic_log_rotate_path: "/var/log/logstash" - tags: - - always - - - include_tasks: common_task_install_elk_repo.yml - - - name: Configure systcl vm.max_map_count=262144 on container hosts - sysctl: - name: "vm.max_map_count" - value: "262144" - state: "present" - reload: "yes" - delegate_to: "{{ physical_host }}" - tags: - - sysctl - - - name: Ensure mount directories exists - file: - path: "/openstack/{{ inventory_hostname }}/logstash" - state: "directory" - delegate_to: "{{ physical_host }}" - - - name: logstash datapath bind mount - lxc_container: - name: "{{ inventory_hostname }}" - container_command: | - [[ ! -d "/var/lib/logstash" ]] && mkdir -p "/var/lib/logstash" - container_config: - - "lxc.mount.entry=/openstack/{{ inventory_hostname }}/logstash var/lib/logstash none bind 0 0" - delegate_to: "{{ physical_host }}" - when: - - physical_host != inventory_hostname - - container_tech | default('lxc') == 'lxc' - - - name: Ensure Java is installed - apt: - name: openjdk-8-jre - state: "{{ elk_package_state | default('present') }}" - install_recommends: yes - update_cache: yes - register: _apt_task - until: _apt_task is success - retries: 3 - delay: 2 - tags: - - package_install - - - name: Ensure Logstash is installed - apt: - name: "{{ item }}" - state: "{{ elk_package_state | default('present') }}" - update_cache: yes - with_items: - - logrotate - - logstash - register: _apt_task - until: _apt_task is success - retries: 3 - delay: 2 - notify: - - Enable and restart logstash - tags: - - package_install - - - name: exit playbook after uninstall - meta: end_play - when: - - (elk_package_state | default('present')) == 'absent' - - post_tasks: - - name: Create logstash systemd service config dir - file: - path: "/etc/systemd/system/logstash.service.d" - state: "directory" - group: "root" - owner: "root" - mode: "0755" - - - name: Apply systemd options - template: - src: "{{ item.src }}" - dest: "/etc/systemd/system/logstash.service.d/{{ item.dest }}" - mode: "0644" - with_items: - - { src: "systemd.general-overrides.conf.j2", dest: "logstash-overrides.conf" } - notify: - - Enable and restart logstash - - - name: Create patterns directory - file: - name: "/opt/logstash/patterns" - owner: "logstash" - group: "logstash" - state: directory - tags: - - logstash-patterns - - - name: Logstash Extra Patterns - template: - src: "{{ item }}" - dest: "/opt/logstash/patterns/{{ item }}" - owner: "logstash" - group: "logstash" - with_items: - - extras - when: - - logstash_deploy_filters - notify: - - Enable and restart logstash - tags: - - logstash-filters - - config - - - name: Run kafka output block - block: - - name: Copy kafka keystore into place - copy: - src: "{{ logstash_kafka_ssl_keystore_location }}" - dest: "/var/lib/logstash/{{ logstash_kafka_ssl_keystore_location | basename }}" - when: - - logstash_kafka_ssl_keystore_location is defined - - - name: Copy kafka truststore into place - copy: - src: "{{ logstash_kafka_ssl_truststore_location }}" - dest: "/var/lib/logstash/{{ logstash_kafka_ssl_truststore_location | basename }}" - when: - - logstash_kafka_ssl_truststore_location is defined - - when: - - logstash_kafka_options is defined - - - name: Drop logstash conf file(s) - template: - src: "{{ item.src }}" - dest: "{{ item.dest }}" - with_items: - - src: templates/jvm.options.j2 - dest: /etc/logstash/jvm.options - - src: templates/logstash.yml.j2 - dest: /etc/logstash/logstash.yml - - src: templates/logstash-pipelines.yml.j2 - dest: /etc/logstash/pipelines.yml - - src: "templates/logrotate.j2" - dest: "/etc/logrotate.d/logstash" - notify: - - Enable and restart logstash - tags: - - config - - - name: Ensure logstash ownership - file: - path: /var/lib/logstash - owner: logstash - group: logstash - recurse: true - register: l_perms - until: l_perms is success - retries: 3 - delay: 1 - - - name: Ensure logstash tmp dir - file: - path: "/var/lib/logstash/tmp" - state: directory - owner: "logstash" - group: "logstash" - mode: "0750" - - - name: Run arcsight output block - block: - - name: Initialise arcsight local facts - ini_file: - dest: "/etc/ansible/facts.d/logstash.fact" - section: arcsight - option: "initialise" - value: true - - - name: refresh local facts - setup: - filter: ansible_local - gather_subset: "!all" - tags: - - nova-config - - - name: Setup arcsight smart connector - shell: >- - /usr/local/bin/logstash --modules arcsight {{ (ansible_local['arcsight'][item.host] is defined) | ternary('', '--setup') }} - -M "arcsight.var.input.smartconnector.bootstrap_servers={{ item.host }}:{{ item.port }}" - -M "arcsight.var.elasticsearch.hosts=localhost:{{ elastic_port }}" - -M "arcsight.var.kibana.host={{ hostvars[groups['kibana'][0]]['ansible_host'] }}:{{ kibana_port }}" - with_items: "{{ logstash_arcsight_smart_connectors }}" - run_once: true - register: smart_connector - until: smart_connector is success - retries: 5 - delay: 5 - - - name: Setup arcsight event broker - shell: >- - /usr/local/bin/logstash --modules arcsight {{ (ansible_local['arcsight'][item.host] is defined) | ternary('', '--setup') }} - -M "arcsight.var.input.eventbroker.bootstrap_servers={{ item.host }}:{{ item.port }}" - -M "arcsight.var.elasticsearch.hosts=localhost:{{ elastic_port }}" - -M "arcsight.var.kibana.host={{ hostvars[groups['kibana'][0]]['ansible_host'] }}:{{ kibana_port }}" - with_items: "{{ logstash_arcsight_event_brokers }}" - run_once: true - register: event_broker - until: event_broker is success - retries: 5 - delay: 5 - - - name: Set arcsight local facts - ini_file: - dest: "/etc/ansible/facts.d/logstash.fact" - section: arcsight - option: "{{ item.host }}" - value: "{{ item.port }}" - with_items: "{{ logstash_arcsight_smart_connectors | union(logstash_arcsight_event_brokers) }}" - when: - - logstash_arcsight_smart_connectors or - logstash_arcsight_event_brokers - - handlers: - - name: Enable and restart logstash - systemd: - name: "logstash" - enabled: true - state: restarted - daemon_reload: true - when: - - (elk_package_state | default('present')) != 'absent' - tags: - - config + roles: + - role: elastic_logstash tags: - server-install diff --git a/elk_metrics_6x/installMetricbeat.yml b/elk_metrics_6x/installMetricbeat.yml index 64629568..d174c1ec 100644 --- a/elk_metrics_6x/installMetricbeat.yml +++ b/elk_metrics_6x/installMetricbeat.yml @@ -1,4 +1,16 @@ --- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + - name: Install Metricsbeat hosts: all become: true @@ -10,226 +22,10 @@ vars_files: - vars/variables.yml - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml - tags: - - always - - tasks: - - include_tasks: common_task_install_elk_repo.yml - - - name: Ensure Metricsbeat is installed - apt: - name: metricbeat - state: "{{ elk_package_state | default('present') }}" - update_cache: true - register: _apt_task - until: _apt_task is success - retries: 3 - delay: 2 - notify: - - Enable and restart metricbeat - tags: - - package_install - - - name: exit playbook after uninstall - meta: end_play - when: - - (elk_package_state | default('present')) == 'absent' - - post_tasks: - - name: Create metricbeat systemd service config dir - file: - path: "/etc/systemd/system/metricbeat.service.d" - state: "directory" - group: "root" - owner: "root" - mode: "0755" - - - name: Apply systemd options - template: - src: "{{ item.src }}" - dest: "/etc/systemd/system/metricbeat.service.d/{{ item.dest }}" - mode: "0644" - with_items: - - { src: "systemd.general-overrides.conf.j2", dest: "metricbeat-overrides.conf" } - notify: - - Enable and restart metricbeat - - - name: Check for apache - stat: - path: /etc/apache2/sites-available - register: apache2 - - - name: Check for ceph - stat: - path: /etc/ceph - register: ceph - - # gather ceph stats from localhost - # except when a list of mons is provided - - name: Set ceph stats hosts - set_fact: - ceph_stats_hosts: |- - {% set ceph_stats = [] %} - {% if (ceph_mons is defined) and (ceph_mons | length > 0) %} - {% for mon in ceph_mons %} - {% set _ = ceph_stats.insert(loop.index, (mon + ":5000")) %} - {% endfor %} - {% else %} - {% set ceph_stats = [ ansible_hostname + ":5000" ] %} - {% endif %} - {{ ceph_stats }} - - - name: Check for etcd - stat: - path: /etc/etcd - register: etcd - - - name: Check for docker - stat: - path: /var/run/docker.sock - register: docker - - - name: Check for haproxy - stat: - path: /etc/haproxy - register: haproxy - - - name: Check for httpd - stat: - path: /etc/httpd - register: httpd - - - name: Check for kvm - stat: - path: /var/run/libvirt/libvirt-sock - register: kvm - - - name: Check for memcached - stat: - path: /etc/memcached.conf - register: memcached - - - name: Check for mysql - stat: - path: /var/lib/mysql - register: mysql - - - name: Check for nginx - stat: - path: /etc/nginx/nginx.conf - register: nginx - - - name: Check for rabbitmq - stat: - path: /var/lib/rabbitmq - register: rabbitmq - - - name: Check for uwsgi - stat: - path: /etc/uwsgi - register: uwsgi - - - name: Check for uwsgi stats sockets - find: - paths: /tmp - file_type: any - patterns: '*uwsgi-stats.sock' - register: uwsgi_find_sockets - - - name: Set discovery facts - set_fact: - apache_enabled: "{{ (apache2.stat.exists | bool) or (httpd.stat.exists | bool) }}" - - # enable ceph on: cinder volume hosts when we have a list of ceph mons - # otherwise: all hosts which have /etc/ceph - ceph_enabled: |- - {% set ceph_detect = false %} - {% if (ceph_mons is defined) and (ceph_mons | length > 0) and (inventory_hostname in groups['cinder_volume']) %} - {% set ceph_detect = true %} - {% else %} - {% set ceph_detect = ceph.stat.exists | bool %} - {% endif %} - {{ ceph_detect }} - - docker_enabled: "{{ docker.stat.exists | bool }}" - etcd_enabled: "{{ etcd.stat.exists | bool }}" - haproxy_enabled: "{{ haproxy.stat.exists | bool }}" - kvm_enabled: "{{ kvm.stat.exists | bool }}" - memcached_enabled: "{{ memcached.stat.exists | bool }}" - mysql_enabled: "{{ mysql.stat.exists | bool }}" - nginx_enabled: "{{ nginx.stat.exists | bool }}" - rabbitmq_enabled: "{{ rabbitmq.stat.exists | bool }}" - uwsgi_enabled: "{{ uwsgi.stat.exists | bool }}" - uwsgi_sockets: "{{ uwsgi_find_sockets }}" - - # Apache 2 stats enablement - - name: Drop apache2 stats site config - template: - src: apache-status.conf.j2 - dest: /etc/apache2/sites-available/apache-status.conf - when: apache_enabled - - - name: Enable apache2 stats site - file: - src: /etc/apache2/sites-available/apache-status.conf - dest: /etc/apache2/sites-enabled/apache-status.conf - state: link - when: apache_enabled - - - name: Ensure apache2 stats mode is enabled - apache2_module: - name: status - state: present - when: apache_enabled - - - name: Reload apache2 - service: - name: apache2 - state: reloaded - when: apache_enabled - - # NGINX stats enablement - - name: Drop nginx stats site config - template: - src: nginx-status.conf.j2 - dest: /etc/nginx/sites-available/nginx-status.conf - when: nginx_enabled - - - name: Enable nginx stats site - file: - src: /etc/nginx/sites-available/nginx-status.conf - dest: /etc/nginx/sites-enabled/nginx-status.conf - state: link - when: nginx_enabled - - - name: Reload nginx - service: - name: nginx - state: reloaded - when: nginx_enabled - - - name: Drop metricbeat conf file - template: - src: templates/metricbeat.yml.j2 - dest: /etc/metricbeat/metricbeat.yml - notify: - - Enable and restart metricbeat - - handlers: - - name: Enable and restart metricbeat - systemd: - name: "metricbeat" - enabled: true - state: restarted - daemon_reload: true - when: - - (elk_package_state | default('present')) != 'absent' - tags: - - config + roles: + - role: elastic_metricbeat + - role: elastic_rollup + index_name: metricbeat tags: - beat-install - -- import_playbook: setupMetricbeat.yml diff --git a/elk_metrics_6x/installPacketbeat.yml b/elk_metrics_6x/installPacketbeat.yml index cc721784..15086c38 100644 --- a/elk_metrics_6x/installPacketbeat.yml +++ b/elk_metrics_6x/installPacketbeat.yml @@ -1,4 +1,16 @@ --- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + - name: Install Packetbeat hosts: hosts become: true @@ -10,75 +22,10 @@ vars_files: - vars/variables.yml - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml - tags: - - always - - tasks: - - include_tasks: common_task_install_elk_repo.yml - - - name: Ensure packetbeat is installed - apt: - name: "{{ item }}" - state: "{{ elk_package_state | default('present') }}" - update_cache: true - with_items: - - tcpdump - - packetbeat - register: _apt_task - until: _apt_task is success - retries: 3 - delay: 2 - notify: - - Enable and restart packetbeat - tags: - - package_install - - - name: exit playbook after uninstall - meta: end_play - when: - - (elk_package_state | default('present')) == 'absent' - - post_tasks: - - name: Create packetbeat systemd service config dir - file: - path: "/etc/systemd/system/packetbeat.service.d" - state: "directory" - group: "root" - owner: "root" - mode: "0755" - - - name: Apply systemd options - template: - src: "{{ item.src }}" - dest: "/etc/systemd/system/packetbeat.service.d/{{ item.dest }}" - mode: "0644" - with_items: - - { src: "systemd.general-overrides.conf.j2", dest: "packetbeat-overrides.conf" } - notify: - - Enable and restart packetbeat - - - name: Drop packetbeat conf file - template: - src: templates/packetbeat.yml.j2 - dest: /etc/packetbeat/packetbeat.yml - notify: - - Enable and restart packetbeat - - handlers: - - name: Enable and restart packetbeat - systemd: - name: "packetbeat" - enabled: true - state: restarted - daemon_reload: true - when: - - (elk_package_state | default('present')) != 'absent' - tags: - - config + roles: + - role: elastic_packetbeat + - role: elastic_rollup + index_name: packetbeat tags: - beat-install - -- import_playbook: setupPacketbeat.yml diff --git a/elk_metrics_6x/roles/elastic_apm_server/handlers/main.yml b/elk_metrics_6x/roles/elastic_apm_server/handlers/main.yml new file mode 100644 index 00000000..69f1a9d5 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_apm_server/handlers/main.yml @@ -0,0 +1,25 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Enable and restart apm server + systemd: + name: "apm-server" + enabled: true + state: restarted + daemon_reload: true + when: + - (elk_package_state | default('present')) != 'absent' + tags: + - config diff --git a/elk_metrics_6x/roles/elastic_apm_server/meta/main.yml b/elk_metrics_6x/roles/elastic_apm_server/meta/main.yml new file mode 100644 index 00000000..d24d370c --- /dev/null +++ b/elk_metrics_6x/roles/elastic_apm_server/meta/main.yml @@ -0,0 +1,33 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v6.x apm-server role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: + - role: elastic_data_hosts + - role: elastic_repositories diff --git a/elk_metrics_6x/roles/elastic_apm_server/tasks/apm_setup.yml b/elk_metrics_6x/roles/elastic_apm_server/tasks/apm_setup.yml new file mode 100644 index 00000000..20c3643f --- /dev/null +++ b/elk_metrics_6x/roles/elastic_apm_server/tasks/apm_setup.yml @@ -0,0 +1,30 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Load templates + shell: >- + apm-server setup + {{ item }} + -E 'apm-server.host=localhost:8200' + -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' + -e -v + with_items: + - "--template" + - "--dashboards" + register: templates + until: templates is success + retries: 5 + delay: 5 + run_once: true diff --git a/elk_metrics_6x/roles/elastic_apm_server/tasks/main.yml b/elk_metrics_6x/roles/elastic_apm_server/tasks/main.yml new file mode 100644 index 00000000..cb249018 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_apm_server/tasks/main.yml @@ -0,0 +1,70 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather variables for each operating system + include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml" + - "{{ ansible_os_family | lower }}.yml" + tags: + - always + +- name: Ensure apm-server is installed + package: + name: "{{ apm_server_distro_packages }}" + state: "{{ elk_package_state | default('present') }}" + update_cache: true + register: _package_task + until: _package_task is success + retries: 3 + delay: 2 + notify: + - Enable and restart apm server + tags: + - package_install + +- name: Create apm-server systemd service config dir + file: + path: "/etc/systemd/system/apm-server.service.d" + state: "directory" + group: "root" + owner: "root" + mode: "0755" + +- name: Apply systemd options + template: + src: "{{ item.src }}" + dest: "/etc/systemd/system/apm-server.service.d/{{ item.dest }}" + mode: "0644" + with_items: + - src: "systemd.general-overrides.conf.j2" + dest: "apm-server-overrides.conf" + notify: + - Enable and restart apm server + +- name: Drop apm-server conf file + template: + src: templates/apm-server.yml.j2 + dest: /etc/apm-server/apm-server.yml + notify: + - Enable and restart apm server + +- include_tasks: apm_setup.yml + tags: + - setup diff --git a/elk_metrics_6x/roles/elastic_apm_server/vars/ubuntu.yml b/elk_metrics_6x/roles/elastic_apm_server/vars/ubuntu.yml new file mode 100644 index 00000000..3fac8c52 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_apm_server/vars/ubuntu.yml @@ -0,0 +1,17 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apm_server_distro_packages: + - apm-server diff --git a/elk_metrics_6x/roles/elastic_auditbeat/handlers/main.yml b/elk_metrics_6x/roles/elastic_auditbeat/handlers/main.yml new file mode 100644 index 00000000..8fdf3a74 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_auditbeat/handlers/main.yml @@ -0,0 +1,25 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Enable and restart auditbeat + systemd: + name: "auditbeat" + enabled: "true" + state: restarted + daemon_reload: true + when: + - (elk_package_state | default('present')) != 'absent' + tags: + - config diff --git a/elk_metrics_6x/roles/elastic_auditbeat/meta/main.yml b/elk_metrics_6x/roles/elastic_auditbeat/meta/main.yml new file mode 100644 index 00000000..19242323 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_auditbeat/meta/main.yml @@ -0,0 +1,33 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v6.x auditbeat role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: + - role: elastic_data_hosts + - role: elastic_repositories diff --git a/elk_metrics_6x/roles/elastic_auditbeat/tasks/auditbeat_setup.yml b/elk_metrics_6x/roles/elastic_auditbeat/tasks/auditbeat_setup.yml new file mode 100644 index 00000000..93542218 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_auditbeat/tasks/auditbeat_setup.yml @@ -0,0 +1,32 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Load templates + shell: >- + auditbeat setup + {{ item }} + -E 'output.logstash.enabled=false' + -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' + -e -v + with_items: + - "--template" + - "--dashboards" + register: templates + until: templates is success + retries: 5 + delay: 5 + run_once: true + tags: + - setup diff --git a/elk_metrics_6x/roles/elastic_auditbeat/tasks/main.yml b/elk_metrics_6x/roles/elastic_auditbeat/tasks/main.yml new file mode 100644 index 00000000..44ee4b3b --- /dev/null +++ b/elk_metrics_6x/roles/elastic_auditbeat/tasks/main.yml @@ -0,0 +1,68 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather variables for each operating system + include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml" + - "{{ ansible_os_family | lower }}.yml" + tags: + - always + +- name: Ensure beat is installed + package: + name: "{{ auditbeat_distro_packages }}" + state: "{{ elk_package_state | default('present') }}" + update_cache: true + register: _package_task + until: _package_task is success + retries: 3 + delay: 2 + notify: + - Enable and restart auditbeat + tags: + - package_install + +- name: Create auditbeat systemd service config dir + file: + path: "/etc/systemd/system/auditbeat.service.d" + state: "directory" + group: "root" + owner: "root" + mode: "0755" + +- name: Apply systemd options + template: + src: "{{ item.src }}" + dest: "/etc/systemd/system/auditbeat.service.d/{{ item.dest }}" + mode: "0644" + with_items: + - src: "systemd.general-overrides.conf.j2" + dest: "auditbeat-overrides.conf" + notify: + - Enable and restart auditbeat + +- name: Drop auditbeat conf file + template: + src: templates/auditbeat.yml.j2 + dest: /etc/auditbeat/auditbeat.yml + notify: + - Enable and restart auditbeat + +- include_tasks: auditbeat_setup.yml diff --git a/elk_metrics_6x/roles/elastic_auditbeat/vars/ubuntu.yml b/elk_metrics_6x/roles/elastic_auditbeat/vars/ubuntu.yml new file mode 100644 index 00000000..59cc129c --- /dev/null +++ b/elk_metrics_6x/roles/elastic_auditbeat/vars/ubuntu.yml @@ -0,0 +1,18 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +auditbeat_distro_packages: + - audispd-plugins + - auditbeat diff --git a/elk_metrics_6x/roles/elastic_curator/handlers/main.yml b/elk_metrics_6x/roles/elastic_curator/handlers/main.yml new file mode 100644 index 00000000..93f43ac3 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_curator/handlers/main.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Enable and restart curator.timer + systemd: + name: "curator.timer" + enabled: true + state: restarted + when: + - (elk_package_state | default('present')) != 'absent' + tags: + - config diff --git a/elk_metrics_6x/roles/elastic_curator/meta/main.yml b/elk_metrics_6x/roles/elastic_curator/meta/main.yml new file mode 100644 index 00000000..0dcd0de2 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_curator/meta/main.yml @@ -0,0 +1,32 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v6.x curator role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: + - role: elastic_retention diff --git a/elk_metrics_6x/roles/elastic_curator/tasks/main.yml b/elk_metrics_6x/roles/elastic_curator/tasks/main.yml new file mode 100644 index 00000000..0cbccce0 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_curator/tasks/main.yml @@ -0,0 +1,111 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather variables for each operating system + include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml" + - "{{ ansible_os_family | lower }}.yml" + tags: + - always + +- name: Ensure virtualenv is installed + package: + name: "{{ curator_distro_packages }}" + state: "{{ elk_package_state | default('present') }}" + update_cache: true + tags: + - package_install + +- name: Ensure curator is installed + pip: + name: "elasticsearch-curator<6" + state: "{{ elk_package_state | default('present') }}" + extra_args: --isolated + virtualenv: /opt/elasticsearch-curator + register: _pip_task + until: _pip_task is success + retries: 3 + delay: 2 + tags: + - package_install + +- name: create the system group + group: + name: "curator" + state: "present" + system: "yes" + +- name: Create the curator system user + user: + name: "curator" + group: "curator" + comment: "curator user" + shell: "/bin/false" + createhome: "yes" + home: "/var/lib/curator" + +- name: Create curator data path + file: + path: "{{ item }}" + state: directory + owner: "curator" + group: "curator" + mode: "0755" + recurse: true + with_items: + - "/var/lib/curator" + - "/var/log/curator" + - "/etc/curator" + +- name: Drop curator conf file + template: + src: templates/curator.yml.j2 + dest: /var/lib/curator/curator.yml + notify: + - Enable and restart curator.timer + +- name: Drop curator action file + template: + src: templates/curator-actions.yml.j2 + dest: /var/lib/curator/actions.yml + notify: + - Enable and restart curator.timer + +- name: Run the systemd service role + include_role: + name: systemd_service + private: true + vars: + systemd_service_enabled: "{{ ((elk_package_state | default('present')) != 'absent') | ternary(true, false) }}" + systemd_service_restart_changed: false + systemd_user_name: curator + systemd_group_name: curator + systemd_services: + - service_name: "curator" + execstarts: + - /opt/elasticsearch-curator/bin/curator + --config /var/lib/curator/curator.yml + /var/lib/curator/actions.yml + timer: + state: "started" + options: + OnBootSec: 30min + OnUnitActiveSec: 24h + Persistent: true diff --git a/elk_metrics_6x/roles/elastic_curator/vars/ubuntu.yml b/elk_metrics_6x/roles/elastic_curator/vars/ubuntu.yml new file mode 100644 index 00000000..363a2210 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_curator/vars/ubuntu.yml @@ -0,0 +1,18 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +curator_distro_packages: + - python-virtualenv + - virtualenv diff --git a/elk_metrics_6x/roles/elastic_data_hosts/meta/main.yml b/elk_metrics_6x/roles/elastic_data_hosts/meta/main.yml new file mode 100644 index 00000000..6b81d61d --- /dev/null +++ b/elk_metrics_6x/roles/elastic_data_hosts/meta/main.yml @@ -0,0 +1,31 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v6.x data hosts role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: [] diff --git a/elk_metrics_6x/roles/elastic_data_hosts/tasks/main.yml b/elk_metrics_6x/roles/elastic_data_hosts/tasks/main.yml new file mode 100644 index 00000000..a2d80925 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_data_hosts/tasks/main.yml @@ -0,0 +1,25 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Refresh minimal facts + setup: + gather_subset: '!all,!any,network,virtual' + tags: + - always + +- name: Load data node variables + include_vars: "data-node-variables.yml" + tags: + - always diff --git a/elk_metrics_6x/roles/elastic_data_hosts/vars/data-node-variables.yml b/elk_metrics_6x/roles/elastic_data_hosts/vars/data-node-variables.yml new file mode 100644 index 00000000..936edccb --- /dev/null +++ b/elk_metrics_6x/roles/elastic_data_hosts/vars/data-node-variables.yml @@ -0,0 +1,117 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# storage node count is equal to the cluster size +storage_node_count: "{{ groups['elastic-logstash'] | length }}" + +# the elasticserch cluster elects one master from all those which are marked as master-eligible +# 1 node cluster can only have one master +# 2 node clusters have 1 master-eligable nodes to avoid split-brain +# 3 node clusters have 3 master-eligable nodes +# >3 node clusters have (nodes // 2) eligable masters rounded up to the next odd number +master_node_count: |- + {% set masters = 0 %} + {% if (storage_node_count | int) < 3 %} + {% set masters = 1 %} + {% elif (storage_node_count | int) == 3 %} + {% set masters = 3 %} + {% else %} + {% set masters = (storage_node_count | int ) // 2 %} + {% if ((masters | int) % 2 == 0) %} + {% set masters = (masters | int) + 1 %} + {% endif %} + {% endif %} + {{ masters }} + +# Assign node roles +# the first 'master_node_count' hosts in groups['elastic-logstash'] become master-eligible nodes +# the first 'master_node_count' and subsequent alternate hosts in groups['elastic-logstash'] becomes data nodes +data_nodes: "{{ (groups['elastic-logstash'][:master_node_count | int] + groups['elastic-logstash'][master_node_count | int::2]) }}" +master_nodes: "{{ groups['elastic-logstash'][:master_node_count | int] }}" +coordination_nodes: >- + {{ + (groups['kibana'] | map('extract', hostvars, 'ansible_host') | list) + | map('regex_replace', '(.*)' ,'\1:' ~ elastic_port) + | list + }} +zen_nodes: >- + {{ + (groups['elastic-logstash'] | union(groups['kibana'])) | map('extract', hostvars, 'ansible_host') | list + }} +elasticserch_interface_speed: |- + {% set default_interface_fact = hostvars[inventory_hostname]['ansible_' + (elastic_data_interface | replace('-', '_'))] %} + {% set speeds = [] %} + {% if default_interface_fact['type'] == 'bridge' %} + {% for interface in default_interface_fact['interfaces'] %} + {% set interface_fact = hostvars[inventory_hostname]['ansible_' + (interface | replace('-', '_'))] %} + {% if 'speed' in interface_fact %} + {% set speed = (interface_fact['speed'] | default(1000)) | string %} + {% if speed == "-1" %} + {% set _ = speeds.append(1000) %} + {% else %} + {% set _ = speeds.append(speed | int) %} + {% endif %} + {% if 'module' in interface_fact %} + {% set _ = speeds.append((interface_fact['speed'] | default(1000)) | int) %} + {% else %} + {% set _ = speeds.append(1000) %} + {% endif %} + {% endif %} + {% endfor %} + {% else %} + {% if ('module' in default_interface_fact) or (default_interface_fact['type'] == 'bond') %} + {% set speed = (default_interface_fact['speed'] | default(1000)) | string %} + {% if speed == "-1" %} + {% set _ = speeds.append(1000) %} + {% else %} + {% set _ = speeds.append(speed | int) %} + {% endif %} + {% else %} + {% set _ = speeds.append(1000) %} + {% endif %} + {% endif %} + {% set interface_speed = ((speeds | min) * 0.20) | int %} + {{ ((interface_speed | int) > 750) | ternary(750, interface_speed) }} +elasticsearch_data_node_details: >- + {{ + (data_nodes | map('extract', hostvars, 'ansible_host') | list) | map('regex_replace', '(.*)' ,'\1:' ~ elastic_port) | list + }} +logstash_data_node_details: >- + {{ + (data_nodes | map('extract', hostvars, 'ansible_host') | list) | map('regex_replace', '(.*)' ,'\1:' ~ logstash_beat_input_port) | list + }} + +# based on the assignment of roles to hosts, set per host booleans +master_node: "{{ (inventory_hostname in master_nodes) | ternary(true, false) }}" +data_node: "{{ (inventory_hostname in data_nodes) | ternary(true, false) }}" +elastic_thread_pool_size: "{{ ((ansible_processor_cores | int) >= 24) | ternary(24, ansible_processor_cores) }}" + +# Set a data node facts. The data nodes, in the case of elasticsearch are also +# ingest nodes. +elasticsearch_number_of_replicas: "{{ ((data_nodes | length) > 2) | ternary('2', ((data_nodes | length) > 1) | ternary('1', '0')) }}" +elasticsearch_data_hosts: |- + {% set nodes = elasticsearch_data_node_details %} + {% if inventory_hostname in data_nodes %} + {% set _ = nodes.insert(0, '127.0.0.1:' ~ elastic_port) %} + {% endif %} + {% set data_hosts = nodes | shuffle(seed=inventory_hostname) %} + {{ data_hosts }} +logstash_data_hosts: |- + {% set nodes = logstash_data_node_details %} + {% if inventory_hostname in data_nodes %} + {% set _ = nodes.insert(0, '127.0.0.1:' ~ logstash_beat_input_port) %} + {% endif %} + {% set data_hosts = nodes | shuffle(seed=inventory_hostname) %} + {{ data_hosts }} diff --git a/elk_metrics_6x/roles/elastic_dependencies/defaults/main.yml b/elk_metrics_6x/roles/elastic_dependencies/defaults/main.yml new file mode 100644 index 00000000..78d93326 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_dependencies/defaults/main.yml @@ -0,0 +1,22 @@ +--- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## Adds option to set the UID/GID of a given service user. +# service_group_gid: 5000 +# service_owner_uid: 5000 + +# Option to define third memory +q_mem: "{{ (ansible_memtotal_mb | int) // 3 }}" + +# Option to define half memory +h_mem: "{{ (ansible_memtotal_mb | int) // 2 }}" diff --git a/elk_metrics_6x/roles/elastic_dependencies/meta/main.yml b/elk_metrics_6x/roles/elastic_dependencies/meta/main.yml new file mode 100644 index 00000000..5d1edce5 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_dependencies/meta/main.yml @@ -0,0 +1,32 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +allow_duplicates: true +galaxy_info: + author: OpenStack + description: Elastic v6.x dependencies role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: [] diff --git a/elk_metrics_6x/roles/elastic_dependencies/tasks/main.yml b/elk_metrics_6x/roles/elastic_dependencies/tasks/main.yml new file mode 100644 index 00000000..36f2db2d --- /dev/null +++ b/elk_metrics_6x/roles/elastic_dependencies/tasks/main.yml @@ -0,0 +1,162 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Check for service_name var + fail: + msg: >- + The required variable [ service_name ] is undefined. + when: + - service_name is undefined + +- name: Check for service_owner var + fail: + msg: >- + The required variable [ service_owner ] is undefined. + when: + - service_owner is undefined + +- name: Check for service_group var + fail: + msg: >- + The required variable [ service_group ] is undefined. + when: + - service_group is undefined + +- name: Load service variables + include_vars: "vars_{{ service_name }}.yml" + +- name: Gather variables for each operating system + include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml" + - "{{ ansible_os_family | lower }}.yml" + tags: + - always + +- name: Set elastic heap defaults + set_fact: + elastic_heap_size_default: "{{ _elastic_heap_size_default }}" + +- name: Configure systcl vm.max_map_count=262144 on container hosts + sysctl: + name: "vm.max_map_count" + value: "262144" + state: "present" + reload: "yes" + delegate_to: "{{ physical_host }}" + tags: + - sysctl + +- name: Physical host block + block: + - name: Check for directory + stat: + path: "/var/lib/{{ service_name }}" + register: service_dir + + - name: Check for existing rollup job + debug: + msg: >- + The service data directory [ /var/lib/{{ service_name }} ] already + exists. To ensure no data is lost, the linked directory path to + [ /openstack/{{ inventory_hostname }}/{{ service_name }} ] will not be + created for this host. + when: + - service_dir.stat.isdir is defined and + service_dir.stat.isdir + + - name: Ensure service directories data-path exists + file: + path: "/openstack/{{ inventory_hostname }}/{{ service_name }}" + state: "directory" + owner: "{{ service_owner }}" + group: "{{ service_group }}" + when: + - service_dir.stat.isdir is defined and + not service_dir.stat.isdir + + - name: Ensure data link exists + file: + src: "/openstack/{{ inventory_hostname }}/{{ service_name }}" + dest: "/var/lib/{{ service_name }}" + owner: "{{ service_owner }}" + group: "{{ service_group }}" + state: link + when: + - service_dir.stat.isdir is defined and + not service_dir.stat.isdir + when: + - physical_host == inventory_hostname + +- name: elasticsearch datapath bind mount + lxc_container: + name: "{{ inventory_hostname }}" + container_command: | + [[ ! -d "/var/lib/{{ service_name }}" ]] && mkdir -p "/var/lib/{{ service_name }}" + container_config: + - "lxc.mount.entry=/openstack/{{ inventory_hostname }}/{{ service_name }} var/lib/{{ service_name }} none bind 0 0" + delegate_to: "{{ physical_host }}" + when: + - physical_host != inventory_hostname + - container_tech | default('lxc') == 'lxc' + +- name: Ensure Java is installed + package: + name: "{{ elastic_distro_packages }}" + state: "{{ elk_package_state | default('present') }}" + install_recommends: yes + update_cache: yes + register: _package_task + until: _package_task is success + retries: 3 + delay: 2 + tags: + - package_install + +- name: Create the system group + group: + name: "{{ service_group }}" + gid: "{{ service_group_gid | default(omit) }}" + state: "present" + system: "yes" + +- name: Create the system user + user: + name: "{{ service_owner }}" + uid: "{{ service_owner_uid | default(omit) }}" + group: "{{ service_group }}" + shell: "/bin/false" + system: "yes" + createhome: "yes" + home: "/var/lib/{{ service_name }}" + +- name: Ensure service directories exists + file: + path: "/etc/{{ service_name }}" + state: "directory" + owner: "{{ service_owner }}" + group: "{{ service_group }}" + +- name: Drop jvm conf file(s) + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + with_items: + - src: templates/jvm.options.j2 + dest: /etc/{{ service_name }}/jvm.options diff --git a/elk_metrics_6x/roles/elastic_dependencies/vars/ubuntu.yml b/elk_metrics_6x/roles/elastic_dependencies/vars/ubuntu.yml new file mode 100644 index 00000000..3e016058 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_dependencies/vars/ubuntu.yml @@ -0,0 +1,17 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +elastic_distro_packages: + - openjdk-8-jre diff --git a/elk_metrics_6x/roles/elastic_dependencies/vars/vars_elasticsearch.yml b/elk_metrics_6x/roles/elastic_dependencies/vars/vars_elasticsearch.yml new file mode 100644 index 00000000..0d0180a2 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_dependencies/vars/vars_elasticsearch.yml @@ -0,0 +1,15 @@ +--- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Set memory fact to half +_elastic_heap_size_default: "{{ ((h_mem | int) > 30720) | ternary(30720, h_mem) }}" diff --git a/elk_metrics_6x/roles/elastic_dependencies/vars/vars_logstash.yml b/elk_metrics_6x/roles/elastic_dependencies/vars/vars_logstash.yml new file mode 100644 index 00000000..8b34cc76 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_dependencies/vars/vars_logstash.yml @@ -0,0 +1,15 @@ +--- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Set quarter memory fact +_elastic_heap_size_default: "{{ ((q_mem | int) > 30720) | ternary(30720, q_mem) }}" diff --git a/elk_metrics_6x/roles/elastic_filebeat/handlers/main.yml b/elk_metrics_6x/roles/elastic_filebeat/handlers/main.yml new file mode 100644 index 00000000..5fdc924a --- /dev/null +++ b/elk_metrics_6x/roles/elastic_filebeat/handlers/main.yml @@ -0,0 +1,25 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Enable and restart filebeat + systemd: + name: "filebeat" + enabled: true + state: restarted + daemon_reload: true + when: + - (elk_package_state | default('present')) != 'absent' + tags: + - config diff --git a/elk_metrics_6x/roles/elastic_filebeat/meta/main.yml b/elk_metrics_6x/roles/elastic_filebeat/meta/main.yml new file mode 100644 index 00000000..95bb8f59 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_filebeat/meta/main.yml @@ -0,0 +1,33 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v6.x filebeat role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: + - role: elastic_data_hosts + - role: elastic_repositories diff --git a/elk_metrics_6x/roles/elastic_filebeat/tasks/filebeat_setup.yml b/elk_metrics_6x/roles/elastic_filebeat/tasks/filebeat_setup.yml new file mode 100644 index 00000000..4b917ed7 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_filebeat/tasks/filebeat_setup.yml @@ -0,0 +1,32 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Load templates + shell: >- + filebeat setup + {{ item }} + -E 'output.logstash.enabled=false' + -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' + -e -v + with_items: + - "--template" + - "--dashboards" + register: templates + until: templates is success + retries: 5 + delay: 5 + run_once: true + tags: + - setup diff --git a/elk_metrics_6x/roles/elastic_filebeat/tasks/main.yml b/elk_metrics_6x/roles/elastic_filebeat/tasks/main.yml new file mode 100644 index 00000000..0d4a1c2e --- /dev/null +++ b/elk_metrics_6x/roles/elastic_filebeat/tasks/main.yml @@ -0,0 +1,177 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather variables for each operating system + include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml" + - "{{ ansible_os_family | lower }}.yml" + tags: + - always + +- name: Ensure beat is installed + package: + name: "{{ filebeat_distro_packages }}" + state: "{{ elk_package_state | default('present') }}" + update_cache: true + register: _package_task + until: _package_task is success + retries: 3 + delay: 2 + notify: + - Enable and restart filebeat + tags: + - package_install + +- name: Create filebeat systemd service config dir + file: + path: "/etc/systemd/system/filebeat.service.d" + state: "directory" + group: "root" + owner: "root" + mode: "0755" + +- name: Apply systemd options + template: + src: "{{ item.src }}" + dest: "/etc/systemd/system/filebeat.service.d/{{ item.dest }}" + mode: "0644" + with_items: + - { src: "systemd.general-overrides.conf.j2", dest: "filebeat-overrides.conf" } + notify: + - Enable and restart filebeat + +- name: Check for apache + stat: + path: /etc/apache2 + register: apache2 + +- name: Check for auditd + stat: + path: /etc/audit + register: audit + +- name: Check for ceph + stat: + path: /var/log/ceph + register: ceph + +- name: Check for cinder + stat: + path: /var/log/cinder + register: cinder + +- name: Check for glance + stat: + path: /var/log/glance + register: glance + +- name: Check for heat + stat: + path: /var/log/heat + register: heat + +- name: Check for horizon + stat: + path: /var/log/horizon + register: horizon + +- name: Check for httpd + stat: + path: /var/log/httpd + register: httpd + +- name: Check for keystone + stat: + path: /var/log/keystone + register: keystone + +- name: Check for mysql + stat: + path: /var/lib/mysql + register: mysql + +- name: Check for neutron + stat: + path: /var/log/neutron + register: neutron + +- name: Check for nginx + stat: + path: /var/log/nginx + register: nginx + +- name: Check for nova + stat: + path: /var/log/nova + register: nova + +- name: Check for octavia + stat: + path: /var/log/octavia + register: octavia + +- name: Check for swift + stat: + path: /var/log/swift + register: swift + +- name: Check for rabbitmq + stat: + path: /var/lib/rabbitmq + register: rabbitmq + +- name: Check for designate + stat: + path: /var/log/designate + register: designate + +- name: Check for osquery + stat: + path: /var/log/osquery/osqueryd.results.log + register: osquery + +- name: Set discovery facts + set_fact: + apache_enabled: "{{ (apache2.stat.exists | bool) or (httpd.stat.exists | bool) }}" + nginx_enabled: "{{ nginx.stat.exists | bool }}" + auditd_enabled: "{{ audit.stat.exists | bool }}" + mysql_enabled: "{{ (mysql.stat.exists | bool) or (inventory_hostname in groups['galera_all'] | default([])) }}" + ceph_enabled: "{{ (ceph.stat.exists | bool) or (inventory_hostname in groups['ceph_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*ceph.*') | list | length) > 0) }}" + cinder_enabled: "{{ (cinder.stat.exists | bool) or (inventory_hostname in groups['cinder_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*cinder.*') | list | length) > 0) }}" + glance_enabled: "{{ (glance.stat.exists | bool) or (inventory_hostname in groups['glance_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*glance.*') | list | length) > 0) }}" + heat_enabled: "{{ (heat.stat.exists | bool) or (inventory_hostname in groups['heat_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*heat.*') | list | length) > 0) }}" + horizon_enabled: "{{ (horizon.stat.exists | bool) or (inventory_hostname in groups['horizon_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*horizon.*') | list | length) > 0) }}" + keystone_enabled: "{{ (keystone.stat.exists | bool) or (inventory_hostname in groups['keystone_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*keystone.*') | list | length) > 0) }}" + neutron_enabled: "{{ (neutron.stat.exists | bool) or (inventory_hostname in groups['neutron_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*neutron.*') | list | length) > 0) }}" + nova_enabled: "{{ (nova.stat.exists | bool) or (inventory_hostname in groups['nova_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*nova.*') | list | length) > 0) }}" + octavia_enabled: "{{ (octavia.stat.exists | bool) or (inventory_hostname in groups['octavia_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*octavia.*') | list | length) > 0) }}" + swift_enabled: "{{ (swift.stat.exists | bool) or (inventory_hostname in groups['swift_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*swift.*') | list | length) > 0) }}" + rabbitmq_enabled: "{{ (rabbitmq.stat.exists | bool) or (inventory_hostname in groups['rabbitmq_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*rabbit.*') | list | length) > 0) }}" + designate_enabled: "{{ (designate.stat.exists | bool) or (inventory_hostname in groups['designate_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*designate.*') | list | length) > 0) }}" + osquery_enabled: "{{ osquery.stat.exists | bool }}" + +- name: Drop Filebeat conf file + template: + src: templates/filebeat.yml.j2 + dest: /etc/filebeat/filebeat.yml + notify: + - Enable and restart filebeat + +- include_tasks: filebeat_setup.yml diff --git a/elk_metrics_6x/roles/elastic_filebeat/vars/ubuntu.yml b/elk_metrics_6x/roles/elastic_filebeat/vars/ubuntu.yml new file mode 100644 index 00000000..2f055205 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_filebeat/vars/ubuntu.yml @@ -0,0 +1,17 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +filebeat_distro_packages: + - filebeat diff --git a/elk_metrics_6x/roles/elastic_heartbeat/handlers/main.yml b/elk_metrics_6x/roles/elastic_heartbeat/handlers/main.yml new file mode 100644 index 00000000..ac87f12c --- /dev/null +++ b/elk_metrics_6x/roles/elastic_heartbeat/handlers/main.yml @@ -0,0 +1,25 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Enable and restart heartbeat + systemd: + name: "heartbeat-elastic" + enabled: true + state: restarted + daemon_reload: true + when: + - (elk_package_state | default('present')) != 'absent' + tags: + - config diff --git a/elk_metrics_6x/roles/elastic_heartbeat/meta/main.yml b/elk_metrics_6x/roles/elastic_heartbeat/meta/main.yml new file mode 100644 index 00000000..761d880e --- /dev/null +++ b/elk_metrics_6x/roles/elastic_heartbeat/meta/main.yml @@ -0,0 +1,33 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v6.x heartbeat role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: + - role: elastic_data_hosts + - role: elastic_repositories diff --git a/elk_metrics_6x/roles/elastic_heartbeat/tasks/heartbeat_setup.yml b/elk_metrics_6x/roles/elastic_heartbeat/tasks/heartbeat_setup.yml new file mode 100644 index 00000000..b6d412a1 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_heartbeat/tasks/heartbeat_setup.yml @@ -0,0 +1,32 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Load templates + shell: >- + heartbeat setup + {{ item }} + -E 'output.logstash.enabled=false' + -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' + -e -v + with_items: + - "--template" + - "--dashboards" + register: templates + until: templates is success + retries: 5 + delay: 5 + run_once: true + tags: + - setup diff --git a/elk_metrics_6x/roles/elastic_heartbeat/tasks/main.yml b/elk_metrics_6x/roles/elastic_heartbeat/tasks/main.yml new file mode 100644 index 00000000..f52185ba --- /dev/null +++ b/elk_metrics_6x/roles/elastic_heartbeat/tasks/main.yml @@ -0,0 +1,63 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather variables for each operating system + include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml" + - "{{ ansible_os_family | lower }}.yml" + tags: + - always + +- name: Ensure beat is installed + package: + name: "{{ heartbeat_distro_packages }}" + state: "{{ elk_package_state | default('present') }}" + update_cache: true + register: _package_task + until: _package_task is success + retries: 3 + delay: 2 + notify: + - Enable and restart heartbeat + tags: + - package_install + +- name: Create heartbeat systemd service config dir + file: + path: "/etc/systemd/system/heartbeat.service.d" + state: "directory" + group: "root" + owner: "root" + mode: "0755" + +- name: Create heartbeat configs + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + mode: "0644" + with_items: + - src: "systemd.general-overrides.conf.j2" + dest: "/etc/systemd/system/heartbeat.service.d/heartbeat-overrides.conf" + - src: templates/heartbeat.yml.j2 + dest: /etc/heartbeat/heartbeat.yml + notify: + - Enable and restart heartbeat + +- include_tasks: heartbeat_setup.yml diff --git a/elk_metrics_6x/roles/elastic_heartbeat/vars/ubuntu.yml b/elk_metrics_6x/roles/elastic_heartbeat/vars/ubuntu.yml new file mode 100644 index 00000000..4a32d6b7 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_heartbeat/vars/ubuntu.yml @@ -0,0 +1,17 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +heartbeat_distro_packages: + - heartbeat-elastic diff --git a/elk_metrics_6x/roles/elastic_journalbeat/handlers/main.yml b/elk_metrics_6x/roles/elastic_journalbeat/handlers/main.yml new file mode 100644 index 00000000..938ba819 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_journalbeat/handlers/main.yml @@ -0,0 +1,25 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Enable and restart journalbeat + systemd: + name: "journalbeat" + enabled: true + state: restarted + daemon_reload: yes + when: + - (elk_package_state | default('present')) != 'absent' + tags: + - config diff --git a/elk_metrics_6x/roles/elastic_journalbeat/meta/main.yml b/elk_metrics_6x/roles/elastic_journalbeat/meta/main.yml new file mode 100644 index 00000000..cc4a5063 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_journalbeat/meta/main.yml @@ -0,0 +1,33 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v6.x journalbeat role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: + - role: elastic_data_hosts + - role: go_install_1.10 diff --git a/elk_metrics_6x/roles/elastic_journalbeat/tasks/journalbeat_setup.yml b/elk_metrics_6x/roles/elastic_journalbeat/tasks/journalbeat_setup.yml new file mode 100644 index 00000000..473dffa0 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_journalbeat/tasks/journalbeat_setup.yml @@ -0,0 +1,40 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Notice + debug: + msg: >- + The setup task for templates and dashboards is broken due to missing + configuration. Once the following issue + [ https://github.com/mheese/journalbeat/issues/136 ] is + resolved, this should be enabled. + +# - name: Load templates +# shell: >- +# /usr/local/bin/journalbeat -setup +# {{ item }} +# -E 'output.logstash.enabled=false' +# -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' +# -e -v +# with_items: +# - "--template" +# - "--dashboards" +# register: templates +# until: templates is success +# retries: 5 +# delay: 5 +# run_once: true +# tags: +# - setup diff --git a/elk_metrics_6x/roles/elastic_journalbeat/tasks/main.yml b/elk_metrics_6x/roles/elastic_journalbeat/tasks/main.yml new file mode 100644 index 00000000..24535b2f --- /dev/null +++ b/elk_metrics_6x/roles/elastic_journalbeat/tasks/main.yml @@ -0,0 +1,134 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather variables for each operating system + include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml" + - "{{ ansible_os_family | lower }}.yml" + tags: + - always + +- name: Check for journal directory + stat: + path: /var/log/journal + register: journal_dir + +- name: exit playbook after uninstall + meta: end_play + when: + - not journal_dir.stat.exists | bool + +- name: Ensure beat is installed + package: + name: "{{ journalbeat_distro_packages }}" + state: "{{ elk_package_state | default('present') }}" + update_cache: true + register: _package_task + until: _package_task is success + retries: 3 + delay: 2 + notify: + - Enable and restart journalbeat + tags: + - package_install + +- name: create the system group + group: + name: "journalbeat" + state: "present" + system: "yes" + +- name: Create the journalbeat user + user: + name: "journalbeat" + group: "journalbeat" + comment: "journalbeat user" + shell: "/bin/false" + createhome: "yes" + home: "/usr/share/journalbeat" + +- name: Create journalbeat data path + file: + path: "{{ item }}" + state: directory + owner: "journalbeat" + group: "journalbeat" + mode: "0755" + with_items: + - "/usr/share/journalbeat" + - "/var/lib/journalbeat" + - "/etc/journalbeat" + +- name: Install journalbeat + shell: /opt/go1.10.1/go/bin/go get -v github.com/mheese/journalbeat + args: + creates: /usr/local/bin/journalbeat + environment: + PATH: "{{ ansible_env.PATH }}:/opt/go1.10.1/go/bin" + GOPATH: /usr/local + notify: + - Enable and restart journalbeat + +- name: Create journalbeat systemd service config dir + file: + path: "/etc/systemd/system/journalbeat.service.d" + state: "directory" + group: "root" + owner: "root" + mode: "0755" + +- name: Drop journalbeat conf files + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + mode: "0644" + with_items: + - src: "systemd.general-overrides.conf.j2" + dest: "/etc/systemd/system/journalbeat.service.d/journalbeat-overrides.conf" + - src: templates/journalbeat.yml.j2 + dest: /etc/journalbeat/journalbeat.yml + notify: + - Enable and restart journalbeat + +- name: Run the systemd service role + include_role: + name: systemd_service + private: true + vars: + systemd_service_enabled: "{{ ((elk_package_state | default('present')) != 'absent') | ternary(true, false) }}" + systemd_service_restart_changed: false + systemd_services: + - service_name: "journalbeat" + execstarts: + - /usr/local/bin/journalbeat + -c /etc/journalbeat/journalbeat.yml + -path.home /usr/share/journalbeat + -path.config /etc/journalbeat + -path.data /var/lib/journalbeat + -path.logs /var/log/beats + config_overrides: + Service: + EnvironmentFile: "-/etc/default/go1.10.1" + Unit: + Documentation: https://github.com/mheese/journalbeat/blob/master/README.md + Wants: network-online.target + After: network-online.target + +- include_tasks: journalbeat_setup.yml diff --git a/elk_metrics_6x/roles/elastic_journalbeat/vars/ubuntu.yml b/elk_metrics_6x/roles/elastic_journalbeat/vars/ubuntu.yml new file mode 100644 index 00000000..bc1ee7bb --- /dev/null +++ b/elk_metrics_6x/roles/elastic_journalbeat/vars/ubuntu.yml @@ -0,0 +1,19 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +journalbeat_distro_packages: + - gcc + - git + - libsystemd-dev diff --git a/elk_metrics_6x/roles/elastic_kibana/defaults/main.yml b/elk_metrics_6x/roles/elastic_kibana/defaults/main.yml new file mode 100644 index 00000000..a197cac3 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_kibana/defaults/main.yml @@ -0,0 +1,16 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kibana_enable_basic_auth: false diff --git a/elk_metrics_6x/roles/elastic_kibana/handlers/main.yml b/elk_metrics_6x/roles/elastic_kibana/handlers/main.yml new file mode 100644 index 00000000..12f4111f --- /dev/null +++ b/elk_metrics_6x/roles/elastic_kibana/handlers/main.yml @@ -0,0 +1,37 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Enable and restart nginx + systemd: + name: "nginx" + enabled: true + state: restarted + when: + - (elk_package_state | default('present')) != 'absent' + listen: Enable and restart services + tags: + - config + +- name: Enable and restart kibana + systemd: + name: "kibana" + enabled: true + state: restarted + daemon_reload: true + when: + - (elk_package_state | default('present')) != 'absent' + listen: Enable and restart services + tags: + - config diff --git a/elk_metrics_6x/roles/elastic_kibana/meta/main.yml b/elk_metrics_6x/roles/elastic_kibana/meta/main.yml new file mode 100644 index 00000000..3214fdf3 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_kibana/meta/main.yml @@ -0,0 +1,32 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v6.x kibana role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: + - role: elastic_repositories diff --git a/elk_metrics_6x/roles/elastic_kibana/tasks/main.yml b/elk_metrics_6x/roles/elastic_kibana/tasks/main.yml new file mode 100644 index 00000000..9391e964 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_kibana/tasks/main.yml @@ -0,0 +1,84 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather variables for each operating system + include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml" + - "{{ ansible_os_family | lower }}.yml" + tags: + - always + +- name: Ensure distro packages are installed + package: + name: "{{ kibana_distro_packages }}" + state: "{{ elk_package_state | default('present') }}" + update_cache: yes + register: _package_task + until: _package_task is success + retries: 3 + delay: 2 + notify: + - Enable and restart services + tags: + - package_install + +- name: create kibana user to access web interface + htpasswd: + path: "/etc/nginx/htpasswd.users" + name: "{{ kibana_username }}" + password: "{{ kibana_password }}" + owner: root + mode: 0644 + when: + - kibana_enable_basic_auth + +- name: Drop Nginx default conf file + template: + src: templates/nginx_default.j2 + dest: /etc/nginx/sites-available/default + notify: + - Enable and restart services + +- name: Create kibana systemd service config dir + file: + path: "/etc/systemd/system/kibana.service.d" + state: "directory" + group: "root" + owner: "root" + mode: "0755" + +- name: Apply systemd options + template: + src: "{{ item.src }}" + dest: "/etc/systemd/system/kibana.service.d/{{ item.dest }}" + mode: "0644" + with_items: + - src: "systemd.general-overrides.conf.j2" + dest: "kibana-overrides.conf" + notify: + - Enable and restart services + +- name: Drop kibana conf file + template: + src: templates/kibana.yml.j2 + dest: /etc/kibana/kibana.yml + mode: "0666" + notify: + - Enable and restart services diff --git a/elk_metrics_6x/roles/elastic_kibana/vars/ubuntu.yml b/elk_metrics_6x/roles/elastic_kibana/vars/ubuntu.yml new file mode 100644 index 00000000..780eb881 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_kibana/vars/ubuntu.yml @@ -0,0 +1,20 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kibana_distro_packages: + - apache2-utils + - kibana + - nginx + - python-passlib diff --git a/elk_metrics_6x/roles/elastic_logstash/defaults/main.yml b/elk_metrics_6x/roles/elastic_logstash/defaults/main.yml new file mode 100644 index 00000000..2c9a92d2 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_logstash/defaults/main.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +temp_dir: /var/lib/logstash/tmp +logstash_pipelines: "{{lookup('template', 'templates/logstash-pipelines.yml.j2') }}" + +# Set processor cores fact +q_storage: "{{ (ansible_processor_cores | int) * (ansible_processor_threads_per_core | int) * 2 }}" + +# Set logstash facts +logstash_queue_size: "{{ ((((q_storage | int) >= 2) | ternary(q_storage, 2) | int) * 1024) // ((logstash_pipelines | from_yaml) | length) }}" +elastic_log_rotate_path: "/var/log/logstash" diff --git a/elk_metrics_6x/roles/elastic_logstash/handlers/main.yml b/elk_metrics_6x/roles/elastic_logstash/handlers/main.yml new file mode 100644 index 00000000..8ea50751 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_logstash/handlers/main.yml @@ -0,0 +1,25 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Enable and restart logstash + systemd: + name: "logstash" + enabled: true + state: restarted + daemon_reload: true + when: + - (elk_package_state | default('present')) != 'absent' + tags: + - config diff --git a/elk_metrics_6x/roles/elastic_logstash/meta/main.yml b/elk_metrics_6x/roles/elastic_logstash/meta/main.yml new file mode 100644 index 00000000..f5684dc8 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_logstash/meta/main.yml @@ -0,0 +1,37 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v6.x logstash role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: + - role: elastic_data_hosts + - role: elastic_repositories + - role: elastic_dependencies + service_name: logstash + service_owner: logstash + service_group: logstash diff --git a/elk_metrics_6x/roles/elastic_logstash/tasks/logstash_arcsight.yml b/elk_metrics_6x/roles/elastic_logstash/tasks/logstash_arcsight.yml new file mode 100644 index 00000000..f9588ca8 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_logstash/tasks/logstash_arcsight.yml @@ -0,0 +1,62 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Initialise arcsight local facts + ini_file: + dest: "/etc/ansible/facts.d/logstash.fact" + section: arcsight + option: "initialise" + value: true + +- name: refresh local facts + setup: + filter: ansible_local + gather_subset: "!all" + tags: + - nova-config + +- name: Setup arcsight smart connector + shell: >- + /usr/local/bin/logstash --modules arcsight {{ (ansible_local['arcsight'][item.host] is defined) | ternary('', '--setup') }} + -M "arcsight.var.input.smartconnector.bootstrap_servers={{ item.host }}:{{ item.port }}" + -M "arcsight.var.elasticsearch.hosts=localhost:{{ elastic_port }}" + -M "arcsight.var.kibana.host={{ hostvars[groups['kibana'][0]]['ansible_host'] }}:{{ kibana_port }}" + with_items: "{{ logstash_arcsight_smart_connectors }}" + run_once: true + register: smart_connector + until: smart_connector is success + retries: 5 + delay: 5 + +- name: Setup arcsight event broker + shell: >- + /usr/local/bin/logstash --modules arcsight {{ (ansible_local['arcsight'][item.host] is defined) | ternary('', '--setup') }} + -M "arcsight.var.input.eventbroker.bootstrap_servers={{ item.host }}:{{ item.port }}" + -M "arcsight.var.elasticsearch.hosts=localhost:{{ elastic_port }}" + -M "arcsight.var.kibana.host={{ hostvars[groups['kibana'][0]]['ansible_host'] }}:{{ kibana_port }}" + with_items: "{{ logstash_arcsight_event_brokers }}" + run_once: true + register: event_broker + until: event_broker is success + retries: 5 + delay: 5 + +- name: Set arcsight local facts + ini_file: + dest: "/etc/ansible/facts.d/logstash.fact" + section: arcsight + option: "{{ item.host }}" + value: "{{ item.port }}" + with_items: "{{ logstash_arcsight_smart_connectors | union(logstash_arcsight_event_brokers) }}" diff --git a/elk_metrics_6x/roles/elastic_logstash/tasks/logstash_kafka_ssl.yml b/elk_metrics_6x/roles/elastic_logstash/tasks/logstash_kafka_ssl.yml new file mode 100644 index 00000000..fdcad549 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_logstash/tasks/logstash_kafka_ssl.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Copy kafka keystore into place + copy: + src: "{{ logstash_kafka_ssl_keystore_location }}" + dest: "/var/lib/logstash/{{ logstash_kafka_ssl_keystore_location | basename }}" + +- name: Copy kafka truststore into place + copy: + src: "{{ logstash_kafka_ssl_truststore_location }}" + dest: "/var/lib/logstash/{{ logstash_kafka_ssl_truststore_location | basename }}" diff --git a/elk_metrics_6x/roles/elastic_logstash/tasks/main.yml b/elk_metrics_6x/roles/elastic_logstash/tasks/main.yml new file mode 100644 index 00000000..dec0dab0 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_logstash/tasks/main.yml @@ -0,0 +1,129 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather variables for each operating system + include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml" + - "{{ ansible_os_family | lower }}.yml" + tags: + - always + +- name: Ensure Logstash is installed + package: + name: "{{ logstash_distro_packages }}" + state: "{{ elk_package_state | default('present') }}" + update_cache: yes + register: _package_task + until: _package_task is success + retries: 3 + delay: 2 + notify: + - Enable and restart logstash + tags: + - package_install + +- name: Create logstash systemd service config dir + file: + path: "/etc/systemd/system/logstash.service.d" + state: "directory" + group: "root" + owner: "root" + mode: "0755" + +- name: Apply systemd options + template: + src: "{{ item.src }}" + dest: "/etc/systemd/system/logstash.service.d/{{ item.dest }}" + mode: "0644" + with_items: + - src: "systemd.general-overrides.conf.j2" + dest: "logstash-overrides.conf" + notify: + - Enable and restart logstash + +- name: Create patterns directory + file: + name: "/opt/logstash/patterns" + owner: "logstash" + group: "logstash" + state: directory + tags: + - logstash-patterns + +- name: Logstash Extra Patterns + template: + src: "extras" + dest: "/opt/logstash/patterns/extras" + owner: "logstash" + group: "logstash" + when: + - logstash_deploy_filters + notify: + - Enable and restart logstash + tags: + - logstash-filters + - config + +- name: Run kafka ssl deployment + include_tasks: logstash_kafka_ssl.yml + when: + - logstash_kafka_options is defined + - logstash_kafka_ssl_keystore_location is defined + +- name: Drop logstash conf file(s) + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + with_items: + - src: templates/logstash.yml.j2 + dest: /etc/logstash/logstash.yml + - src: templates/logstash-pipelines.yml.j2 + dest: /etc/logstash/pipelines.yml + - src: "templates/logrotate.j2" + dest: "/etc/logrotate.d/logstash" + notify: + - Enable and restart logstash + tags: + - config + +- name: Ensure logstash ownership + file: + path: "/var/lib/logstash/" + owner: logstash + group: logstash + recurse: true + register: l_perms + until: l_perms is success + retries: 3 + delay: 1 + +- name: Ensure logstash tmp dir + file: + path: "/var/lib/logstash/tmp" + state: directory + owner: "logstash" + group: "logstash" + mode: "0750" + +- name: Deploy arcsight collector + include_tasks: logstash_arcsight.yml + when: + - logstash_arcsight_smart_connectors or + logstash_arcsight_event_brokers diff --git a/elk_metrics_6x/roles/elastic_logstash/vars/ubuntu.yml b/elk_metrics_6x/roles/elastic_logstash/vars/ubuntu.yml new file mode 100644 index 00000000..716310e6 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_logstash/vars/ubuntu.yml @@ -0,0 +1,18 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +logstash_distro_packages: + - logrotate + - logstash diff --git a/elk_metrics_6x/roles/elastic_metricbeat/handlers/main.yml b/elk_metrics_6x/roles/elastic_metricbeat/handlers/main.yml new file mode 100644 index 00000000..9493df01 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_metricbeat/handlers/main.yml @@ -0,0 +1,25 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Enable and restart metricbeat + systemd: + name: "metricbeat" + enabled: true + state: restarted + daemon_reload: true + when: + - (elk_package_state | default('present')) != 'absent' + tags: + - config diff --git a/elk_metrics_6x/roles/elastic_metricbeat/meta/main.yml b/elk_metrics_6x/roles/elastic_metricbeat/meta/main.yml new file mode 100644 index 00000000..73585980 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_metricbeat/meta/main.yml @@ -0,0 +1,33 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v6.x metricbeat role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: + - role: elastic_data_hosts + - role: elastic_repositories diff --git a/elk_metrics_6x/roles/elastic_metricbeat/tasks/main.yml b/elk_metrics_6x/roles/elastic_metricbeat/tasks/main.yml new file mode 100644 index 00000000..2229e242 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_metricbeat/tasks/main.yml @@ -0,0 +1,236 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather variables for each operating system + include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml" + - "{{ ansible_os_family | lower }}.yml" + tags: + - always + +- name: Ensure beat is installed + package: + name: "{{ metricbeat_distro_packages }}" + state: "{{ elk_package_state | default('present') }}" + update_cache: true + register: _package_task + until: _package_task is success + retries: 3 + delay: 2 + notify: + - Enable and restart metricbeat + tags: + - package_install + +- name: Check for apache + stat: + path: /etc/apache2/sites-available + register: apache2 + +- name: Check for ceph + stat: + path: /etc/ceph + register: ceph + +# gather ceph stats from localhost +# except when a list of mons is provided +- name: Set ceph stats hosts + set_fact: + ceph_stats_hosts: |- + {% set ceph_stats = [] %} + {% if (ceph_mons is defined) and (ceph_mons | length > 0) %} + {% for mon in ceph_mons %} + {% set _ = ceph_stats.insert(loop.index, (mon + ":5000")) %} + {% endfor %} + {% else %} + {% set ceph_stats = [ ansible_hostname + ":5000" ] %} + {% endif %} + {{ ceph_stats }} + +- name: Check for etcd + stat: + path: /etc/etcd + register: etcd + +- name: Check for docker + stat: + path: /var/run/docker.sock + register: docker + +- name: Check for haproxy + stat: + path: /etc/haproxy + register: haproxy + +- name: Check for httpd + stat: + path: /etc/httpd + register: httpd + +- name: Check for kvm + stat: + path: /var/run/libvirt/libvirt-sock + register: kvm + +- name: Check for memcached + stat: + path: /etc/memcached.conf + register: memcached + +- name: Check for mysql + stat: + path: /var/lib/mysql + register: mysql + +- name: Check for nginx + stat: + path: /etc/nginx/nginx.conf + register: nginx + +- name: Check for rabbitmq + stat: + path: /var/lib/rabbitmq + register: rabbitmq + +- name: Check for uwsgi + stat: + path: /etc/uwsgi + register: uwsgi + +- name: Check for uwsgi stats sockets + find: + paths: /tmp + file_type: any + patterns: '*uwsgi-stats.sock' + register: uwsgi_find_sockets + +- name: Set discovery facts + set_fact: + apache_enabled: "{{ (apache2.stat.exists | bool) or (httpd.stat.exists | bool) }}" + + # enable ceph on: cinder volume hosts when we have a list of ceph mons + # otherwise: all hosts which have /etc/ceph + ceph_enabled: |- + {% set ceph_detect = false %} + {% if (ceph_mons is defined) and (ceph_mons | length > 0) and (inventory_hostname in groups['cinder_volume']) %} + {% set ceph_detect = true %} + {% else %} + {% set ceph_detect = ceph.stat.exists | bool %} + {% endif %} + {{ ceph_detect }} + + docker_enabled: "{{ docker.stat.exists | bool }}" + etcd_enabled: "{{ etcd.stat.exists | bool }}" + haproxy_enabled: "{{ haproxy.stat.exists | bool }}" + kvm_enabled: "{{ kvm.stat.exists | bool }}" + memcached_enabled: "{{ memcached.stat.exists | bool }}" + mysql_enabled: "{{ mysql.stat.exists | bool }}" + nginx_enabled: "{{ nginx.stat.exists | bool }}" + rabbitmq_enabled: "{{ rabbitmq.stat.exists | bool }}" + uwsgi_enabled: "{{ uwsgi.stat.exists | bool }}" + uwsgi_sockets: "{{ uwsgi_find_sockets }}" + +# Apache 2 stats enablement +- name: Drop apache2 stats site config + template: + src: apache-status.conf.j2 + dest: /etc/apache2/sites-available/apache-status.conf + when: apache_enabled + +- name: Enable apache2 stats site + file: + src: /etc/apache2/sites-available/apache-status.conf + dest: /etc/apache2/sites-enabled/apache-status.conf + state: link + when: apache_enabled + +- name: Ensure apache2 stats mode is enabled + apache2_module: + name: status + state: present + register: apache_status_mod + when: apache_enabled + +- name: Reload apache2 + service: + name: apache2 + state: reloaded + when: + - apache_enabled + - apache_status_mod is changed + +# NGINX stats enablement +- name: Drop nginx stats site config + template: + src: nginx-status.conf.j2 + dest: /etc/nginx/sites-available/nginx-status.conf + when: nginx_enabled + +- name: Enable nginx stats site + file: + src: /etc/nginx/sites-available/nginx-status.conf + dest: /etc/nginx/sites-enabled/nginx-status.conf + state: link + register: nginx_status + when: nginx_enabled + +- name: Reload nginx + service: + name: nginx + state: reloaded + when: + - nginx_enabled + - nginx_status is changed + +- name: Create metricbeat systemd service config dir + file: + path: "/etc/systemd/system/metricbeat.service.d" + state: "directory" + group: "root" + owner: "root" + mode: "0755" + when: + - ansible_service_mgr == 'systemd' + +- name: Apply systemd options + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + mode: "0644" + when: + - ansible_service_mgr == 'systemd' + with_items: + - src: "systemd.general-overrides.conf.j2" + dest: "/etc/systemd/system/metricbeat.service.d/metricbeat-overrides.conf" + notify: + - Enable and restart metricbeat + +- name: Drop metricbeat conf file + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + mode: "0644" + with_items: + - src: templates/metricbeat.yml.j2 + dest: /etc/metricbeat/metricbeat.yml + notify: + - Enable and restart metricbeat + +- include_tasks: metricbeat_setup.yml diff --git a/elk_metrics_6x/roles/elastic_metricbeat/tasks/metricbeat_setup.yml b/elk_metrics_6x/roles/elastic_metricbeat/tasks/metricbeat_setup.yml new file mode 100644 index 00000000..86d56690 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_metricbeat/tasks/metricbeat_setup.yml @@ -0,0 +1,32 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Load templates + shell: >- + metricbeat setup + {{ item }} + -E 'output.logstash.enabled=false' + -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' + -e -v + with_items: + - "--template" + - "--dashboards" + register: templates + until: templates is success + retries: 5 + delay: 5 + run_once: true + tags: + - setup diff --git a/elk_metrics_6x/roles/elastic_metricbeat/vars/ubuntu.yml b/elk_metrics_6x/roles/elastic_metricbeat/vars/ubuntu.yml new file mode 100644 index 00000000..2eecca08 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_metricbeat/vars/ubuntu.yml @@ -0,0 +1,17 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +metricbeat_distro_packages: + - metricbeat diff --git a/elk_metrics_6x/roles/elastic_packetbeat/handlers/main.yml b/elk_metrics_6x/roles/elastic_packetbeat/handlers/main.yml new file mode 100644 index 00000000..068f373c --- /dev/null +++ b/elk_metrics_6x/roles/elastic_packetbeat/handlers/main.yml @@ -0,0 +1,25 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Enable and restart packetbeat + systemd: + name: "packetbeat" + enabled: true + state: restarted + daemon_reload: true + when: + - (elk_package_state | default('present')) != 'absent' + tags: + - config diff --git a/elk_metrics_6x/roles/elastic_packetbeat/meta/main.yml b/elk_metrics_6x/roles/elastic_packetbeat/meta/main.yml new file mode 100644 index 00000000..c403c434 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_packetbeat/meta/main.yml @@ -0,0 +1,33 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v6.x packetbeat role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: + - role: elastic_data_hosts + - role: elastic_repositories diff --git a/elk_metrics_6x/roles/elastic_packetbeat/tasks/main.yml b/elk_metrics_6x/roles/elastic_packetbeat/tasks/main.yml new file mode 100644 index 00000000..d634ea48 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_packetbeat/tasks/main.yml @@ -0,0 +1,63 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather variables for each operating system + include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml" + - "{{ ansible_os_family | lower }}.yml" + tags: + - always + +- name: Ensure beat is installed + package: + name: "{{ packetbeat_distro_packages }}" + state: "{{ elk_package_state | default('present') }}" + update_cache: true + register: _package_task + until: _package_task is success + retries: 3 + delay: 2 + notify: + - Enable and restart packetbeat + tags: + - package_install + +- name: Create packetbeat systemd service config dir + file: + path: "/etc/systemd/system/packetbeat.service.d" + state: "directory" + group: "root" + owner: "root" + mode: "0755" + +- name: Drop packetbeat conf files + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + mode: "0644" + with_items: + - src: "systemd.general-overrides.conf.j2" + dest: "/etc/systemd/system/packetbeat.service.d/packetbeat-overrides.conf" + - src: templates/packetbeat.yml.j2 + dest: /etc/packetbeat/packetbeat.yml + notify: + - Enable and restart packetbeat + +- include_tasks: packetbeat_setup.yml diff --git a/elk_metrics_6x/roles/elastic_packetbeat/tasks/packetbeat_setup.yml b/elk_metrics_6x/roles/elastic_packetbeat/tasks/packetbeat_setup.yml new file mode 100644 index 00000000..33dc703d --- /dev/null +++ b/elk_metrics_6x/roles/elastic_packetbeat/tasks/packetbeat_setup.yml @@ -0,0 +1,32 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Load templates + shell: >- + packetbeat setup + {{ item }} + -E 'output.logstash.enabled=false' + -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' + -e -v + with_items: + - "--template" + - "--dashboards" + register: templates + until: templates is success + retries: 5 + delay: 5 + run_once: true + tags: + - setup diff --git a/elk_metrics_6x/roles/elastic_packetbeat/vars/ubuntu.yml b/elk_metrics_6x/roles/elastic_packetbeat/vars/ubuntu.yml new file mode 100644 index 00000000..e949478a --- /dev/null +++ b/elk_metrics_6x/roles/elastic_packetbeat/vars/ubuntu.yml @@ -0,0 +1,18 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +packetbeat_distro_packages: + - tcpdump + - packetbeat diff --git a/elk_metrics_6x/roles/elastic_repositories/meta/main.yml b/elk_metrics_6x/roles/elastic_repositories/meta/main.yml new file mode 100644 index 00000000..c3a7f802 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_repositories/meta/main.yml @@ -0,0 +1,31 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v6.x repositories role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: [] diff --git a/elk_metrics_6x/common_task_install_elk_repo.yml b/elk_metrics_6x/roles/elastic_repositories/tasks/elastic_apt_repos.yml similarity index 84% rename from elk_metrics_6x/common_task_install_elk_repo.yml rename to elk_metrics_6x/roles/elastic_repositories/tasks/elastic_apt_repos.yml index 774e5ae6..9bbd6389 100644 --- a/elk_metrics_6x/common_task_install_elk_repo.yml +++ b/elk_metrics_6x/roles/elastic_repositories/tasks/elastic_apt_repos.yml @@ -24,18 +24,6 @@ tags: - package_install -- name: enable apt-transport-https - apt: - name: apt-transport-https - state: present - update_cache: yes - register: _apt_task - until: _apt_task is success - retries: 3 - delay: 2 - tags: - - package_install - - name: add elk repo to apt sources list apt_repository: repo: "{{ elk_repo.repo }}" diff --git a/elk_metrics_6x/roles/elastic_repositories/tasks/main.yml b/elk_metrics_6x/roles/elastic_repositories/tasks/main.yml new file mode 100644 index 00000000..0708ca46 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_repositories/tasks/main.yml @@ -0,0 +1,40 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather variables for each operating system + include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml" + - "{{ ansible_os_family | lower }}.yml" + tags: + - always + +- name: Install repo disto packages + package: + name: "{{ elastic_repo_distro_packages }}" + state: present + update_cache: yes + register: _apt_task + until: _apt_task is success + retries: 3 + delay: 2 + tags: + - package_install + +- include_tasks: "elastic_{{ ansible_pkg_mgr }}_repos.yml" diff --git a/elk_metrics_6x/roles/elastic_repositories/vars/ubuntu.yml b/elk_metrics_6x/roles/elastic_repositories/vars/ubuntu.yml new file mode 100644 index 00000000..8752df14 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_repositories/vars/ubuntu.yml @@ -0,0 +1,17 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +elastic_repo_distro_packages: + - apt-transport-https diff --git a/elk_metrics_6x/roles/elastic_retention/defaults/main.yml b/elk_metrics_6x/roles/elastic_retention/defaults/main.yml new file mode 100644 index 00000000..a3ae847f --- /dev/null +++ b/elk_metrics_6x/roles/elastic_retention/defaults/main.yml @@ -0,0 +1,16 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +elastic_index_retention_algorithm: default diff --git a/elk_metrics_6x/roles/elastic_retention/meta/main.yml b/elk_metrics_6x/roles/elastic_retention/meta/main.yml new file mode 100644 index 00000000..f360643a --- /dev/null +++ b/elk_metrics_6x/roles/elastic_retention/meta/main.yml @@ -0,0 +1,32 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v6.x retention role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: + - role: elastic_data_hosts diff --git a/elk_metrics_6x/roles/elastic_retention/tasks/main.yml b/elk_metrics_6x/roles/elastic_retention/tasks/main.yml new file mode 100644 index 00000000..3b7a52d3 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_retention/tasks/main.yml @@ -0,0 +1,35 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Query es storage + uri: + url: "http://{{ coordination_nodes[0] }}/_nodes/{{ (data_nodes | map('extract', hostvars, 'ansible_host') | list) | join(',') }}/stats/fs" + method: GET + register: elk_data + until: elk_data is success + retries: 5 + delay: 5 + run_once: true + +- name: Load data node variables + include_vars: "calculate_index_retention_{{ elastic_index_retention_algorithm }}.yml" + tags: + - always + +- name: Set retention facts + set_fact: "elastic_{{ item.key }}_retention={{ (es_assumed_usable_storage_per_node | int) // ((item.value | int) * (es_storage_multiplier | int)) }}" + when: + - hostvars[inventory_hostname]["elastic_" + item.key + "_retention"] is undefined + with_dict: "{{ elastic_beat_retention_policy_hosts }}" diff --git a/elk_metrics_6x/roles/elastic_retention/vars/calculate_index_retention_default.yml b/elk_metrics_6x/roles/elastic_retention/vars/calculate_index_retention_default.yml new file mode 100644 index 00000000..4a060a8a --- /dev/null +++ b/elk_metrics_6x/roles/elastic_retention/vars/calculate_index_retention_default.yml @@ -0,0 +1,30 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Set available storage fact +es_total_available_storage: "{{ ((elk_data['json']['nodes'].values() | list) | map(attribute='fs.total.total_in_bytes') | list | sum) // 1024 // 1024 }}" + +# Set assumed buffer storage fact +es_assumed_buffer_storage: "{{ ((es_total_available_storage | int) * 0.25) | round | int }}" + +# Set usable buffer storage fact(s) +es_usable_buffer_storage: "{{ (es_total_available_storage | int) - (es_assumed_buffer_storage | int) }}" +es_expected_storage: "{{ ((elastic_beat_retention_policy_hosts.values() | map('int') | list) | sum) * (elastic_beat_storage_constant | int) }}" + +# Set buffer storage fact +es_assumed_usable_storage_per_node: "{{ (es_usable_buffer_storage | int) // (data_nodes | length | int) }}" + +# Set storage the mulitplier +es_storage_multiplier: "{{ ((es_usable_buffer_storage | int) < (es_expected_storage | int)) | ternary(((elastic_beat_storage_constant | int) * 2), elastic_beat_storage_constant | int) }}" diff --git a/elk_metrics_6x/roles/elastic_rollup/defaults/main.yml b/elk_metrics_6x/roles/elastic_rollup/defaults/main.yml new file mode 100644 index 00000000..f72162fd --- /dev/null +++ b/elk_metrics_6x/roles/elastic_rollup/defaults/main.yml @@ -0,0 +1,16 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +elastic_allow_rollup_purge: false diff --git a/elk_metrics_6x/roles/elastic_rollup/meta/main.yml b/elk_metrics_6x/roles/elastic_rollup/meta/main.yml new file mode 100644 index 00000000..2a2aed2c --- /dev/null +++ b/elk_metrics_6x/roles/elastic_rollup/meta/main.yml @@ -0,0 +1,32 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v6.x rollup role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: + - role: elastic_retention diff --git a/elk_metrics_6x/roles/elastic_rollup/tasks/main.yml b/elk_metrics_6x/roles/elastic_rollup/tasks/main.yml new file mode 100644 index 00000000..0ce194d9 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_rollup/tasks/main.yml @@ -0,0 +1,96 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Get rollup job + uri: + url: "http://{{ coordination_nodes[0] }}/_xpack/rollup/job/rollup_{{ index_name }}" + method: GET + register: check_rollup + until: check_rollup is success + retries: 3 + delay: 5 + run_once: true + +- name: Check for existing rollup job + debug: + msg: >- + An existing rollup job was found for {{ index_name }}. In order to + re-create this rollup job the old job will need to be purged. If you're + OK with the old rollup job being purged, add the following option + `elastic_allow_rollup_purge=yes` to the command line and rerun the + playbook. + when: + - check_rollup['json']['jobs'] | length > 0 + - not elastic_allow_rollup_purge | bool + +- name: Create rollup block + block: + - name: Set retention days fact + set_fact: + days_until_rollup: |- + {% set index_retention = [] %} + {% for item in ansible_play_hosts %} + {% set _ = index_retention.append((hostvars[item]['elastic_' + index_name + '_retention'] | int) // 3) %} + {% endfor %} + {{ index_retention | min }} + run_once: true + + - name: Create rollup job + uri: + url: "{{ item.url }}" + method: "{{ item.method }}" + body: "{{ item.index_options | to_json }}" + status_code: "{{ item.status_code }}" + body_format: json + register: elk_indexes + until: elk_indexes is success + retries: 3 + delay: 5 + when: + - hostvars[inventory_hostname]['elastic_' + index_name + '_retention'] > days_until_rollup + with_items: + - url: "http://{{ coordination_nodes[0] }}/_xpack/rollup/job/rollup_{{ index_name }}/_stop" + method: POST + status_code: 200,404 + index_options: {} + - url: "http://{{ coordination_nodes[0] }}/_xpack/rollup/job/rollup_{{ index_name }}" + method: DELETE + status_code: 200,404 + index_options: {} + - url: "http://{{ coordination_nodes[0] }}/rollup_{{ index_name }}" + method: DELETE + status_code: 200,404 + index_options: {} + - url: "http://{{ coordination_nodes[0] }}/_xpack/rollup/job/rollup_{{ index_name }}" + method: PUT + status_code: 200 + index_options: + index_pattern: "{{ index_name }}-*" + rollup_index: "rollup_{{ index_name }}" + cron: "*/30 * * * * ?" + page_size: 1000 + groups: + date_histogram: + field: "@timestamp" + interval: "1h" + delay: "{{ days_until_rollup }}d" + - url: "http://{{ coordination_nodes[0] }}/_xpack/rollup/job/rollup_{{ index_name }}/_start" + method: POST + status_code: 200 + index_options: {} + run_once: true + when: + - check_rollup['json']['jobs'] | length < 1 or + elastic_allow_rollup_purge | bool diff --git a/elk_metrics_6x/roles/elasticsearch/defaults/main.yml b/elk_metrics_6x/roles/elasticsearch/defaults/main.yml new file mode 100644 index 00000000..aedbb705 --- /dev/null +++ b/elk_metrics_6x/roles/elasticsearch/defaults/main.yml @@ -0,0 +1,19 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +elastic_log_rotate_path: "/var/log/elasticsearch" + +temp_dir: /var/lib/elasticsearch/tmp +nfs_query: "[?fstype=='nfs' || fstype=='nfs4']" diff --git a/elk_metrics_6x/roles/elasticsearch/handlers/main.yml b/elk_metrics_6x/roles/elasticsearch/handlers/main.yml new file mode 100644 index 00000000..f1403d58 --- /dev/null +++ b/elk_metrics_6x/roles/elasticsearch/handlers/main.yml @@ -0,0 +1,25 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Enable and restart elastic + systemd: + name: "elasticsearch" + enabled: true + state: restarted + daemon_reload: true + when: + - (elk_package_state | default('present')) != 'absent' + tags: + - config diff --git a/elk_metrics_6x/roles/elasticsearch/meta/main.yml b/elk_metrics_6x/roles/elasticsearch/meta/main.yml new file mode 100644 index 00000000..6010c208 --- /dev/null +++ b/elk_metrics_6x/roles/elasticsearch/meta/main.yml @@ -0,0 +1,37 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v6.x elasticsearch role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: + - role: elastic_data_hosts + - role: elastic_repositories + - role: elastic_dependencies + service_name: elasticsearch + service_owner: elasticsearch + service_group: elasticsearch diff --git a/elk_metrics_6x/roles/elasticsearch/tasks/elasticsearch_nfs_setup.yml b/elk_metrics_6x/roles/elasticsearch/tasks/elasticsearch_nfs_setup.yml new file mode 100644 index 00000000..2234f482 --- /dev/null +++ b/elk_metrics_6x/roles/elasticsearch/tasks/elasticsearch_nfs_setup.yml @@ -0,0 +1,29 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Ensure nfs client is available if necessary + package: + name: nfs-common + state: present + +- name: Ensure backup filesystems are mounted + mount: + fstype: "{{ item.fstype }}" + src: "{{ item.src }}" + opts: "{{ item.opts }}" + path: "{{ item.path }}" + state: "{{ item.state }}" + with_items: + - "{{ elastic_shared_fs_repos }}" diff --git a/elk_metrics_6x/roles/elasticsearch/tasks/main.yml b/elk_metrics_6x/roles/elasticsearch/tasks/main.yml new file mode 100644 index 00000000..b650f9ec --- /dev/null +++ b/elk_metrics_6x/roles/elasticsearch/tasks/main.yml @@ -0,0 +1,102 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather variables for each operating system + include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml" + - "{{ ansible_os_family | lower }}.yml" + tags: + - always + +- name: Set elasticsearch variables + include_vars: "vars_{{ ((inventory_hostname in (groups['kibana'] | default([])) and not inventory_hostname in (groups['elastic-logstash']) | default([]))) | ternary('kibana', 'default') }}.yml" + tags: + - always + +- name: Ensure elasticsearch is installed + package: + name: "{{ elasticsearch_distro_packages }}" + state: "{{ elk_package_state | default('present') }}" + update_cache: yes + register: _package_task + until: _package_task is success + retries: 3 + delay: 2 + notify: + - Enable and restart elastic + tags: + - package_install + +- name: Create elasticsearch systemd service config dir + file: + path: "/etc/systemd/system/elasticsearch.service.d" + state: "directory" + group: "root" + owner: "root" + mode: "0755" + +- name: Drop elasticsearch conf file + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + with_items: + - src: templates/elasticsearch.yml.j2 + dest: /etc/elasticsearch/elasticsearch.yml + - src: templates/jvm.options.j2 + dest: /etc/elasticsearch/jvm.options + - src: templates/es-log4j2.properties.j2 + dest: /etc/elasticsearch/log4j2.properties + - src: "systemd.elasticsearch-overrides.conf.j2" + dest: "/etc/systemd/system/elasticsearch.service.d/elasticsearch-overrides.conf" + notify: + - Enable and restart elastic + tags: + - config + +- name: Ensure elasticsearch ownership + file: + path: "/var/lib/elasticsearch/" + owner: elasticsearch + group: elasticsearch + recurse: true + register: e_perms + until: e_perms is success + retries: 3 + delay: 1 + tags: + - config + +- name: Ensure elasticsearch tmp dir + file: + path: "/var/lib/elasticsearch/tmp" + state: directory + owner: "elasticsearch" + group: "elasticsearch" + mode: "0750" + +- name: Create logrotate config + template: + src: "templates/logrotate.j2" + dest: "/etc/logrotate.d/elasticsearch" + +- include_tasks: "elasticsearch_nfs_setup.yml" + when: + - elastic_shared_fs_repos is defined + - (elastic_shared_fs_repos | json_query(nfs_query)) | length > 0 diff --git a/elk_metrics_6x/roles/elasticsearch/vars/ubuntu.yml b/elk_metrics_6x/roles/elasticsearch/vars/ubuntu.yml new file mode 100644 index 00000000..a5948d12 --- /dev/null +++ b/elk_metrics_6x/roles/elasticsearch/vars/ubuntu.yml @@ -0,0 +1,18 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +elasticsearch_distro_packages: + - logrotate + - elasticsearch diff --git a/elk_metrics_6x/roles/elasticsearch/vars/vars_default.yml b/elk_metrics_6x/roles/elasticsearch/vars/vars_default.yml new file mode 100644 index 00000000..147f27f2 --- /dev/null +++ b/elk_metrics_6x/roles/elasticsearch/vars/vars_default.yml @@ -0,0 +1,17 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Set elasticsearch facts +elastic_heap_size: "{{ elastic_heap_size_default }}" diff --git a/elk_metrics_6x/roles/elasticsearch/vars/vars_kibana.yml b/elk_metrics_6x/roles/elasticsearch/vars/vars_kibana.yml new file mode 100644 index 00000000..65056b9b --- /dev/null +++ b/elk_metrics_6x/roles/elasticsearch/vars/vars_kibana.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Set kibana as elasticsearch coordinators +elasticsearch_node_master: false +elasticsearch_node_data: false +elasticsearch_node_ingest: false +elastic_coordination_node: true +elastic_heap_size: "{{ (elastic_heap_size_default | int) // 3 }}" +elastic_thread_pool_size: "{{ ((ansible_processor_cores | int) > 4) | ternary(4, (ansible_processor_cores // 2)) }}" diff --git a/elk_metrics_6x/roles/go_install_1.10/handlers/main.yml b/elk_metrics_6x/roles/go_install_1.10/handlers/main.yml new file mode 100644 index 00000000..4bc61095 --- /dev/null +++ b/elk_metrics_6x/roles/go_install_1.10/handlers/main.yml @@ -0,0 +1,14 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/elk_metrics_6x/roles/go_install_1.10/meta/main.yml b/elk_metrics_6x/roles/go_install_1.10/meta/main.yml new file mode 100644 index 00000000..3998985c --- /dev/null +++ b/elk_metrics_6x/roles/go_install_1.10/meta/main.yml @@ -0,0 +1,31 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v6.x go install role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: [] diff --git a/elk_metrics_6x/roles/go_install_1.10/tasks/go_install.yml b/elk_metrics_6x/roles/go_install_1.10/tasks/go_install.yml new file mode 100644 index 00000000..904bb619 --- /dev/null +++ b/elk_metrics_6x/roles/go_install_1.10/tasks/go_install.yml @@ -0,0 +1,45 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: GET go + get_url: + url: "https://dl.google.com/go/go1.10.1.linux-amd64.tar.gz" + dest: "/opt/go1.10.1.linux-amd64.tar.gz" + checksum: "sha256:72d820dec546752e5a8303b33b009079c15c2390ce76d67cf514991646c6127b" + register: _get_task + until: _get_task is success + retries: 3 + delay: 2 + tags: + - package_install + +- name: Create go directory + file: + path: "/opt/go1.10.1" + state: directory + +- name: Unarchive go + unarchive: + src: "/opt/go1.10.1.linux-amd64.tar.gz" + dest: "/opt/go1.10.1" + remote_src: yes + +- name: Create go defaults file + copy: + content: | + GOROOT=/opt/go1.10.1/go + GOPATH=/usr/local + PATH=${PATH}:${GOROOT}/bin + dest: /etc/default/go1.10.1 diff --git a/elk_metrics_6x/roles/go_install_1.10/tasks/main.yml b/elk_metrics_6x/roles/go_install_1.10/tasks/main.yml new file mode 100644 index 00000000..c7940ca1 --- /dev/null +++ b/elk_metrics_6x/roles/go_install_1.10/tasks/main.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Check for go + stat: + path: /opt/go1.10.1/go/bin/go + register: go_path + +- include_tasks: go_install.yml + when: + - not go_path.stat.exists | bool diff --git a/elk_metrics_6x/setupAPMserver.yml b/elk_metrics_6x/setupAPMserver.yml deleted file mode 100644 index ba3f76f1..00000000 --- a/elk_metrics_6x/setupAPMserver.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- - -- name: Load apm-server Dashboards - hosts: apm-server[0] - gather_facts: true - vars_files: - - vars/variables.yml - - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml - tags: - - always - - tasks: - - name: Load templates - shell: >- - apm-server setup - {{ item }} - -E 'apm-server.host=localhost:8200' - -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' - -e -v - with_items: - - "--template" - - "--dashboards" - register: templates - until: templates is success - retries: 5 - delay: 5 - - tags: - - beat-setup diff --git a/elk_metrics_6x/setupAuditbeat.yml b/elk_metrics_6x/setupAuditbeat.yml deleted file mode 100644 index e5e100cc..00000000 --- a/elk_metrics_6x/setupAuditbeat.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- - -- name: Load Auditbeat Dashboards - hosts: hosts[0] - gather_facts: true - vars_files: - - vars/variables.yml - - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml - tags: - - always - - tasks: - - name: Load templates - shell: >- - auditbeat setup - {{ item }} - -E 'output.logstash.enabled=false' - -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' - -e -v - with_items: - - "--template" - - "--dashboards" - register: templates - until: templates is success - retries: 5 - delay: 5 - - tags: - - beat-setup diff --git a/elk_metrics_6x/setupFilebeat.yml b/elk_metrics_6x/setupFilebeat.yml deleted file mode 100644 index ec67d7e0..00000000 --- a/elk_metrics_6x/setupFilebeat.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- - -- name: Load Filebeat Dashboards - hosts: hosts[0] - gather_facts: true - vars_files: - - vars/variables.yml - - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml - tags: - - always - - tasks: - - name: Load templates - shell: >- - filebeat setup - {{ item }} - -E 'output.logstash.enabled=false' - -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' - -e -v - with_items: - - "--template" - - "--dashboards" - register: templates - until: templates is success - retries: 5 - delay: 5 - - tags: - - beat-setup diff --git a/elk_metrics_6x/setupHeartbeat.yml b/elk_metrics_6x/setupHeartbeat.yml deleted file mode 100644 index 9d648867..00000000 --- a/elk_metrics_6x/setupHeartbeat.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- - -- name: Load Heartbeat Dashboards - hosts: kibana[0] - gather_facts: true - vars_files: - - vars/variables.yml - - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml - tags: - - always - - tasks: - - name: Load templates - shell: >- - heartbeat setup - {{ item }} - -E 'output.logstash.enabled=false' - -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' - -e -v - with_items: - - "--template" - - "--dashboards" - register: templates - until: templates is success - retries: 5 - delay: 5 - - tags: - - beat-setup diff --git a/elk_metrics_6x/setupJournalbeat.yml b/elk_metrics_6x/setupJournalbeat.yml deleted file mode 100644 index ecf84f17..00000000 --- a/elk_metrics_6x/setupJournalbeat.yml +++ /dev/null @@ -1,47 +0,0 @@ ---- -# Copyright 2018, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Load Journalbeat Dashboards - hosts: hosts[0] - gather_facts: true - vars_files: - - vars/variables.yml - - pre_tasks: - # NOTE(cloudnull): This task is broken at this point due to missing - # configuration. Once the following issue - # [ https://github.com/mheese/journalbeat/issues/136 ] is - # resolved, this should be removed. - - name: exit playbook - meta: end_play - - - include_tasks: common_task_data_node_hosts.yml - tags: - - always - - tasks: - - name: Load templates - shell: >- - /usr/local/bin/journalbeat -setup - -E 'output.logstash.enabled=false' - -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' - -e -v - register: templates - until: templates is success - retries: 5 - delay: 5 - - tags: - - beat-setup diff --git a/elk_metrics_6x/setupKibanaDashboard.yml b/elk_metrics_6x/setupKibanaDashboard.yml index 47d6c3f6..bc9fabeb 100644 --- a/elk_metrics_6x/setupKibanaDashboard.yml +++ b/elk_metrics_6x/setupKibanaDashboard.yml @@ -1,4 +1,16 @@ --- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + - name: Install Kibana Dashboards hosts: "elastic-logstash" become: true diff --git a/elk_metrics_6x/setupMetricbeat.yml b/elk_metrics_6x/setupMetricbeat.yml deleted file mode 100644 index 302b9321..00000000 --- a/elk_metrics_6x/setupMetricbeat.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- - -- name: Load Metricsbeat Dashboards - hosts: all[0] - gather_facts: true - vars_files: - - vars/variables.yml - - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml - tags: - - always - - tasks: - - name: Load templates - shell: >- - metricbeat setup - {{ item }} - -E 'output.logstash.enabled=false' - -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' - -e -v - with_items: - - "--template" - - "--dashboards" - register: templates - until: templates is success - retries: 5 - delay: 5 - - tags: - - beat-setup diff --git a/elk_metrics_6x/setupPacketbeat.yml b/elk_metrics_6x/setupPacketbeat.yml deleted file mode 100644 index 8312a34e..00000000 --- a/elk_metrics_6x/setupPacketbeat.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- - -- name: Load Packetbeat Dashboards - hosts: hosts[0] - gather_facts: true - vars_files: - - vars/variables.yml - - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml - tags: - - always - - tasks: - - name: Load templates - shell: >- - packetbeat setup - {{ item }} - -E 'output.logstash.enabled=false' - -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' - -e -v - with_items: - - "--template" - - "--dashboards" - register: templates - until: templates is success - retries: 5 - delay: 5 - - tags: - - beat-setup diff --git a/elk_metrics_6x/showElasticCluster.yml b/elk_metrics_6x/showElasticCluster.yml index 0ecaa29e..5bf38ca6 100644 --- a/elk_metrics_6x/showElasticCluster.yml +++ b/elk_metrics_6x/showElasticCluster.yml @@ -1,4 +1,16 @@ --- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + - name: Dump cluster node info hosts: elastic-logstash:kibana become: true @@ -8,11 +20,11 @@ environment: "{{ deployment_environment_variables | default({}) }}" - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml + roles: + - role: elastic_data_hosts # display overall cluster setup - tasks: + post_tasks: - block: - name: "Report number of storage nodes" debug: diff --git a/elk_metrics_6x/site-beats-community.yml b/elk_metrics_6x/site-beats-community.yml index 6eb444ff..86891276 100644 --- a/elk_metrics_6x/site-beats-community.yml +++ b/elk_metrics_6x/site-beats-community.yml @@ -1,6 +1,4 @@ --- -# Copyright 2018, Rackspace US, Inc. -# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/elk_metrics_6x/site-beats-core.yml b/elk_metrics_6x/site-beats-core.yml index b506abd5..266ab657 100644 --- a/elk_metrics_6x/site-beats-core.yml +++ b/elk_metrics_6x/site-beats-core.yml @@ -1,6 +1,4 @@ --- -# Copyright 2018, Rackspace US, Inc. -# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/elk_metrics_6x/site-elka.yml b/elk_metrics_6x/site-elka.yml index 7052a204..bc02fce9 100644 --- a/elk_metrics_6x/site-elka.yml +++ b/elk_metrics_6x/site-elka.yml @@ -1,6 +1,4 @@ --- -# Copyright 2018, Rackspace US, Inc. -# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/elk_metrics_6x/site.yml b/elk_metrics_6x/site.yml index f820a587..01b23aa1 100644 --- a/elk_metrics_6x/site.yml +++ b/elk_metrics_6x/site.yml @@ -1,6 +1,4 @@ --- -# Copyright 2018, Rackspace US, Inc. -# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/elk_metrics_6x/templates/jvm.options.j2 b/elk_metrics_6x/templates/jvm.options.j2 index 712699c7..c0130f32 100644 --- a/elk_metrics_6x/templates/jvm.options.j2 +++ b/elk_metrics_6x/templates/jvm.options.j2 @@ -1,8 +1,8 @@ ## JVM configuration {% if (not (elasticsearch_node_master | default(master_node)) | bool) and (not (elasticsearch_node_data | default(data_node)) | bool) %} -{% set heap_size = (elastic_heap_size | int) // 2 %} +{% set heap_size = (elastic_heap_size_default | int) // 2 %} {% else %} -{% set heap_size = (elastic_heap_size | int) %} +{% set heap_size = (elastic_heap_size_default | int) %} {% endif %} # Xms represents the initial size of total heap space -Xms{{ heap_size }}m diff --git a/elk_metrics_6x/tests/testLayout.yml b/elk_metrics_6x/tests/testLayout.yml index 629381c9..418a3457 100644 --- a/elk_metrics_6x/tests/testLayout.yml +++ b/elk_metrics_6x/tests/testLayout.yml @@ -1,4 +1,15 @@ --- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. - import_playbook: run-setup.yml