openstack-ansible-ops/elk_metrics_6x/bootstrap-embedded-ansible.sh
Kevin Carter 0d4a4a92c7
Converg the logstash pipelines and enhance memory backed queues
The multi-logstash pipeline setup, while amazingly fast, was crashing
and causing index errors when under high load for a long period of time.
Because of the crashing behavior and the fact that the folks from
Elastic describe multi-pipeline queues to be "beta" at this time the
logstash pipelines have been converted back into a single pipeline.

The memory backed queue options are now limited by a ram disk (tmpfs)
which will ensure that a burst within the queue does not cause OOM
issues and ensures a highly performant deployment and limiting memory
usage at the same time. Memory backed queues will be enabled when the
underlying system is using "rotational" media as detected by ansible
facts. This will ensure a fast and consistent experience across all
deployment types.

Pipeline/ml/template/dashboard setup has been added to the beat
configurations which will ensure beats are properly configured even
when running in an isolated deployment and outside of normal operations
where beats are generally configured on the first data node.

Change-Id: Ie3c775f98b14f71bcbed05db9cb1c5aa46d9c436
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>
2018-09-16 23:44:58 -05:00

128 lines
5.1 KiB
Bash
Executable File

#!/usr/bin/env bash
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
export OPTS=()
export ANSIBLE_EMBED_HOME="${HOME}/ansible25"
OPTS+=('ANSIBLE_EMBED_HOME')
source /etc/os-release
if [[ ! -e "${ANSIBLE_EMBED_HOME}/bin/ansible" ]]; then
if [ ${ID} = "ubuntu" ]; then
apt-get update
apt-get -y install python-virtualenv
elif [ ${ID} = "opensuse" ]; then
zypper install -y python-virtualenv
elif [ ${ID} = "centos" ] || [ ${ID} ="redhat" ]; then
yum install -y python-virtualenv
else
echo "Unknown operating system"
exit 99
fi
echo "done installing python-virtualenv"
if [[ -f "/usr/bin/python2" ]]; then
virtualenv --python="/usr/bin/python2" "${ANSIBLE_EMBED_HOME}"
elif [[ -f "/usr/bin/python3" ]]; then
virtualenv --python="/usr/bin/python3" "${ANSIBLE_EMBED_HOME}"
else
virtualenv "${ANSIBLE_EMBED_HOME}"
fi
eval "${ANSIBLE_EMBED_HOME}/bin/pip install --upgrade --force pip"
eval "${ANSIBLE_EMBED_HOME}/bin/pip install --upgrade ansible==2.5.5.0 --isolated"
eval "${ANSIBLE_EMBED_HOME}/bin/pip install --upgrade jmespath --isolated"
echo "Ansible can be found here: ${ANSIBLE_EMBED_HOME}/bin"
fi
if [[ ! -d "${ANSIBLE_EMBED_HOME}/repositories/ansible-config_template" ]]; then
mkdir -p "${ANSIBLE_EMBED_HOME}/repositories"
git clone https://git.openstack.org/openstack/ansible-config_template "${ANSIBLE_EMBED_HOME}/repositories/ansible-config_template"
pushd "${ANSIBLE_EMBED_HOME}/repositories/ansible-config_template"
git checkout a5c9d97e18683f0fdf9769d94ba174c72e2d093c # HEAD of master from 20-06-18
popd
fi
if [[ ! -d "${ANSIBLE_EMBED_HOME}/repositories/openstack_ansible_plugins" ]]; then
mkdir -p "${ANSIBLE_EMBED_HOME}/repositories"
git clone https://git.openstack.org/openstack/openstack-ansible-plugins "${ANSIBLE_EMBED_HOME}/repositories/openstack-ansible-plugins"
pushd "${ANSIBLE_EMBED_HOME}/repositories/openstack-ansible-plugins"
git checkout cef7946b3b3b3e4d02406c228741985a94b72cff # HEAD of master from 20-06-18
popd
fi
if [[ ! -d "${ANSIBLE_EMBED_HOME}/repositories/roles/systemd_service" ]]; then
mkdir -p "${ANSIBLE_EMBED_HOME}/repositories"
git clone https://git.openstack.org/openstack/ansible-role-systemd_service "${ANSIBLE_EMBED_HOME}/repositories/roles/systemd_service"
pushd "${ANSIBLE_EMBED_HOME}/repositories/roles/systemd_service"
git checkout 02f5ff1c0e073af53bed2141a045e608162970ea # HEAD of master from 20-06-18
popd
fi
if [[ ! -d "${ANSIBLE_EMBED_HOME}/repositories/roles/systemd_mount" ]]; then
mkdir -p "${ANSIBLE_EMBED_HOME}/repositories"
git clone https://git.openstack.org/openstack/ansible-role-systemd_mount "${ANSIBLE_EMBED_HOME}/repositories/roles/systemd_mount"
pushd "${ANSIBLE_EMBED_HOME}/repositories/roles/systemd_mount"
git checkout 0cca0b06e20a4e3d2b6b4ca19172717b6b37b68a # HEAD of master from 20-06-18
popd
fi
if [[ -f "/etc/openstack_deploy/openstack_inventory.json" ]]; then
if [[ ! -f "${ANSIBLE_EMBED_HOME}/inventory/openstack_inventory.sh" ]]; then
mkdir -p "${ANSIBLE_EMBED_HOME}/inventory"
cat > "${ANSIBLE_EMBED_HOME}/inventory/openstack_inventory.sh" <<EOF
#!/usr/bin/env bash
cat /etc/openstack_deploy/openstack_inventory.json
EOF
chmod +x "${ANSIBLE_EMBED_HOME}/inventory/openstack_inventory.sh"
fi
export USER_VARS="$(for i in $(ls -1 /etc/openstack_deploy/user_*secret*.yml); do echo -n "-e@$i "; done)"
OPTS+=('USER_VARS')
echo "env USER_VARS set"
echo "Extra users variables can be expanded by including the option \$USER_VARS on a playbook run."
export ANSIBLE_INVENTORY="${ANSIBLE_EMBED_HOME}/inventory/openstack_inventory.sh"
OPTS+=('ANSIBLE_INVENTORY')
echo "env ANSIBLE_INVENTORY set"
fi
export ANSIBLE_HOST_KEY_CHECKING="False"
OPTS+=('ANSIBLE_HOST_KEY_CHECKING')
echo "env ANSIBLE_HOST_KEY_CHECKING set"
export ANSIBLE_ROLES_PATH="${ANSIBLE_EMBED_HOME}/repositories/roles"
OPTS+=('ANSIBLE_ROLES_PATH')
echo "env ANSIBLE_ROLES_PATH set"
export ANSIBLE_ACTION_PLUGINS="${ANSIBLE_EMBED_HOME}/repositories/ansible-config_template/action"
OPTS+=('ANSIBLE_ACTION_PLUGINS')
echo "env ANSIBLE_ACTION_PLUGINS set"
export ANSIBLE_CONNECTION_PLUGINS="${ANSIBLE_EMBED_HOME}/repositories/openstack-ansible-plugins/connection/"
OPTS+=('ANSIBLE_CONNECTION_PLUGINS')
echo "env ANSIBLE_CONNECTION_PLUGINS set"
source ${ANSIBLE_EMBED_HOME}/bin/activate
echo "Embedded Ansible has been activated. Run 'deactivate' to leave the embedded environment".
function deactivate_embedded_venv {
deactivate
for i in ${OPTS[@]}; do
unset ${i}
done
unset deactivate_embedded_venv
unalias deactivate
}
alias deactivate=deactivate_embedded_venv