From 573ac49939031aba0db6de222252b77a64a31d9b Mon Sep 17 00:00:00 2001 From: "Huang, Sophie (sh879n)" Date: Tue, 5 May 2020 22:37:13 +0000 Subject: [PATCH] Mariadb backup/restore enhancements Below enhancements are made to Mariadb backup: 1) Used new helm-toolkit function to send/retrieve Mariadb backups to/from RGW via OpenStack Swift API. 2) Modified the backup script such that the database backup tarball can be sent to RGW. 3) Added a keystone user for RGW access. 4) Added a secret for OpenStack Swift API access. 5) Changed the cronjob image and runAsUser 6) Modified the restore script so that archives stored remotely on RGW can be used for the restore data source. 7) Added functions to the restore script to retrieve data from an archive for tables, table rows and table schema of a databse 8) Added a secret containing all the backup/restore related configuration needed for invoking the backup/restore operation from a different application or namespace. Change-Id: Iadb9438fe419cded374897b43337039609077e61 --- mariadb/templates/bin/_backup_mariadb.sh.tpl | 156 +++---- mariadb/templates/bin/_restore_mariadb.sh.tpl | 409 ++++++++---------- mariadb/templates/configmap-bin.yaml | 8 + .../templates/cron-job-backup-mariadb.yaml | 51 ++- mariadb/templates/job-ks-user.yaml | 20 + mariadb/templates/secret-backup-restore.yaml | 27 ++ mariadb/templates/secret-rgw.yaml | 78 ++++ mariadb/values.yaml | 89 +++- 8 files changed, 504 insertions(+), 334 deletions(-) create mode 100644 mariadb/templates/job-ks-user.yaml create mode 100644 mariadb/templates/secret-backup-restore.yaml create mode 100644 mariadb/templates/secret-rgw.yaml diff --git a/mariadb/templates/bin/_backup_mariadb.sh.tpl b/mariadb/templates/bin/_backup_mariadb.sh.tpl index b0bea9b163..00517de17f 100644 --- a/mariadb/templates/bin/_backup_mariadb.sh.tpl +++ b/mariadb/templates/bin/_backup_mariadb.sh.tpl @@ -12,104 +12,76 @@ # License for the specific language governing permissions and limitations # under the License. set -x -BACKUPS_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/mariadb/current -ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/mariadb/archive -MYSQL="mysql \ - --defaults-file=/etc/mysql/admin_user.cnf \ - --connect-timeout 10" +source /tmp/backup_main.sh -MYSQLDUMP="mysqldump \ - --defaults-file=/etc/mysql/admin_user.cnf" +# Export the variables required by the framework +# Note: REMOTE_BACKUP_ENABLED, STORAGE_POLICY and CONTAINER_NAME are already +# exported. +export DB_NAMESPACE=${MARIADB_POD_NAMESPACE} +export DB_NAME="mariadb" +export LOCAL_DAYS_TO_KEEP=${MARIADB_LOCAL_BACKUP_DAYS_TO_KEEP} +export REMOTE_DAYS_TO_KEEP=${MARIADB_REMOTE_BACKUP_DAYS_TO_KEEP} +export ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${DB_NAMESPACE}/${DB_NAME}/archive -seconds_difference() { - archive_date=$( date --date="$1" +%s ) - if [ "$?" -ne 0 ] +# Dump all the database files to existing $TMP_DIR and save logs to $LOG_FILE +dump_databases_to_directory() { + TMP_DIR=$1 + LOG_FILE=$2 + + MYSQL="mysql \ + --defaults-file=/etc/mysql/admin_user.cnf \ + --connect-timeout 10" + + MYSQLDUMP="mysqldump \ + --defaults-file=/etc/mysql/admin_user.cnf" + + MYSQL_DBNAMES=( $($MYSQL --silent --skip-column-names -e \ + "show databases;" | \ + egrep -vi 'information_schema|performance_schema') ) + + #check if there is a database to backup, otherwise exit + if [[ -z "${MYSQL_DBNAMES// }" ]] then - second_delta=0 + log INFO "There is no database to backup" + return 0 fi - current_date=$( date +%s ) - second_delta=$(($current_date-$archive_date)) - if [ "$second_delta" -lt 0 ] + + #Create a list of Databases + printf "%s\n" "${MYSQL_DBNAMES[@]}" > $TMP_DIR/db.list + + #Retrieve and create the GRANT files per DB + for db in "${MYSQL_DBNAMES[@]}" + do + echo $($MYSQL --skip-column-names -e "select concat('show grants for ',user,';') \ + from mysql.db where ucase(db)=ucase('$db');") | \ + sed -r "s/show grants for ([a-zA-Z0-9_-]*)/show grants for '\1'/" | \ + $MYSQL --silent --skip-column-names 2>>$LOG_FILE > $TMP_DIR/${db}_grant.sql + if [ "$?" -eq 0 ] + then + sed -i 's/$/;/' $TMP_DIR/${db}_grant.sql + else + log ERROR "Failed to create GRANT files for ${db}" + fi + done + + #Dumping the database + DATE=$(date +'%Y-%m-%dT%H:%M:%SZ') + + SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all + TARBALL_FILE=${SQL_FILE}.${DATE}.tar.gz + + $MYSQLDUMP $MYSQL_BACKUP_MYSQLDUMP_OPTIONS "${MYSQL_DBNAMES[@]}" \ + > $TMP_DIR/${SQL_FILE}.sql 2>>$LOG_FILE + if [[ $? -eq 0 && -s $TMP_DIR/${SQL_FILE}.sql ]] then - second_delta=0 + log INFO "Databases dumped successfully." + return 0 + else + log ERROR "Backup failed and need attention." + return 1 fi - echo $second_delta } -DBNAME=( $($MYSQL --silent --skip-column-names -e \ - "show databases;" | \ - egrep -vi 'information_schema|performance_schema|mysql') ) - -#check if there is a database to backup, otherwise exit -if [[ -z "${DBNAME// }" ]] -then - echo "There is no database to backup" - exit 0 -fi - -#Create archive and backup directories. -mkdir -p $BACKUPS_DIR $ARCHIVE_DIR - -#Create a list of Databases -printf "%s\n" "${DBNAME[@]}" > $BACKUPS_DIR/db.list - -#Retrieve and create the GRANT files per DB -for db in "${DBNAME[@]}" -do - echo $($MYSQL --skip-column-names -e "select concat('show grants for ',user,';') \ - from mysql.db where ucase(db)=ucase('$db');") | \ - sed -r "s/show grants for ([a-zA-Z0-9_-]*)/show grants for '\1'/" | \ - $MYSQL --silent --skip-column-names 2>grant_err.log > $BACKUPS_DIR/${db}_grant.sql - if [ "$?" -eq 0 ] - then - sed -i 's/$/;/' $BACKUPS_DIR/${db}_grant.sql - else - cat grant_err.log - fi -done - -#Dumping the database -#DATE=$(date +"%Y_%m_%d_%H_%M_%S") -DATE=$(date +'%Y-%m-%dT%H:%M:%SZ') -$MYSQLDUMP $MYSQL_BACKUP_MYSQLDUMP_OPTIONS "${DBNAME[@]}" \ - > $BACKUPS_DIR/mariadb.all.sql 2>dberror.log -if [[ $? -eq 0 && -s $BACKUPS_DIR/mariadb.all.sql ]] -then - #Archive the current db files - pushd $BACKUPS_DIR 1>/dev/null - tar zcvf $ARCHIVE_DIR/mariadb.all.${DATE}.tar.gz * - ARCHIVE_RET=$? - popd 1>/dev/null -else - #TODO: This can be convert into mail alert of alert send to a monitoring system - echo "Backup failed and need attention." - cat dberror.log - exit 1 -fi - -#Remove the current backup -if [ -d $BACKUPS_DIR ] -then - rm -rf $BACKUPS_DIR/*.sql -fi - -#Only delete the old archive after a successful archive -if [ $ARCHIVE_RET -eq 0 ] - then - if [ "$MARIADB_BACKUP_DAYS_TO_KEEP" -gt 0 ] - then - echo "Deleting backups older than $MARIADB_BACKUP_DAYS_TO_KEEP days" - if [ -d $ARCHIVE_DIR ] - then - for archive_file in $(ls -1 $ARCHIVE_DIR/*.gz) - do - archive_date=$( echo $archive_file | awk -F/ '{print $NF}' | cut -d'.' -f 3) - if [ "$(seconds_difference $archive_date)" -gt "$(($MARIADB_BACKUP_DAYS_TO_KEEP*86400))" ] - then - rm -rf $archive_file - fi - done - fi - fi -fi +# Call main program to start the database backup +backup_databases diff --git a/mariadb/templates/bin/_restore_mariadb.sh.tpl b/mariadb/templates/bin/_restore_mariadb.sh.tpl index 3d4d2394ab..1e8841189c 100755 --- a/mariadb/templates/bin/_restore_mariadb.sh.tpl +++ b/mariadb/templates/bin/_restore_mariadb.sh.tpl @@ -12,22 +12,26 @@ # License for the specific language governing permissions and limitations # under the License. -log_error() { - echo $1 - exit 1 -} - -ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/mariadb/archive -RESTORE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/mariadb/restore +# Capture the user's command line arguments ARGS=("$@") + +if [[ -s /tmp/restore_main.sh ]]; then + source /tmp/restore_main.sh +else + echo "File /tmp/restore_main.sh does not exist." + exit 1 +fi + +# Export the variables needed by the framework +export DB_NAME="mariadb" +export DB_NAMESPACE=${MARIADB_POD_NAMESPACE} +export ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${DB_NAMESPACE}/${DB_NAME}/archive + RESTORE_USER='restoreuser' RESTORE_PW=$(pwgen 16 1) RESTORE_LOG='/tmp/restore_error.log' rm -f $RESTORE_LOG -#Create Restore Directory -mkdir -p $RESTORE_DIR - # This is for commands which require admin access MYSQL="mysql \ --defaults-file=/etc/mysql/admin_user.cnf \ @@ -42,67 +46,89 @@ RESTORE_CMD="mysql \ --host=$MARIADB_SERVER_SERVICE_HOST \ --connect-timeout 10" -#Delete file -delete_files() { - files_to_delete=("$@") - for f in "${files_to_delete[@]}" - do - if [ -f $f ] - then - rm -rf $f - fi - done -} - -#Display all archives -list_archives() { - if [ -d ${ARCHIVE_DIR} ] - then - archives=$(find ${ARCHIVE_DIR}/ -iname "*.gz" -print) - echo "All Archives" - echo "==================================" - for archive in $archives - do - echo $archive | cut -d '/' -f 8 - done - else - log_error "Archive directory is not available." - fi +# Get a single database data from the SQL file. +# $1 - database name +# $2 - sql file path +current_db_desc() { + PATTERN="-- Current Database:" + sed -n "/${PATTERN} \`$1\`/,/${PATTERN}/p" $2 } #Return all database from an archive get_databases() { - archive_file=$1 - if [ -e ${ARCHIVE_DIR}/${archive_file} ] + TMP_DIR=$1 + DB_FILE=$2 + + if [[ -e ${TMP_DIR}/db.list ]] then - files_to_purge=$(find $RESTORE_DIR/ -iname "*.sql" -print) - delete_files $files_to_purge - tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null - if [ -e ${RESTORE_DIR}/db.list ] - then - DBS=$(cat ${RESTORE_DIR}/db.list ) - else - DBS=" " - fi + DBS=$(cat ${TMP_DIR}/db.list ) else DBS=" " fi + + echo $DBS > $DB_FILE } -#Display all database from an archive -list_databases() { - archive_file=$1 - get_databases $archive_file - #echo $DBS - if [ -n "$DBS" ] - then - echo " " - echo "Databases in the archive $archive_file" - echo "=================================================================" - for db in $DBS - do - echo $db - done +# Extract all tables of a database from an archive and put them in the requested +# file. +get_tables() { + DATABASE=$1 + TMP_DIR=$2 + TABLE_FILE=$3 + + SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql + if [[ -e $TMP_DIR/$SQL_FILE ]]; then + current_db_desc ${DATABASE} ${TMP_DIR}/${SQL_FILE} \ + | grep "^CREATE TABLE" | awk -F '`' '{print $2}' \ + > $TABLE_FILE + else + # Error, cannot report the tables + echo "No SQL file found - cannot extract the tables" + return 1 + fi +} + +# Extract all rows in the given table of a database from an archive and put +# them in the requested file. +get_rows() { + DATABASE=$1 + TABLE=$2 + TMP_DIR=$3 + ROW_FILE=$4 + + SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql + if [[ -e $TMP_DIR/$SQL_FILE ]]; then + current_db_desc ${DATABASE} ${TMP_DIR}/${SQL_FILE} \ + | grep "INSERT INTO \`${TABLE}\` VALUES" > $ROW_FILE + else + # Error, cannot report the rows + echo "No SQL file found - cannot extract the rows" + return 1 + fi +} + +# Extract the schema for the given table in the given database belonging to +# the archive file found in the TMP_DIR. +get_schema() { + DATABASE=$1 + TABLE=$2 + TMP_DIR=$3 + SCHEMA_FILE=$4 + + SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql + if [[ -e $TMP_DIR/$SQL_FILE ]]; then + DB_FILE=$(mktemp -p /tmp) + current_db_desc ${DATABASE} ${TMP_DIR}/${SQL_FILE} > ${DB_FILE} + sed -n /'CREATE TABLE `'$TABLE'`'/,/'--'/p ${DB_FILE} > ${SCHEMA_FILE} + if [[ ! (-s ${SCHEMA_FILE}) ]]; then + sed -n /'CREATE TABLE IF NOT EXISTS `'$TABLE'`'/,/'--'/p ${DB_FILE} \ + > ${SCHEMA_FILE} + fi + rm -f ${DB_FILE} + else + # Error, cannot report the rows + echo "No SQL file found - cannot extract the schema" + return 1 fi } @@ -116,17 +142,19 @@ create_restore_user() { delete_restore_user "dont_exit_on_error" $MYSQL --execute="GRANT SELECT ON *.* TO ${RESTORE_USER}@'%' IDENTIFIED BY '${RESTORE_PW}';" 2>>$RESTORE_LOG - if [ "$?" -eq 0 ] + if [[ "$?" -eq 0 ]] then $MYSQL --execute="GRANT ALL ON ${restore_db}.* TO ${RESTORE_USER}@'%' IDENTIFIED BY '${RESTORE_PW}';" 2>>$RESTORE_LOG - if [ "$?" -ne 0 ] + if [[ "$?" -ne 0 ]] then cat $RESTORE_LOG - log_error "Failed to grant restore user ALL permissions on database ${restore_db}" + echo "Failed to grant restore user ALL permissions on database ${restore_db}" + return 1 fi else cat $RESTORE_LOG - log_error "Failed to grant restore user select permissions on all databases" + echo "Failed to grant restore user select permissions on all databases" + return 1 fi } @@ -135,208 +163,125 @@ delete_restore_user() { error_handling=$1 $MYSQL --execute="DROP USER ${RESTORE_USER}@'%';" 2>>$RESTORE_LOG - if [ "$?" -ne 0 ] + if [[ "$?" -ne 0 ]] then if [ "$error_handling" == "exit_on_error" ] then cat $RESTORE_LOG - log_error "Failed to delete temporary restore user - needs attention to avoid a security hole" + echo "Failed to delete temporary restore user - needs attention to avoid a security hole" + return 1 fi fi } #Restore a single database restore_single_db() { - single_db_name=$1 - if [ -z "$single_db_name" ] + SINGLE_DB_NAME=$1 + TMP_DIR=$2 + + if [[ -z "$SINGLE_DB_NAME" ]] then - log_error "Restore single DB called but with wrong parameter." + echo "Restore single DB called but with wrong parameter." + return 1 fi - if [ -f ${ARCHIVE_DIR}/${archive_file} ] + + SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql + if [[ -f ${TMP_DIR}/$SQL_FILE ]] then - files_to_purge=$(find $RESTORE_DIR/ -iname "*.sql" -print) - delete_files $files_to_purge - tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null - if [ -f ${RESTORE_DIR}/mariadb.all.sql ] + # Restoring a single database requires us to create a temporary user + # which has capability to only restore that ONE database. One gotcha + # is that the mysql command to restore the database is going to throw + # errors because of all the other databases that it cannot access. So + # because of this reason, the --force option is used to prevent the + # command from stopping on an error. + create_restore_user $SINGLE_DB_NAME + if [[ $? -ne 0 ]] then - # Restoring a single database requires us to create a temporary user - # which has capability to only restore that ONE database. One gotcha - # is that the mysql command to restore the database is going to throw - # errors because of all the other databases that it cannot access. So - # because of this reason, the --force option is used to prevent the - # command from stopping on an error. - create_restore_user $single_db_name - $RESTORE_CMD --force < ${RESTORE_DIR}/mariadb.all.sql 2>>$RESTORE_LOG - if [ "$?" -eq 0 ] + echo "Restore $SINGLE_DB_NAME failed create restore user." + return 1 + fi + $RESTORE_CMD --force < ${TMP_DIR}/$SQL_FILE 2>>$RESTORE_LOG + if [[ "$?" -eq 0 ]] + then + echo "Database $SINGLE_DB_NAME Restore successful." + else + cat $RESTORE_LOG + delete_restore_user "exit_on_error" + echo "Database $SINGLE_DB_NAME Restore failed." + return 1 + fi + delete_restore_user "exit_on_error" + if [[ $? -ne 0 ]] + then + echo "Restore $SINGLE_DB_NAME failed delete restore user." + return 1 + fi + if [ -f ${TMP_DIR}/${SINGLE_DB_NAME}_grant.sql ] + then + $MYSQL < ${TMP_DIR}/${SINGLE_DB_NAME}_grant.sql 2>>$RESTORE_LOG + if [[ "$?" -eq 0 ]] then - echo "Database $single_db_name Restore successful." + echo "Database $SINGLE_DB_NAME Permission Restore successful." else cat $RESTORE_LOG - delete_restore_user "exit_on_error" - log_error "Database $single_db_name Restore failed." - fi - delete_restore_user "exit_on_error" - - if [ -f ${RESTORE_DIR}/${single_db_name}_grant.sql ] - then - $MYSQL < ${RESTORE_DIR}/${single_db_name}_grant.sql 2>>$RESTORE_LOG - if [ "$?" -eq 0 ] - then - echo "Database $single_db_name Permission Restore successful." - else - cat $RESTORE_LOG - log_error "Database $single_db_name Permission Restore failed." - fi - else - log_error "There is no permission file available for $single_db_name" + echo "Database $SINGLE_DB_NAME Permission Restore failed." + return 1 fi else - log_error "There is no database file available to restore from" + echo "There is no permission file available for $SINGLE_DB_NAME" + return 1 fi else - log_error "Archive does not exist" + echo "There is no database file available to restore from" + return 1 fi + return 0 } #Restore all the databases restore_all_dbs() { - if [ -f ${ARCHIVE_DIR}/${archive_file} ] + TMP_DIR=$1 + + SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql + if [[ -f ${TMP_DIR}/$SQL_FILE ]] then - files_to_purge=$(find $RESTORE_DIR/ -iname "*.sql" -print) - delete_files $files_to_purge - tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null - if [ -f ${RESTORE_DIR}/mariadb.all.sql ] + $MYSQL < ${TMP_DIR}/$SQL_FILE 2>$RESTORE_LOG + if [[ "$?" -eq 0 ]] then - $MYSQL < ${RESTORE_DIR}/mariadb.all.sql 2>$RESTORE_LOG - if [ "$?" -eq 0 ] - then - echo "Databases $( echo $DBS | tr -d '\n') Restore successful." - else - cat $RESTORE_LOG - log_error "Databases $( echo $DBS | tr -d '\n') Restore failed." - fi - if [ -n "$DBS" ] - then - for db in $DBS - do - if [ -f ${RESTORE_DIR}/${db}_grant.sql ] - then - $MYSQL < ${RESTORE_DIR}/${db}_grant.sql 2>>$RESTORE_LOG - if [ "$?" -eq 0 ] - then - echo "Database $db Permission Restore successful." - else - cat $RESTORE_LOG - log_error "Database $db Permission Restore failed." - fi - else - log_error "There is no permission file available for $db" - fi - done + echo "Databases $( echo $DBS | tr -d '\n') Restore successful." else - log_error "There is no database file available to restore from" + cat $RESTORE_LOG + echo "Databases $( echo $DBS | tr -d '\n') Restore failed." + return 1 fi - else - log_error "Archive does not exist" - fi - fi -} - -usage() { - ret_val=$1 - echo "Usage:" - echo "Restore command options" - echo "=============================" - echo "help" - echo "list_archives" - echo "list_databases " - echo "restore [ | ALL]" - exit $ret_val -} - -is_Option() { - opts=$1 - param=$2 - find=0 - for opt in $opts - do - if [ "$opt" == "$param" ] + if [ -n "$DBS" ] then - find=1 - fi - done - echo $find -} - -#Main -if [ ${#ARGS[@]} -gt 3 ] -then - usage 1 -elif [ ${#ARGS[@]} -eq 1 ] -then - if [ "${ARGS[0]}" == "list_archives" ] - then - list_archives - elif [ "${ARGS[0]}" == "help" ] - then - usage 0 - else - usage 1 - fi -elif [ ${#ARGS[@]} -eq 2 ] -then - if [ "${ARGS[0]}" == "list_databases" ] - then - list_databases ${ARGS[1]} - else - usage 1 - fi -elif [ ${#ARGS[@]} -eq 3 ] -then - if [ "${ARGS[0]}" != "restore" ] - then - usage 1 - else - if [ -f ${ARCHIVE_DIR}/${ARGS[1]} ] - then - #Get all the databases in that archive - get_databases ${ARGS[1]} - - #check if the requested database is available in the archive - if [ $(is_Option "$DBS" ${ARGS[2]}) -eq 1 ] - then - echo "Creating database ${ARGS[2]} if it does not exist" - $MYSQL -e "CREATE DATABASE IF NOT EXISTS \`${ARGS[2]}\`" 2>>$RESTORE_LOG - if [ "$?" -ne 0 ] + for db in $DBS + do + if [ -f ${TMP_DIR}/${db}_grant.sql ] then - cat $RESTORE_LOG - log_error "Database ${ARGS[2]} could not be created." - fi - echo "Restoring database ${ARGS[2]} and grants...this could take a few minutes." - restore_single_db ${ARGS[2]} - elif [ "$( echo ${ARGS[2]} | tr '[a-z]' '[A-Z]')" == "ALL" ] - then - echo "Creating databases if they do not exist" - for db in $DBS - do - $MYSQL -e "CREATE DATABASE IF NOT EXISTS \`$db\`" - if [ "$?" -ne 0 ] + $MYSQL < ${TMP_DIR}/${db}_grant.sql 2>>$RESTORE_LOG + if [[ "$?" -eq 0 ]] then + echo "Database $db Permission Restore successful." + else cat $RESTORE_LOG - log_error "Database ${db} could not be created." + echo "Database $db Permission Restore failed." + return 1 fi - done - echo "Restoring all databases and grants...this could take a few minutes." - restore_all_dbs - else - echo "Database ${ARGS[2]} does not exist." - fi - else - echo "Archive file not found" + else + echo "There is no permission file available for $db" + return 1 + fi + done fi + else + echo "There is no database file available to restore from" + return 1 fi -else - usage 1 -fi + return 0 +} -exit 0 +# Call the CLI interpreter, providing the archive directory path and the +# user arguments passed in +cli_main ${ARGS[@]} diff --git a/mariadb/templates/configmap-bin.yaml b/mariadb/templates/configmap-bin.yaml index 81e1713db4..4af240877d 100644 --- a/mariadb/templates/configmap-bin.yaml +++ b/mariadb/templates/configmap-bin.yaml @@ -42,5 +42,13 @@ data: {{ tuple "bin/_backup_mariadb.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} restore_mariadb.sh: | {{ tuple "bin/_restore_mariadb.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + backup_main.sh: | +{{ include "helm-toolkit.scripts.db-backup-restore.backup_main" . | indent 4 }} + restore_main.sh: | +{{ include "helm-toolkit.scripts.db-backup-restore.restore_main" . | indent 4 }} +{{- end }} +{{- if .Values.manifests.job_ks_user }} + ks-user.sh: | +{{ include "helm-toolkit.scripts.keystone_user" . | indent 4 }} {{- end }} {{- end }} diff --git a/mariadb/templates/cron-job-backup-mariadb.yaml b/mariadb/templates/cron-job-backup-mariadb.yaml index e226c268e3..80ecdfa2e5 100644 --- a/mariadb/templates/cron-job-backup-mariadb.yaml +++ b/mariadb/templates/cron-job-backup-mariadb.yaml @@ -27,6 +27,12 @@ metadata: labels: {{ tuple $envAll "mariadb-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} spec: +{{- if .Values.jobs.backup_mariadb.backoffLimit }} + backoffLimit: {{ .Values.jobs.backup_mariadb.backoffLimit }} +{{- end }} +{{- if .Values.jobs.backup_mariadb.activeDeadlineSeconds }} + activeDeadlineSeconds: {{ .Values.jobs.backup_mariadb.activeDeadlineSeconds }} +{{- end }} schedule: {{ .Values.jobs.backup_mariadb.cron | quote }} successfulJobsHistoryLimit: {{ .Values.jobs.backup_mariadb.history.success }} failedJobsHistoryLimit: {{ .Values.jobs.backup_mariadb.history.failed }} @@ -36,7 +42,7 @@ spec: labels: {{ tuple $envAll "mariadb-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} annotations: -{{ dict "envAll" $envAll "podName" "mariadb-backup" "containerNames" (list "init" "mariadb-backup") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} +{{ dict "envAll" $envAll "podName" "mariadb-backup" "containerNames" (list "init" "backup-perms" "mariadb-backup") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} spec: template: metadata: @@ -48,7 +54,24 @@ spec: nodeSelector: {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} initContainers: -{{ tuple $envAll "mariadb_backup" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 10 }} +{{ tuple $envAll "mariadb_backup" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 12 }} + - name: backup-perms +{{ tuple $envAll "mariadb_backup" | include "helm-toolkit.snippets.image" | indent 14 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.mariadb_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }} +{{ dict "envAll" $envAll "application" "mariadb_backup" "container" "backup_perms" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }} + command: + - chown + - -R + - "65534:65534" + - $(MARIADB_BACKUP_BASE_DIR) + env: + - name: MARIADB_BACKUP_BASE_DIR + value: {{ .Values.conf.backup.base_path | quote }} + volumeMounts: + - mountPath: /tmp + name: pod-tmp + - mountPath: {{ .Values.conf.backup.base_path }} + name: mariadb-backup-dir containers: - name: mariadb-backup command: @@ -58,14 +81,28 @@ spec: value: {{ .Values.conf.backup.base_path | quote }} - name: MYSQL_BACKUP_MYSQLDUMP_OPTIONS value: {{ .Values.conf.backup.mysqldump_options | quote }} - - name: MARIADB_BACKUP_DAYS_TO_KEEP - value: {{ .Values.conf.backup.days_of_backup_to_keep | quote }} + - name: MARIADB_LOCAL_BACKUP_DAYS_TO_KEEP + value: {{ .Values.conf.backup.days_to_keep | quote }} - name: MARIADB_POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace + - name: REMOTE_BACKUP_ENABLED + value: "{{ .Values.conf.backup.remote_backup.enabled }}" +{{- if .Values.conf.backup.remote_backup.enabled }} + - name: MARIADB_REMOTE_BACKUP_DAYS_TO_KEEP + value: {{ .Values.conf.backup.remote_backup.days_to_keep | quote }} + - name: CONTAINER_NAME + value: {{ .Values.conf.backup.remote_backup.container_name | quote }} + - name: STORAGE_POLICY + value: "{{ .Values.conf.backup.remote_backup.storage_policy }}" +{{- with $env := dict "ksUserSecret" $envAll.Values.secrets.identity.remote_rgw_user }} +{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 16 }} +{{- end }} +{{- end }} {{ tuple $envAll "mariadb_backup" | include "helm-toolkit.snippets.image" | indent 14 }} {{ tuple $envAll $envAll.Values.pod.resources.jobs.mariadb_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }} +{{ dict "envAll" $envAll "application" "mariadb_backup" "container" "mariadb_backup" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }} volumeMounts: - name: pod-tmp mountPath: /tmp @@ -73,6 +110,10 @@ spec: name: mariadb-bin readOnly: true subPath: backup_mariadb.sh + - mountPath: /tmp/backup_main.sh + name: mariadb-bin + readOnly: true + subPath: backup_main.sh - mountPath: {{ .Values.conf.backup.base_path }} name: mariadb-backup-dir - name: mariadb-secrets @@ -88,7 +129,7 @@ spec: - name: mariadb-secrets secret: secretName: mariadb-secrets - defaultMode: 384 + defaultMode: 420 - configMap: defaultMode: 365 name: mariadb-bin diff --git a/mariadb/templates/job-ks-user.yaml b/mariadb/templates/job-ks-user.yaml new file mode 100644 index 0000000000..99b384d6c8 --- /dev/null +++ b/mariadb/templates/job-ks-user.yaml @@ -0,0 +1,20 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.job_ks_user }} +{{- $backoffLimit := .Values.jobs.ks_user.backoffLimit }} +{{- $activeDeadlineSeconds := .Values.jobs.ks_user.activeDeadlineSeconds }} +{{- $ksUserJob := dict "envAll" . "serviceName" "mariadb" "configMapBin" "mariadb-bin" "backoffLimit" $backoffLimit "activeDeadlineSeconds" $activeDeadlineSeconds -}} +{{ $ksUserJob | include "helm-toolkit.manifests.job_ks_user" }} +{{- end }} diff --git a/mariadb/templates/secret-backup-restore.yaml b/mariadb/templates/secret-backup-restore.yaml new file mode 100644 index 0000000000..7886b1a7e8 --- /dev/null +++ b/mariadb/templates/secret-backup-restore.yaml @@ -0,0 +1,27 @@ +{{/* +This manifest results a secret being created which has the key information +needed for backing up and restoring the Mariadb databases. +*/}} + +{{- if and .Values.conf.backup.enabled .Values.manifests.secret_backup_restore }} + +{{- $envAll := . }} +{{- $userClass := "backup_restore" }} +{{- $secretName := index $envAll.Values.secrets.mariadb $userClass }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: + BACKUP_ENABLED: {{ $envAll.Values.conf.backup.enabled | quote | b64enc }} + BACKUP_BASE_PATH: {{ $envAll.Values.conf.backup.base_path | b64enc }} + LOCAL_DAYS_TO_KEEP: {{ $envAll.Values.conf.backup.days_to_keep | quote | b64enc }} + MYSQLDUMP_OPTIONS: {{ $envAll.Values.conf.backup.mysqldump_options | b64enc }} + REMOTE_BACKUP_ENABLED: {{ $envAll.Values.conf.backup.remote_backup.enabled | quote | b64enc }} + REMOTE_BACKUP_CONTAINER: {{ $envAll.Values.conf.backup.remote_backup.container_name | b64enc }} + REMOTE_BACKUP_DAYS_TO_KEEP: {{ $envAll.Values.conf.backup.remote_backup.days_to_keep | quote | b64enc }} + REMOTE_BACKUP_STORAGE_POLICY: {{ $envAll.Values.conf.backup.remote_backup.storage_policy | b64enc }} +... +{{- end }} diff --git a/mariadb/templates/secret-rgw.yaml b/mariadb/templates/secret-rgw.yaml new file mode 100644 index 0000000000..7b960f8ac4 --- /dev/null +++ b/mariadb/templates/secret-rgw.yaml @@ -0,0 +1,78 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +This manifest results in two secrets being created: + 1) Keystone "remote_rgw_user" secret, which is needed to access the cluster + (remote or same cluster) for storing mariadb backups. If the + cluster is remote, the auth_url would be non-null. + 2) Keystone "remote_ks_admin" secret, which is needed to create the + "remote_rgw_user" keystone account mentioned above. This may not + be needed if the account is in a remote cluster (auth_url is non-null + in that case). +*/}} + +{{- if .Values.conf.backup.remote_backup.enabled }} + +{{- $envAll := . }} +{{- $userClass := "remote_rgw_user" }} +{{- $secretName := index $envAll.Values.secrets.identity $userClass }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: +{{- $identityClass := index .Values.endpoints.identity.auth $userClass }} +{{- if $identityClass.auth_url }} + OS_AUTH_URL: {{ $identityClass.auth_url | b64enc }} +{{- else }} + OS_AUTH_URL: {{ tuple "identity" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | b64enc }} +{{- end }} + OS_REGION_NAME: {{ $identityClass.region_name | b64enc }} + OS_INTERFACE: {{ $identityClass.interface | default "internal" | b64enc }} + OS_PROJECT_DOMAIN_NAME: {{ $identityClass.project_domain_name | b64enc }} + OS_PROJECT_NAME: {{ $identityClass.project_name | b64enc }} + OS_USER_DOMAIN_NAME: {{ $identityClass.user_domain_name | b64enc }} + OS_USERNAME: {{ $identityClass.username | b64enc }} + OS_PASSWORD: {{ $identityClass.password | b64enc }} + OS_DEFAULT_DOMAIN: {{ $identityClass.default_domain_id | default "default" | b64enc }} +... +{{- if .Values.manifests.job_ks_user }} +{{- $userClass := "remote_ks_admin" }} +{{- $secretName := index $envAll.Values.secrets.identity $userClass }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: +{{- $identityClass := index .Values.endpoints.identity.auth $userClass }} +{{- if $identityClass.auth_url }} + OS_AUTH_URL: {{ $identityClass.auth_url | b64enc }} +{{- else }} + OS_AUTH_URL: {{ tuple "identity" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | b64enc }} +{{- end }} + OS_REGION_NAME: {{ $identityClass.region_name | b64enc }} + OS_INTERFACE: {{ $identityClass.interface | default "internal" | b64enc }} + OS_PROJECT_DOMAIN_NAME: {{ $identityClass.project_domain_name | b64enc }} + OS_PROJECT_NAME: {{ $identityClass.project_name | b64enc }} + OS_USER_DOMAIN_NAME: {{ $identityClass.user_domain_name | b64enc }} + OS_USERNAME: {{ $identityClass.username | b64enc }} + OS_PASSWORD: {{ $identityClass.password | b64enc }} + OS_DEFAULT_DOMAIN: {{ $identityClass.default_domain_id | default "default" | b64enc }} +... +{{- end }} +{{- end }} diff --git a/mariadb/values.yaml b/mariadb/values.yaml index ff5ab41730..cabf6d136e 100644 --- a/mariadb/values.yaml +++ b/mariadb/values.yaml @@ -29,7 +29,8 @@ images: prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:newton-ubuntu_xenial dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0 image_repo_sync: docker.io/docker:17.07.0 - mariadb_backup: docker.io/openstackhelm/mariadb:ubuntu_xenial-20191031 + mariadb_backup: quay.io/airshipit/porthole-mysqlclient-utility:latest-ubuntu_bionic + ks_user: docker.io/openstackhelm/heat:stein-ubuntu_bionic scripted_test: docker.io/openstackhelm/mariadb:ubuntu_xenial-20191031 pull_policy: "IfNotPresent" local_registry: @@ -109,6 +110,17 @@ pod: main: allowPrivilegeEscalation: false readOnlyRootFilesystem: true + mariadb_backup: + pod: + runAsUser: 65534 + container: + backup_perms: + runAsUser: 0 + readOnlyRootFilesystem: true + mariadb_backup: + runAsUser: 65534 + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false tests: pod: runAsUser: 999 @@ -190,6 +202,13 @@ pod: limits: memory: "1024Mi" cpu: "2000m" + ks_user: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" dependencies: dynamic: @@ -208,8 +227,9 @@ dependencies: services: - endpoint: error_pages service: oslo_db - mariadb: - jobs: null + backup_mariadb: + jobs: + - mariadb-ks-user services: null prometheus_create_mysql_user: services: @@ -260,10 +280,17 @@ jobs: backoffLimit: 87600 activeDeadlineSeconds: 3600 backup_mariadb: + # activeDeadlineSeconds == 0 means no deadline + activeDeadlineSeconds: 0 + backoffLimit: 6 cron: "0 0 * * *" history: success: 3 failed: 1 + ks_user: + # activeDeadlineSeconds == 0 means no deadline + activeDeadlineSeconds: 0 + backoffLimit: 6 conf: tests: @@ -284,12 +311,17 @@ conf: ingress_conf: worker-processes: "auto" backup: - enabled: true + enabled: false base_path: /var/backup mysqldump_options: > --single-transaction --quick --add-drop-database --add-drop-table --add-locks --databases - days_of_backup_to_keep: 3 + days_to_keep: 3 + remote_backup: + enabled: false + container_name: mariadb + days_to_keep: 14 + storage_policy: default-placement database: my: | [mysqld] @@ -407,6 +439,13 @@ monitoring: mysqld_exporter: scrape: true +secrets: + identity: + remote_ks_admin: keystone-admin-user + remote_rgw_user: mariadb-backup-user + mariadb: + backup_restore: mariadb-backup-restore + # typically overridden by environmental # values, but should include all endpoints # required by this chart @@ -498,6 +537,44 @@ endpoints: dns: default: 53 protocol: UDP + identity: + name: backup-storage-auth + namespace: openstack + auth: + remote_ks_admin: + # Auth URL of null indicates local authentication + # HTK will form the URL unless specified here + auth_url: null + region_name: RegionOne + username: admin + password: password + project_name: admin + user_domain_name: default + project_domain_name: default + remote_rgw_user: + # Auth URL of null indicates local authentication + # HTK will form the URL unless specified here + auth_url: null + role: admin + region_name: RegionOne + username: mariadb-backup-user + password: password + project_name: service + user_domain_name: service + project_domain_name: service + hosts: + default: keystone + internal: keystone-api + host_fqdn_override: + default: null + path: + default: /v3 + scheme: + default: 'http' + port: + api: + default: 80 + internal: 5000 network_policy: mariadb: @@ -521,6 +598,7 @@ manifests: deployment_ingress: true job_image_repo_sync: true cron_job_mariadb_backup: false + job_ks_user: false pvc_backup: false monitoring: prometheus: @@ -536,6 +614,7 @@ manifests: secret_dbadmin_password: true secret_sst_password: true secret_dbaudit_password: true + secret_backup_restore: false secret_etc: true service_discovery: true service_ingress: true