From 33c5791782c9e2a01462c6e16809c12dbacaf55e Mon Sep 17 00:00:00 2001 From: mirocow Date: Mon, 21 Mar 2016 14:16:19 +0400 Subject: [PATCH] add default character fore backup data --- restore.sh | 405 +++++++++++++++++++++++++++++------------------------ 1 file changed, 219 insertions(+), 186 deletions(-) diff --git a/restore.sh b/restore.sh index 43f9dbe..c2d5418 100755 --- a/restore.sh +++ b/restore.sh @@ -1,25 +1,22 @@ #!/bin/bash # === CONFIG === -CONFIG_CHUNK=1000000 VERBOSE=0 +COMPRESS='bzip2' +USER='mysql' +GROUP='mysql' +DIRECTORYATTRIBUTES=0770 +FILEATTRIBUTES=640 +TIME_REMOVED_DUMP_FILES='1 week ago' +BACKUP_DIR='/var/backups/mysql' +CONFIG_FILE='/etc/mysql/debian.cnf' # === DO NOT EDIT BELOW THIS LINE === if [ ! -n "$BASH" ] ;then echo Please run this script $0 with bash; exit 1; fi -# === FUNCTIONS === -database_exists() -{ - RESULT=`mysqlshow --defaults-extra-file=$CONFIG_FILE $@| grep -v Wildcard | grep -o $@` - if [ "$RESULT" == "$@" ]; then - echo YES - else - echo NO - fi -} - -f_log() +# === FUNCTION === +f_log() { local bold=$(tput bold) local yellow=$(tput setf 6) @@ -27,183 +24,188 @@ f_log() local green=$(tput setf 2) local reset=$(tput sgr0) local toend=$(tput hpa $(tput cols))$(tput cub 6) - - logger "RESTORE: $@" - + + logger "BACKUP: $@" + if [ $VERBOSE -eq 1 ]; then - echo "RESTORE: $@" - fi + echo "BACKUP: $@" + fi } -restore() +prepaire_skip_expression() { - DIR=$@ + local array_skip=( "${@}" ) + for skip in "${array_skip[@]}"; do + if [ -x $return ]; then + local return="^$skip\$" + else + return="$return|^$skip\$" + fi + done + echo ${return} +} - f_log "Check path $DIR" +backup() +{ + f_log " START " + + query="SHOW databases;" + + local default_databases_exclude=( + 'information_schema' + 'performance_schema' + ) + + local array_views=() + + database_exclude=( ${default_databases_exclude[@]} ${EXCLUDE_DATABASES[@]} ) + database_exclude_expression=`prepaire_skip_expression "${database_exclude[@]}"` + f_log "Exclude databases: $database_exclude_expression" + + for BDD in $(mysql --defaults-extra-file=$CONFIG_FILE --skip-column-names -B -e "$query" | egrep -v "$database_exclude_expression"); do - f_log "** START **" - - f_log "Check runtime" - for BIN in $BIN_DEPS; do - which $BIN 1>/dev/null 2>&1 - if [ $? -ne 0 ]; then - f_log "Error: Required file could not be found: $BIN" - exit 1 - fi - done + touch $DST/$BDD/error.log - f_log "Check backups folder" - if [ "$(ls -1 $DIR/*/__create.sql 2>/dev/null | wc -l)" -le "0" ]; then - f_log "Your must run script from backup directory" - exit 1 - fi + mkdir -p $DST/$BDD 2>/dev/null 1>&2 + chown $USER:$GROUP $DST/$BDD + chmod $DIRECTORYATTRIBUTES $DST/$BDD - for i in $(ls -1 -d $DIR/*); do + query="SHOW CREATE DATABASE \`$BDD\`;" + mysql --defaults-extra-file=$CONFIG_FILE --skip-column-names -B -e "$query" | awk -F"\t" '{ print $2 }' > $DST/$BDD/__create.sql + if [ -f $DST/$BDD/__create.sql ]; then + f_log " > Export create" + fi - BDD=$(basename $i) + query="SHOW FULL TABLES WHERE Table_type = 'VIEW';" + for viewName in $(mysql --defaults-extra-file=$CONFIG_FILE $BDD -N -e "$query" | sed 's/|//' | awk '{print $1}'); do + mysqldump --defaults-file=$CONFIG_FILE $BDD $viewName >> $DST/$BDD/__views.sql 2>> $DST/$BDD/error.log + array_views+=($viewName) + done + if [ -f $DST/$BDD/__views.sql ]; then + f_log " > Exports views" + fi + + mysqldump --defaults-file=$CONFIG_FILE --routines --no-create-info --no-data --no-create-db --skip-opt $BDD 2>> $DST/$BDD/error.log | sed -e 's/DEFINER=[^*]*\*/\*/' > $DST/$BDD/__routines.sql + if [ -f $DST/$BDD/__routines.sql ]; then + f_log " > Exports Routines" + fi + + local default_tables_exclude=( + 'slow_log' + 'general_log' + ) + + tables_exclude=( ${default_tables_exclude[@]} ${array_views[@]} ${EXCLUDE_TABLES[@]} ) + tables_exclude_expression=`prepaire_skip_expression "${tables_exclude[@]}"` + f_log "Exclude tables: $tables_exclude_expression" + + data_tables_exclude=( ${EXCLUDE_DATA_TABLES[@]} ) + data_tables_exclude_expression=`prepaire_skip_expression "${data_tables_exclude[@]}"` + f_log "Exclude data tables: $data_tables_exclude_expression" - for skip in "${DATABASES_SKIP[@]}"; do - if [ $BDD = $skip ]; then - f_log "Skip database $BDD" - unset BDD - break - fi - done - - for select in "${DATABASES_SELECTED[@]}"; do - if [ $BDD != $select ]; then - f_log "Skip database $BDD" - unset BDD - break - fi - done - - if [ $BDD ]; then - - if [ -f $DIR/$BDD/__create.sql ]; then - f_log "Create database $BDD" - mysql --defaults-extra-file=$CONFIG_FILE < $DIR/$BDD/__create.sql 2>/dev/null - fi + query="SHOW TABLES;" + for TABLE in $(mysql --defaults-extra-file=$CONFIG_FILE --skip-column-names -B $BDD -e "$query" | egrep -v "$tables_exclude_expression"); do + f_log " ** Dump $BDD.$TABLE" - if [ $(database_exists $BDD) != "YES" ]; then - f_log "Error: Database $BDD dose not exists"; - else - - tables=$(ls -1 $DIR/$BDD | grep -v __ | grep .sql | awk -F. '{print $1}' | sort | uniq) - - f_log "Create tables in $BDD" - for TABLE in $tables; do - f_log "Create table: $BDD/$TABLE" - mysql --defaults-extra-file=$CONFIG_FILE $BDD -e "SET foreign_key_checks = 0; - DROP TABLE IF EXISTS $TABLE; - SOURCE $DIR/$BDD/$TABLE.sql; - SET foreign_key_checks = 1;" - done - - f_log "Import data into $BDD" - for TABLE in $tables; do - f_log "Import data into $BDD/$TABLE" - - if [ -f "$DIR/$BDD/$TABLE.txt.bz2" ]; then - f_log "< $TABLE" - if [ -f "$DIR/$BDD/$TABLE.txt" ]; then - rm $DIR/$BDD/$TABLE.txt - fi - bunzip2 -k $DIR/$BDD/$TABLE.txt.bz2 - fi - - if [ -s "$DIR/$BDD/$TABLE.txt" ]; then - - if [ ! -z "$(cat $DIR/$BDD/$TABLE.sql | grep -i 'DEFAULT CHARSET=CP1251')" ]; then - charset='cp1251' - else - charset='utf8' - fi - - f_log "+ $TABLE, Set default charset: $charset" - - split -l $CONFIG_CHUNK "$DIR/$BDD/$TABLE.txt" "$DIR/$BDD/${TABLE}_part_" - for segment in "$DIR/$BDD/${TABLE}"_part_*; do - f_log "Restore from $segment" - mysql --defaults-extra-file=$CONFIG_FILE $BDD --local-infile -e "SET foreign_key_checks = 0; SET unique_checks = 0; SET sql_log_bin = 0; - SET character_set_database = $charset; - LOAD DATA LOCAL INFILE '$segment' - INTO TABLE $TABLE; - SET foreign_key_checks = 1; SET unique_checks = 1; SET sql_log_bin = 1;" - - if [ -f "$segment" ]; then - f_log "Delete segment $segment" - rm "$segment" - fi - done - - fi - - if [ -f "$DIR/$BDD/$TABLE.txt" ]; then - f_log "Delete source: $TABLE.txt" - rm $DIR/$BDD/$TABLE.txt - fi - - if [ $DATABASES_TABLE_CHECK ]; then - if [ -f "$DIR/$BDD/$TABLE.ibd" ]; then - if [ ! $(innochecksum $DIR/$BDD/$TABLE.ibd) ]; then - f_log "$TABLE [OK]" - else - f_log "$TABLE [ERR]" - fi - fi - fi - - done - - if [ -f "$DIR/$BDD/__routines.sql" ]; then - f_log "Import routines into $BDD" - mysql --defaults-extra-file=$CONFIG_FILE $BDD < $DIR/$BDD/__routines.sql 2>/dev/null - fi - - if [ -f "$DIR/$BDD/__views.sql" ]; then - f_log "Import views into $BDD" - mysql --defaults-extra-file=$CONFIG_FILE $BDD < $DIR/$BDD/__views.sql 2>/dev/null - fi - - fi - fi - done + f_log "+ $TABLE, get default charset: $charset" - f_log "Flush privileges;" - mysql --defaults-extra-file=$CONFIG_FILE -e "flush privileges;" + if [ $(echo $data_tables_exclude_expression| grep $TABLE) ]; then + f_log "Exclude data from table $TABLE" + mysqldump --defaults-file=$CONFIG_FILE --no-data --add-drop-table --tab=$DST/$BDD/ $BDD $TABLE 2>> $DST/$BDD/error.log + else + mysqldump --defaults-file=$CONFIG_FILE --default-character-set=utf8 --add-drop-table --quick --tab=$DST/$BDD/ $BDD $TABLE 2>> $DST/$BDD/error.log + fi - f_log "** END **" + if [ -f "$DST/$BDD/$TABLE.sql" ]; then + chmod $FILEATTRIBUTES $DST/$BDD/$TABLE.sql + chown $USER:$GROUP $DST/$BDD/$TABLE.sql + f_log " ** set perm on $BDD/$TABLE.sql" + else + f_log " ** WARNING : $DST/$BDD/$TABLE.sql not found" + fi + + if [ -f "$DST/$BDD/$TABLE.txt" ]; then + + if [ $COMPRESS ]; then + + f_log " ** $COMPRESS $BDD/$TABLE.txt in background" + + if [ $COMPRESS == 'bzip2' ]; then + + if [ -f "$DST/$BDD/$TABLE.txt.bz2" ]; then + rm $DST/$BDD/$TABLE.txt.bz2 + fi + + ($COMPRESS $DST/$BDD/$TABLE.txt && chown $USER:$GROUP $DST/$BDD/$TABLE.txt.bz2 && chmod $FILEATTRIBUTES $DST/$BDD/$TABLE.txt.bz2) & + + elif [ $COMPRESS == 'gzip' ]; then + + if [ -f "$DST/$BDD/$TABLE.txt.gz" ]; then + rm $DST/$BDD/$TABLE.txt.gz + fi + + ($COMPRESS $DST/$BDD/$TABLE.txt && chown $USER:$GROUP $DST/$BDD/$TABLE.txt.gz && chmod $FILEATTRIBUTES $DST/$BDD/$TABLE.txt.gz) & + + fi + + fi + + else + f_log " ** WARNING : $DST/$BDD/$TABLE.txt not found" + fi + + done + + done + + f_log " END " } usage() { -cat << EOF -usage: $0 options + cat << EOF + + This mysql backup engine. + + Usage: $0 <[options]> or bash $0 <[options]> -This script restore databases. +Options: + -e= | --exclude= Exclude databases + --exclude-tables= Exclude tables + --exclude-data-tables= Exclude data tables + -c= | --compress= Compress with gzip or bzip2 + -v | --verbose Add verbose into output + -l | --lifetime= Lifetime for dump files + --config= Config file of Debian format + --dir= Backup directory + -h | --help This text -OPTIONS: - -e Exclude databases - -s Selected databases - -c Check innochecksum of table after import - - +Examples: + backup.sh --verbose --compress= + backup.sh --verbose --compress=gzip + backup.sh --verbose --compress=bzip2 + backup.sh --verbose --compress= --exclude="mysql" + backup.sh --verbose --compress= --exclude="mysql" --lifetime="3 day ago" + backup.sh --verbose --config="/etc/mysql/debian.cnf" --exclude="mysql" --lifetime="1 day ago" + backup.sh --verbose --dir="/var/backups/mysql" --config="/etc/mysql/debian.cnf" --exclude="mysql" --lifetime="1 day ago" + backup.sh --verbose --dir="/home/backups/mysql" --exclude="mysql" --lifetime="1 day ago" + backup.sh --verbose --dir="/home/backups/mysql" --exclude="mysql" --exclude-tables="tbl_template" --lifetime="1 day ago" + + EOF } -# === CHECKS === - -BACKUP_DIR=$(pwd) - -BIN_DEPS="ls grep awk sort uniq bunzip2 bzip2 mysql" - -if [ -f '/etc/debian_version' ]; then - CONFIG_FILE='/etc/mysql/debian.cnf' -else - CONFIG_FILE='~/mysql_utils/etc/mysql/debian.cnf' +if [ $# = 0 ]; then + usage; + exit; fi +EXCLUDE_DATABASES='' +EXCLUDE_TABLES='' +EXCLUDE_DATA_TABLES='' +BIN_DEPS="mysql mysqldump $COMPRESS" + +# === CHECKS === for BIN in $BIN_DEPS; do which $BIN 1>/dev/null 2>&1 if [ $? -ne 0 ]; then @@ -212,29 +214,43 @@ for BIN in $BIN_DEPS; do fi done +if [ -f '/etc/debian_version' ]; then + CONFIG_FILE='/etc/mysql/debian.cnf' +else + CONFIG_FILE='~/mysql_utils/etc/mysql/debian.cnf' +fi + for i in "$@" do case $i in - -e) - DATABASES_SKIP=( "${i#*=}" ) - shift - ;; - -s) - DATABASES_SELECTED=( "${i#*=}" ) - shift - ;; - -c) - DATABASES_TABLE_CHECK=1 - shift - ;; + -e=* | --exclude=*) + EXCLUDE_DATABASES=( "${i#*=}" ) + shift # past argument=value + ;; + --exclude-tables=*) + EXCLUDE_TABLES=( "${i#*=}" ) + shift # past argument=value + ;; + --exclude-data-tables=*) + EXCLUDE_DATA_TABLES=( "${i#*=}" ) + shift # past argument=value + ;; + -c=* | --compress=*) + COMPRESS=( "${i#*=}" ) + shift # past argument=value + ;; + -l=* | --lifetime=*) + TIME_REMOVED_DUMP_FILES=( "${i#*=}" ) + shift # past argument=value + ;; + --dir=*) + BACKUP_DIR=( "${i#*=}" ) + shift # past argument=value + ;; --config=*) CONFIG_FILE=( "${i#*=}" ) shift # past argument=value ;; - --chunk=*) - CONFIG_CHUNK=( "${i#*=}" ) - shift # past argument=value - ;; -v | --verbose) VERBOSE=1 shift # past argument=value @@ -249,15 +265,32 @@ do esac done +DATE=`date '+%Y.%m.%d'` +DATEOLD=`date --date="$TIME_REMOVED_DUMP_FILES" +%Y.%m.%d` +DST=$BACKUP_DIR/$DATE +DSTOLD=$BACKUP_DIR/$DATEOLD + +if [ ! -d "$DST" ]; then + mkdir -p $DST; + chmod $DIRECTORYATTRIBUTES $DST; + chown $USER:$GROUP $DST; +fi + +if [ -d "$DSTOLD" ]; then + rm -fr $DSTOLD; +fi + # === SETTINGS === f_log "============================================" -f_log "Restore from: $BACKUP_DIR" +f_log "Dump into: $BACKUP_DIR" f_log "Config file: $CONFIG_FILE" f_log "Verbose: $VERBOSE" -f_log "Selected databases: $DATABASES_SELECTED" +f_log "Compress: $COMPRESS" +f_log "Exclude databases: $EXCLUDE_DATABASES" +f_log "Exclude tables: $EXCLUDE_TABLES" +f_log "Life time: $TIME_REMOVED_DUMP_FILES" f_log "============================================" f_log "" # === AUTORUN === -restore $(pwd) - +backup