remove bad commit

This commit is contained in:
mirocow 2016-03-21 14:18:23 +04:00
parent 33c5791782
commit fd432fb80a

View file

@ -1,22 +1,25 @@
#!/bin/bash #!/bin/bash
# === CONFIG === # === CONFIG ===
CONFIG_CHUNK=1000000
VERBOSE=0 VERBOSE=0
COMPRESS='bzip2'
USER='mysql'
GROUP='mysql'
DIRECTORYATTRIBUTES=0770
FILEATTRIBUTES=640
TIME_REMOVED_DUMP_FILES='1 week ago'
BACKUP_DIR='/var/backups/mysql'
CONFIG_FILE='/etc/mysql/debian.cnf'
# === DO NOT EDIT BELOW THIS LINE === # === DO NOT EDIT BELOW THIS LINE ===
if [ ! -n "$BASH" ] ;then echo Please run this script $0 with bash; exit 1; fi if [ ! -n "$BASH" ] ;then echo Please run this script $0 with bash; exit 1; fi
# === FUNCTION === # === FUNCTIONS ===
f_log() database_exists()
{
RESULT=`mysqlshow --defaults-extra-file=$CONFIG_FILE $@| grep -v Wildcard | grep -o $@`
if [ "$RESULT" == "$@" ]; then
echo YES
else
echo NO
fi
}
f_log()
{ {
local bold=$(tput bold) local bold=$(tput bold)
local yellow=$(tput setf 6) local yellow=$(tput setf 6)
@ -24,188 +27,183 @@ f_log()
local green=$(tput setf 2) local green=$(tput setf 2)
local reset=$(tput sgr0) local reset=$(tput sgr0)
local toend=$(tput hpa $(tput cols))$(tput cub 6) local toend=$(tput hpa $(tput cols))$(tput cub 6)
logger "BACKUP: $@"
if [ $VERBOSE -eq 1 ]; then
echo "BACKUP: $@"
fi
}
prepaire_skip_expression()
{
local array_skip=( "${@}" )
for skip in "${array_skip[@]}"; do
if [ -x $return ]; then
local return="^$skip\$"
else
return="$return|^$skip\$"
fi
done
echo ${return}
}
backup()
{
f_log " START "
query="SHOW databases;"
local default_databases_exclude=(
'information_schema'
'performance_schema'
)
local array_views=()
database_exclude=( ${default_databases_exclude[@]} ${EXCLUDE_DATABASES[@]} )
database_exclude_expression=`prepaire_skip_expression "${database_exclude[@]}"`
f_log "Exclude databases: $database_exclude_expression"
for BDD in $(mysql --defaults-extra-file=$CONFIG_FILE --skip-column-names -B -e "$query" | egrep -v "$database_exclude_expression"); do
touch $DST/$BDD/error.log logger "RESTORE: $@"
if [ $VERBOSE -eq 1 ]; then
echo "RESTORE: $@"
fi
}
mkdir -p $DST/$BDD 2>/dev/null 1>&2 restore()
chown $USER:$GROUP $DST/$BDD {
chmod $DIRECTORYATTRIBUTES $DST/$BDD DIR=$@
query="SHOW CREATE DATABASE \`$BDD\`;" f_log "Check path $DIR"
mysql --defaults-extra-file=$CONFIG_FILE --skip-column-names -B -e "$query" | awk -F"\t" '{ print $2 }' > $DST/$BDD/__create.sql
if [ -f $DST/$BDD/__create.sql ]; then f_log "** START **"
f_log " > Export create"
fi f_log "Check runtime"
for BIN in $BIN_DEPS; do
which $BIN 1>/dev/null 2>&1
if [ $? -ne 0 ]; then
f_log "Error: Required file could not be found: $BIN"
exit 1
fi
done
query="SHOW FULL TABLES WHERE Table_type = 'VIEW';" f_log "Check backups folder"
for viewName in $(mysql --defaults-extra-file=$CONFIG_FILE $BDD -N -e "$query" | sed 's/|//' | awk '{print $1}'); do if [ "$(ls -1 $DIR/*/__create.sql 2>/dev/null | wc -l)" -le "0" ]; then
mysqldump --defaults-file=$CONFIG_FILE $BDD $viewName >> $DST/$BDD/__views.sql 2>> $DST/$BDD/error.log f_log "Your must run script from backup directory"
array_views+=($viewName) exit 1
done fi
if [ -f $DST/$BDD/__views.sql ]; then
f_log " > Exports views"
fi
mysqldump --defaults-file=$CONFIG_FILE --routines --no-create-info --no-data --no-create-db --skip-opt $BDD 2>> $DST/$BDD/error.log | sed -e 's/DEFINER=[^*]*\*/\*/' > $DST/$BDD/__routines.sql for i in $(ls -1 -d $DIR/*); do
if [ -f $DST/$BDD/__routines.sql ]; then
f_log " > Exports Routines"
fi
local default_tables_exclude=( BDD=$(basename $i)
'slow_log'
'general_log'
)
tables_exclude=( ${default_tables_exclude[@]} ${array_views[@]} ${EXCLUDE_TABLES[@]} )
tables_exclude_expression=`prepaire_skip_expression "${tables_exclude[@]}"`
f_log "Exclude tables: $tables_exclude_expression"
data_tables_exclude=( ${EXCLUDE_DATA_TABLES[@]} )
data_tables_exclude_expression=`prepaire_skip_expression "${data_tables_exclude[@]}"`
f_log "Exclude data tables: $data_tables_exclude_expression"
query="SHOW TABLES;" for skip in "${DATABASES_SKIP[@]}"; do
for TABLE in $(mysql --defaults-extra-file=$CONFIG_FILE --skip-column-names -B $BDD -e "$query" | egrep -v "$tables_exclude_expression"); do if [ $BDD = $skip ]; then
f_log " ** Dump $BDD.$TABLE" f_log "Skip database $BDD"
unset BDD
break
fi
done
for select in "${DATABASES_SELECTED[@]}"; do
if [ $BDD != $select ]; then
f_log "Skip database $BDD"
unset BDD
break
fi
done
if [ $BDD ]; then
if [ -f $DIR/$BDD/__create.sql ]; then
f_log "Create database $BDD"
mysql --defaults-extra-file=$CONFIG_FILE < $DIR/$BDD/__create.sql 2>/dev/null
fi
f_log "+ $TABLE, get default charset: $charset" if [ $(database_exists $BDD) != "YES" ]; then
f_log "Error: Database $BDD dose not exists";
if [ $(echo $data_tables_exclude_expression| grep $TABLE) ]; then else
f_log "Exclude data from table $TABLE"
mysqldump --defaults-file=$CONFIG_FILE --no-data --add-drop-table --tab=$DST/$BDD/ $BDD $TABLE 2>> $DST/$BDD/error.log tables=$(ls -1 $DIR/$BDD | grep -v __ | grep .sql | awk -F. '{print $1}' | sort | uniq)
else
mysqldump --defaults-file=$CONFIG_FILE --default-character-set=utf8 --add-drop-table --quick --tab=$DST/$BDD/ $BDD $TABLE 2>> $DST/$BDD/error.log f_log "Create tables in $BDD"
fi for TABLE in $tables; do
f_log "Create table: $BDD/$TABLE"
if [ -f "$DST/$BDD/$TABLE.sql" ]; then mysql --defaults-extra-file=$CONFIG_FILE $BDD -e "SET foreign_key_checks = 0;
chmod $FILEATTRIBUTES $DST/$BDD/$TABLE.sql DROP TABLE IF EXISTS $TABLE;
chown $USER:$GROUP $DST/$BDD/$TABLE.sql SOURCE $DIR/$BDD/$TABLE.sql;
f_log " ** set perm on $BDD/$TABLE.sql" SET foreign_key_checks = 1;"
else done
f_log " ** WARNING : $DST/$BDD/$TABLE.sql not found"
fi f_log "Import data into $BDD"
for TABLE in $tables; do
if [ -f "$DST/$BDD/$TABLE.txt" ]; then f_log "Import data into $BDD/$TABLE"
if [ $COMPRESS ]; then
f_log " ** $COMPRESS $BDD/$TABLE.txt in background"
if [ $COMPRESS == 'bzip2' ]; then
if [ -f "$DST/$BDD/$TABLE.txt.bz2" ]; then
rm $DST/$BDD/$TABLE.txt.bz2
fi
($COMPRESS $DST/$BDD/$TABLE.txt && chown $USER:$GROUP $DST/$BDD/$TABLE.txt.bz2 && chmod $FILEATTRIBUTES $DST/$BDD/$TABLE.txt.bz2) &
elif [ $COMPRESS == 'gzip' ]; then if [ -f "$DIR/$BDD/$TABLE.txt.bz2" ]; then
f_log "< $TABLE"
if [ -f "$DIR/$BDD/$TABLE.txt" ]; then
rm $DIR/$BDD/$TABLE.txt
fi
bunzip2 -k $DIR/$BDD/$TABLE.txt.bz2
fi
if [ -f "$DST/$BDD/$TABLE.txt.gz" ]; then if [ -s "$DIR/$BDD/$TABLE.txt" ]; then
rm $DST/$BDD/$TABLE.txt.gz
fi
($COMPRESS $DST/$BDD/$TABLE.txt && chown $USER:$GROUP $DST/$BDD/$TABLE.txt.gz && chmod $FILEATTRIBUTES $DST/$BDD/$TABLE.txt.gz) &
fi if [ ! -z "$(cat $DIR/$BDD/$TABLE.sql | grep -i 'DEFAULT CHARSET=CP1251')" ]; then
charset='cp1251'
else
charset='utf8'
fi
f_log "+ $TABLE, Set default charset: $charset"
split -l $CONFIG_CHUNK "$DIR/$BDD/$TABLE.txt" "$DIR/$BDD/${TABLE}_part_"
for segment in "$DIR/$BDD/${TABLE}"_part_*; do
f_log "Restore from $segment"
mysql --defaults-extra-file=$CONFIG_FILE $BDD --local-infile -e "SET foreign_key_checks = 0; SET unique_checks = 0; SET sql_log_bin = 0;
SET character_set_database = $charset;
LOAD DATA LOCAL INFILE '$segment'
INTO TABLE $TABLE;
SET foreign_key_checks = 1; SET unique_checks = 1; SET sql_log_bin = 1;"
if [ -f "$segment" ]; then
f_log "Delete segment $segment"
rm "$segment"
fi
done
fi
if [ -f "$DIR/$BDD/$TABLE.txt" ]; then
f_log "Delete source: $TABLE.txt"
rm $DIR/$BDD/$TABLE.txt
fi
if [ $DATABASES_TABLE_CHECK ]; then
if [ -f "$DIR/$BDD/$TABLE.ibd" ]; then
if [ ! $(innochecksum $DIR/$BDD/$TABLE.ibd) ]; then
f_log "$TABLE [OK]"
else
f_log "$TABLE [ERR]"
fi
fi
fi
done
if [ -f "$DIR/$BDD/__routines.sql" ]; then
f_log "Import routines into $BDD"
mysql --defaults-extra-file=$CONFIG_FILE $BDD < $DIR/$BDD/__routines.sql 2>/dev/null
fi
if [ -f "$DIR/$BDD/__views.sql" ]; then
f_log "Import views into $BDD"
mysql --defaults-extra-file=$CONFIG_FILE $BDD < $DIR/$BDD/__views.sql 2>/dev/null
fi
fi
fi
done
fi f_log "Flush privileges;"
mysql --defaults-extra-file=$CONFIG_FILE -e "flush privileges;"
else f_log "** END **"
f_log " ** WARNING : $DST/$BDD/$TABLE.txt not found"
fi
done
done
f_log " END "
} }
usage() usage()
{ {
cat << EOF cat << EOF
usage: $0 options
This mysql backup engine.
Usage: $0 <[options]> or bash $0 <[options]>
Options: This script restore databases.
-e= | --exclude= Exclude databases
--exclude-tables= Exclude tables
--exclude-data-tables= Exclude data tables
-c= | --compress= Compress with gzip or bzip2
-v | --verbose Add verbose into output
-l | --lifetime= Lifetime for dump files
--config= Config file of Debian format
--dir= Backup directory
-h | --help This text
Examples: OPTIONS:
backup.sh --verbose --compress= -e Exclude databases
backup.sh --verbose --compress=gzip -s Selected databases
backup.sh --verbose --compress=bzip2 -c Check innochecksum of table after import
backup.sh --verbose --compress= --exclude="mysql"
backup.sh --verbose --compress= --exclude="mysql" --lifetime="3 day ago"
backup.sh --verbose --config="/etc/mysql/debian.cnf" --exclude="mysql" --lifetime="1 day ago"
backup.sh --verbose --dir="/var/backups/mysql" --config="/etc/mysql/debian.cnf" --exclude="mysql" --lifetime="1 day ago"
backup.sh --verbose --dir="/home/backups/mysql" --exclude="mysql" --lifetime="1 day ago"
backup.sh --verbose --dir="/home/backups/mysql" --exclude="mysql" --exclude-tables="tbl_template" --lifetime="1 day ago"
EOF EOF
} }
if [ $# = 0 ]; then # === CHECKS ===
usage;
exit; BACKUP_DIR=$(pwd)
BIN_DEPS="ls grep awk sort uniq bunzip2 bzip2 mysql"
if [ -f '/etc/debian_version' ]; then
CONFIG_FILE='/etc/mysql/debian.cnf'
else
CONFIG_FILE='~/mysql_utils/etc/mysql/debian.cnf'
fi fi
EXCLUDE_DATABASES=''
EXCLUDE_TABLES=''
EXCLUDE_DATA_TABLES=''
BIN_DEPS="mysql mysqldump $COMPRESS"
# === CHECKS ===
for BIN in $BIN_DEPS; do for BIN in $BIN_DEPS; do
which $BIN 1>/dev/null 2>&1 which $BIN 1>/dev/null 2>&1
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
@ -214,43 +212,29 @@ for BIN in $BIN_DEPS; do
fi fi
done done
if [ -f '/etc/debian_version' ]; then
CONFIG_FILE='/etc/mysql/debian.cnf'
else
CONFIG_FILE='~/mysql_utils/etc/mysql/debian.cnf'
fi
for i in "$@" for i in "$@"
do do
case $i in case $i in
-e=* | --exclude=*) -e)
EXCLUDE_DATABASES=( "${i#*=}" ) DATABASES_SKIP=( "${i#*=}" )
shift # past argument=value shift
;; ;;
--exclude-tables=*) -s)
EXCLUDE_TABLES=( "${i#*=}" ) DATABASES_SELECTED=( "${i#*=}" )
shift # past argument=value shift
;; ;;
--exclude-data-tables=*) -c)
EXCLUDE_DATA_TABLES=( "${i#*=}" ) DATABASES_TABLE_CHECK=1
shift # past argument=value shift
;; ;;
-c=* | --compress=*)
COMPRESS=( "${i#*=}" )
shift # past argument=value
;;
-l=* | --lifetime=*)
TIME_REMOVED_DUMP_FILES=( "${i#*=}" )
shift # past argument=value
;;
--dir=*)
BACKUP_DIR=( "${i#*=}" )
shift # past argument=value
;;
--config=*) --config=*)
CONFIG_FILE=( "${i#*=}" ) CONFIG_FILE=( "${i#*=}" )
shift # past argument=value shift # past argument=value
;; ;;
--chunk=*)
CONFIG_CHUNK=( "${i#*=}" )
shift # past argument=value
;;
-v | --verbose) -v | --verbose)
VERBOSE=1 VERBOSE=1
shift # past argument=value shift # past argument=value
@ -265,32 +249,14 @@ do
esac esac
done done
DATE=`date '+%Y.%m.%d'`
DATEOLD=`date --date="$TIME_REMOVED_DUMP_FILES" +%Y.%m.%d`
DST=$BACKUP_DIR/$DATE
DSTOLD=$BACKUP_DIR/$DATEOLD
if [ ! -d "$DST" ]; then
mkdir -p $DST;
chmod $DIRECTORYATTRIBUTES $DST;
chown $USER:$GROUP $DST;
fi
if [ -d "$DSTOLD" ]; then
rm -fr $DSTOLD;
fi
# === SETTINGS === # === SETTINGS ===
f_log "============================================" f_log "============================================"
f_log "Dump into: $BACKUP_DIR" f_log "Restore from: $BACKUP_DIR"
f_log "Config file: $CONFIG_FILE" f_log "Config file: $CONFIG_FILE"
f_log "Verbose: $VERBOSE" f_log "Verbose: $VERBOSE"
f_log "Compress: $COMPRESS" f_log "Selected databases: $DATABASES_SELECTED"
f_log "Exclude databases: $EXCLUDE_DATABASES"
f_log "Exclude tables: $EXCLUDE_TABLES"
f_log "Life time: $TIME_REMOVED_DUMP_FILES"
f_log "============================================" f_log "============================================"
f_log "" f_log ""
# === AUTORUN === # === AUTORUN ===
backup restore $(pwd)