init commit
This commit is contained in:
Vassiliy Yegorov
@@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
apt-get update && \
|
||||
apt-get install -y ca-certificates mysql-client nfs-common git wget python-setuptools cron iputils-ping curl nano gnupg
|
||||
|
||||
# postgres 11
|
||||
# TODO добавить поддержку нескольких версий и по кейсу понимать какую надо подставлять
|
||||
curl -s https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
|
||||
|
||||
echo "deb http://apt.postgresql.org/pub/repos/apt/ bionic"-pgdg main | tee /etc/apt/sources.list.d/pgdg.list
|
||||
|
||||
apt update && apt -y install postgresql-client-common postgresql-client-11 -y
|
||||
|
||||
# mongo
|
||||
wget -qO - https://www.mongodb.org/static/pgp/server-4.2.asc | apt-key add -
|
||||
echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.2 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.2.list
|
||||
apt-get update
|
||||
apt-get install mongodb-org-tools -y
|
||||
|
||||
# mo
|
||||
curl -sSL https://git.io/get-mo -o mo
|
||||
chmod +x mo
|
||||
mv mo /usr/local/bin/
|
||||
|
||||
# s3
|
||||
echo -e "\n====> Install S3"
|
||||
wget -O s3tools.tar.gz https://sourceforge.net/projects/s3tools/files/s3cmd/2.0.2/s3cmd-2.0.2.tar.gz/download && \
|
||||
( tar xvfz s3tools.tar.gz && cd s3cmd-2.0.2 && \
|
||||
python setup.py install && s3cmd --configure ) && rm -rf s3cmd-2.0.2
|
||||
|
||||
# b2
|
||||
echo -e "\n====> Install B2"
|
||||
git clone https://github.com/Backblaze/B2_Command_Line_Tool.git && \
|
||||
( cd B2_Command_Line_Tool && python setup.py install ) && rm -rf B2_Command_Line_Tool
|
||||
|
||||
# # git-lfs
|
||||
# echo -e "\n====> Install git-lfs"
|
||||
# curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash && \
|
||||
# apt-get install git-lfs && \
|
||||
|
||||
# # Docker
|
||||
# echo -e "\n====> Install docker"
|
||||
# apt update && \
|
||||
# apt install -y apt-transport-https ca-certificates curl software-properties-common
|
||||
|
||||
# curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \
|
||||
# apt-key fingerprint 0EBFCD88 && \
|
||||
# add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
|
||||
|
||||
# apt update && \
|
||||
# apt install -y docker-ce
|
||||
|
||||
# #docker version
|
||||
|
||||
# # Docker compose
|
||||
# echo -e "\n====> Install docker-compose"
|
||||
# curl -L "https://github.com/docker/compose/releases/download/1.23.1/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose && \
|
||||
# chmod +x /usr/local/bin/docker-compose
|
||||
|
||||
# #docker-compose version
|
||||
|
||||
# Post install tasks
|
||||
mkdir -p /srv/backup
|
||||
find /etc/cron.* -type f ! -name .placeholder -delete
|
||||
apt-get clean
|
||||
Executable
+58
@@ -0,0 +1,58 @@
|
||||
#!/bin/bash
|
||||
|
||||
source lib/lib.sh
|
||||
|
||||
ENVS=(PROJECT_NAME DBTYPE DBHOST DBUSER DBPASS DBTABLE STORAGE_TYPE STORAGE_HOST STORAGE_PATH ACCOUNT_ID APPLICATION_KEY)
|
||||
|
||||
for ENV in "${ENVS[@]}"; do
|
||||
if [[ -z "${!ENV+x}" ]]; then
|
||||
log_error "Not set $ENV env"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
case $DBTYPE in
|
||||
mysql)
|
||||
log "DB type is <mysql>"
|
||||
source lib/mysql.sh
|
||||
;;
|
||||
postgresql|psql|postgres)
|
||||
log "DB type is <psql>"
|
||||
source lib/postgresql.sh
|
||||
;;
|
||||
mongo|mongod|mongodb)
|
||||
log "DB type is <mongodb>"
|
||||
source lib/mongo.sh
|
||||
;;
|
||||
*)
|
||||
log_error "Unknown DB type, known <mysql>, <psql>"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
case $STORAGE_TYPE in
|
||||
b2)
|
||||
log "Storage type is <b2>"
|
||||
source storage/b2.sh
|
||||
;;
|
||||
s3)
|
||||
log "Storage type is <s3>"
|
||||
source storage/s3.sh
|
||||
;;
|
||||
*)
|
||||
log_error "Unknown storage, known <s3>, <b2>"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
ARCHIVE="/srv/backup/""$PROJECT_NAME"_`date +%Y-%m-%d-%H-%M`".dump"
|
||||
|
||||
log "Backuping.."
|
||||
db_backup
|
||||
|
||||
log "Copy to ${STORAGE}"
|
||||
storage_upload
|
||||
|
||||
log "Completed"
|
||||
|
||||
exit 0
|
||||
@@ -0,0 +1,29 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Логирование сообщения со статусом
|
||||
log() {
|
||||
local res=$?
|
||||
local mes=$@
|
||||
|
||||
case "${res}" in
|
||||
0 ) mes="[ Ok ] "${mes} ;;
|
||||
* ) mes="[ Fail ] "${mes} ;;
|
||||
esac
|
||||
|
||||
echo -e ${mes}
|
||||
|
||||
return ${res}
|
||||
}
|
||||
|
||||
log_error() {
|
||||
false
|
||||
log $@
|
||||
return 1
|
||||
}
|
||||
|
||||
# Сообщение с подтверждением
|
||||
confirm() {
|
||||
mes="$@"
|
||||
read -p "$mes (y/n): " res
|
||||
if [[ "${res}" == "y" || "${res}" == "yes" ]]; then return 0; else return 1; fi
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/bin/bash
|
||||
|
||||
db_backup() {
|
||||
cd /tmp
|
||||
mkdir 1/
|
||||
mongodump -v --host="${DBHOST}" --port=27017 --out=/tmp/1
|
||||
cd /tmp/1
|
||||
tar zcfv ../mongo_backup.tgz .
|
||||
cd ..
|
||||
mv mongo_backup.tgz ${ARCHIVE}
|
||||
rm -rf /tmp/1
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# TODO:
|
||||
# - Проверка существования БД
|
||||
|
||||
db_connect() {
|
||||
export DB_CONNECTION="-h ${DB_HOST} -u ${DB_USER} ${DB_NAME}"
|
||||
|
||||
cat > ~/.my.cnf << EOF
|
||||
[mysql]
|
||||
user=${DB_USER}
|
||||
password=${DB_PASS}
|
||||
|
||||
[mysqldump]
|
||||
user=${DB_USER}
|
||||
password=${DB_PASS}
|
||||
EOF
|
||||
|
||||
chmod 600 ~/.my.cnf
|
||||
}
|
||||
|
||||
db_drop() {
|
||||
[[ -z "${DB_CONNECTION}" ]] && db_connect
|
||||
|
||||
mysql ${DB_CONNECTION} -e "drop database ${DB_NAME}"
|
||||
}
|
||||
|
||||
db_backup() {
|
||||
[[ -z "${DB_CONNECTION}" ]] && db_connect
|
||||
|
||||
mysqldump --routines ${DB_CONNECTION} > ${ARCHIVE}
|
||||
}
|
||||
|
||||
db_restore() {
|
||||
[[ -z "${DB_CONNECTION}" ]] && db_connect
|
||||
|
||||
mysql ${DB_CONNECTION} < ${ARCHIVE}
|
||||
}
|
||||
|
||||
db_create() {
|
||||
# [[ -z "${DB_CONNECTION}" ]] && db_connect
|
||||
|
||||
mysql -h ${DB_HOST} -u ${DB_USER} -e "create database if not exist ${DB_NAME} charset utf8"
|
||||
}
|
||||
|
||||
db_isExist() {
|
||||
[[ -z "${DB_CONNECTION}" ]] && db_connect
|
||||
|
||||
false
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# TODO:
|
||||
# - Проверка существования БД
|
||||
|
||||
DB_CONNECTION=
|
||||
|
||||
db_connect() {
|
||||
export DB_CONNECTION="-h ${DBHOST} -d ${DBTABLE} -U ${DBUSER}"
|
||||
|
||||
echo "${DBHOST}:5432:${DBTABLE}:${DBUSER}:${DBPASS}" > ~/.pgpass
|
||||
#cat ~/.pgpass
|
||||
chmod 600 ~/.pgpass
|
||||
}
|
||||
|
||||
db_drop() {
|
||||
[[ -z "${DB_CONNECTION}" ]] && db_connect
|
||||
|
||||
# psql ${DB_CONNECTION} -c "drop schema public cascade"
|
||||
PGSQL_ARGS="--clean"
|
||||
}
|
||||
|
||||
db_create() {
|
||||
# [[ -z "${DB_CONNECTION}" ]] && db_connect
|
||||
|
||||
echo "SELECT 'CREATE DATABASE ${DBTABLE}' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = '${DBTABLE}')\gexec" | psql -h ${DBHOST} -U ${DBUSER}
|
||||
|
||||
# psql ${DB_CONNECTION} -c "create schema public"
|
||||
}
|
||||
|
||||
db_isExist() {
|
||||
[[ -z "${DB_CONNECTION}" ]] && db_connect
|
||||
|
||||
false
|
||||
}
|
||||
|
||||
db_backup() {
|
||||
[[ -z "${DB_CONNECTION}" ]] && db_connect
|
||||
|
||||
pg_dump -F c -O -b ${DB_CONNECTION} -f ${ARCHIVE}
|
||||
}
|
||||
|
||||
db_restore() {
|
||||
[[ -z "${DB_CONNECTION}" ]] && db_connect
|
||||
|
||||
local log_file=${ARCHIVE%.*}.log
|
||||
echo "[ Info ] pg_restore log file: ${log_file}"
|
||||
pg_restore --no-owner ${PGSQL_ARGS} ${DB_CONNECTION} ${ARCHIVE} > ${log_file}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
S3_BUCKET=amega
|
||||
S3_HOST_BASE=s3.amegaserver.com
|
||||
S3_ACCOUNT_ID=
|
||||
S3_APPLICATION_KEY=
|
||||
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
B2_BUCKET=${B2_BUCKET:-${REMOTE_DIR%%/*}}
|
||||
B2_REMOTE_DIR=${REMOTE_DIR#*/}
|
||||
Var::check B2_BUCKET
|
||||
|
||||
# Добавляем в конце '/'
|
||||
if [[ -n "${B2_REMOTE_DIR}" ]] && [[ ${B2_REMOTE_DIR} != */ ]]; then B2_REMOTE_DIR="${B2_REMOTE_DIR}/"; fi
|
||||
|
||||
# Полное имя архива на хранилище
|
||||
B2_ARCHIVE=${B2_REMOTE_DIR}${REMOTE_ARCHIVE}
|
||||
B2_ACCOUNT_ID=${B2_ACCOUNT_ID}
|
||||
B2_APPLICATION_KEY=${B2_APPLICATION_KEY}
|
||||
|
||||
B2_CONNECTION=
|
||||
|
||||
if [[ -n "${DEBUG}" ]]; then
|
||||
echo B2_BUCKET=${B2_BUCKET}
|
||||
echo B2_REMOTE_DIR=${B2_REMOTE_DIR}
|
||||
echo B2_ARCHIVE=${B2_ARCHIVE}
|
||||
fi
|
||||
|
||||
storage_login() {
|
||||
b2 authorize-account ${B2_ACCOUNT_ID} ${B2_APPLICATION_KEY}
|
||||
log "Login to B2 by ${B2_ACCOUNT_ID}" && \
|
||||
export B2_CONNECTION=true
|
||||
|
||||
return $?
|
||||
}
|
||||
|
||||
storage_download() {
|
||||
if Var::isFalse ${B2_CONNECTION}; then storage_login; fi
|
||||
|
||||
b2 download-file-by-name ${B2_BUCKET} ${B2_ARCHIVE} ${ARCHIVE}
|
||||
log "Download \"${B2_ARCHIVE}\" from busket \"${B2_BUCKET}\" \n\t as \"${ARCHIVE}\""
|
||||
|
||||
return $?
|
||||
}
|
||||
|
||||
storage_upload() {
|
||||
if Var::isFalse ${B2_CONNECTION}; then storage_login; fi
|
||||
|
||||
if Var::isFalse ${DRY_RUN}; then
|
||||
b2 upload-file ${B2_BUCKET} ${ARCHIVE} ${B2_ARCHIVE}
|
||||
fi
|
||||
log "Upload \"${ARCHIVE}\" \n\t as \"${B2_ARCHIVE}\" in \"${B2_BUCKET}\""
|
||||
|
||||
return $?
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
S3_HOST_BASE=${STORAGE_HOST:-s3.amazonaws.com}
|
||||
S3_REMOTE_DIR=${STORAGE_PATH#*/}
|
||||
S3_BUCKET=${STORAGE_PATH%%/*}
|
||||
|
||||
# Добавляем в конце '/'
|
||||
if [[ -n "${S3_REMOTE_DIR}" ]] && [[ ${S3_REMOTE_DIR} != */ ]]; then S3_REMOTE_DIR="${S3_REMOTE_DIR}/"; fi
|
||||
|
||||
# Полное имя архива на хранилище
|
||||
S3_ARCHIVE=${S3_REMOTE_DIR}${REMOTE_ARCHIVE}
|
||||
|
||||
S3_CONNECTION=
|
||||
|
||||
if [[ -n "${DEBUG}" ]]; then
|
||||
echo S3_BUCKET=${S3_BUCKET}
|
||||
echo S3_REMOTE_DIR=${S3_REMOTE_DIR}
|
||||
echo S3_ARCHIVE=${S3_ARCHIVE}
|
||||
fi
|
||||
|
||||
storage_login() {
|
||||
[ -f /root/.s3cfg ] && rm /root/.s3cfg
|
||||
|
||||
echo S3_BUCKET=${S3_BUCKET} > .env
|
||||
echo S3_HOST_BASE=${S3_HOST_BASE} >> .env
|
||||
echo S3_ACCOUNT_ID=${ACCOUNT_ID} >> .env
|
||||
echo S3_APPLICATION_KEY=${APPLICATION_KEY} >> .env
|
||||
|
||||
mo -s=.env -u /usr/local/sbin/storage/s3.template > /root/.s3cfg
|
||||
|
||||
log "Setup to S3 by ${S3_ACCOUNT_ID}" && \
|
||||
export S3_CONNECTION=true
|
||||
|
||||
return $?
|
||||
}
|
||||
|
||||
storage_download() {
|
||||
[ -z ${S3_CONNECTION} ] && storage_login
|
||||
|
||||
s3cmd get s3://${S3_BUCKET}/${S3_ARCHIVE} ${ARCHIVE}
|
||||
log "Download \"${S3_ARCHIVE}\" from bucket \"${S3_BUCKET}\" \n\t as \"${ARCHIVE}\""
|
||||
|
||||
return $?
|
||||
}
|
||||
|
||||
storage_upload() {
|
||||
[ -z ${S3_CONNECTION} ] && storage_login
|
||||
|
||||
log "Upload \"${ARCHIVE}\" \n\t as \"${S3_ARCHIVE}\" in \"${S3_BUCKET}\""
|
||||
s3cmd put ${ARCHIVE} s3://${S3_BUCKET}/${S3_ARCHIVE}
|
||||
|
||||
return $?
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
[default]
|
||||
access_key = {{S3_ACCOUNT_ID}}
|
||||
access_token =
|
||||
add_encoding_exts =
|
||||
add_headers =
|
||||
bucket_location = US
|
||||
ca_certs_file =
|
||||
cache_file =
|
||||
check_ssl_certificate = True
|
||||
check_ssl_hostname = True
|
||||
cloudfront_host = cloudfront.amazonaws.com
|
||||
content_disposition =
|
||||
content_type =
|
||||
default_mime_type = binary/octet-stream
|
||||
delay_updates = False
|
||||
delete_after = False
|
||||
delete_after_fetch = False
|
||||
delete_removed = False
|
||||
dry_run = False
|
||||
enable_multipart = True
|
||||
encoding = UTF-8
|
||||
encrypt = False
|
||||
expiry_date =
|
||||
expiry_days =
|
||||
expiry_prefix =
|
||||
follow_symlinks = False
|
||||
force = False
|
||||
get_continue = False
|
||||
gpg_command = /usr/local/bin/gpg
|
||||
gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
|
||||
gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
|
||||
gpg_passphrase =
|
||||
guess_mime_type = True
|
||||
host_base = {{S3_HOST_BASE}}
|
||||
host_bucket = {{S3_BUCKET}}.{{S3_HOST_BASE}}
|
||||
human_readable_sizes = False
|
||||
invalidate_default_index_on_cf = False
|
||||
invalidate_default_index_root_on_cf = True
|
||||
invalidate_on_cf = False
|
||||
kms_key =
|
||||
limit = -1
|
||||
limitrate = 0
|
||||
list_md5 = False
|
||||
log_target_prefix =
|
||||
long_listing = False
|
||||
max_delete = -1
|
||||
mime_type =
|
||||
multipart_chunk_size_mb = 15
|
||||
multipart_max_chunks = 10000
|
||||
preserve_attrs = True
|
||||
progress_meter = True
|
||||
proxy_host =
|
||||
proxy_port = 0
|
||||
put_continue = False
|
||||
recursive = False
|
||||
recv_chunk = 65536
|
||||
reduced_redundancy = False
|
||||
requester_pays = False
|
||||
restore_days = 1
|
||||
restore_priority = Standard
|
||||
secret_key = {{S3_APPLICATION_KEY}}
|
||||
send_chunk = 65536
|
||||
server_side_encryption = False
|
||||
signature_v2 = False
|
||||
signurl_use_https = False
|
||||
simpledb_host = sdb.amazonaws.com
|
||||
skip_existing = False
|
||||
socket_timeout = 300
|
||||
stats = False
|
||||
stop_on_error = False
|
||||
storage_class =
|
||||
throttle_max = 100
|
||||
upload_id =
|
||||
urlencoding_mode = normal
|
||||
use_http_expect = False
|
||||
use_https = True
|
||||
use_mime_magic = True
|
||||
verbosity = WARNING
|
||||
website_endpoint = http://%(bucket)s.s3-website-%(location)s.amazonaws.com/
|
||||
website_error =
|
||||
website_index = index.html
|
||||
Reference in New Issue
Block a user