diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0660d4f..b16411f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,6 @@ image: docker.bildme.ru/images/docker variables: - DOCKER_BASEIMAGE: ubuntu:18.04 + DOCKER_BASEIMAGE: ubuntu:22.04 DOCKER_MAINTAINER: vasyakrg@gmail.com DOCKER_IMAGE_NAME: backup BUILD_SCRIPT: ci/build-image/build.sh @@ -14,4 +14,3 @@ build: - export DOCKER_IMAGE_NAME=${DOCKER_IMAGE_NAME} - export DOCKER_IMAGE_TAG=$VERSION - chmod +x "${BUILD_SCRIPT}" && "./${BUILD_SCRIPT}" - when: manual diff --git a/Dockerfile b/Dockerfile index 20d9793..673629a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,21 +4,6 @@ FROM ${DOCKER_BASEIMAGE} ARG DOCKER_MAINTAINER LABEL maintainer="${DOCKER_MAINTAINER}" -ARG PROJECT_NAME -ARG DBTYPE -ARG DBHOST -ARG DBUSER -ARG DBPASS -ARG DBTABLE -ARG STORAGE - -ARG STORAGE_TYPE -ARG STORAGE_HOST -ARG STORAGE_PATH -ARG ACCOUNT_ID -ARG APPLICATION_KEY - - COPY assets/build /build WORKDIR /build RUN chmod +x *.sh && \ diff --git a/README.md b/README.md index e76e9f6..cf3b4b0 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,35 @@ # Backup под для cronjobs + поддерживает бекапирование баз типа: + - postgresql - mongo - mysql умеет заливать на: -- backblaze + - s3 всегда передаем следующие переменные: + - PROJECT_NAME - на основе имени проекта будет сформирован файли бекапа вида `"$PROJECT_NAME"_<дата+время>".dump"` -- DBTYPE - , , +- DBTYPE - ``, ``, `` - DBHOST - на службу нужной базы - DBUSER - юзер (если не используется, как например в монге, ставим "1") - DBPASS - пасс (если не используется, как например в монге, ставим "1") - DBTABLE - таблица (если не используется, как например в монге, ставим "1") -- STORAGE_TYPE - , -- STORAGE_HOST - (если не используется, как например в b2, ставим "1"), можно указать свое s3 хранилище вида `s3.domain.com` либо амазоновское -- STORAGE_PATH - путь, куда складывать архив вида +- STORAGE_HOST - можно указать свое s3 хранилище вида `s3.domain.com` либо амазоновское +- STORAGE_REGION - по умолчанию: us-west +- STORAGE_PATH - путь, куда складывать архив вида `` - ACCOUNT_ID - логин в хранилище - APPLICATION_KEY - пароль в хранилище доп параметры: - - CHUNK_SIZE - размер одного чанка при заливке больших файлов (по дефолту 15 Мб) + +- CHUNK_SIZE - размер одного чанка при заливке больших файлов (по умолчанию 100 Мб) ##### Автор -- **Vassiliy Yegorov** - *Initial work* - [vasyakrg](https://github.com/vasyakrg) -- [сайт](vk.com/realmanual) -- [youtube](youtube.com/realmanual) + +- **Vassiliy Yegorov** +- [сайт](https://realmanual.ru) +- [youtube](https://youtube.com/realmanual) diff --git a/assets/build/install.sh b/assets/build/install.sh index 1a91d94..f4a0df5 100644 --- a/assets/build/install.sh +++ b/assets/build/install.sh @@ -5,45 +5,41 @@ set -e DEBIAN_FRONTEND=noninteractive apt-get update && \ -apt-get install -y gnupg2 git wget python-setuptools cron iputils-ping curl nano lsb-release apt-utils software-properties-common && \ +apt-get install -y gnupg2 wget iputils-ping curl nano ca-certificates apt-transport-https lsb-release && \ + +# echo deb http://archive.ubuntu.com/ubuntu $(lsb_release -cs) main restricted universe multiverse > /etc/apt/sources.list && \ +# echo deb http://archive.ubuntu.com/ubuntu $(lsb_release -cs)-updates main restricted universe multiverse >> /etc/apt/sources.list && \ +# echo deb http://archive.ubuntu.com/ubuntu $(lsb_release -cs)-security main restricted universe multiverse >> /etc/apt/sources.list && \ # psql -wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \ -echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list && \ +echo -e "\n====> Install PGSQL + Mysql" +wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/postgresql.gpg > /dev/null && \ +echo deb [arch=amd64 signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main | tee -a /etc/apt/sources.list.d/postgresql.list && \ apt-get update && \ -apt-get install -y postgresql-client-10 postgresql-client-11 postgresql-client-12 mysql-client nfs-common +apt-cache search postgresql-client-* && \ +apt-get install -y postgresql-client-10 postgresql-client-12 postgresql-client-14 mysql-client # mongo -wget -qO - https://www.mongodb.org/static/pgp/server-4.2.asc | apt-key add - -echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.2 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.2.list +echo -e "\n====> Install Mongo" +wget -qO - https://www.mongodb.org/static/pgp/server-4.2.asc | gpg --dearmor | tee /usr/share/keyrings/mongodb.gpg > /dev/null && \ +echo deb [arch=amd64 signed-by=/usr/share/keyrings/mongodb.gpg] https://repo.mongodb.org/apt/ubuntu $(lsb_release -cs)/mongodb-org/4.2 multiverse | tee -a /etc/apt/sources.list.d/mongodb-org-4.2.list && \ apt-get update apt-get install mongodb-org-tools -y # mo +echo -e "\n====> Install Mo" curl -sSL https://git.io/get-mo -o mo chmod +x mo mv mo /usr/local/bin/ # s3 echo -e "\n====> Install S3" -wget -O s3tools.tar.gz https://github.com/s3tools/s3cmd/releases/download/v2.1.0/s3cmd-2.1.0.tar.gz && \ -( tar xvfz s3tools.tar.gz && cd s3cmd-2.1.0 && \ - python setup.py install && s3cmd --configure ) && rm -rf s3cmd-2.1.0 +apt-get install rclone -# b2 -echo -e "\n====> Install B2" -apt-get install python-pip python-dev build-essential -y -pip install --upgrade pip -pip install --upgrade b2 #==1.4.2 - - -# # git-lfs -# echo -e "\n====> Install git-lfs" -# curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash && \ -# apt-get install git-lfs && \ # Post install tasks +echo -e "\n====> Post install tasks" mkdir -p /srv/backup find /etc/cron.* -type f ! -name .placeholder -delete apt-get clean diff --git a/assets/usr/local/sbin/backup.sh b/assets/usr/local/sbin/backup.sh index 934d27b..478c92f 100755 --- a/assets/usr/local/sbin/backup.sh +++ b/assets/usr/local/sbin/backup.sh @@ -2,7 +2,7 @@ source lib/lib.sh -ENVS=(PROJECT_NAME DBTYPE DBHOST DBUSER DBPASS DBTABLE STORAGE_TYPE STORAGE_HOST STORAGE_PATH ACCOUNT_ID APPLICATION_KEY) +ENVS=(PROJECT_NAME DBTYPE DBHOST DBUSER DBPASS DBTABLE STORAGE_HOST STORAGE_REGION STORAGE_PATH ACCOUNT_ID APPLICATION_KEY) for ENV in "${ENVS[@]}"; do if [[ -z "${!ENV+x}" ]]; then @@ -30,25 +30,22 @@ case $DBTYPE in source lib/mongo.sh ;; *) - log_error "Unknown DB type, known , " + log_error "Unknown DB type, known , , mongodb" exit 1 ;; esac -case $STORAGE_TYPE in - b2) - log "Storage type is " - source storage/b2.sh - ;; - s3) - log "Storage type is " - source storage/s3.sh - ;; - *) - log_error "Unknown storage, known , " - exit 1 - ;; -esac +# case $STORAGE_TYPE in +# s3) +# log "Storage type is " +# source storage/s3.sh +# ;; +# *) +# log_error "Unknown storage, known " +# exit 1 +# ;; +# esac +STORAGE_TYPE=s3 TIME=`date +%Y-%m-%d-%H-%M` ARCHIVE="/srv/backup/""$PROJECT_NAME"_$TIME".$EXT_DB" diff --git a/assets/usr/local/sbin/storage/.env b/assets/usr/local/sbin/storage/.env index d4b8509..b26ca21 100644 --- a/assets/usr/local/sbin/storage/.env +++ b/assets/usr/local/sbin/storage/.env @@ -1,4 +1,5 @@ S3_BUCKET=amega S3_HOST_BASE=s3.amegaserver.com +S3_REGION=ru-nsk S3_ACCOUNT_ID= S3_APPLICATION_KEY= diff --git a/assets/usr/local/sbin/storage/b2.sh b/assets/usr/local/sbin/storage/b2.sh deleted file mode 100644 index 957f0de..0000000 --- a/assets/usr/local/sbin/storage/b2.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -storage_login() { - B2_REMOTE_DIR=${STORAGE_PATH#*/} - B2_BUCKET=${STORAGE_PATH%%/*} - - B2_ARCHIVE=${B2_REMOTE_DIR}/$(basename ${ARCHIVE}) - B2_ACCOUNT_ID=${ACCOUNT_ID} - B2_APPLICATION_KEY=${APPLICATION_KEY} - - if [[ -n "${DEBUG}" ]]; then - echo B2_BUCKET=${B2_BUCKET} - echo B2_REMOTE_DIR=${B2_REMOTE_DIR} - echo B2_ARCHIVE=${B2_ARCHIVE} - echo ARCHIVE=${ARCHIVE} - fi - - B2_CONNECTION= - b2 authorize-account ${B2_ACCOUNT_ID} ${B2_APPLICATION_KEY} - log "Login to B2 by ${B2_ACCOUNT_ID}" && export B2_CONNECTION=true - - return $? -} - -storage_download() { - [ -z ${B2_CONNECTION} ] && storage_login - b2 download-file-by-name ${B2_BUCKET} ${B2_ARCHIVE} ${ARCHIVE} - log "Download in \"${B2_ARCHIVE}\" from bucket \"${B2_BUCKET}\" \n\t as \"$(basename ${ARCHIVE})\"" - - return $? -} - -storage_upload() { - [ -z ${B2_CONNECTION} ] && storage_login - b2 upload-file ${B2_BUCKET} ${ARCHIVE} ${B2_ARCHIVE} - log "Upload \"$(basename ${ARCHIVE})\" \n\t to \"${B2_ARCHIVE}\" in bucket \"${B2_BUCKET}\"" - - return $? -} diff --git a/assets/usr/local/sbin/storage/s3.sh b/assets/usr/local/sbin/storage/s3.sh index f01ed5b..3f27a26 100644 --- a/assets/usr/local/sbin/storage/s3.sh +++ b/assets/usr/local/sbin/storage/s3.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash S3_HOST_BASE=${STORAGE_HOST:-s3.amazonaws.com} +S3_REGION=${S3_REGION:-us-west} S3_REMOTE_DIR=${STORAGE_PATH#*/} S3_BUCKET=${STORAGE_PATH%%/*} @@ -14,39 +15,34 @@ S3_CONNECTION= if [[ -n "${DEBUG}" ]]; then echo S3_BUCKET=${S3_BUCKET} + echo S3_REGION=${S3_REGION} echo S3_REMOTE_DIR=${S3_REMOTE_DIR} echo S3_ARCHIVE=${S3_ARCHIVE} fi storage_login() { - [ -f /root/.s3cfg ] && rm /root/.s3cfg + config=/root/.config/rclone/rclone.config + + [ -f $config ] && rm $config echo S3_BUCKET=${S3_BUCKET} > .env + echo S3_REGION=${S3_REGION} >> .env echo S3_HOST_BASE=${S3_HOST_BASE} >> .env echo S3_ACCOUNT_ID=${ACCOUNT_ID} >> .env echo S3_APPLICATION_KEY=${APPLICATION_KEY} >> .env - mo -s=.env -u /usr/local/sbin/storage/s3.template > /root/.s3cfg + mo -s=.env -u /usr/local/sbin/storage/s3.template > $config - log "Setup to S3 by ${S3_ACCOUNT_ID}" && \ + log "Setup to S3 by ${S3_ACCOUNT_ID} to ${S3_HOST_BASE}" && \ export S3_CONNECTION=true return $? } -storage_download() { - [ -z ${S3_CONNECTION} ] && storage_login - - s3cmd get s3://${S3_BUCKET}/${S3_ARCHIVE} ${ARCHIVE} - log "Download \"${S3_ARCHIVE}\" from bucket \"${S3_BUCKET}\" \n\t as \"${ARCHIVE}\"" - - return $? -} - storage_upload() { [ -z ${S3_CONNECTION} ] && storage_login log "Upload \"${ARCHIVE}\" \n\t as \"${S3_ARCHIVE}\" in \"${S3_BUCKET}\"" - s3cmd put --multipart-chunk-size-mb=${CHUNK_SIZE:-15} ${ARCHIVE} s3://${S3_BUCKET}/${S3_ARCHIVE} + rclone copy --progress --transfers=5 --update --s3-chunk-size=${CHUNK_SIZE:-100m} ${ARCHIVE} s3:/${S3_BUCKET}/${S3_ARCHIVE} return $? } diff --git a/assets/usr/local/sbin/storage/s3.template b/assets/usr/local/sbin/storage/s3.template index 419018c..e5db3e4 100644 --- a/assets/usr/local/sbin/storage/s3.template +++ b/assets/usr/local/sbin/storage/s3.template @@ -1,81 +1,8 @@ -[default] -access_key = {{S3_ACCOUNT_ID}} -access_token = -add_encoding_exts = -add_headers = -bucket_location = US -ca_certs_file = -cache_file = -check_ssl_certificate = True -check_ssl_hostname = True -cloudfront_host = cloudfront.amazonaws.com -content_disposition = -content_type = -default_mime_type = binary/octet-stream -delay_updates = False -delete_after = False -delete_after_fetch = False -delete_removed = False -dry_run = False -enable_multipart = True -encoding = UTF-8 -encrypt = False -expiry_date = -expiry_days = -expiry_prefix = -follow_symlinks = False -force = False -get_continue = False -gpg_command = /usr/local/bin/gpg -gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s -gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s -gpg_passphrase = -guess_mime_type = True -host_base = {{S3_HOST_BASE}} -host_bucket = {{S3_BUCKET}}.{{S3_HOST_BASE}} -human_readable_sizes = False -invalidate_default_index_on_cf = False -invalidate_default_index_root_on_cf = True -invalidate_on_cf = False -kms_key = -limit = -1 -limitrate = 0 -list_md5 = False -log_target_prefix = -long_listing = False -max_delete = -1 -mime_type = -multipart_chunk_size_mb = 15 -multipart_max_chunks = 10000 -preserve_attrs = True -progress_meter = True -proxy_host = -proxy_port = 0 -put_continue = False -recursive = False -recv_chunk = 65536 -reduced_redundancy = False -requester_pays = False -restore_days = 1 -restore_priority = Standard -secret_key = {{S3_APPLICATION_KEY}} -send_chunk = 65536 -server_side_encryption = False -signature_v2 = False -signurl_use_https = False -simpledb_host = sdb.amazonaws.com -skip_existing = False -socket_timeout = 300 -stats = False -stop_on_error = False -storage_class = -throttle_max = 100 -upload_id = -urlencoding_mode = normal -use_http_expect = False -use_https = True -use_mime_magic = True -verbosity = WARNING -website_endpoint = http://%(bucket)s.s3-website-%(location)s.amazonaws.com/ -website_error = -website_index = index.html +[s3] +type = s3 +provider = Minio +acl = private +access_key_id = {{S3_ACCOUNT_ID}} +secret_access_key = {{S3_APPLICATION_KEY}} +region = {{S3_REGION}} +endpoint = https://{{S3_HOST_BASE}} diff --git a/build-docker.sh b/build-docker.sh index 826cbe0..4ce5f0e 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -1,4 +1,10 @@ #!/bin/bash -docker build -t vasyakrg/backup:v2 . -docker push vasyakrg/backup:v2 +DOCKER_BASEIMAGE=ubuntu:22.04 +DOCKER_MAINTAINER=vasyakrg@gmail.com + +docker buildx create --name cibuilder --driver docker-container --use + +docker buildx build --progress=plain --platform linux/amd64,linux/arm64 --push -t hub.realmanual.ru/images/backup-2:$(cat VERSION) \ + --build-arg DOCKER_BASEIMAGE=${DOCKER_BASEIMAGE} \ + --build-arg DOCKER_MAINTAINER=${DOCKER_MAINTAINER} . diff --git a/run-test.sh b/run-test.sh new file mode 100755 index 0000000..3d25b1c --- /dev/null +++ b/run-test.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +source .env + +docker run --rm --env-file .env --network dalarmbot_net --name test_backup hub.realmanual.ru/images/backup-2:latest +echo "Backup create and save to ${STORAGE_HOST}/${STORAGE_PATH}"