del legacy b2, add rclone

.gitlab-ci.yml

@@ -1,6 +1,6 @@
 image: docker.bildme.ru/images/docker
 variables:
-  DOCKER_BASEIMAGE: ubuntu:18.04
+  DOCKER_BASEIMAGE: ubuntu:22.04
   DOCKER_MAINTAINER: vasyakrg@gmail.com
   DOCKER_IMAGE_NAME: backup
   BUILD_SCRIPT: ci/build-image/build.sh
@@ -14,4 +14,3 @@ build:
     - export DOCKER_IMAGE_NAME=${DOCKER_IMAGE_NAME}
     - export DOCKER_IMAGE_TAG=$VERSION
     - chmod +x "${BUILD_SCRIPT}" && "./${BUILD_SCRIPT}"
-  when: manual

Dockerfile

@@ -4,21 +4,6 @@ FROM ${DOCKER_BASEIMAGE}
 ARG DOCKER_MAINTAINER
 LABEL maintainer="${DOCKER_MAINTAINER}"
 
-ARG PROJECT_NAME
-ARG DBTYPE
-ARG DBHOST
-ARG DBUSER
-ARG DBPASS
-ARG DBTABLE
-ARG STORAGE
-
-ARG STORAGE_TYPE
-ARG STORAGE_HOST
-ARG STORAGE_PATH
-ARG ACCOUNT_ID
-ARG APPLICATION_KEY
-
-
 COPY assets/build /build
 WORKDIR /build
 RUN chmod +x *.sh && \

README.md

@@ -1,30 +1,35 @@
 # Backup под для cronjobs
+
 поддерживает бекапирование баз типа:
+
 - postgresql
 - mongo
 - mysql
 
 умеет заливать на:
-- backblaze
+
 - s3
 
 всегда передаем следующие переменные:
+
 - PROJECT_NAME - на основе имени проекта будет сформирован файли бекапа вида `"$PROJECT_NAME"_<дата+время>".dump"`
-- DBTYPE - <mysql>, <postgres>, <mongo>
+- DBTYPE - `<mysql>`, `<postgres>`, `<mongo>`
 - DBHOST - <host_service> на службу нужной базы
 - DBUSER - юзер (если не используется, как например в монге, ставим "1")
 - DBPASS - пасс (если не используется, как например в монге, ставим "1")
 - DBTABLE - таблица (если не используется, как например в монге, ставим "1")
-- STORAGE_TYPE - <s3>, <b2>
+- STORAGE_HOST - можно указать свое s3 хранилище вида `s3.domain.com` либо амазоновское
-- STORAGE_HOST - (если не используется, как например в b2, ставим "1"), можно указать свое s3 хранилище вида `s3.domain.com` либо амазоновское
+- STORAGE_REGION - по умолчанию: us-west
-- STORAGE_PATH - путь, куда складывать архив вида <bucket/folder>
+- STORAGE_PATH - путь, куда складывать архив вида `<bucket/folder>`
 - ACCOUNT_ID - логин в хранилище
 - APPLICATION_KEY - пароль в хранилище
 
 доп параметры:
- - CHUNK_SIZE - размер одного чанка при заливке больших файлов (по дефолту 15 Мб)
+
+- CHUNK_SIZE - размер одного чанка при заливке больших файлов (по умолчанию 100 Мб)
 
 ##### Автор
-- **Vassiliy Yegorov** - *Initial work* - [vasyakrg](https://github.com/vasyakrg)
+
-- [сайт](vk.com/realmanual)
+- **Vassiliy Yegorov**
-- [youtube](youtube.com/realmanual)
+- [сайт](https://realmanual.ru)
+- [youtube](https://youtube.com/realmanual)

assets/build/install.sh

@@ -5,45 +5,41 @@ set -e
 DEBIAN_FRONTEND=noninteractive
 
 apt-get update && \
-apt-get install -y gnupg2 git wget python-setuptools cron iputils-ping curl nano lsb-release apt-utils software-properties-common && \
+apt-get install -y gnupg2 wget iputils-ping curl nano ca-certificates apt-transport-https lsb-release && \
+
+# echo deb http://archive.ubuntu.com/ubuntu $(lsb_release -cs) main restricted universe multiverse > /etc/apt/sources.list && \
+# echo deb http://archive.ubuntu.com/ubuntu $(lsb_release -cs)-updates main restricted universe multiverse >> /etc/apt/sources.list && \
+# echo deb http://archive.ubuntu.com/ubuntu $(lsb_release -cs)-security main restricted universe multiverse >> /etc/apt/sources.list && \
 
 # psql
-wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \
+echo -e "\n====> Install PGSQL + Mysql"
-echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" | tee  /etc/apt/sources.list.d/pgdg.list && \
+wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/postgresql.gpg > /dev/null && \
+echo deb [arch=amd64 signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main | tee -a /etc/apt/sources.list.d/postgresql.list && \
 apt-get update && \
-apt-get install -y postgresql-client-10 postgresql-client-11 postgresql-client-12 mysql-client nfs-common
+apt-cache search postgresql-client-* && \
+apt-get install -y postgresql-client-10 postgresql-client-12 postgresql-client-14 mysql-client
 
 
 # mongo
-wget -qO - https://www.mongodb.org/static/pgp/server-4.2.asc | apt-key add -
+echo -e "\n====> Install Mongo"
-echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.2 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.2.list
+wget -qO - https://www.mongodb.org/static/pgp/server-4.2.asc | gpg --dearmor | tee /usr/share/keyrings/mongodb.gpg > /dev/null && \
+echo deb [arch=amd64 signed-by=/usr/share/keyrings/mongodb.gpg] https://repo.mongodb.org/apt/ubuntu $(lsb_release -cs)/mongodb-org/4.2 multiverse | tee -a /etc/apt/sources.list.d/mongodb-org-4.2.list && \
 apt-get update
 apt-get install mongodb-org-tools -y
 
 # mo
+echo -e "\n====> Install Mo"
 curl -sSL https://git.io/get-mo -o mo
 chmod +x mo
 mv mo /usr/local/bin/
 
 # s3
 echo -e "\n====> Install S3"
-wget -O s3tools.tar.gz https://github.com/s3tools/s3cmd/releases/download/v2.1.0/s3cmd-2.1.0.tar.gz && \
+apt-get install rclone
-( tar xvfz s3tools.tar.gz && cd s3cmd-2.1.0 && \
-  python setup.py install && s3cmd --configure ) && rm -rf s3cmd-2.1.0
 
-# b2
-echo -e "\n====> Install B2"
-apt-get install python-pip python-dev build-essential -y
-pip install --upgrade pip
-pip install --upgrade b2 #==1.4.2
-
-
-# # git-lfs
-# echo -e "\n====> Install git-lfs"
-# curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash && \
-# apt-get install git-lfs && \
 
 # Post install tasks
+echo -e "\n====> Post install tasks"
 mkdir -p /srv/backup
 find /etc/cron.* -type f ! -name .placeholder -delete
 apt-get clean

assets/usr/local/sbin/backup.sh

@@ -2,7 +2,7 @@
 
 source lib/lib.sh
 
-ENVS=(PROJECT_NAME DBTYPE DBHOST DBUSER DBPASS DBTABLE STORAGE_TYPE STORAGE_HOST STORAGE_PATH ACCOUNT_ID APPLICATION_KEY)
+ENVS=(PROJECT_NAME DBTYPE DBHOST DBUSER DBPASS DBTABLE STORAGE_HOST STORAGE_REGION STORAGE_PATH ACCOUNT_ID APPLICATION_KEY)
 
 for ENV in "${ENVS[@]}"; do
     if [[ -z "${!ENV+x}" ]]; then
@@ -30,25 +30,22 @@ case $DBTYPE in
         source lib/mongo.sh
     ;;
     *)
-        log_error "Unknown DB type, known <mysql>, <psql>"
+        log_error "Unknown DB type, known <mysql>, <psql>, mongodb"
         exit 1
     ;;
 esac
 
-case $STORAGE_TYPE in
+# case $STORAGE_TYPE in
-    b2)
+#     s3)
-        log "Storage type is <b2>"
+#         log "Storage type is <s3>"
-        source storage/b2.sh
+#         source storage/s3.sh
-    ;;
+#     ;;
-    s3)
+#     *)
-        log "Storage type is <s3>"
+#         log_error "Unknown storage, known <s3>"
-        source storage/s3.sh
+#         exit 1
-    ;;
+#     ;;
-    *)
+# esac
-        log_error "Unknown storage, known <s3>, <b2>"
+STORAGE_TYPE=s3
-        exit 1
-    ;;
-esac
 
 TIME=`date +%Y-%m-%d-%H-%M`
 ARCHIVE="/srv/backup/""$PROJECT_NAME"_$TIME".$EXT_DB"

assets/usr/local/sbin/storage/.env

@@ -1,4 +1,5 @@
 S3_BUCKET=amega
 S3_HOST_BASE=s3.amegaserver.com
+S3_REGION=ru-nsk
 S3_ACCOUNT_ID=
 S3_APPLICATION_KEY=

assets/usr/local/sbin/storage/b2.sh

@@ -1,39 +0,0 @@
-#!/usr/bin/env bash
-
-storage_login() {
-    B2_REMOTE_DIR=${STORAGE_PATH#*/}
-    B2_BUCKET=${STORAGE_PATH%%/*}
-
-    B2_ARCHIVE=${B2_REMOTE_DIR}/$(basename ${ARCHIVE})
-    B2_ACCOUNT_ID=${ACCOUNT_ID}
-    B2_APPLICATION_KEY=${APPLICATION_KEY}
-
-    if [[ -n "${DEBUG}" ]]; then
-        echo B2_BUCKET=${B2_BUCKET}
-        echo B2_REMOTE_DIR=${B2_REMOTE_DIR}
-        echo B2_ARCHIVE=${B2_ARCHIVE}
-        echo ARCHIVE=${ARCHIVE}
-    fi
-
-    B2_CONNECTION=
-    b2 authorize-account ${B2_ACCOUNT_ID} ${B2_APPLICATION_KEY}
-    log "Login to B2 by ${B2_ACCOUNT_ID}" && export B2_CONNECTION=true
-
-    return $?
-}
-
-storage_download() {
-    [ -z ${B2_CONNECTION} ] && storage_login
-    b2 download-file-by-name ${B2_BUCKET} ${B2_ARCHIVE} ${ARCHIVE}
-    log "Download in \"${B2_ARCHIVE}\" from bucket \"${B2_BUCKET}\" \n\t as \"$(basename ${ARCHIVE})\""
-
-    return $?
-}
-
-storage_upload() {
-    [ -z ${B2_CONNECTION} ] && storage_login
-    b2 upload-file ${B2_BUCKET} ${ARCHIVE} ${B2_ARCHIVE}
-    log "Upload \"$(basename ${ARCHIVE})\" \n\t to \"${B2_ARCHIVE}\" in bucket \"${B2_BUCKET}\""
-
-    return $?
-}

assets/usr/local/sbin/storage/s3.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 S3_HOST_BASE=${STORAGE_HOST:-s3.amazonaws.com}
+S3_REGION=${S3_REGION:-us-west}
 S3_REMOTE_DIR=${STORAGE_PATH#*/}
 S3_BUCKET=${STORAGE_PATH%%/*}
 
@@ -14,39 +15,34 @@ S3_CONNECTION=
 
 if [[ -n "${DEBUG}" ]]; then
     echo S3_BUCKET=${S3_BUCKET}
+    echo S3_REGION=${S3_REGION}
     echo S3_REMOTE_DIR=${S3_REMOTE_DIR}
     echo S3_ARCHIVE=${S3_ARCHIVE}
 fi
 
 storage_login() {
-    [ -f /root/.s3cfg ] && rm /root/.s3cfg
+    config=/root/.config/rclone/rclone.config
+
+    [ -f $config ] && rm $config
 
     echo S3_BUCKET=${S3_BUCKET} > .env
+    echo S3_REGION=${S3_REGION} >> .env
     echo S3_HOST_BASE=${S3_HOST_BASE} >> .env
     echo S3_ACCOUNT_ID=${ACCOUNT_ID} >> .env
     echo S3_APPLICATION_KEY=${APPLICATION_KEY} >> .env
 
-    mo -s=.env -u /usr/local/sbin/storage/s3.template > /root/.s3cfg
+    mo -s=.env -u /usr/local/sbin/storage/s3.template > $config
 
-    log "Setup to S3 by ${S3_ACCOUNT_ID}" && \
+    log "Setup to S3 by ${S3_ACCOUNT_ID} to ${S3_HOST_BASE}" && \
     export S3_CONNECTION=true
 
     return $?
 }
 
-storage_download() {
-    [ -z ${S3_CONNECTION} ] && storage_login
-
-    s3cmd get s3://${S3_BUCKET}/${S3_ARCHIVE} ${ARCHIVE}
-    log "Download \"${S3_ARCHIVE}\" from bucket \"${S3_BUCKET}\" \n\t as \"${ARCHIVE}\""
-
-    return $?
-}
-
 storage_upload() {
     [ -z ${S3_CONNECTION} ] && storage_login
 
     log "Upload \"${ARCHIVE}\" \n\t as \"${S3_ARCHIVE}\" in \"${S3_BUCKET}\""
-    s3cmd put --multipart-chunk-size-mb=${CHUNK_SIZE:-15} ${ARCHIVE} s3://${S3_BUCKET}/${S3_ARCHIVE}
+    rclone copy --progress --transfers=5 --update --s3-chunk-size=${CHUNK_SIZE:-100m} ${ARCHIVE} s3:/${S3_BUCKET}/${S3_ARCHIVE}
     return $?
 }

assets/usr/local/sbin/storage/s3.template

@@ -1,81 +1,8 @@
-[default]
+[s3]
-access_key = {{S3_ACCOUNT_ID}}
+type = s3
-access_token =
+provider = Minio
-add_encoding_exts =
+acl = private
-add_headers =
+access_key_id = {{S3_ACCOUNT_ID}}
-bucket_location = US
+secret_access_key = {{S3_APPLICATION_KEY}}
-ca_certs_file =
+region = {{S3_REGION}}
-cache_file =
+endpoint = https://{{S3_HOST_BASE}}
-check_ssl_certificate = True
-check_ssl_hostname = True
-cloudfront_host = cloudfront.amazonaws.com
-content_disposition =
-content_type =
-default_mime_type = binary/octet-stream
-delay_updates = False
-delete_after = False
-delete_after_fetch = False
-delete_removed = False
-dry_run = False
-enable_multipart = True
-encoding = UTF-8
-encrypt = False
-expiry_date =
-expiry_days =
-expiry_prefix =
-follow_symlinks = False
-force = False
-get_continue = False
-gpg_command = /usr/local/bin/gpg
-gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
-gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
-gpg_passphrase =
-guess_mime_type = True
-host_base = {{S3_HOST_BASE}}
-host_bucket = {{S3_BUCKET}}.{{S3_HOST_BASE}}
-human_readable_sizes = False
-invalidate_default_index_on_cf = False
-invalidate_default_index_root_on_cf = True
-invalidate_on_cf = False
-kms_key =
-limit = -1
-limitrate = 0
-list_md5 = False
-log_target_prefix =
-long_listing = False
-max_delete = -1
-mime_type =
-multipart_chunk_size_mb = 15
-multipart_max_chunks = 10000
-preserve_attrs = True
-progress_meter = True
-proxy_host =
-proxy_port = 0
-put_continue = False
-recursive = False
-recv_chunk = 65536
-reduced_redundancy = False
-requester_pays = False
-restore_days = 1
-restore_priority = Standard
-secret_key = {{S3_APPLICATION_KEY}}
-send_chunk = 65536
-server_side_encryption = False
-signature_v2 = False
-signurl_use_https = False
-simpledb_host = sdb.amazonaws.com
-skip_existing = False
-socket_timeout = 300
-stats = False
-stop_on_error = False
-storage_class =
-throttle_max = 100
-upload_id =
-urlencoding_mode = normal
-use_http_expect = False
-use_https = True
-use_mime_magic = True
-verbosity = WARNING
-website_endpoint = http://%(bucket)s.s3-website-%(location)s.amazonaws.com/
-website_error =
-website_index = index.html

build-docker.sh

@@ -1,4 +1,10 @@
 #!/bin/bash
 
-docker build -t vasyakrg/backup:v2 .
+DOCKER_BASEIMAGE=ubuntu:22.04
-docker push vasyakrg/backup:v2
+DOCKER_MAINTAINER=vasyakrg@gmail.com
+
+docker buildx create --name cibuilder --driver docker-container --use
+
+docker buildx build --progress=plain --platform linux/amd64,linux/arm64 --push -t hub.realmanual.ru/images/backup-2:$(cat VERSION) \
+	--build-arg DOCKER_BASEIMAGE=${DOCKER_BASEIMAGE} \
+	--build-arg DOCKER_MAINTAINER=${DOCKER_MAINTAINER} .

run-test.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+source .env
+
+docker run --rm --env-file .env --network dalarmbot_net --name test_backup hub.realmanual.ru/images/backup-2:latest
+echo "Backup create and save to ${STORAGE_HOST}/${STORAGE_PATH}"