Minio 数据迁移

minio由3节点迁移到1节点，rclone是minio的迁移工具

服务器列表如下:

主机	IP	下线/保留
5gxx-2-32	192.168.64.32	保留，部署单机minio
5gxx-2-33	192.168.64.33	下线
5gxx-2-34	192.168.64.34	下线

部署单节点 minio

##安装 supervisor 管理minio启动
[root@5gxx-2-32 ~]# yum -y install epel-release
[root@5gxx-2-32 ~]# yum -y install python-pip
[root@5gxx-2-32 ~]# pip install supervisor

##创建supervisor配置文件
[root@5gxx-2-32 ~]# vim /etc/supervisord.conf
[unix_http_server]
file=/var/run/supervisor/supervisor.sock   ; (the path to the socket file)
[supervisord]
logfile=/var/log/supervisor/supervisord.log  ; (main log file;default $CWD/supervisord.log)
logfile_maxbytes=50MB       ; (max main logfile bytes b4 rotation;default 50MB)
logfile_backups=10          ; (num of main logfile rotation backups;default 10)
loglevel=info               ; (log level;default info; others: debug,warn,trace)
pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
nodaemon=false              ; (start in foreground if true;default false)
minfds=1024                 ; (min. avail startup file descriptors;default 1024)
minprocs=200                ; (min. avail process descriptors;default 200)
[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
[supervisorctl]
serverurl=unix:///var/run/supervisor/supervisor.sock ; use a unix:// URL  for a unix socket
[include]
files = supervisord.d/*.ini

##创建supervisor使用的目录
[root@5gxx-2-32 ~]# mkdir /var/run/supervisor /var/log/supervisor /etc/supervisord.d
##配置开启自动创建目录/var/run/supervisor
[root@5gxx-2-32 ~]# vim /etc/tmpfiles.d/supervisor.conf
D /var/run/supervisor 0775 root root -

##配置supervisor服务开机启动
[root@5gxx-2-32 ~]# vim /usr/lib/systemd/system/supervisord.service
[Unit]
Description=Process Monitoring and Control Daemon
After=rc-local.service nss-user-lookup.target

[Service]
Type=forking
ExecStart=/usr/bin/supervisord -c /etc/supervisord.conf

[Install]
WantedBy=multi-user.target

[root@5gxx-2-32 ~]# systemctl enable supervisord
[root@5gxx-2-32 ~]# systemctl start supervisord
##部署单节点minio，使用原版本的minio程序，不用安装这里就直接启动
##配置superviosr启动文件
[root@5gxx-2-32 ~]# vim /etc/supervisord.d/minio.ini 
[program:minio]
command = /home/czz/minio/minio server --console-address ":9001" /home/czz/minio/data
user = czz
;supervisor启动的时候是否随着同时启动，默认True
autostart = true
;程序退出后自动重启,可选值：[unexpected,true,false]，默认为unexpected，表示进程意外杀死后才重启
autorestart = true
;这个选项是子进程启动多少秒之后，此时状态如果是running，则我们认为启动成功了。默认值为1
startsecs = 5
;;把stderr重定向到stdout，默认 false
redirect_stderr = true
;;默认为false,进程被杀死时，是否向这个进程组发送stop信号，包括子进程
stopasgroup = true
;默认为false，向进程组发送kill信号，包括子进程
killasgroup = true
;stdout日志文件大小，默认 50MB
stdout_logfile_maxbytes = 50MB
;;stdout日志文件备份数
stdout_logfile_backups = 10
;日志输出
stdout_logfile = /home/czz/minio/logs/minio.log
environment=MINIO_ROOT_USER=minio,MINIO_ROOT_PASSWORD=FuvI5O4X6W2i3hxc,MINIO_PROMETHEUS_AUTH_TYPE="public",MINIO_PROMETHEUS_URL="http://192.168.64.58:9090"

##启动minio
[root@5gxx-2-32 ~]# supervisorctl 
supervisor> update
minio: stopped
minio: updated process group

使用 rclone 迁移minio

安装rclone

[root@5gxx-2-32 ~]# yum install unzip
[root@5gxx-2-32 ~]# curl https://rclone.org/install.sh | sudo bash

install.sh

#!/usr/bin/env bash

# error codes
# 0 - exited without problems
# 1 - parameters not supported were used or some unexpected error occurred
# 2 - OS not supported by this script
# 3 - installed version of rclone is up to date
# 4 - supported unzip tools are not available

set -e

#when adding a tool to the list make sure to also add its corresponding command further in the script
unzip_tools_list=('unzip' '7z' 'busybox')

usage() { echo "Usage: sudo -v ; curl https://rclone.org/install.sh | sudo bash [-s beta]" 1>&2; exit 1; }

#check for beta flag
if [ -n "$1" ] && [ "$1" != "beta" ]; then
    usage
fi

if [ -n "$1" ]; then
    install_beta="beta "
fi


#create tmp directory and move to it with macOS compatibility fallback
tmp_dir=$(mktemp -d 2>/dev/null || mktemp -d -t 'rclone-install.XXXXXXXXXX')
cd "$tmp_dir"


#make sure unzip tool is available and choose one to work with
set +e
for tool in ${unzip_tools_list[*]}; do
    trash=$(hash "$tool" 2>>errors)
    if [ "$?" -eq 0 ]; then
        unzip_tool="$tool"
        break
    fi
done  
set -e

# exit if no unzip tools available
if [ -z "$unzip_tool" ]; then
    printf "\nNone of the supported tools for extracting zip archives (${unzip_tools_list[*]}) were found. "
    printf "Please install one of them and try again.\n\n"
    exit 4
fi

# Make sure we don't create a root owned .config/rclone directory #2127
export XDG_CONFIG_HOME=config

#check installed version of rclone to determine if update is necessary
version=$(rclone --version 2>>errors | head -n 1)
if [ -z "$install_beta" ]; then
    current_version=$(curl -fsS https://downloads.rclone.org/version.txt)
else
    current_version=$(curl -fsS https://beta.rclone.org/version.txt)
fi

if [ "$version" = "$current_version" ]; then
    printf "\nThe latest ${install_beta}version of rclone ${version} is already installed.\n\n"
    exit 3
fi


#detect the platform
OS="$(uname)"
case $OS in
  Linux)
    OS='linux'
    ;;
  FreeBSD)
    OS='freebsd'
    ;;
  NetBSD)
    OS='netbsd'
    ;;
  OpenBSD)
    OS='openbsd'
    ;;  
  Darwin)
    OS='osx'
    binTgtDir=/usr/local/bin
    man1TgtDir=/usr/local/share/man/man1
    ;;
  SunOS)
    OS='solaris'
    echo 'OS not supported'
    exit 2
    ;;
  *)
    echo 'OS not supported'
    exit 2
    ;;
esac

OS_type="$(uname -m)"
case "$OS_type" in
  x86_64|amd64)
    OS_type='amd64'
    ;;
  i?86|x86)
    OS_type='386'
    ;;
  aarch64|arm64)
    OS_type='arm64'
    ;;
  armv7*)
    OS_type='arm-v7'
    ;;
  armv6*)
    OS_type='arm-v6'
    ;;
  arm*)
    OS_type='arm'
    ;;
  *)
    echo 'OS type not supported'
    exit 2
    ;;
esac


#download and unzip
if [ -z "$install_beta" ]; then
    download_link="https://downloads.rclone.org/rclone-current-${OS}-${OS_type}.zip"
    rclone_zip="rclone-current-${OS}-${OS_type}.zip"
else
    download_link="https://beta.rclone.org/rclone-beta-latest-${OS}-${OS_type}.zip"
    rclone_zip="rclone-beta-latest-${OS}-${OS_type}.zip"
fi

curl -OfsS "$download_link"
unzip_dir="tmp_unzip_dir_for_rclone"
# there should be an entry in this switch for each element of unzip_tools_list
case "$unzip_tool" in
  'unzip')
    unzip -a "$rclone_zip" -d "$unzip_dir"
    ;;
  '7z')
    7z x "$rclone_zip" "-o$unzip_dir"
    ;;
  'busybox')
    mkdir -p "$unzip_dir"
    busybox unzip "$rclone_zip" -d "$unzip_dir"
    ;;
esac

cd $unzip_dir/*

#mounting rclone to environment

case "$OS" in
  'linux')
    #binary
    cp rclone /usr/bin/rclone.new
    chmod 755 /usr/bin/rclone.new
    chown root:root /usr/bin/rclone.new
    mv /usr/bin/rclone.new /usr/bin/rclone
    #manual
    if ! [ -x "$(command -v mandb)" ]; then
        echo 'mandb not found. The rclone man docs will not be installed.'
    else 
        mkdir -p /usr/local/share/man/man1
        cp rclone.1 /usr/local/share/man/man1/
        mandb
    fi
    ;;
  'freebsd'|'openbsd'|'netbsd')
    #binary
    cp rclone /usr/bin/rclone.new
    chown root:wheel /usr/bin/rclone.new
    mv /usr/bin/rclone.new /usr/bin/rclone
    #manual
    mkdir -p /usr/local/man/man1
    cp rclone.1 /usr/local/man/man1/
    makewhatis
    ;;
  'osx')
    #binary
    mkdir -m 0555 -p ${binTgtDir}
    cp rclone ${binTgtDir}/rclone.new
    mv ${binTgtDir}/rclone.new ${binTgtDir}/rclone
    chmod a=x ${binTgtDir}/rclone
    #manual
    mkdir -m 0555 -p ${man1TgtDir}
    cp rclone.1 ${man1TgtDir}    
    chmod a=r ${man1TgtDir}/rclone.1
    ;;
  *)
    echo 'OS not supported'
    exit 2
esac


#update version variable post install
version=$(rclone --version 2>>errors | head -n 1)

printf "\n${version} has successfully installed."
printf '\nNow run "rclone config" for setup. Check https://rclone.org/docs/ for more details.\n\n'
exit 0

配置rclone

[root@5gxx-2-32 ~]# vim /root/.config/rclone/rclone.conf
/root/.config/rclone/rclone.conf
[minio-old]
type = s3
provider = Minio
env_auth = false
access_key_id = minio
secret_access_key = FuvI5O4X6W2i3hxc
endpoint = http://192.168.64.32:9000
[minio-new]
type = s3
provider = Minio
env_auth = false
access_key_id = minio
secret_access_key = FuvI5O4X6W2i3hxc11
endpoint = http://192.168.223.128:9000

使用rclone迁移数据到新的minio上

##迁移所有数据
[root@5gxx-2-32 ~]# rclone sync minio-old:/ minio-new:/

##如果想迁移单个Bucket，指定源和目标Bucket名称即可
[root@5gxx-2-32 ~]# rclone sync minio-old:/bj-manager-sjb minio-new:/bj-manager-sjb