rsync – 水色（仮）

How to get the script directory, this method is good. update 8 Sep 2018.

use FindBin;
$SDIR = $FindBin::Bin; # script directory

1 2	use FindBin; $SDIR = $FindBin::Bin; # script directory

Get a snapshot by the output of rsync. This way is enough for my practical use.

#
# config.pl
#
# 2016-03-02
#

$TARGET_BASE = '/home/user/Desktop/snapshot';
$LOG_DIR = $TARGET_BASE . '/log';
$RSYNC_LOG = $LOG_DIR . '/rsync.log';
$TARGET_OF_SNAPSHOT = $TARGET_BASE . '/share';
$SNAPSHOT_DIR = $TARGET_BASE . '/snapshot';

# backup from $TARGET_OF_SNAPSHOT to $BACKUP_DIR
$BACKUP_DIR = $TARGET_BASE . '/backup';

#

$SELF_DIR = '/home/user/Desktop/snapshot/bin';
# push(@INC, $SELF_DIR);
$EXCLUDE_FROM = $SELF_DIR . '/exclude-from.txt';

#

$RSYNC = "/usr/bin/rsync";
$FIND = "/usr/bin/find";
$MKDIR = "/bin/mkdir";

#

$GMT = -9; # JST

$MAX_SIZE = '256M';

$KEEP_NUMBER_OF_SNAPSHOT = 30;

$DELETE_IF_DAY_AFTER_N_DAYS = 400;

# end of config.pl

# config.pl

# 2016-03-02

$TARGET_BASE = '/home/user/Desktop/snapshot';

$LOG_DIR = $TARGET_BASE . '/log';

$RSYNC_LOG = $LOG_DIR . '/rsync.log';

$TARGET_OF_SNAPSHOT = $TARGET_BASE . '/share';

$SNAPSHOT_DIR = $TARGET_BASE . '/snapshot';

# backup from $TARGET_OF_SNAPSHOT to $BACKUP_DIR

$BACKUP_DIR = $TARGET_BASE . '/backup';

$SELF_DIR = '/home/user/Desktop/snapshot/bin';

# push(@INC, $SELF_DIR);

$EXCLUDE_FROM = $SELF_DIR . '/exclude-from.txt';

$RSYNC = "/usr/bin/rsync";

$FIND = "/usr/bin/find";

$MKDIR = "/bin/mkdir";

$GMT = -9; # JST

$MAX_SIZE = '256M';

$KEEP_NUMBER_OF_SNAPSHOT = 30;

$DELETE_IF_DAY_AFTER_N_DAYS = 400;

# end of config.pl

#
# exclude-from.txt
#

- .DS_Store
- .git
- .gitignore

# temporary

- tmp/
- *.tmp

# ms office temporary file

- ~$*.*

# + *

# exclude-from.txt

- .DS_Store

- .git

- .gitignore

# temporary

- tmp/

- *.tmp

# ms office temporary file

- ~$*.*

# + *

#!/usr/bin/perl
#
# backup.pl
# 
# test script
#
# 2014-04-14
#

use File::Basename 'basename', 'dirname';
use FindBin;

# get script directory
$SDIR = $FindBin::Bin;
# print $SDIR."\n";

require "$SDIR/config.pl";

$cmd = sprintf("%s -azv '%s' '%s' >>'%s'", $RSYNC, $TARGET_OF_SNAPSHOT, $BACKUP_DIR, $RSYNC_LOG);
print("$cmd\n");
exit(system($cmd));

#!/usr/bin/perl

# backup.pl

# test script

# 2014-04-14

use File::Basename 'basename', 'dirname';

use FindBin;

# get script directory

$SDIR = $FindBin::Bin;

# print $SDIR."\n";

require "$SDIR/config.pl";

$cmd = sprintf("%s -azv '%s' '%s' >>'%s'", $RSYNC, $TARGET_OF_SNAPSHOT, $BACKUP_DIR, $RSYNC_LOG);

print("$cmd\n");

exit(system($cmd));

#!/usr/bin/perl
#
# get_snapshot.pl
#
# 2017-04-14
#

# use File::Copy;
use File::Basename 'basename', 'dirname';
use FindBin;

# get script directory
$SDIR = $FindBin::Bin;
# print $SDIR."\n";

require "$SDIR/config.pl";

$return_code = 0;

if (!open(IN, "< $RSYNC_LOG")) {
    printf(STDERR "$0: Cannot open log file: $RSYNC_LOG.\n");
    exit(1);
}
@logs = <IN>;
close(IN);
# print @logs;

local($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time + $GMT * 60 * 60);
$year += 1900;
$mon  += 1;
# /path/@GMT-2017.04.12-00.53.59
$dst_dir = sprintf("%s/\@GMT-%04d.%02d.%02d-%02d.%02d.%02d", $SNAPSHOT_DIR, $year, $mon, $mday, $hour, $min, $sec);

$now_int = sprintf("%04d%02d%02d", $year, $mon, $mday) + 0;

$copy_count = 0;
%srcs;
$is_list = 0;
$src_dir = dirname($TARGET_OF_SNAPSHOT);
foreach $fpath (@logs) {
    chomp $fpath;
    # printf("fpath: %s\n", $fpath);

    if ($is_list && !$fpath) {
        $is_list = 0;
        next;
    }
    if (!$is_list && $fpath eq 'sending incremental file list') {
        $is_list = 1;
        next;
    }
    if (!$is_list) {
        next;
    }

    if ($fpath =~ m/^deleting /) {
        next;
    }

    $src = sprintf("%s/%s", $src_dir, $fpath);

    if (! -f $src) {
        printf("%s: skip: file not found: '%s'\n", $0, $src);
        next;
    }

    if ($srcs{$src} == 1) {
        printf("%s: skip: already did snapshot: '%s'\n", $0, $src);
        next;
    }

    # do snapshot ------------------------------
    $dst = sprintf("%s/%s", $dst_dir, $fpath);
    # printf("%s: snapshot: %s => %s\n", $0, $src, $dst);
    $dir = dirname $dst;
    # printf("mkdir %s\n", $dir);
    $cmd = sprintf("%s -p '%s'", $MKDIR, $dir);
    if (system($cmd)) {
        $return_code++;
        printf("%s: failed to make directory: '%s'\n", $0, $dir);
        next;
    }
    # copy the one file to snapshot directory.
    $cmd = sprintf("%s -av --max-size=%s --exclude-from='%s' '%s' '%s' >/dev/null", $RSYNC, $MAX_SIZE, $EXCLUDE_FROM, $src, $dst);
    if (system($cmd)) {
        $return_code++;
        printf(STDERR "%s: failed to snapshot: '%s' => '%s'\n", $0, $src, $dst);
    } else {
        $srcs{$src} = 1;
        $copy_count++;
        printf("%s: did snapshot: '%s' => '%s'\n", $0, $src, $dst);
    }
}

if (!unlink($RSYNC_LOG)) {
    printf(STDERR "%s: failed to delete log: '%s'\n", $0, $RSYNC_LOG);
}

exit($return_code);

100

#!/usr/bin/perl

# get_snapshot.pl

# 2017-04-14

# use File::Copy;

use File::Basename 'basename', 'dirname';

use FindBin;

# get script directory

$SDIR = $FindBin::Bin;

# print $SDIR."\n";

require "$SDIR/config.pl";

$return_code = 0;

if (!open(IN, "< $RSYNC_LOG")) {

printf(STDERR "$0: Cannot open log file: $RSYNC_LOG.\n");

exit(1);

}

@logs = <IN>;

close(IN);

# print @logs;

local($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time + $GMT * 60 * 60);

$year += 1900;

$mon += 1;

# /path/@GMT-2017.04.12-00.53.59

$dst_dir = sprintf("%s/\@GMT-%04d.%02d.%02d-%02d.%02d.%02d", $SNAPSHOT_DIR, $year, $mon, $mday, $hour, $min, $sec);

$now_int = sprintf("%04d%02d%02d", $year, $mon, $mday) + 0;

$copy_count = 0;

%srcs;

$is_list = 0;

$src_dir = dirname($TARGET_OF_SNAPSHOT);

foreach $fpath (@logs) {

chomp $fpath;

# printf("fpath: %s\n", $fpath);

if ($is_list && !$fpath) {

$is_list = 0;

next;

}

if (!$is_list && $fpath eq 'sending incremental file list') {

$is_list = 1;

next;

}

if (!$is_list) {

next;

}

if ($fpath =~ m/^deleting /) {

next;

}

$src = sprintf("%s/%s", $src_dir, $fpath);

if (! -f $src) {

printf("%s: skip: file not found: '%s'\n", $0, $src);

next;

}

if ($srcs{$src} == 1) {

printf("%s: skip: already did snapshot: '%s'\n", $0, $src);

next;

}

# do snapshot ------------------------------

$dst = sprintf("%s/%s", $dst_dir, $fpath);

# printf("%s: snapshot: %s => %s\n", $0, $src, $dst);

$dir = dirname $dst;

# printf("mkdir %s\n", $dir);

$cmd = sprintf("%s -p '%s'", $MKDIR, $dir);

if (system($cmd)) {

$return_code++;

printf("%s: failed to make directory: '%s'\n", $0, $dir);

next;

}

# copy the one file to snapshot directory.

$cmd = sprintf("%s -av --max-size=%s --exclude-from='%s' '%s' '%s' >/dev/null", $RSYNC, $MAX_SIZE, $EXCLUDE_FROM, $src, $dst);

if (system($cmd)) {

$return_code++;

printf(STDERR "%s: failed to snapshot: '%s' => '%s'\n", $0, $src, $dst);

} else {

$srcs{$src} = 1;

$copy_count++;

printf("%s: did snapshot: '%s' => '%s'\n", $0, $src, $dst);

}

if (!unlink($RSYNC_LOG)) {

printf(STDERR "%s: failed to delete log: '%s'\n", $0, $RSYNC_LOG);

}

exit($return_code);

#!/usr/bin/perl
#
# purge_snapshot.pl
#
# 2014-04-14
#

use File::Basename 'basename', 'dirname';
use File::Path 'rmtree';
# use File::Copy;
use FindBin;

# get script directory
$SDIR = $FindBin::Bin;
# print $SDIR."\n";

require "$SDIR/config.pl";

$return_code = 0;

#
# Get the snapshot directory list.
#

if (!opendir(DIRHANDLE, $SNAPSHOT_DIR)) {
    printf(STDERR "%s: Cannot open snapshot directory: %s.\n", $0, $SNAPSHOT_DIR);
    exit 1;
}
@dirs;
foreach(readdir(DIRHANDLE)){
    # print "$_\n";
    next if /^\.{1,2}$/;    # skip '.' and '..'
    $dir = sprintf("%s/%s", $SNAPSHOT_DIR, $_);
    if (! $dir =~ /^\@GMT\-/) { next; }   # @GMT-
    if (! -d $dir) { next; }
    push(@dirs, $dir);
    # print "$dir\n";
}
closedir(DIRHANDLE);

@dirs = sort @dirs;
# print join(',',@dirs),"\n";

#
# Keep constant the number of snapshots.
#

$cut = $#dirs - $KEEP_NUMBER_OF_SNAPSHOT + 1;

if ($cut < 1) {
    printf("%s: No need to purge.\n", $0);
    exit($return_code);
}

# Merge from new snapshots to old,
# put snapshots together to $dst.
for ($i = $cut; $i > 0; $i--) {
    $src = sprintf("%s", $dirs[$i]);
    $dst = sprintf("%s", $dirs[$i - 1]);
    printf("%s: %s => %s\n", $0, $src, $dst);
    $cmd = sprintf("%s -av --max-size=%s --exclude-from='%s' '%s/' '%s/'", $RSYNC, $MAX_SIZE, $EXCLUDE_FROM, $src, $dst);
    printf("%s: execute: %s\n", $0, $cmd);
    if (system($cmd) > 0) {
        printf(STDERR "%s: failed to execute: %s\n", $0, $cmd);
        $return_code++;
        next;
    }
    rmtree($src);
}

if ($return_code) {
    exit($return_code);
}

# Move from the merged snapshot to new.
# from $dst to $dirs[$cut]
# don't use rename and move
$save = sprintf("%s", $dirs[$cut]);
printf("%s: %s => %s\n", $0, $dst, $save);
$cmd = sprintf("%s -av --max-size=%s --exclude-from='%s' '%s/' '%s/'", $RSYNC, $MAX_SIZE, $EXCLUDE_FROM, $dst, $save);
printf("%s: execute: %s\n", $0, $cmd);
if (system($cmd) > 0) {
    $return_code++;
    printf("%s: failed to execute: %s\n", $0, $cmd);
} else {
    rmtree($dst);
}

if ($SNAPSHOT_DIR =~ /snapshot$/ && $DELETE_IF_DAY_AFTER_N_DAYS > 14) {
    printf("%s: delete the snapshot files that is %d days old from the changed.\n", $0, $DELETE_IF_DAY_AFTER_N_DAYS);
    $files = `$FIND '$SNAPSHOT_DIR' -mtime +$DELETE_IF_DAY_AFTER_N_DAYS -delete -print`;
    if ($files) {
        printf("%s: deleted old snapshot files: %s\n", $0, $DELETE_IF_DAY_AFTER_N_DAYS, join(", ", $files));
    }

    # delete empty snapshot directories
    $files = `$FIND '$SNAPSHOT_DIR' -type d -empty -delete -print`;
    if ($files) {
        printf("%s: delete empty snapshot directories: %s\n", $0, join(", ", $files));
    }
}

exit($return_code);

100

101

102

103

#!/usr/bin/perl

# purge_snapshot.pl

# 2014-04-14

use File::Basename 'basename', 'dirname';

use File::Path 'rmtree';

# use File::Copy;

use FindBin;

# get script directory

$SDIR = $FindBin::Bin;

# print $SDIR."\n";

require "$SDIR/config.pl";

$return_code = 0;

# Get the snapshot directory list.

if (!opendir(DIRHANDLE, $SNAPSHOT_DIR)) {

printf(STDERR "%s: Cannot open snapshot directory: %s.\n", $0, $SNAPSHOT_DIR);

exit 1;

}

@dirs;

foreach(readdir(DIRHANDLE)){

# print "$_\n";

next if /^\.{1,2}$/; # skip '.' and '..'

$dir = sprintf("%s/%s", $SNAPSHOT_DIR, $_);

if (! $dir =~ /^\@GMT\-/) { next; } # @GMT-

if (! -d $dir) { next; }

push(@dirs, $dir);

# print "$dir\n";

}

closedir(DIRHANDLE);

@dirs = sort @dirs;

# print join(',',@dirs),"\n";

# Keep constant the number of snapshots.

$cut = $#dirs - $KEEP_NUMBER_OF_SNAPSHOT + 1;

if ($cut < 1) {

printf("%s: No need to purge.\n", $0);

exit($return_code);

}

# Merge from new snapshots to old,

# put snapshots together to $dst.

for ($i = $cut; $i > 0; $i--) {

$src = sprintf("%s", $dirs[$i]);

$dst = sprintf("%s", $dirs[$i - 1]);

printf("%s: %s => %s\n", $0, $src, $dst);

$cmd = sprintf("%s -av --max-size=%s --exclude-from='%s' '%s/' '%s/'", $RSYNC, $MAX_SIZE, $EXCLUDE_FROM, $src, $dst);

printf("%s: execute: %s\n", $0, $cmd);

if (system($cmd) > 0) {

printf(STDERR "%s: failed to execute: %s\n", $0, $cmd);

$return_code++;

next;

}

rmtree($src);

}

if ($return_code) {

exit($return_code);

}

# Move from the merged snapshot to new.

# from $dst to $dirs[$cut]

# don't use rename and move

$save = sprintf("%s", $dirs[$cut]);

printf("%s: %s => %s\n", $0, $dst, $save);

$cmd = sprintf("%s -av --max-size=%s --exclude-from='%s' '%s/' '%s/'", $RSYNC, $MAX_SIZE, $EXCLUDE_FROM, $dst, $save);

printf("%s: execute: %s\n", $0, $cmd);

if (system($cmd) > 0) {

$return_code++;

printf("%s: failed to execute: %s\n", $0, $cmd);

} else {

rmtree($dst);

}

if ($SNAPSHOT_DIR =~ /snapshot$/ && $DELETE_IF_DAY_AFTER_N_DAYS > 14) {

printf("%s: delete the snapshot files that is %d days old from the changed.\n", $0, $DELETE_IF_DAY_AFTER_N_DAYS);

$files = `$FIND '$SNAPSHOT_DIR' -mtime +$DELETE_IF_DAY_AFTER_N_DAYS -delete -print`;

if ($files) {

printf("%s: deleted old snapshot files: %s\n", $0, $DELETE_IF_DAY_AFTER_N_DAYS, join(", ", $files));

}

# delete empty snapshot directories

$files = `$FIND '$SNAPSHOT_DIR' -type d -empty -delete -print`;

if ($files) {

printf("%s: delete empty snapshot directories: %s\n", $0, join(", ", $files));

}

exit($return_code);

lsyncd でサーバー間ファイル同期してみるメモ / synchronize files between servers with lsyncd

see: https://memo.silverpigeon.jp/roundcube-webmail-server-install-memo-23-nov-2017/

from server A (rcadmin@192.168.11.9:/var/nas/samba) to server B (rcadmin@192.168.11.10:/var/nas/backup)
username: rcadmin

server A:

# vi /etc/default/rsync

1	# vi /etc/default/rsync

RSYNC_ENABLE=true

1	RSYNC_ENABLE=true

lsyncd が利用している inotify が監視できるファイル数上限の設定
この上限を超えると lsyncd は動作を停止または、起動してくれない

下記の様なエラーが発生する

lsyncd: Error, Terminating since out of inotify watches.#012Consider increasing /proc/sys/fs/inotify/max_user_watches

1	lsyncd: Error, Terminating since out of inotify watches.#012Consider increasing /proc/sys/fs/inotify/max_user_watches

値変更

# vi /etc/sysctl.conf

1	# vi /etc/sysctl.conf

設定の追加／変更

fs.inotify.max_user_watches = 960000

1	fs.inotify.max_user_watches = 960000

値の設定

# /sbin/sysctl -p

1	# /sbin/sysctl -p

値の確認

$ cat /proc/sys/fs/inotify/max_user_watches
960000

1 2	$ cat /proc/sys/fs/inotify/max_user_watches 960000

鍵の生成

$ ssh-keygen
Enter passphrase (empty for no passphrase):

1 2	$ ssh-keygen Enter passphrase (empty for no passphrase):

ssh-keygen を実行すると Enter passphrase (empty for no passphrase): と
パスフレーズを聞かれるので
空のままにしてパスフレーズ無しで生成する

すると、 .ssh/ 以下に2ファイルが生成される

id_rsa 秘密鍵
id_rsa.pub 公開鍵

$ cd .ssh
$ chmod 600 id_rsa

1 2	$ cd .ssh $ chmod 600 id_rsa

server B に id_rsa.pub (公開鍵) をコピーする

server B:

$ mkdir .ssh
$ chmod 700 .ssh
$ cd .ssh
$ cat id_rsa.pub >> authorized_keys
$ chmod 600 authorized_keys

$ mkdir .ssh

$ chmod 700 .ssh

$ cd .ssh

$ cat id_rsa.pub >> authorized_keys

$ chmod 600 authorized_keys

server A:

server A から B (192.168.11.10) へ接続テスト

$ /usr/bin/ssh -i /home/rcadmin/.ssh/id_rsa rcadmin@192.168.11.10

1	$ /usr/bin/ssh -i /home/rcadmin/.ssh/id_rsa rcadmin@192.168.11.10

$ vi /var/nas/bin/lsyncd.conf.lua

1	$ vi /var/nas/bin/lsyncd.conf.lua

--
-- User configuration file for lsyncd.
--
-- Simple example for default rsync.
-- /etc/lsyncd/lsyncd.conf.lua
-- /var/nas/bin/lsyncd.conf.lua
--
-- version 1.0 
--
settings{
  insist = true,
  logfile = "/var/log/lsyncd.log",
  statusFile = "/tmp/lsyncd.stat",
  statusInterval = 1,
  rsyncOpts="-az",
}

-- 同一サーバー内
sync{
  default.rsync,
  source="/var/nas/samba",
  target="/var/nas/backup",
  delete="running",
  rsync = {
    binary = '/var/nas/bin/rsync.pl'
  }
}

sync{
  default.rsync,
  delay=0,
  source="/var/nas/samba",
  target="rcadmin@192.168.11.10:/var/nas/backup",
  delete="running",
  rsync = {
    rsh = "/usr/bin/ssh -i /home/rcadmin/.ssh/id_rsa -o StrictHostKeyChecking=no"
  }
}

-- User configuration file for lsyncd.

-- Simple example for default rsync.

-- /etc/lsyncd/lsyncd.conf.lua

-- /var/nas/bin/lsyncd.conf.lua

-- version 1.0

settings{

insist = true,

logfile = "/var/log/lsyncd.log",

statusFile = "/tmp/lsyncd.stat",

statusInterval = 1,

rsyncOpts="-az",

}

-- 同一サーバー内

sync{

default.rsync,

source="/var/nas/samba",

target="/var/nas/backup",

delete="running",

rsync = {

binary = '/var/nas/bin/rsync.pl'

}

sync{

default.rsync,

delay=0,

source="/var/nas/samba",

target="rcadmin@192.168.11.10:/var/nas/backup",

delete="running",

rsync = {

rsh = "/usr/bin/ssh -i /home/rcadmin/.ssh/id_rsa -o StrictHostKeyChecking=no"

}

server B:

# vi /etc/rsyncd.conf

1	# vi /etc/rsyncd.conf

uid = root
gid = root
read only = no

[nas]
# コピー先対象ディレクトリー
path = /var/nas/backup
# hosts allow = localhost, 192.168.11.9
hosts allow = 192.168.11.9/24
hosts deny = *
read only = false

uid = root

gid = root

read only = no

[nas]

# コピー先対象ディレクトリー

path = /var/nas/backup

# hosts allow = localhost, 192.168.11.9

hosts allow = 192.168.11.9/24

hosts deny = *

read only = false

server A:

# /usr/bin/lsyncd -pidfile /var/run/lsyncd.pid /var/nas/bin/lsyncd.conf.lua

1	# /usr/bin/lsyncd -pidfile /var/run/lsyncd.pid /var/nas/bin/lsyncd.conf.lua

タグ: rsync

Get and manage snapshot for samba file sharing

lsyncd でサーバー間ファイル同期してみるメモ / synchronize files between servers with lsyncd